clang 24.0.0git
CIRGenBuiltinNVPTX.cpp
Go to the documentation of this file.
1//===---- CIRGenBuiltinNVPTX.cpp - Emit CIR for NVPTX builtins ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit NVPTX Builtin calls.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CIRGenFunction.h"
14
15#include "mlir/IR/Value.h"
16#include "clang/Basic/TargetBuiltins.h"
17#include "clang/CIR/Dialect/IR/CIRDialect.h"
18#include "llvm/Support/NVPTXAddrSpace.h"
19
20using namespace clang;
21using namespace clang::CIRGen;
22
23static mlir::Value makeLdu(CIRGenFunction &cgf, const CallExpr *expr,
24 llvm::StringRef intrinsicName) {
25 auto &builder = cgf.getBuilder();
26 Address ptr = cgf.emitPointerWithAlignment(expr->getArg(0));
27 QualType argType = expr->getArg(0)->getType();
28 mlir::Type elemTy = cgf.convertTypeForMem(argType->getPointeeType());
29 clang::CharUnits align = ptr.getAlignment();
30 mlir::Location loc = cgf.getLoc(expr->getExprLoc());
31 mlir::Value alignVal =
32 builder.getConstantInt(loc, builder.getSInt32Ty(), align.getQuantity());
33 return cir::LLVMIntrinsicCallOp::create(
34 builder, loc, builder.getStringAttr(intrinsicName), elemTy,
35 {ptr.emitRawPointer(), alignVal})
36 .getResult();
37}
38
39static mlir::Value makeLdg(CIRGenFunction &cgf, const CallExpr *expr) {
40 auto &builder = cgf.getBuilder();
41 Address ptr = cgf.emitPointerWithAlignment(expr->getArg(0));
42 QualType argType = expr->getArg(0)->getType();
43 mlir::Type elemTy = cgf.convertTypeForMem(argType->getPointeeType());
44 mlir::Location loc = cgf.getLoc(expr->getExprLoc());
45
46 // Use addrspace(1) for NVPTX ADDRESS_SPACE_GLOBAL.
47 mlir::Type globalPtrTy = cir::PointerType::get(
48 elemTy, cir::TargetAddressSpaceAttr::get(
49 builder.getContext(), llvm::NVPTXAS::ADDRESS_SPACE_GLOBAL));
50 mlir::Value asc =
51 builder.createAddrSpaceCast(loc, ptr.getPointer(), globalPtrTy);
52 cir::LoadOp load =
53 builder.createAlignedLoad(loc, elemTy, asc, ptr.getAlignment());
54 load.setInvariant(true);
55 return load.getResult();
56}
57
58/// Emit a CIR LLVMIntrinsicCallOp for a unary NVVM intrinsic.
59/// The result type is inferred from the single argument.
60static mlir::Value emitUnaryNVVMIntrinsic(CIRGenFunction &cgf,
61 const CallExpr *expr,
62 llvm::StringRef intrinsicName) {
63 auto &builder = cgf.getBuilder();
64 mlir::Value arg = cgf.emitScalarExpr(expr->getArg(0));
65 return cir::LLVMIntrinsicCallOp::create(
66 builder, cgf.getLoc(expr->getExprLoc()),
67 builder.getStringAttr(intrinsicName), arg.getType(), {arg})
68 .getResult();
69}
70
71/// Emit a CIR LLVMIntrinsicCallOp for an NVVM fadd/fmul intrinsic, which takes
72/// the rounding mode as a trailing operand.
73static mlir::Value emitNVVMFPArith(CIRGenFunction &cgf, const CallExpr *expr,
74 llvm::StringRef intrinsicName,
75 llvm::APFloat::roundingMode rm) {
76 auto &builder = cgf.getBuilder();
77 mlir::Location loc = cgf.getLoc(expr->getExprLoc());
78 mlir::Value lhs = cgf.emitScalarExpr(expr->getArg(0));
79 mlir::Value rhs = cgf.emitScalarExpr(expr->getArg(1));
80 mlir::Value rnd =
81 builder.getConstInt(loc, builder.getSInt32Ty(), static_cast<int>(rm));
82 return cir::LLVMIntrinsicCallOp::create(builder, loc,
83 builder.getStringAttr(intrinsicName),
84 lhs.getType(), {lhs, rhs, rnd})
85 .getResult();
86}
87
88/// Emit a CIR LLVMIntrinsicCallOp for an NVVM fma intrinsic, which takes the
89/// rounding mode as a trailing operand.
90static mlir::Value emitNVVMFMA(CIRGenFunction &cgf, const CallExpr *expr,
91 llvm::StringRef intrinsicName,
92 llvm::APFloat::roundingMode rm) {
93 auto &builder = cgf.getBuilder();
94 mlir::Location loc = cgf.getLoc(expr->getExprLoc());
95 mlir::Value a = cgf.emitScalarExpr(expr->getArg(0));
96 mlir::Value b = cgf.emitScalarExpr(expr->getArg(1));
97 mlir::Value c = cgf.emitScalarExpr(expr->getArg(2));
98 mlir::Value rnd =
99 builder.getConstInt(loc, builder.getSInt32Ty(), static_cast<int>(rm));
100 return cir::LLVMIntrinsicCallOp::create(builder, loc,
101 builder.getStringAttr(intrinsicName),
102 a.getType(), {a, b, c, rnd})
103 .getResult();
104}
105
106static mlir::Value emitBar0Reduction(CIRGenFunction &cgf, const CallExpr *expr,
107 llvm::StringRef intrinsicName,
108 bool returnsPred) {
109 CIRGenBuilderTy &builder = cgf.getBuilder();
110 mlir::Location loc = cgf.getLoc(expr->getExprLoc());
111 mlir::Type si32Ty = builder.getSInt32Ty();
112 mlir::Value zero = builder.getNullValue(si32Ty, loc);
113 mlir::Value pred = builder.createCompare(
114 loc, cir::CmpOpKind::ne, cgf.emitScalarExpr(expr->getArg(0)), zero);
115 mlir::Type resultTy = returnsPred ? mlir::Type(builder.getBoolTy()) : si32Ty;
116 mlir::Value result = builder.emitIntrinsicCallOp(
117 loc, intrinsicName, resultTy, mlir::ValueRange{zero, pred});
118 if (returnsPred)
119 result = builder.createBoolToInt(result, si32Ty);
120 return result;
121}
122
123static mlir::Value makeScopedAtomicRMW(CIRGenFunction &cgf,
124 const CallExpr *expr,
125 cir::AtomicFetchKind kind,
126 cir::SyncScopeKind scope) {
127 auto &builder = cgf.getBuilder();
128 Address destAddr = cgf.emitPointerWithAlignment(expr->getArg(0));
129 mlir::Value destValue = destAddr.emitRawPointer();
130 mlir::Value val = cgf.emitScalarExpr(expr->getArg(1));
131 auto rmwi = cir::AtomicFetchOp::create(
132 builder, cgf.getLoc(expr->getSourceRange()), destValue, val, kind,
133 cir::MemOrder::Relaxed, scope, /*is_volatile=*/false,
134 /*fetch_first=*/true);
135 return rmwi->getResult(0);
136}
137
138static mlir::Value makeScopedAtomicXchg(CIRGenFunction &cgf,
139 const CallExpr *expr,
140 cir::SyncScopeKind scope) {
141 auto &builder = cgf.getBuilder();
142 Address destAddr = cgf.emitPointerWithAlignment(expr->getArg(0));
143 mlir::Value destValue = destAddr.emitRawPointer();
144 mlir::Value val = cgf.emitScalarExpr(expr->getArg(1));
145 auto xchg = cir::AtomicXchgOp::create(
146 builder, cgf.getLoc(expr->getSourceRange()), destValue, val,
147 cir::MemOrder::Relaxed, scope, /*is_volatile=*/false);
148 return xchg.getResult();
149}
150
151std::optional<mlir::Value>
152CIRGenFunction::emitNVPTXBuiltinExpr(unsigned builtinId, const CallExpr *expr) {
153 switch (builtinId) {
154 case NVPTX::BI__nvvm_atom_add_gen_i:
155 case NVPTX::BI__nvvm_atom_add_gen_l:
156 case NVPTX::BI__nvvm_atom_add_gen_ll:
157 return makeBinaryAtomicValue(cir::AtomicFetchKind::Add, expr,
158 /*originalArgType=*/nullptr,
159 /*emittedArgValue=*/nullptr,
160 cir::MemOrder::Relaxed);
161 case NVPTX::BI__nvvm_atom_sub_gen_i:
162 case NVPTX::BI__nvvm_atom_sub_gen_l:
163 case NVPTX::BI__nvvm_atom_sub_gen_ll:
164 return makeBinaryAtomicValue(cir::AtomicFetchKind::Sub, expr,
165 /*originalArgType=*/nullptr,
166 /*emittedArgValue=*/nullptr,
167 cir::MemOrder::Relaxed);
168 case NVPTX::BI__nvvm_atom_and_gen_i:
169 case NVPTX::BI__nvvm_atom_and_gen_l:
170 case NVPTX::BI__nvvm_atom_and_gen_ll:
171 return makeBinaryAtomicValue(cir::AtomicFetchKind::And, expr,
172 /*originalArgType=*/nullptr,
173 /*emittedArgValue=*/nullptr,
174 cir::MemOrder::Relaxed);
175 case NVPTX::BI__nvvm_atom_or_gen_i:
176 case NVPTX::BI__nvvm_atom_or_gen_l:
177 case NVPTX::BI__nvvm_atom_or_gen_ll:
178 return makeBinaryAtomicValue(cir::AtomicFetchKind::Or, expr,
179 /*originalArgType=*/nullptr,
180 /*emittedArgValue=*/nullptr,
181 cir::MemOrder::Relaxed);
182 case NVPTX::BI__nvvm_atom_xor_gen_i:
183 case NVPTX::BI__nvvm_atom_xor_gen_l:
184 case NVPTX::BI__nvvm_atom_xor_gen_ll:
185 return makeBinaryAtomicValue(cir::AtomicFetchKind::Xor, expr,
186 /*originalArgType=*/nullptr,
187 /*emittedArgValue=*/nullptr,
188 cir::MemOrder::Relaxed);
189 case NVPTX::BI__nvvm_atom_xchg_gen_i:
190 case NVPTX::BI__nvvm_atom_xchg_gen_l:
191 case NVPTX::BI__nvvm_atom_xchg_gen_ll:
192 return makeScopedAtomicXchg(*this, expr, cir::SyncScopeKind::System);
193 case NVPTX::BI__nvvm_atom_max_gen_i:
194 case NVPTX::BI__nvvm_atom_max_gen_l:
195 case NVPTX::BI__nvvm_atom_max_gen_ll:
196 return makeBinaryAtomicValue(cir::AtomicFetchKind::Max, expr,
197 /*originalArgType=*/nullptr,
198 /*emittedArgValue=*/nullptr,
199 cir::MemOrder::Relaxed);
200 case NVPTX::BI__nvvm_atom_max_gen_ui:
201 case NVPTX::BI__nvvm_atom_max_gen_ul:
202 case NVPTX::BI__nvvm_atom_max_gen_ull:
203 return makeBinaryAtomicValue(cir::AtomicFetchKind::Max, expr,
204 /*originalArgType=*/nullptr,
205 /*emittedArgValue=*/nullptr,
206 cir::MemOrder::Relaxed);
207 case NVPTX::BI__nvvm_atom_min_gen_i:
208 case NVPTX::BI__nvvm_atom_min_gen_l:
209 case NVPTX::BI__nvvm_atom_min_gen_ll:
210 return makeBinaryAtomicValue(cir::AtomicFetchKind::Min, expr,
211 /*originalArgType=*/nullptr,
212 /*emittedArgValue=*/nullptr,
213 cir::MemOrder::Relaxed);
214 case NVPTX::BI__nvvm_atom_min_gen_ui:
215 case NVPTX::BI__nvvm_atom_min_gen_ul:
216 case NVPTX::BI__nvvm_atom_min_gen_ull:
217 return makeBinaryAtomicValue(cir::AtomicFetchKind::Min, expr,
218 /*originalArgType=*/nullptr,
219 /*emittedArgValue=*/nullptr,
220 cir::MemOrder::Relaxed);
221 case NVPTX::BI__nvvm_atom_cas_gen_us:
222 case NVPTX::BI__nvvm_atom_cas_gen_i:
223 case NVPTX::BI__nvvm_atom_cas_gen_l:
224 case NVPTX::BI__nvvm_atom_cas_gen_ll:
225 return emitAtomicCmpXchg(expr, /*returnBool=*/false, cir::MemOrder::Relaxed,
226 cir::MemOrder::Relaxed,
227 cir::SyncScopeKind::System);
228 case NVPTX::BI__nvvm_atom_add_gen_f:
229 case NVPTX::BI__nvvm_atom_add_gen_d:
230 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Add,
231 cir::SyncScopeKind::System);
232 case NVPTX::BI__nvvm_atom_inc_gen_ui:
233 return makeBinaryAtomicValue(cir::AtomicFetchKind::UIncWrap, expr,
234 /*originalArgType=*/nullptr,
235 /*emittedArgValue=*/nullptr,
236 cir::MemOrder::Relaxed);
237 case NVPTX::BI__nvvm_atom_dec_gen_ui:
238 return makeBinaryAtomicValue(cir::AtomicFetchKind::UDecWrap, expr,
239 /*originalArgType=*/nullptr,
240 /*emittedArgValue=*/nullptr,
241 cir::MemOrder::Relaxed);
242 case NVPTX::BI__nvvm_ldg_c:
243 case NVPTX::BI__nvvm_ldg_sc:
244 case NVPTX::BI__nvvm_ldg_c2:
245 case NVPTX::BI__nvvm_ldg_sc2:
246 case NVPTX::BI__nvvm_ldg_c4:
247 case NVPTX::BI__nvvm_ldg_sc4:
248 case NVPTX::BI__nvvm_ldg_s:
249 case NVPTX::BI__nvvm_ldg_s2:
250 case NVPTX::BI__nvvm_ldg_s4:
251 case NVPTX::BI__nvvm_ldg_i:
252 case NVPTX::BI__nvvm_ldg_i2:
253 case NVPTX::BI__nvvm_ldg_i4:
254 case NVPTX::BI__nvvm_ldg_l:
255 case NVPTX::BI__nvvm_ldg_l2:
256 case NVPTX::BI__nvvm_ldg_ll:
257 case NVPTX::BI__nvvm_ldg_ll2:
258 case NVPTX::BI__nvvm_ldg_uc:
259 case NVPTX::BI__nvvm_ldg_uc2:
260 case NVPTX::BI__nvvm_ldg_uc4:
261 case NVPTX::BI__nvvm_ldg_us:
262 case NVPTX::BI__nvvm_ldg_us2:
263 case NVPTX::BI__nvvm_ldg_us4:
264 case NVPTX::BI__nvvm_ldg_ui:
265 case NVPTX::BI__nvvm_ldg_ui2:
266 case NVPTX::BI__nvvm_ldg_ui4:
267 case NVPTX::BI__nvvm_ldg_ul:
268 case NVPTX::BI__nvvm_ldg_ul2:
269 case NVPTX::BI__nvvm_ldg_ull:
270 case NVPTX::BI__nvvm_ldg_ull2:
271 case NVPTX::BI__nvvm_ldg_f:
272 case NVPTX::BI__nvvm_ldg_f2:
273 case NVPTX::BI__nvvm_ldg_f4:
274 case NVPTX::BI__nvvm_ldg_d:
275 case NVPTX::BI__nvvm_ldg_d2:
276 return makeLdg(*this, expr);
277 case NVPTX::BI__nvvm_ldu_c:
278 case NVPTX::BI__nvvm_ldu_sc:
279 case NVPTX::BI__nvvm_ldu_c2:
280 case NVPTX::BI__nvvm_ldu_sc2:
281 case NVPTX::BI__nvvm_ldu_c4:
282 case NVPTX::BI__nvvm_ldu_sc4:
283 case NVPTX::BI__nvvm_ldu_s:
284 case NVPTX::BI__nvvm_ldu_s2:
285 case NVPTX::BI__nvvm_ldu_s4:
286 case NVPTX::BI__nvvm_ldu_i:
287 case NVPTX::BI__nvvm_ldu_i2:
288 case NVPTX::BI__nvvm_ldu_i4:
289 case NVPTX::BI__nvvm_ldu_l:
290 case NVPTX::BI__nvvm_ldu_l2:
291 case NVPTX::BI__nvvm_ldu_ll:
292 case NVPTX::BI__nvvm_ldu_ll2:
293 case NVPTX::BI__nvvm_ldu_uc:
294 case NVPTX::BI__nvvm_ldu_uc2:
295 case NVPTX::BI__nvvm_ldu_uc4:
296 case NVPTX::BI__nvvm_ldu_us:
297 case NVPTX::BI__nvvm_ldu_us2:
298 case NVPTX::BI__nvvm_ldu_us4:
299 case NVPTX::BI__nvvm_ldu_ui:
300 case NVPTX::BI__nvvm_ldu_ui2:
301 case NVPTX::BI__nvvm_ldu_ui4:
302 case NVPTX::BI__nvvm_ldu_ul:
303 case NVPTX::BI__nvvm_ldu_ul2:
304 case NVPTX::BI__nvvm_ldu_ull:
305 case NVPTX::BI__nvvm_ldu_ull2:
306 return makeLdu(*this, expr, "nvvm.ldu.global.i");
307 case NVPTX::BI__nvvm_ldu_f:
308 case NVPTX::BI__nvvm_ldu_f2:
309 case NVPTX::BI__nvvm_ldu_f4:
310 case NVPTX::BI__nvvm_ldu_d:
311 case NVPTX::BI__nvvm_ldu_d2:
312 return makeLdu(*this, expr, "nvvm.ldu.global.f");
313 case NVPTX::BI__nvvm_atom_cta_add_gen_i:
314 case NVPTX::BI__nvvm_atom_cta_add_gen_l:
315 case NVPTX::BI__nvvm_atom_cta_add_gen_ll:
316 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Add,
317 cir::SyncScopeKind::Workgroup);
318 case NVPTX::BI__nvvm_atom_sys_add_gen_i:
319 case NVPTX::BI__nvvm_atom_sys_add_gen_l:
320 case NVPTX::BI__nvvm_atom_sys_add_gen_ll:
321 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Add,
322 cir::SyncScopeKind::System);
323 case NVPTX::BI__nvvm_atom_cta_add_gen_f:
324 case NVPTX::BI__nvvm_atom_cta_add_gen_d:
325 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Add,
326 cir::SyncScopeKind::Workgroup);
327 case NVPTX::BI__nvvm_atom_sys_add_gen_f:
328 case NVPTX::BI__nvvm_atom_sys_add_gen_d:
329 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Add,
330 cir::SyncScopeKind::System);
331 case NVPTX::BI__nvvm_atom_cta_xchg_gen_i:
332 case NVPTX::BI__nvvm_atom_cta_xchg_gen_l:
333 case NVPTX::BI__nvvm_atom_cta_xchg_gen_ll:
334 return makeScopedAtomicXchg(*this, expr, cir::SyncScopeKind::Workgroup);
335 case NVPTX::BI__nvvm_atom_sys_xchg_gen_i:
336 case NVPTX::BI__nvvm_atom_sys_xchg_gen_l:
337 case NVPTX::BI__nvvm_atom_sys_xchg_gen_ll:
338 return makeScopedAtomicXchg(*this, expr, cir::SyncScopeKind::System);
339 case NVPTX::BI__nvvm_atom_cta_max_gen_i:
340 case NVPTX::BI__nvvm_atom_cta_max_gen_ui:
341 case NVPTX::BI__nvvm_atom_cta_max_gen_l:
342 case NVPTX::BI__nvvm_atom_cta_max_gen_ul:
343 case NVPTX::BI__nvvm_atom_cta_max_gen_ll:
344 case NVPTX::BI__nvvm_atom_cta_max_gen_ull:
345 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Max,
346 cir::SyncScopeKind::Workgroup);
347 case NVPTX::BI__nvvm_atom_sys_max_gen_i:
348 case NVPTX::BI__nvvm_atom_sys_max_gen_ui:
349 case NVPTX::BI__nvvm_atom_sys_max_gen_l:
350 case NVPTX::BI__nvvm_atom_sys_max_gen_ul:
351 case NVPTX::BI__nvvm_atom_sys_max_gen_ll:
352 case NVPTX::BI__nvvm_atom_sys_max_gen_ull:
353 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Max,
354 cir::SyncScopeKind::System);
355 case NVPTX::BI__nvvm_atom_cta_min_gen_i:
356 case NVPTX::BI__nvvm_atom_cta_min_gen_ui:
357 case NVPTX::BI__nvvm_atom_cta_min_gen_l:
358 case NVPTX::BI__nvvm_atom_cta_min_gen_ul:
359 case NVPTX::BI__nvvm_atom_cta_min_gen_ll:
360 case NVPTX::BI__nvvm_atom_cta_min_gen_ull:
361 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Min,
362 cir::SyncScopeKind::Workgroup);
363 case NVPTX::BI__nvvm_atom_sys_min_gen_i:
364 case NVPTX::BI__nvvm_atom_sys_min_gen_ui:
365 case NVPTX::BI__nvvm_atom_sys_min_gen_l:
366 case NVPTX::BI__nvvm_atom_sys_min_gen_ul:
367 case NVPTX::BI__nvvm_atom_sys_min_gen_ll:
368 case NVPTX::BI__nvvm_atom_sys_min_gen_ull:
369 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Min,
370 cir::SyncScopeKind::System);
371 case NVPTX::BI__nvvm_atom_cta_inc_gen_ui:
372 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::UIncWrap,
373 cir::SyncScopeKind::Workgroup);
374 case NVPTX::BI__nvvm_atom_cta_dec_gen_ui:
375 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::UDecWrap,
376 cir::SyncScopeKind::Workgroup);
377 case NVPTX::BI__nvvm_atom_sys_inc_gen_ui:
378 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::UIncWrap,
379 cir::SyncScopeKind::System);
380 case NVPTX::BI__nvvm_atom_sys_dec_gen_ui:
381 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::UDecWrap,
382 cir::SyncScopeKind::System);
383 case NVPTX::BI__nvvm_atom_cta_and_gen_i:
384 case NVPTX::BI__nvvm_atom_cta_and_gen_l:
385 case NVPTX::BI__nvvm_atom_cta_and_gen_ll:
386 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::And,
387 cir::SyncScopeKind::Workgroup);
388 case NVPTX::BI__nvvm_atom_sys_and_gen_i:
389 case NVPTX::BI__nvvm_atom_sys_and_gen_l:
390 case NVPTX::BI__nvvm_atom_sys_and_gen_ll:
391 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::And,
392 cir::SyncScopeKind::System);
393 case NVPTX::BI__nvvm_atom_cta_or_gen_i:
394 case NVPTX::BI__nvvm_atom_cta_or_gen_l:
395 case NVPTX::BI__nvvm_atom_cta_or_gen_ll:
396 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Or,
397 cir::SyncScopeKind::Workgroup);
398 case NVPTX::BI__nvvm_atom_sys_or_gen_i:
399 case NVPTX::BI__nvvm_atom_sys_or_gen_l:
400 case NVPTX::BI__nvvm_atom_sys_or_gen_ll:
401 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Or,
402 cir::SyncScopeKind::System);
403 case NVPTX::BI__nvvm_atom_cta_xor_gen_i:
404 case NVPTX::BI__nvvm_atom_cta_xor_gen_l:
405 case NVPTX::BI__nvvm_atom_cta_xor_gen_ll:
406 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Xor,
407 cir::SyncScopeKind::Workgroup);
408 case NVPTX::BI__nvvm_atom_sys_xor_gen_i:
409 case NVPTX::BI__nvvm_atom_sys_xor_gen_l:
410 case NVPTX::BI__nvvm_atom_sys_xor_gen_ll:
411 return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Xor,
412 cir::SyncScopeKind::System);
413 case NVPTX::BI__nvvm_atom_cta_cas_gen_us:
414 case NVPTX::BI__nvvm_atom_cta_cas_gen_i:
415 case NVPTX::BI__nvvm_atom_cta_cas_gen_l:
416 case NVPTX::BI__nvvm_atom_cta_cas_gen_ll:
417 return emitAtomicCmpXchg(expr, /*returnBool=*/false, cir::MemOrder::Relaxed,
418 cir::MemOrder::Relaxed,
419 cir::SyncScopeKind::Workgroup);
420 case NVPTX::BI__nvvm_atom_sys_cas_gen_us:
421 case NVPTX::BI__nvvm_atom_sys_cas_gen_i:
422 case NVPTX::BI__nvvm_atom_sys_cas_gen_l:
423 case NVPTX::BI__nvvm_atom_sys_cas_gen_ll:
424 return emitAtomicCmpXchg(expr, /*returnBool=*/false, cir::MemOrder::Relaxed,
425 cir::MemOrder::Relaxed,
426 cir::SyncScopeKind::System);
427 case NVPTX::BI__nvvm_match_all_sync_i32p:
428 case NVPTX::BI__nvvm_match_all_sync_i64p:
429 cgm.errorNYI(expr->getSourceRange(),
430 std::string("unimplemented NVPTX builtin call: ") +
431 getContext().BuiltinInfo.getName(builtinId));
432 return mlir::Value{};
433 // FP MMA loads
434 case NVPTX::BI__hmma_m16n16k16_ld_a:
435 case NVPTX::BI__hmma_m16n16k16_ld_b:
436 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
437 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
438 case NVPTX::BI__hmma_m32n8k16_ld_a:
439 case NVPTX::BI__hmma_m32n8k16_ld_b:
440 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
441 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
442 case NVPTX::BI__hmma_m8n32k16_ld_a:
443 case NVPTX::BI__hmma_m8n32k16_ld_b:
444 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
445 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
446 cgm.errorNYI(expr->getSourceRange(),
447 std::string("unimplemented NVPTX builtin call: ") +
448 getContext().BuiltinInfo.getName(builtinId));
449 return mlir::Value{};
450 case NVPTX::BI__imma_m16n16k16_ld_a_s8:
451 case NVPTX::BI__imma_m16n16k16_ld_a_u8:
452 case NVPTX::BI__imma_m16n16k16_ld_b_s8:
453 case NVPTX::BI__imma_m16n16k16_ld_b_u8:
454 case NVPTX::BI__imma_m16n16k16_ld_c:
455 case NVPTX::BI__imma_m32n8k16_ld_a_s8:
456 case NVPTX::BI__imma_m32n8k16_ld_a_u8:
457 case NVPTX::BI__imma_m32n8k16_ld_b_s8:
458 case NVPTX::BI__imma_m32n8k16_ld_b_u8:
459 case NVPTX::BI__imma_m32n8k16_ld_c:
460 case NVPTX::BI__imma_m8n32k16_ld_a_s8:
461 case NVPTX::BI__imma_m8n32k16_ld_a_u8:
462 case NVPTX::BI__imma_m8n32k16_ld_b_s8:
463 case NVPTX::BI__imma_m8n32k16_ld_b_u8:
464 case NVPTX::BI__imma_m8n32k16_ld_c:
465 cgm.errorNYI(expr->getSourceRange(),
466 std::string("unimplemented NVPTX builtin call: ") +
467 getContext().BuiltinInfo.getName(builtinId));
468 return mlir::Value{};
469 case NVPTX::BI__imma_m8n8k32_ld_a_s4:
470 case NVPTX::BI__imma_m8n8k32_ld_a_u4:
471 case NVPTX::BI__imma_m8n8k32_ld_b_s4:
472 case NVPTX::BI__imma_m8n8k32_ld_b_u4:
473 case NVPTX::BI__imma_m8n8k32_ld_c:
474 case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
475 case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
476 case NVPTX::BI__bmma_m8n8k128_ld_c:
477 cgm.errorNYI(expr->getSourceRange(),
478 std::string("unimplemented NVPTX builtin call: ") +
479 getContext().BuiltinInfo.getName(builtinId));
480 return mlir::Value{};
481 case NVPTX::BI__dmma_m8n8k4_ld_a:
482 case NVPTX::BI__dmma_m8n8k4_ld_b:
483 case NVPTX::BI__dmma_m8n8k4_ld_c:
484 cgm.errorNYI(expr->getSourceRange(),
485 std::string("unimplemented NVPTX builtin call: ") +
486 getContext().BuiltinInfo.getName(builtinId));
487 return mlir::Value{};
488 case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
489 case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
490 case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
491 case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
492 case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
493 case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
494 case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
495 case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
496 case NVPTX::BI__mma_tf32_m16n16k8_ld_c:
497 cgm.errorNYI(expr->getSourceRange(),
498 std::string("unimplemented NVPTX builtin call: ") +
499 getContext().BuiltinInfo.getName(builtinId));
500 return mlir::Value{};
501 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
502 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
503 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
504 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
505 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
506 case NVPTX::BI__hmma_m8n32k16_st_c_f32:
507 case NVPTX::BI__imma_m16n16k16_st_c_i32:
508 case NVPTX::BI__imma_m32n8k16_st_c_i32:
509 case NVPTX::BI__imma_m8n32k16_st_c_i32:
510 case NVPTX::BI__imma_m8n8k32_st_c_i32:
511 case NVPTX::BI__bmma_m8n8k128_st_c_i32:
512 case NVPTX::BI__dmma_m8n8k4_st_c_f64:
513 case NVPTX::BI__mma_m16n16k8_st_c_f32:
514 cgm.errorNYI(expr->getSourceRange(),
515 std::string("unimplemented NVPTX builtin call: ") +
516 getContext().BuiltinInfo.getName(builtinId));
517 return mlir::Value{};
518 // BI__hmma_m16n16k16_mma_<Dtype><CType>(d, a, b, c, layout, satf) -->
519 // Intrinsic::nvvm_wmma_m16n16k16_mma_sync<layout A,B><DType><CType><Satf>
520 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
521 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
522 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
523 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
524 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
525 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
526 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
527 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
528 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
529 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
530 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
531 case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
532 case NVPTX::BI__imma_m16n16k16_mma_s8:
533 case NVPTX::BI__imma_m16n16k16_mma_u8:
534 case NVPTX::BI__imma_m32n8k16_mma_s8:
535 case NVPTX::BI__imma_m32n8k16_mma_u8:
536 case NVPTX::BI__imma_m8n32k16_mma_s8:
537 case NVPTX::BI__imma_m8n32k16_mma_u8:
538 case NVPTX::BI__imma_m8n8k32_mma_s4:
539 case NVPTX::BI__imma_m8n8k32_mma_u4:
540 case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1:
541 case NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1:
542 case NVPTX::BI__dmma_m8n8k4_mma_f64:
543 case NVPTX::BI__mma_bf16_m16n16k16_mma_f32:
544 case NVPTX::BI__mma_bf16_m8n32k16_mma_f32:
545 case NVPTX::BI__mma_bf16_m32n8k16_mma_f32:
546 case NVPTX::BI__mma_tf32_m16n16k8_mma_f32:
547 cgm.errorNYI(expr->getSourceRange(),
548 std::string("unimplemented NVPTX builtin call: ") +
549 getContext().BuiltinInfo.getName(builtinId));
550 return mlir::Value{};
551 // The following builtins require half type support
552 case NVPTX::BI__nvvm_ff2f16x2_rn:
553 cgm.errorNYI(expr->getSourceRange(),
554 std::string("unimplemented NVPTX builtin call: ") +
555 getContext().BuiltinInfo.getName(builtinId));
556 return mlir::Value{};
557 case NVPTX::BI__nvvm_ff2f16x2_rn_relu:
558 cgm.errorNYI(expr->getSourceRange(),
559 std::string("unimplemented NVPTX builtin call: ") +
560 getContext().BuiltinInfo.getName(builtinId));
561 return mlir::Value{};
562 case NVPTX::BI__nvvm_ff2f16x2_rz:
563 cgm.errorNYI(expr->getSourceRange(),
564 std::string("unimplemented NVPTX builtin call: ") +
565 getContext().BuiltinInfo.getName(builtinId));
566 return mlir::Value{};
567 case NVPTX::BI__nvvm_ff2f16x2_rz_relu:
568 cgm.errorNYI(expr->getSourceRange(),
569 std::string("unimplemented NVPTX builtin call: ") +
570 getContext().BuiltinInfo.getName(builtinId));
571 return mlir::Value{};
572 case NVPTX::BI__nvvm_fma_rn_f16:
573 case NVPTX::BI__nvvm_fma_rn_f16x2:
574 case NVPTX::BI__nvvm_fma_rn_bf16:
575 case NVPTX::BI__nvvm_fma_rn_bf16x2:
576 case NVPTX::BI__nvvm_fma_rn_f:
577 case NVPTX::BI__nvvm_fma_rn_d:
578 return emitNVVMFMA(*this, expr, "nvvm.fma",
579 llvm::APFloat::rmNearestTiesToEven);
580 case NVPTX::BI__nvvm_fma_rz_f:
581 case NVPTX::BI__nvvm_fma_rz_d:
582 return emitNVVMFMA(*this, expr, "nvvm.fma", llvm::APFloat::rmTowardZero);
583 case NVPTX::BI__nvvm_fma_rm_f:
584 case NVPTX::BI__nvvm_fma_rm_d:
585 return emitNVVMFMA(*this, expr, "nvvm.fma",
586 llvm::APFloat::rmTowardNegative);
587 case NVPTX::BI__nvvm_fma_rp_f:
588 case NVPTX::BI__nvvm_fma_rp_d:
589 return emitNVVMFMA(*this, expr, "nvvm.fma",
590 llvm::APFloat::rmTowardPositive);
591 case NVPTX::BI__nvvm_fma_rn_ftz_f16:
592 case NVPTX::BI__nvvm_fma_rn_ftz_f16x2:
593 case NVPTX::BI__nvvm_fma_rn_ftz_f:
594 return emitNVVMFMA(*this, expr, "nvvm.fma.ftz",
595 llvm::APFloat::rmNearestTiesToEven);
596 case NVPTX::BI__nvvm_fma_rz_ftz_f:
597 return emitNVVMFMA(*this, expr, "nvvm.fma.ftz",
598 llvm::APFloat::rmTowardZero);
599 case NVPTX::BI__nvvm_fma_rm_ftz_f:
600 return emitNVVMFMA(*this, expr, "nvvm.fma.ftz",
601 llvm::APFloat::rmTowardNegative);
602 case NVPTX::BI__nvvm_fma_rp_ftz_f:
603 return emitNVVMFMA(*this, expr, "nvvm.fma.ftz",
604 llvm::APFloat::rmTowardPositive);
605 case NVPTX::BI__nvvm_fma_rn_sat_f16:
606 case NVPTX::BI__nvvm_fma_rn_sat_f16x2:
607 case NVPTX::BI__nvvm_fma_rn_sat_f:
608 return emitNVVMFMA(*this, expr, "nvvm.fma.sat",
609 llvm::APFloat::rmNearestTiesToEven);
610 case NVPTX::BI__nvvm_fma_rz_sat_f:
611 return emitNVVMFMA(*this, expr, "nvvm.fma.sat",
612 llvm::APFloat::rmTowardZero);
613 case NVPTX::BI__nvvm_fma_rm_sat_f:
614 return emitNVVMFMA(*this, expr, "nvvm.fma.sat",
615 llvm::APFloat::rmTowardNegative);
616 case NVPTX::BI__nvvm_fma_rp_sat_f:
617 return emitNVVMFMA(*this, expr, "nvvm.fma.sat",
618 llvm::APFloat::rmTowardPositive);
619 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16:
620 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16x2:
621 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f:
622 return emitNVVMFMA(*this, expr, "nvvm.fma.ftz.sat",
623 llvm::APFloat::rmNearestTiesToEven);
624 case NVPTX::BI__nvvm_fma_rz_ftz_sat_f:
625 return emitNVVMFMA(*this, expr, "nvvm.fma.ftz.sat",
626 llvm::APFloat::rmTowardZero);
627 case NVPTX::BI__nvvm_fma_rm_ftz_sat_f:
628 return emitNVVMFMA(*this, expr, "nvvm.fma.ftz.sat",
629 llvm::APFloat::rmTowardNegative);
630 case NVPTX::BI__nvvm_fma_rp_ftz_sat_f:
631 return emitNVVMFMA(*this, expr, "nvvm.fma.ftz.sat",
632 llvm::APFloat::rmTowardPositive);
633 case NVPTX::BI__nvvm_fma_rn_relu_f16:
634 case NVPTX::BI__nvvm_fma_rn_relu_f16x2:
635 case NVPTX::BI__nvvm_fma_rn_relu_bf16:
636 case NVPTX::BI__nvvm_fma_rn_relu_bf16x2:
637 return emitNVVMFMA(*this, expr, "nvvm.fma.relu",
638 llvm::APFloat::rmNearestTiesToEven);
639 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16:
640 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16x2:
641 return emitNVVMFMA(*this, expr, "nvvm.fma.ftz.relu",
642 llvm::APFloat::rmNearestTiesToEven);
643 case NVPTX::BI__nvvm_fma_rn_oob_f16:
644 case NVPTX::BI__nvvm_fma_rn_oob_f16x2:
645 case NVPTX::BI__nvvm_fma_rn_oob_bf16:
646 case NVPTX::BI__nvvm_fma_rn_oob_bf16x2:
647 return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.oob")
648 .getValue();
649 case NVPTX::BI__nvvm_fma_rn_oob_relu_f16:
650 case NVPTX::BI__nvvm_fma_rn_oob_relu_f16x2:
651 case NVPTX::BI__nvvm_fma_rn_oob_relu_bf16:
652 case NVPTX::BI__nvvm_fma_rn_oob_relu_bf16x2:
653 return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.oob.relu")
654 .getValue();
655 case NVPTX::BI__nvvm_fmax_f16:
656 return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmax.f16")
657 .getValue();
658 case NVPTX::BI__nvvm_fmax_f16x2:
659 return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmax.f16x2")
660 .getValue();
661 case NVPTX::BI__nvvm_fmax_ftz_f16:
662 return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmax.ftz.f16")
663 .getValue();
664 case NVPTX::BI__nvvm_fmax_ftz_f16x2:
665 return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmax.ftz.f16x2")
666 .getValue();
667 case NVPTX::BI__nvvm_fmax_ftz_nan_f16:
668 return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmax.ftz.nan.f16")
669 .getValue();
670 case NVPTX::BI__nvvm_fmax_ftz_nan_f16x2:
671 return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmax.ftz.nan.f16x2")
672 .getValue();
673 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16:
674 return emitBuiltinWithOneOverloadedType<2>(
675 expr, "nvvm.fmax.ftz.nan.xorsign.abs.f16")
676 .getValue();
677 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16x2:
678 return emitBuiltinWithOneOverloadedType<2>(
679 expr, "nvvm.fmax.ftz.nan.xorsign.abs.f16x2")
680 .getValue();
681 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16:
682 return emitBuiltinWithOneOverloadedType<2>(expr,
683 "nvvm.fmax.ftz.xorsign.abs.f16")
684 .getValue();
685 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16x2:
686 return emitBuiltinWithOneOverloadedType<2>(
687 expr, "nvvm.fmax.ftz.xorsign.abs.f16x2")
688 .getValue();
689 case NVPTX::BI__nvvm_fmax_nan_f16:
690 return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmax.nan.f16")
691 .getValue();
692 case NVPTX::BI__nvvm_fmax_nan_f16x2:
693 return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmax.nan.f16x2")
694 .getValue();
695 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16:
696 return emitBuiltinWithOneOverloadedType<2>(expr,
697 "nvvm.fmax.nan.xorsign.abs.f16")
698 .getValue();
699 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16x2:
700 return emitBuiltinWithOneOverloadedType<2>(
701 expr, "nvvm.fmax.nan.xorsign.abs.f16x2")
702 .getValue();
703 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16:
704 return emitBuiltinWithOneOverloadedType<2>(expr,
705 "nvvm.fmax.xorsign.abs.f16")
706 .getValue();
707 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16x2:
708 return emitBuiltinWithOneOverloadedType<2>(expr,
709 "nvvm.fmax.xorsign.abs.f16x2")
710 .getValue();
711 case NVPTX::BI__nvvm_fmin_f16:
712 return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmin.f16")
713 .getValue();
714 case NVPTX::BI__nvvm_fmin_f16x2:
715 return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmin.f16x2")
716 .getValue();
717 case NVPTX::BI__nvvm_fmin_ftz_f16:
718 return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmin.ftz.f16")
719 .getValue();
720 case NVPTX::BI__nvvm_fmin_ftz_f16x2:
721 return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmin.ftz.f16x2")
722 .getValue();
723 case NVPTX::BI__nvvm_fmin_ftz_nan_f16:
724 return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmin.ftz.nan.f16")
725 .getValue();
726 case NVPTX::BI__nvvm_fmin_ftz_nan_f16x2:
727 return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmin.ftz.nan.f16x2")
728 .getValue();
729 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16:
730 return emitBuiltinWithOneOverloadedType<2>(
731 expr, "nvvm.fmin.ftz.nan.xorsign.abs.f16")
732 .getValue();
733 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16x2:
734 return emitBuiltinWithOneOverloadedType<2>(
735 expr, "nvvm.fmin.ftz.nan.xorsign.abs.f16x2")
736 .getValue();
737 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16:
738 return emitBuiltinWithOneOverloadedType<2>(expr,
739 "nvvm.fmin.ftz.xorsign.abs.f16")
740 .getValue();
741 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16x2:
742 return emitBuiltinWithOneOverloadedType<2>(
743 expr, "nvvm.fmin.ftz.xorsign.abs.f16x2")
744 .getValue();
745 case NVPTX::BI__nvvm_fmin_nan_f16:
746 return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmin.nan.f16")
747 .getValue();
748 case NVPTX::BI__nvvm_fmin_nan_f16x2:
749 return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmin.nan.f16x2")
750 .getValue();
751 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16:
752 return emitBuiltinWithOneOverloadedType<2>(expr,
753 "nvvm.fmin.nan.xorsign.abs.f16")
754 .getValue();
755 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16x2:
756 return emitBuiltinWithOneOverloadedType<2>(
757 expr, "nvvm.fmin.nan.xorsign.abs.f16x2")
758 .getValue();
759 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16:
760 return emitBuiltinWithOneOverloadedType<2>(expr,
761 "nvvm.fmin.xorsign.abs.f16")
762 .getValue();
763 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16x2:
764 return emitBuiltinWithOneOverloadedType<2>(expr,
765 "nvvm.fmin.xorsign.abs.f16x2")
766 .getValue();
767 case NVPTX::BI__nvvm_fabs_f:
768 case NVPTX::BI__nvvm_abs_bf16:
769 case NVPTX::BI__nvvm_abs_bf16x2:
770 case NVPTX::BI__nvvm_fabs_f16:
771 case NVPTX::BI__nvvm_fabs_f16x2:
772 return emitUnaryNVVMIntrinsic(*this, expr, "nvvm.fabs");
773 case NVPTX::BI__nvvm_fabs_ftz_f:
774 case NVPTX::BI__nvvm_fabs_ftz_f16:
775 case NVPTX::BI__nvvm_fabs_ftz_f16x2:
776 return emitUnaryNVVMIntrinsic(*this, expr, "nvvm.fabs.ftz");
777 case NVPTX::BI__nvvm_fabs_d:
778 return emitUnaryNVVMIntrinsic(*this, expr, "fabs");
779 case NVPTX::BI__nvvm_ex2_approx_d:
780 case NVPTX::BI__nvvm_ex2_approx_f:
781 case NVPTX::BI__nvvm_ex2_approx_f16:
782 case NVPTX::BI__nvvm_ex2_approx_f16x2:
783 return emitUnaryNVVMIntrinsic(*this, expr, "nvvm.ex2.approx");
784 case NVPTX::BI__nvvm_ex2_approx_ftz_f:
785 return emitUnaryNVVMIntrinsic(*this, expr, "nvvm.ex2.approx.ftz");
786 case NVPTX::BI__nvvm_add_rn_f:
787 case NVPTX::BI__nvvm_add_rn_d:
788 return emitNVVMFPArith(*this, expr, "nvvm.fadd",
789 llvm::APFloat::rmNearestTiesToEven);
790 case NVPTX::BI__nvvm_add_rz_f:
791 case NVPTX::BI__nvvm_add_rz_d:
792 return emitNVVMFPArith(*this, expr, "nvvm.fadd",
793 llvm::APFloat::rmTowardZero);
794 case NVPTX::BI__nvvm_add_rm_f:
795 case NVPTX::BI__nvvm_add_rm_d:
796 return emitNVVMFPArith(*this, expr, "nvvm.fadd",
797 llvm::APFloat::rmTowardNegative);
798 case NVPTX::BI__nvvm_add_rp_f:
799 case NVPTX::BI__nvvm_add_rp_d:
800 return emitNVVMFPArith(*this, expr, "nvvm.fadd",
801 llvm::APFloat::rmTowardPositive);
802 case NVPTX::BI__nvvm_add_rn_ftz_f:
803 return emitNVVMFPArith(*this, expr, "nvvm.fadd.ftz",
804 llvm::APFloat::rmNearestTiesToEven);
805 case NVPTX::BI__nvvm_add_rz_ftz_f:
806 return emitNVVMFPArith(*this, expr, "nvvm.fadd.ftz",
807 llvm::APFloat::rmTowardZero);
808 case NVPTX::BI__nvvm_add_rm_ftz_f:
809 return emitNVVMFPArith(*this, expr, "nvvm.fadd.ftz",
810 llvm::APFloat::rmTowardNegative);
811 case NVPTX::BI__nvvm_add_rp_ftz_f:
812 return emitNVVMFPArith(*this, expr, "nvvm.fadd.ftz",
813 llvm::APFloat::rmTowardPositive);
814 case NVPTX::BI__nvvm_add_rn_sat_f:
815 case NVPTX::BI__nvvm_add_rn_sat_f16:
816 case NVPTX::BI__nvvm_add_rn_sat_v2f16:
817 return emitNVVMFPArith(*this, expr, "nvvm.fadd.sat",
818 llvm::APFloat::rmNearestTiesToEven);
819 case NVPTX::BI__nvvm_add_rz_sat_f:
820 return emitNVVMFPArith(*this, expr, "nvvm.fadd.sat",
821 llvm::APFloat::rmTowardZero);
822 case NVPTX::BI__nvvm_add_rm_sat_f:
823 return emitNVVMFPArith(*this, expr, "nvvm.fadd.sat",
824 llvm::APFloat::rmTowardNegative);
825 case NVPTX::BI__nvvm_add_rp_sat_f:
826 return emitNVVMFPArith(*this, expr, "nvvm.fadd.sat",
827 llvm::APFloat::rmTowardPositive);
828 case NVPTX::BI__nvvm_add_rn_ftz_sat_f:
829 case NVPTX::BI__nvvm_add_rn_ftz_sat_f16:
830 case NVPTX::BI__nvvm_add_rn_ftz_sat_v2f16:
831 return emitNVVMFPArith(*this, expr, "nvvm.fadd.ftz.sat",
832 llvm::APFloat::rmNearestTiesToEven);
833 case NVPTX::BI__nvvm_add_rz_ftz_sat_f:
834 return emitNVVMFPArith(*this, expr, "nvvm.fadd.ftz.sat",
835 llvm::APFloat::rmTowardZero);
836 case NVPTX::BI__nvvm_add_rm_ftz_sat_f:
837 return emitNVVMFPArith(*this, expr, "nvvm.fadd.ftz.sat",
838 llvm::APFloat::rmTowardNegative);
839 case NVPTX::BI__nvvm_add_rp_ftz_sat_f:
840 return emitNVVMFPArith(*this, expr, "nvvm.fadd.ftz.sat",
841 llvm::APFloat::rmTowardPositive);
842 case NVPTX::BI__nvvm_mul_rn_f:
843 case NVPTX::BI__nvvm_mul_rn_d:
844 return emitNVVMFPArith(*this, expr, "nvvm.fmul",
845 llvm::APFloat::rmNearestTiesToEven);
846 case NVPTX::BI__nvvm_mul_rz_f:
847 case NVPTX::BI__nvvm_mul_rz_d:
848 return emitNVVMFPArith(*this, expr, "nvvm.fmul",
849 llvm::APFloat::rmTowardZero);
850 case NVPTX::BI__nvvm_mul_rm_f:
851 case NVPTX::BI__nvvm_mul_rm_d:
852 return emitNVVMFPArith(*this, expr, "nvvm.fmul",
853 llvm::APFloat::rmTowardNegative);
854 case NVPTX::BI__nvvm_mul_rp_f:
855 case NVPTX::BI__nvvm_mul_rp_d:
856 return emitNVVMFPArith(*this, expr, "nvvm.fmul",
857 llvm::APFloat::rmTowardPositive);
858 case NVPTX::BI__nvvm_mul_rn_ftz_f:
859 return emitNVVMFPArith(*this, expr, "nvvm.fmul.ftz",
860 llvm::APFloat::rmNearestTiesToEven);
861 case NVPTX::BI__nvvm_mul_rz_ftz_f:
862 return emitNVVMFPArith(*this, expr, "nvvm.fmul.ftz",
863 llvm::APFloat::rmTowardZero);
864 case NVPTX::BI__nvvm_mul_rm_ftz_f:
865 return emitNVVMFPArith(*this, expr, "nvvm.fmul.ftz",
866 llvm::APFloat::rmTowardNegative);
867 case NVPTX::BI__nvvm_mul_rp_ftz_f:
868 return emitNVVMFPArith(*this, expr, "nvvm.fmul.ftz",
869 llvm::APFloat::rmTowardPositive);
870 case NVPTX::BI__nvvm_mul_rn_sat_f16:
871 case NVPTX::BI__nvvm_mul_rn_sat_v2f16:
872 return emitNVVMFPArith(*this, expr, "nvvm.fmul.sat",
873 llvm::APFloat::rmNearestTiesToEven);
874 case NVPTX::BI__nvvm_mul_rn_ftz_sat_f16:
875 case NVPTX::BI__nvvm_mul_rn_ftz_sat_v2f16:
876 return emitNVVMFPArith(*this, expr, "nvvm.fmul.ftz.sat",
877 llvm::APFloat::rmNearestTiesToEven);
878 case NVPTX::BI__nvvm_ldg_h:
879 case NVPTX::BI__nvvm_ldg_h2:
880 cgm.errorNYI(expr->getSourceRange(),
881 std::string("unimplemented NVPTX builtin call: ") +
882 getContext().BuiltinInfo.getName(builtinId));
883 return mlir::Value{};
884 case NVPTX::BI__nvvm_ldu_h:
885 case NVPTX::BI__nvvm_ldu_h2:
886 return makeLdu(*this, expr, "nvvm.ldu.global.f");
887 case NVPTX::BI__nvvm_cp_async_ca_shared_global_4:
888 cgm.errorNYI(expr->getSourceRange(),
889 std::string("unimplemented NVPTX builtin call: ") +
890 getContext().BuiltinInfo.getName(builtinId));
891 return mlir::Value{};
892 case NVPTX::BI__nvvm_cp_async_ca_shared_global_8:
893 cgm.errorNYI(expr->getSourceRange(),
894 std::string("unimplemented NVPTX builtin call: ") +
895 getContext().BuiltinInfo.getName(builtinId));
896 return mlir::Value{};
897 case NVPTX::BI__nvvm_cp_async_ca_shared_global_16:
898 cgm.errorNYI(expr->getSourceRange(),
899 std::string("unimplemented NVPTX builtin call: ") +
900 getContext().BuiltinInfo.getName(builtinId));
901 return mlir::Value{};
902 case NVPTX::BI__nvvm_cp_async_cg_shared_global_16:
903 cgm.errorNYI(expr->getSourceRange(),
904 std::string("unimplemented NVPTX builtin call: ") +
905 getContext().BuiltinInfo.getName(builtinId));
906 return mlir::Value{};
907 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_x:
908 cgm.errorNYI(expr->getSourceRange(),
909 std::string("unimplemented NVPTX builtin call: ") +
910 getContext().BuiltinInfo.getName(builtinId));
911 return mlir::Value{};
912 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_y:
913 cgm.errorNYI(expr->getSourceRange(),
914 std::string("unimplemented NVPTX builtin call: ") +
915 getContext().BuiltinInfo.getName(builtinId));
916 return mlir::Value{};
917 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_z:
918 cgm.errorNYI(expr->getSourceRange(),
919 std::string("unimplemented NVPTX builtin call: ") +
920 getContext().BuiltinInfo.getName(builtinId));
921 return mlir::Value{};
922 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_w:
923 cgm.errorNYI(expr->getSourceRange(),
924 std::string("unimplemented NVPTX builtin call: ") +
925 getContext().BuiltinInfo.getName(builtinId));
926 return mlir::Value{};
927 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_x:
928 cgm.errorNYI(expr->getSourceRange(),
929 std::string("unimplemented NVPTX builtin call: ") +
930 getContext().BuiltinInfo.getName(builtinId));
931 return mlir::Value{};
932 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_y:
933 cgm.errorNYI(expr->getSourceRange(),
934 std::string("unimplemented NVPTX builtin call: ") +
935 getContext().BuiltinInfo.getName(builtinId));
936 return mlir::Value{};
937 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_z:
938 cgm.errorNYI(expr->getSourceRange(),
939 std::string("unimplemented NVPTX builtin call: ") +
940 getContext().BuiltinInfo.getName(builtinId));
941 return mlir::Value{};
942 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_w:
943 cgm.errorNYI(expr->getSourceRange(),
944 std::string("unimplemented NVPTX builtin call: ") +
945 getContext().BuiltinInfo.getName(builtinId));
946 return mlir::Value{};
947 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_x:
948 cgm.errorNYI(expr->getSourceRange(),
949 std::string("unimplemented NVPTX builtin call: ") +
950 getContext().BuiltinInfo.getName(builtinId));
951 return mlir::Value{};
952 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_y:
953 cgm.errorNYI(expr->getSourceRange(),
954 std::string("unimplemented NVPTX builtin call: ") +
955 getContext().BuiltinInfo.getName(builtinId));
956 return mlir::Value{};
957 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_z:
958 cgm.errorNYI(expr->getSourceRange(),
959 std::string("unimplemented NVPTX builtin call: ") +
960 getContext().BuiltinInfo.getName(builtinId));
961 return mlir::Value{};
962 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_w:
963 cgm.errorNYI(expr->getSourceRange(),
964 std::string("unimplemented NVPTX builtin call: ") +
965 getContext().BuiltinInfo.getName(builtinId));
966 return mlir::Value{};
967 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_x:
968 cgm.errorNYI(expr->getSourceRange(),
969 std::string("unimplemented NVPTX builtin call: ") +
970 getContext().BuiltinInfo.getName(builtinId));
971 return mlir::Value{};
972 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_y:
973 cgm.errorNYI(expr->getSourceRange(),
974 std::string("unimplemented NVPTX builtin call: ") +
975 getContext().BuiltinInfo.getName(builtinId));
976 return mlir::Value{};
977 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_z:
978 cgm.errorNYI(expr->getSourceRange(),
979 std::string("unimplemented NVPTX builtin call: ") +
980 getContext().BuiltinInfo.getName(builtinId));
981 return mlir::Value{};
982 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_w:
983 cgm.errorNYI(expr->getSourceRange(),
984 std::string("unimplemented NVPTX builtin call: ") +
985 getContext().BuiltinInfo.getName(builtinId));
986 return mlir::Value{};
987 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctarank:
988 cgm.errorNYI(expr->getSourceRange(),
989 std::string("unimplemented NVPTX builtin call: ") +
990 getContext().BuiltinInfo.getName(builtinId));
991 return mlir::Value{};
992 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctarank:
993 cgm.errorNYI(expr->getSourceRange(),
994 std::string("unimplemented NVPTX builtin call: ") +
995 getContext().BuiltinInfo.getName(builtinId));
996 return mlir::Value{};
997 case NVPTX::BI__nvvm_is_explicit_cluster:
998 cgm.errorNYI(expr->getSourceRange(),
999 std::string("unimplemented NVPTX builtin call: ") +
1000 getContext().BuiltinInfo.getName(builtinId));
1001 return mlir::Value{};
1002 case NVPTX::BI__nvvm_isspacep_shared_cluster:
1003 cgm.errorNYI(expr->getSourceRange(),
1004 std::string("unimplemented NVPTX builtin call: ") +
1005 getContext().BuiltinInfo.getName(builtinId));
1006 return mlir::Value{};
1007 case NVPTX::BI__nvvm_mapa:
1008 cgm.errorNYI(expr->getSourceRange(),
1009 std::string("unimplemented NVPTX builtin call: ") +
1010 getContext().BuiltinInfo.getName(builtinId));
1011 return mlir::Value{};
1012 case NVPTX::BI__nvvm_mapa_shared_cluster:
1013 cgm.errorNYI(expr->getSourceRange(),
1014 std::string("unimplemented NVPTX builtin call: ") +
1015 getContext().BuiltinInfo.getName(builtinId));
1016 return mlir::Value{};
1017 case NVPTX::BI__nvvm_getctarank:
1018 cgm.errorNYI(expr->getSourceRange(),
1019 std::string("unimplemented NVPTX builtin call: ") +
1020 getContext().BuiltinInfo.getName(builtinId));
1021 return mlir::Value{};
1022 case NVPTX::BI__nvvm_getctarank_shared_cluster:
1023 cgm.errorNYI(expr->getSourceRange(),
1024 std::string("unimplemented NVPTX builtin call: ") +
1025 getContext().BuiltinInfo.getName(builtinId));
1026 return mlir::Value{};
1027 case NVPTX::BI__nvvm_barrier_cluster_arrive:
1028 return builder.emitIntrinsicCallOp(getLoc(expr->getExprLoc()),
1029 "nvvm.barrier.cluster.arrive",
1030 builder.getVoidTy());
1031 case NVPTX::BI__nvvm_barrier_cluster_arrive_relaxed:
1032 return builder.emitIntrinsicCallOp(getLoc(expr->getExprLoc()),
1033 "nvvm.barrier.cluster.arrive.relaxed",
1034 builder.getVoidTy());
1035 case NVPTX::BI__nvvm_barrier_cluster_wait:
1036 return builder.emitIntrinsicCallOp(getLoc(expr->getExprLoc()),
1037 "nvvm.barrier.cluster.wait",
1038 builder.getVoidTy());
1039 case NVPTX::BI__nvvm_fence_sc_cluster:
1040 return builder.emitIntrinsicCallOp(getLoc(expr->getExprLoc()),
1041 "nvvm.fence.sc.cluster",
1042 builder.getVoidTy());
1043 case NVPTX::BI__nvvm_bar_sync:
1044 return builder.emitIntrinsicCallOp(
1045 getLoc(expr->getExprLoc()), "nvvm.barrier.cta.sync.aligned.all",
1046 builder.getVoidTy(), mlir::ValueRange{emitScalarExpr(expr->getArg(0))});
1047 case NVPTX::BI__syncthreads:
1048 return builder.emitIntrinsicCallOp(
1049 getLoc(expr->getExprLoc()), "nvvm.barrier.cta.sync.aligned.all",
1050 builder.getVoidTy(),
1051 mlir::ValueRange{builder.getConstInt(getLoc(expr->getExprLoc()),
1052 builder.getSInt32Ty(), 0)});
1053 case NVPTX::BI__nvvm_barrier_sync:
1054 return builder.emitIntrinsicCallOp(
1055 getLoc(expr->getExprLoc()), "nvvm.barrier.cta.sync.all",
1056 builder.getVoidTy(), mlir::ValueRange{emitScalarExpr(expr->getArg(0))});
1057 case NVPTX::BI__nvvm_barrier_sync_cnt:
1058 return builder.emitIntrinsicCallOp(
1059 getLoc(expr->getExprLoc()), "nvvm.barrier.cta.sync.count",
1060 builder.getVoidTy(),
1061 mlir::ValueRange{emitScalarExpr(expr->getArg(0)),
1062 emitScalarExpr(expr->getArg(1))});
1063 case NVPTX::BI__nvvm_bar0_and:
1064 return emitBar0Reduction(*this, expr,
1065 "nvvm.barrier.cta.red.and.aligned.all",
1066 /*returnsPred=*/true);
1067 case NVPTX::BI__nvvm_bar0_or:
1068 return emitBar0Reduction(*this, expr, "nvvm.barrier.cta.red.or.aligned.all",
1069 /*returnsPred=*/true);
1070 case NVPTX::BI__nvvm_bar0_popc:
1071 return emitBar0Reduction(*this, expr,
1072 "nvvm.barrier.cta.red.popc.aligned.all",
1073 /*returnsPred=*/false);
1074
1075 default:
1076 return std::nullopt;
1077 }
1078}
makeScopedAtomicRMW
static mlir::Value makeScopedAtomicRMW(CIRGenFunction &cgf, const CallExpr *expr, cir::AtomicFetchKind kind, cir::SyncScopeKind scope)
Definition CIRGenBuiltinNVPTX.cpp:123
makeLdg
static mlir::Value makeLdg(CIRGenFunction &cgf, const CallExpr *expr)
Definition CIRGenBuiltinNVPTX.cpp:39
emitBar0Reduction
static mlir::Value emitBar0Reduction(CIRGenFunction &cgf, const CallExpr *expr, llvm::StringRef intrinsicName, bool returnsPred)
Definition CIRGenBuiltinNVPTX.cpp:106
emitNVVMFPArith
static mlir::Value emitNVVMFPArith(CIRGenFunction &cgf, const CallExpr *expr, llvm::StringRef intrinsicName, llvm::APFloat::roundingMode rm)
Emit a CIR LLVMIntrinsicCallOp for an NVVM fadd/fmul intrinsic, which takes the rounding mode as a tr...
Definition CIRGenBuiltinNVPTX.cpp:73
makeScopedAtomicXchg
static mlir::Value makeScopedAtomicXchg(CIRGenFunction &cgf, const CallExpr *expr, cir::SyncScopeKind scope)
Definition CIRGenBuiltinNVPTX.cpp:138
makeLdu
static mlir::Value makeLdu(CIRGenFunction &cgf, const CallExpr *expr, llvm::StringRef intrinsicName)
Definition CIRGenBuiltinNVPTX.cpp:23
emitUnaryNVVMIntrinsic
static mlir::Value emitUnaryNVVMIntrinsic(CIRGenFunction &cgf, const CallExpr *expr, llvm::StringRef intrinsicName)
Emit a CIR LLVMIntrinsicCallOp for a unary NVVM intrinsic.
Definition CIRGenBuiltinNVPTX.cpp:60
emitNVVMFMA
static mlir::Value emitNVVMFMA(CIRGenFunction &cgf, const CallExpr *expr, llvm::StringRef intrinsicName, llvm::APFloat::roundingMode rm)
Emit a CIR LLVMIntrinsicCallOp for an NVVM fma intrinsic, which takes the rounding mode as a trailing...
Definition CIRGenBuiltinNVPTX.cpp:90
kind
*collection of selector each with an associated kind and an ordered *collection of selectors A selector has a kind
Definition OpenMPClause.h:10136
TargetBuiltins.h
Enumerates target-specific builtins in their own namespaces within namespace clang.
cir::CIRBaseBuilderTy::getNullValue
cir::ConstantOp getNullValue(mlir::Type ty, mlir::Location loc)
Definition CIRBaseBuilder.h:116
cir::CIRBaseBuilderTy::createBoolToInt
mlir::Value createBoolToInt(mlir::Value src, mlir::Type newTy)
Definition CIRBaseBuilder.h:757
cir::CIRBaseBuilderTy::createCompare
cir::CmpOp createCompare(mlir::Location loc, cir::CmpOpKind kind, mlir::Value lhs, mlir::Value rhs)
Definition CIRBaseBuilder.h:968
cir::CIRBaseBuilderTy::getBoolTy
cir::BoolType getBoolTy()
Definition CIRBaseBuilder.h:170
clang::CIRGen::Address::getPointer
mlir::Value getPointer() const
Definition Address.h:98
clang::CIRGen::Address::getAlignment
clang::CharUnits getAlignment() const
Definition Address.h:138
clang::CIRGen::Address::emitRawPointer
mlir::Value emitRawPointer() const
Return the pointer contained in this class after authenticating it and adding offset to it if necessa...
Definition Address.h:112
clang::CIRGen::CIRGenBuilderTy::emitIntrinsicCallOp
mlir::Value emitIntrinsicCallOp(mlir::Location loc, const llvm::StringRef str, const mlir::Type &resTy, Operands &&...op)
Definition CIRGenBuilder.h:901
clang::CIRGen::CIRGenFunction::emitPointerWithAlignment
Address emitPointerWithAlignment(const clang::Expr *expr, LValueBaseInfo *baseInfo=nullptr)
Given an expression with a pointer type, emit the value and compute our best estimate of the alignmen...
Definition CIRGenExpr.cpp:128
clang::CIRGen::CIRGenFunction::getLoc
mlir::Location getLoc(clang::SourceLocation srcLoc)
Helpers to convert Clang's SourceLocation to a MLIR Location.
Definition CIRGenFunction.cpp:147
clang::CIRGen::CIRGenFunction::makeBinaryAtomicValue
mlir::Value makeBinaryAtomicValue(cir::AtomicFetchKind kind, const clang::CallExpr *expr, mlir::Type *originalArgType=nullptr, mlir::Value *emittedArgValue=nullptr, cir::MemOrder ordering=cir::MemOrder::SequentiallyConsistent)
Utility to insert an atomic instruction based on Intrinsic::ID and the expression node.
Definition CIRGenBuiltin.cpp:327
clang::CIRGen::CIRGenFunction::convertTypeForMem
mlir::Type convertTypeForMem(QualType t)
Definition CIRGenFunction.cpp:139
clang::CIRGen::CIRGenFunction::emitBuiltinWithOneOverloadedType
RValue emitBuiltinWithOneOverloadedType(const CallExpr *e, llvm::StringRef intrinName, mlir::Type resultType={})
Emit a simple LLVM intrinsic that takes N scalar arguments.
Definition CIRGenFunction.h:2044
clang::CIRGen::CIRGenFunction::emitScalarExpr
mlir::Value emitScalarExpr(const clang::Expr *e, bool ignoreResultAssign=false)
Emit the computation of the specified expression of scalar type.
Definition CIRGenExprScalar.cpp:1908
clang::CIRGen::CIRGenFunction::emitAtomicCmpXchg
mlir::Value emitAtomicCmpXchg(const clang::CallExpr *expr, bool returnBool, cir::MemOrder successOrder=cir::MemOrder::SequentiallyConsistent, cir::MemOrder failureOrder=cir::MemOrder::SequentiallyConsistent, cir::SyncScopeKind scope=cir::SyncScopeKind::System)
Emit cir.atomic.cmpxchg.
Definition CIRGenBuiltin.cpp:411
clang::CIRGen::CIRGenFunction::getBuilder
CIRGenBuilderTy & getBuilder()
Definition CIRGenFunction.h:325
clang::CIRGen::CIRGenFunction::emitNVPTXBuiltinExpr
std::optional< mlir::Value > emitNVPTXBuiltinExpr(unsigned builtinID, const CallExpr *expr)
Emit a call to an NVPTX builtin function.
Definition CIRGenBuiltinNVPTX.cpp:152
clang::CIRGen::CIRGenFunction::getContext
clang::ASTContext & getContext() const
Definition CIRGenFunction.h:323
clang::CIRGen::RValue::getValue
mlir::Value getValue() const
Return the value of this scalar value.
Definition CIRGenValue.h:57
clang::CallExpr
CallExpr - Represents a function call (C99 6.5.2.2, C++ [expr.call]).
Definition Expr.h:3012
clang::CharUnits
This is an opaque type for sizes expressed in character units.
Definition CharUnits.h:38
clang::QualType
A (possibly-)qualified type.
Definition TypeBase.h:938
clang::Type::getPointeeType
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
Definition Type.cpp:885
clang::ast_matchers::expr
const internal::VariadicDynCastAllOfMatcher< Stmt, Expr > expr
Matches expressions.
Definition ASTMatchersInternal.cpp:937
clang
Top level wrappers for InstallAPI frontend operations.
Definition CalledOnceCheck.h:17
Generated on for clang by doxygen 1.14.0