clang 24.0.0git
NVPTX.cpp
Go to the documentation of this file.
1//===-------- NVPTX.cpp - Emit LLVM Code for builtins ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Builtin calls as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGBuiltin.h"
14#include "TargetInfo.h"
15#include "clang/Basic/TargetBuiltins.h"
16#include "llvm/IR/IntrinsicsNVPTX.h"
17#include "llvm/TargetParser/AtomicScope.h"
18
19using namespace clang;
20using namespace CodeGen;
21using namespace llvm;
22
23namespace {
24// Helper classes for mapping MMA builtins to particular LLVM intrinsic variant.
25struct NVPTXMmaLdstInfo {
26 unsigned NumResults; // Number of elements to load/store
27 // Intrinsic IDs for row/col variants. 0 if particular layout is unsupported.
28 unsigned IID_col;
29 unsigned IID_row;
30};
31
32#define MMA_INTR(geom_op_type, layout) \
33 Intrinsic::nvvm_wmma_##geom_op_type##_##layout##_stride
34#define MMA_LDST(n, geom_op_type) \
35 { n, MMA_INTR(geom_op_type, col), MMA_INTR(geom_op_type, row) }
36
37static NVPTXMmaLdstInfo getNVPTXMmaLdstInfo(unsigned BuiltinID) {
38 switch (BuiltinID) {
39 // FP MMA loads
40 case NVPTX::BI__hmma_m16n16k16_ld_a:
41 return MMA_LDST(8, m16n16k16_load_a_f16);
42 case NVPTX::BI__hmma_m16n16k16_ld_b:
43 return MMA_LDST(8, m16n16k16_load_b_f16);
44 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
45 return MMA_LDST(4, m16n16k16_load_c_f16);
46 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
47 return MMA_LDST(8, m16n16k16_load_c_f32);
48 case NVPTX::BI__hmma_m32n8k16_ld_a:
49 return MMA_LDST(8, m32n8k16_load_a_f16);
50 case NVPTX::BI__hmma_m32n8k16_ld_b:
51 return MMA_LDST(8, m32n8k16_load_b_f16);
52 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
53 return MMA_LDST(4, m32n8k16_load_c_f16);
54 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
55 return MMA_LDST(8, m32n8k16_load_c_f32);
56 case NVPTX::BI__hmma_m8n32k16_ld_a:
57 return MMA_LDST(8, m8n32k16_load_a_f16);
58 case NVPTX::BI__hmma_m8n32k16_ld_b:
59 return MMA_LDST(8, m8n32k16_load_b_f16);
60 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
61 return MMA_LDST(4, m8n32k16_load_c_f16);
62 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
63 return MMA_LDST(8, m8n32k16_load_c_f32);
64
65 // Integer MMA loads
66 case NVPTX::BI__imma_m16n16k16_ld_a_s8:
67 return MMA_LDST(2, m16n16k16_load_a_s8);
68 case NVPTX::BI__imma_m16n16k16_ld_a_u8:
69 return MMA_LDST(2, m16n16k16_load_a_u8);
70 case NVPTX::BI__imma_m16n16k16_ld_b_s8:
71 return MMA_LDST(2, m16n16k16_load_b_s8);
72 case NVPTX::BI__imma_m16n16k16_ld_b_u8:
73 return MMA_LDST(2, m16n16k16_load_b_u8);
74 case NVPTX::BI__imma_m16n16k16_ld_c:
75 return MMA_LDST(8, m16n16k16_load_c_s32);
76 case NVPTX::BI__imma_m32n8k16_ld_a_s8:
77 return MMA_LDST(4, m32n8k16_load_a_s8);
78 case NVPTX::BI__imma_m32n8k16_ld_a_u8:
79 return MMA_LDST(4, m32n8k16_load_a_u8);
80 case NVPTX::BI__imma_m32n8k16_ld_b_s8:
81 return MMA_LDST(1, m32n8k16_load_b_s8);
82 case NVPTX::BI__imma_m32n8k16_ld_b_u8:
83 return MMA_LDST(1, m32n8k16_load_b_u8);
84 case NVPTX::BI__imma_m32n8k16_ld_c:
85 return MMA_LDST(8, m32n8k16_load_c_s32);
86 case NVPTX::BI__imma_m8n32k16_ld_a_s8:
87 return MMA_LDST(1, m8n32k16_load_a_s8);
88 case NVPTX::BI__imma_m8n32k16_ld_a_u8:
89 return MMA_LDST(1, m8n32k16_load_a_u8);
90 case NVPTX::BI__imma_m8n32k16_ld_b_s8:
91 return MMA_LDST(4, m8n32k16_load_b_s8);
92 case NVPTX::BI__imma_m8n32k16_ld_b_u8:
93 return MMA_LDST(4, m8n32k16_load_b_u8);
94 case NVPTX::BI__imma_m8n32k16_ld_c:
95 return MMA_LDST(8, m8n32k16_load_c_s32);
96
97 // Sub-integer MMA loads.
98 // Only row/col layout is supported by A/B fragments.
99 case NVPTX::BI__imma_m8n8k32_ld_a_s4:
100 return {1, 0, MMA_INTR(m8n8k32_load_a_s4, row)};
101 case NVPTX::BI__imma_m8n8k32_ld_a_u4:
102 return {1, 0, MMA_INTR(m8n8k32_load_a_u4, row)};
103 case NVPTX::BI__imma_m8n8k32_ld_b_s4:
104 return {1, MMA_INTR(m8n8k32_load_b_s4, col), 0};
105 case NVPTX::BI__imma_m8n8k32_ld_b_u4:
106 return {1, MMA_INTR(m8n8k32_load_b_u4, col), 0};
107 case NVPTX::BI__imma_m8n8k32_ld_c:
108 return MMA_LDST(2, m8n8k32_load_c_s32);
109 case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
110 return {1, 0, MMA_INTR(m8n8k128_load_a_b1, row)};
111 case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
112 return {1, MMA_INTR(m8n8k128_load_b_b1, col), 0};
113 case NVPTX::BI__bmma_m8n8k128_ld_c:
114 return MMA_LDST(2, m8n8k128_load_c_s32);
115
116 // Double MMA loads
117 case NVPTX::BI__dmma_m8n8k4_ld_a:
118 return MMA_LDST(1, m8n8k4_load_a_f64);
119 case NVPTX::BI__dmma_m8n8k4_ld_b:
120 return MMA_LDST(1, m8n8k4_load_b_f64);
121 case NVPTX::BI__dmma_m8n8k4_ld_c:
122 return MMA_LDST(2, m8n8k4_load_c_f64);
123
124 // Alternate float MMA loads
125 case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
126 return MMA_LDST(4, m16n16k16_load_a_bf16);
127 case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
128 return MMA_LDST(4, m16n16k16_load_b_bf16);
129 case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
130 return MMA_LDST(2, m8n32k16_load_a_bf16);
131 case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
132 return MMA_LDST(8, m8n32k16_load_b_bf16);
133 case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
134 return MMA_LDST(8, m32n8k16_load_a_bf16);
135 case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
136 return MMA_LDST(2, m32n8k16_load_b_bf16);
137 case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
138 return MMA_LDST(4, m16n16k8_load_a_tf32);
139 case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
140 return MMA_LDST(4, m16n16k8_load_b_tf32);
141 case NVPTX::BI__mma_tf32_m16n16k8_ld_c:
142 return MMA_LDST(8, m16n16k8_load_c_f32);
143
144 // NOTE: We need to follow inconsitent naming scheme used by NVCC. Unlike
145 // PTX and LLVM IR where stores always use fragment D, NVCC builtins always
146 // use fragment C for both loads and stores.
147 // FP MMA stores.
148 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
149 return MMA_LDST(4, m16n16k16_store_d_f16);
150 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
151 return MMA_LDST(8, m16n16k16_store_d_f32);
152 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
153 return MMA_LDST(4, m32n8k16_store_d_f16);
154 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
155 return MMA_LDST(8, m32n8k16_store_d_f32);
156 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
157 return MMA_LDST(4, m8n32k16_store_d_f16);
158 case NVPTX::BI__hmma_m8n32k16_st_c_f32:
159 return MMA_LDST(8, m8n32k16_store_d_f32);
160
161 // Integer and sub-integer MMA stores.
162 // Another naming quirk. Unlike other MMA builtins that use PTX types in the
163 // name, integer loads/stores use LLVM's i32.
164 case NVPTX::BI__imma_m16n16k16_st_c_i32:
165 return MMA_LDST(8, m16n16k16_store_d_s32);
166 case NVPTX::BI__imma_m32n8k16_st_c_i32:
167 return MMA_LDST(8, m32n8k16_store_d_s32);
168 case NVPTX::BI__imma_m8n32k16_st_c_i32:
169 return MMA_LDST(8, m8n32k16_store_d_s32);
170 case NVPTX::BI__imma_m8n8k32_st_c_i32:
171 return MMA_LDST(2, m8n8k32_store_d_s32);
172 case NVPTX::BI__bmma_m8n8k128_st_c_i32:
173 return MMA_LDST(2, m8n8k128_store_d_s32);
174
175 // Double MMA store
176 case NVPTX::BI__dmma_m8n8k4_st_c_f64:
177 return MMA_LDST(2, m8n8k4_store_d_f64);
178
179 // Alternate float MMA store
180 case NVPTX::BI__mma_m16n16k8_st_c_f32:
181 return MMA_LDST(8, m16n16k8_store_d_f32);
182
183 default:
184 llvm_unreachable("Unknown MMA builtin");
185 }
186}
187#undef MMA_LDST
188#undef MMA_INTR
189
190
191struct NVPTXMmaInfo {
192 unsigned NumEltsA;
193 unsigned NumEltsB;
194 unsigned NumEltsC;
195 unsigned NumEltsD;
196
197 // Variants are ordered by layout-A/layout-B/satf, where 'row' has priority
198 // over 'col' for layout. The index of non-satf variants is expected to match
199 // the undocumented layout constants used by CUDA's mma.hpp.
200 std::array<unsigned, 8> Variants;
201
202 unsigned getMMAIntrinsic(int Layout, bool Satf) {
203 unsigned Index = Layout + 4 * Satf;
204 if (Index >= Variants.size())
205 return 0;
206 return Variants[Index];
207 }
208};
209
210 // Returns an intrinsic that matches Layout and Satf for valid combinations of
211 // Layout and Satf, 0 otherwise.
212static NVPTXMmaInfo getNVPTXMmaInfo(unsigned BuiltinID) {
213 // clang-format off
214#define MMA_VARIANTS(geom, type) \
215 Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type, \
216 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type, \
217 Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type, \
218 Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type
219#define MMA_SATF_VARIANTS(geom, type) \
220 MMA_VARIANTS(geom, type), \
221 Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type##_satfinite, \
222 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type##_satfinite, \
223 Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type##_satfinite, \
224 Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type##_satfinite
225// Sub-integer MMA only supports row.col layout.
226#define MMA_VARIANTS_I4(geom, type) \
227 0, \
228 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type, \
229 0, \
230 0, \
231 0, \
232 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type##_satfinite, \
233 0, \
234 0
235// b1 MMA does not support .satfinite.
236#define MMA_VARIANTS_B1_XOR(geom, type) \
237 0, \
238 Intrinsic::nvvm_wmma_##geom##_mma_xor_popc_row_col_##type, \
239 0, \
240 0, \
241 0, \
242 0, \
243 0, \
244 0
245#define MMA_VARIANTS_B1_AND(geom, type) \
246 0, \
247 Intrinsic::nvvm_wmma_##geom##_mma_and_popc_row_col_##type, \
248 0, \
249 0, \
250 0, \
251 0, \
252 0, \
253 0
254 // clang-format on
255 switch (BuiltinID) {
256 // FP MMA
257 // Note that 'type' argument of MMA_SATF_VARIANTS uses D_C notation, while
258 // NumEltsN of return value are ordered as A,B,C,D.
259 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
260 return {8, 8, 4, 4, {{MMA_SATF_VARIANTS(m16n16k16, f16_f16)}}};
261 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
262 return {8, 8, 4, 8, {{MMA_SATF_VARIANTS(m16n16k16, f32_f16)}}};
263 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
264 return {8, 8, 8, 4, {{MMA_SATF_VARIANTS(m16n16k16, f16_f32)}}};
265 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
266 return {8, 8, 8, 8, {{MMA_SATF_VARIANTS(m16n16k16, f32_f32)}}};
267 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
268 return {8, 8, 4, 4, {{MMA_SATF_VARIANTS(m32n8k16, f16_f16)}}};
269 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
270 return {8, 8, 4, 8, {{MMA_SATF_VARIANTS(m32n8k16, f32_f16)}}};
271 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
272 return {8, 8, 8, 4, {{MMA_SATF_VARIANTS(m32n8k16, f16_f32)}}};
273 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
274 return {8, 8, 8, 8, {{MMA_SATF_VARIANTS(m32n8k16, f32_f32)}}};
275 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
276 return {8, 8, 4, 4, {{MMA_SATF_VARIANTS(m8n32k16, f16_f16)}}};
277 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
278 return {8, 8, 4, 8, {{MMA_SATF_VARIANTS(m8n32k16, f32_f16)}}};
279 case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
280 return {8, 8, 8, 4, {{MMA_SATF_VARIANTS(m8n32k16, f16_f32)}}};
281 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
282 return {8, 8, 8, 8, {{MMA_SATF_VARIANTS(m8n32k16, f32_f32)}}};
283
284 // Integer MMA
285 case NVPTX::BI__imma_m16n16k16_mma_s8:
286 return {2, 2, 8, 8, {{MMA_SATF_VARIANTS(m16n16k16, s8)}}};
287 case NVPTX::BI__imma_m16n16k16_mma_u8:
288 return {2, 2, 8, 8, {{MMA_SATF_VARIANTS(m16n16k16, u8)}}};
289 case NVPTX::BI__imma_m32n8k16_mma_s8:
290 return {4, 1, 8, 8, {{MMA_SATF_VARIANTS(m32n8k16, s8)}}};
291 case NVPTX::BI__imma_m32n8k16_mma_u8:
292 return {4, 1, 8, 8, {{MMA_SATF_VARIANTS(m32n8k16, u8)}}};
293 case NVPTX::BI__imma_m8n32k16_mma_s8:
294 return {1, 4, 8, 8, {{MMA_SATF_VARIANTS(m8n32k16, s8)}}};
295 case NVPTX::BI__imma_m8n32k16_mma_u8:
296 return {1, 4, 8, 8, {{MMA_SATF_VARIANTS(m8n32k16, u8)}}};
297
298 // Sub-integer MMA
299 case NVPTX::BI__imma_m8n8k32_mma_s4:
300 return {1, 1, 2, 2, {{MMA_VARIANTS_I4(m8n8k32, s4)}}};
301 case NVPTX::BI__imma_m8n8k32_mma_u4:
302 return {1, 1, 2, 2, {{MMA_VARIANTS_I4(m8n8k32, u4)}}};
303 case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1:
304 return {1, 1, 2, 2, {{MMA_VARIANTS_B1_XOR(m8n8k128, b1)}}};
305 case NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1:
306 return {1, 1, 2, 2, {{MMA_VARIANTS_B1_AND(m8n8k128, b1)}}};
307
308 // Double MMA
309 case NVPTX::BI__dmma_m8n8k4_mma_f64:
310 return {1, 1, 2, 2, {{MMA_VARIANTS(m8n8k4, f64)}}};
311
312 // Alternate FP MMA
313 case NVPTX::BI__mma_bf16_m16n16k16_mma_f32:
314 return {4, 4, 8, 8, {{MMA_VARIANTS(m16n16k16, bf16)}}};
315 case NVPTX::BI__mma_bf16_m8n32k16_mma_f32:
316 return {2, 8, 8, 8, {{MMA_VARIANTS(m8n32k16, bf16)}}};
317 case NVPTX::BI__mma_bf16_m32n8k16_mma_f32:
318 return {8, 2, 8, 8, {{MMA_VARIANTS(m32n8k16, bf16)}}};
319 case NVPTX::BI__mma_tf32_m16n16k8_mma_f32:
320 return {4, 4, 8, 8, {{MMA_VARIANTS(m16n16k8, tf32)}}};
321 default:
322 llvm_unreachable("Unexpected builtin ID.");
323 }
324#undef MMA_VARIANTS
325#undef MMA_SATF_VARIANTS
326#undef MMA_VARIANTS_I4
327#undef MMA_VARIANTS_B1_AND
328#undef MMA_VARIANTS_B1_XOR
329}
330
331static Value *MakeLdu(unsigned IntrinsicID, CodeGenFunction &CGF,
332 const CallExpr *E) {
333 Value *Ptr = CGF.EmitScalarExpr(E->getArg(0));
334 QualType ArgType = E->getArg(0)->getType();
335 clang::CharUnits Align = CGF.CGM.getNaturalPointeeTypeAlignment(ArgType);
336 llvm::Type *ElemTy = CGF.ConvertTypeForMem(ArgType->getPointeeType());
337 return CGF.Builder.CreateCall(
338 CGF.CGM.getIntrinsic(IntrinsicID, {ElemTy, Ptr->getType()}),
339 {Ptr, ConstantInt::get(CGF.Builder.getInt32Ty(), Align.getQuantity())});
340}
341
342static Value *MakeLdg(CodeGenFunction &CGF, const CallExpr *E) {
343 Value *Ptr = CGF.EmitScalarExpr(E->getArg(0));
344 QualType ArgType = E->getArg(0)->getType();
345 clang::CharUnits AlignV = CGF.CGM.getNaturalPointeeTypeAlignment(ArgType);
346 llvm::Type *ElemTy = CGF.ConvertTypeForMem(ArgType->getPointeeType());
347
348 // Use addrspace(1) for NVPTX ADDRESS_SPACE_GLOBAL
349 auto *ASC = CGF.Builder.CreateAddrSpaceCast(Ptr, CGF.Builder.getPtrTy(1));
350 auto *LD = CGF.Builder.CreateAlignedLoad(ElemTy, ASC, AlignV.getAsAlign());
351 MDNode *MD = MDNode::get(CGF.Builder.getContext(), {});
352 LD->setMetadata(LLVMContext::MD_invariant_load, MD);
353
354 return LD;
355}
356
357// `Scope` is AtomicScope::Workgroup for _cta builtins and AtomicScope::System
358// for _sys builtins.
359static Value *MakeScopedAtomicRMW(CodeGenFunction &CGF, const CallExpr *E,
360 llvm::AtomicRMWInst::BinOp Kind,
361 llvm::AtomicScope Scope) {
362 Address Ptr = CGF.EmitPointerWithAlignment(E->getArg(0));
363 Value *Val = CGF.EmitScalarExpr(E->getArg(1));
364 llvm::SyncScope::ID SSID = CGF.getLLVMContext().getOrInsertSyncScopeID(
365 *llvm::getAtomicScopeIRString(CGF.getTarget().getTriple(), Scope));
366 llvm::AtomicRMWInst *RMW = CGF.Builder.CreateAtomicRMW(
367 Kind, Ptr, Val, llvm::AtomicOrdering::Monotonic, SSID);
368
369 // CUDA's atomicAdd_block()/atomicAdd_system() reach this through the scoped
370 // add builtins, and are defined in terms of the native atom.add just as the
371 // unscoped atomicAdd() is. The helper ignores everything that is not a
372 // floating-point add, so the other scoped operations are unaffected.
373 addAtomicIgnoreDenormalModeMetadata(CGF, *RMW, /*AllowHalf=*/true);
374 return RMW;
375}
376
377// `Scope` is AtomicScope::Workgroup for _cta builtins and AtomicScope::System
378// for _sys builtins.
379static Value *MakeScopedAtomicCAS(CodeGenFunction &CGF, const CallExpr *E,
380 llvm::AtomicScope Scope) {
381 Address Ptr = CGF.EmitPointerWithAlignment(E->getArg(0));
382 Value *Cmp = CGF.EmitScalarExpr(E->getArg(1));
383 Value *New = CGF.EmitScalarExpr(E->getArg(2));
384 llvm::SyncScope::ID SSID = CGF.getLLVMContext().getOrInsertSyncScopeID(
385 *llvm::getAtomicScopeIRString(CGF.getTarget().getTriple(), Scope));
386 Value *Pair = CGF.Builder.CreateAtomicCmpXchg(
387 Ptr, Cmp, New, llvm::AtomicOrdering::Monotonic,
388 llvm::AtomicOrdering::Monotonic, SSID);
389 return CGF.Builder.CreateExtractValue(Pair, 0);
390}
391
392static Value *MakeCpAsync(unsigned IntrinsicID, unsigned IntrinsicIDS,
393 CodeGenFunction &CGF, const CallExpr *E,
394 int SrcSize) {
395 return E->getNumArgs() == 3
396 ? CGF.Builder.CreateCall(CGF.CGM.getIntrinsic(IntrinsicIDS),
397 {CGF.EmitScalarExpr(E->getArg(0)),
398 CGF.EmitScalarExpr(E->getArg(1)),
399 CGF.EmitScalarExpr(E->getArg(2))})
400 : CGF.Builder.CreateCall(CGF.CGM.getIntrinsic(IntrinsicID),
401 {CGF.EmitScalarExpr(E->getArg(0)),
402 CGF.EmitScalarExpr(E->getArg(1))});
403}
404
405static Value *MakeHalfType(Function *Intrinsic, unsigned BuiltinID,
406 const CallExpr *E, CodeGenFunction &CGF,
407 ArrayRef<Value *> TrailingArgs = {}) {
408 SmallVector<Value *, 16> Args;
409 auto *FTy = Intrinsic->getFunctionType();
410 unsigned ICEArguments = 0;
411 ASTContext::GetBuiltinTypeError Error;
412 CGF.CGM.getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
413 assert(Error == ASTContext::GE_None && "Should not codegen an error");
414 for (unsigned i = 0, e = E->getNumArgs(); i != e; ++i) {
415 assert((ICEArguments & (1 << i)) == 0);
416 auto *ArgValue = CGF.EmitScalarExpr(E->getArg(i));
417 auto *PTy = FTy->getParamType(i);
418 if (PTy != ArgValue->getType())
419 ArgValue = CGF.Builder.CreateBitCast(ArgValue, PTy);
420 Args.push_back(ArgValue);
421 }
422
423 llvm::append_range(Args, TrailingArgs);
424 appendDefaultIntrinsicArgs(Args, Intrinsic);
425
426 return CGF.Builder.CreateCall(Intrinsic, Args);
427}
428
429static Value *MakeHalfType(unsigned IntrinsicID, unsigned BuiltinID,
430 const CallExpr *E, CodeGenFunction &CGF,
431 ArrayRef<Value *> TrailingArgs = {}) {
432 return MakeHalfType(CGF.CGM.getIntrinsic(IntrinsicID), BuiltinID, E, CGF,
433 TrailingArgs);
434}
435
436static Value *MakeFMAOOB(unsigned IntrinsicID, llvm::Type *Ty,
437 const CallExpr *E, CodeGenFunction &CGF) {
438 return CGF.Builder.CreateCall(CGF.CGM.getIntrinsic(IntrinsicID, {Ty}),
439 {CGF.EmitScalarExpr(E->getArg(0)),
440 CGF.EmitScalarExpr(E->getArg(1)),
441 CGF.EmitScalarExpr(E->getArg(2))});
442}
443
444static Value *MakeFPArith(unsigned IntrinsicID, APFloat::roundingMode RM,
445 unsigned BuiltinID, const CallExpr *E,
446 CodeGenFunction &CGF) {
447 llvm::Type *Ty = CGF.ConvertType(E->getType());
448 return MakeHalfType(CGF.CGM.getIntrinsic(IntrinsicID, Ty), BuiltinID, E, CGF,
449 {CGF.Builder.getInt32(static_cast<int>(RM))});
450}
451
452static Value *MakeIdp2a(unsigned IntrinsicID, bool IsHi, const CallExpr *E,
453 CodeGenFunction &CGF) {
454 return CGF.Builder.CreateCall(
455 CGF.CGM.getIntrinsic(IntrinsicID),
456 {CGF.EmitScalarExpr(E->getArg(0)), CGF.EmitScalarExpr(E->getArg(1)),
457 CGF.Builder.getInt1(IsHi), CGF.EmitScalarExpr(E->getArg(2))});
458}
459
460} // namespace
461
462Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
463 const CallExpr *E) {
464 switch (BuiltinID) {
465 case NVPTX::BI__nvvm_mulhi_i:
466 case NVPTX::BI__nvvm_mulhi_ui:
467 case NVPTX::BI__nvvm_mulhi_ll:
468 case NVPTX::BI__nvvm_mulhi_ull:
469 return Builder.CreateBinaryIntrinsic(
470 E->getType()->hasSignedIntegerRepresentation() ? Intrinsic::smulh
471 : Intrinsic::umulh,
472 EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1)));
473 case NVPTX::BI__nvvm_atom_add_gen_i:
474 case NVPTX::BI__nvvm_atom_add_gen_l:
475 case NVPTX::BI__nvvm_atom_add_gen_ll:
476 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Add, E,
477 AtomicOrdering::Monotonic);
478
479 case NVPTX::BI__nvvm_atom_sub_gen_i:
480 case NVPTX::BI__nvvm_atom_sub_gen_l:
481 case NVPTX::BI__nvvm_atom_sub_gen_ll:
482 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Sub, E,
483 AtomicOrdering::Monotonic);
484
485 case NVPTX::BI__nvvm_atom_and_gen_i:
486 case NVPTX::BI__nvvm_atom_and_gen_l:
487 case NVPTX::BI__nvvm_atom_and_gen_ll:
488 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::And, E,
489 AtomicOrdering::Monotonic);
490
491 case NVPTX::BI__nvvm_atom_or_gen_i:
492 case NVPTX::BI__nvvm_atom_or_gen_l:
493 case NVPTX::BI__nvvm_atom_or_gen_ll:
494 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Or, E,
495 AtomicOrdering::Monotonic);
496
497 case NVPTX::BI__nvvm_atom_xor_gen_i:
498 case NVPTX::BI__nvvm_atom_xor_gen_l:
499 case NVPTX::BI__nvvm_atom_xor_gen_ll:
500 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Xor, E,
501 AtomicOrdering::Monotonic);
502
503 case NVPTX::BI__nvvm_atom_xchg_gen_i:
504 case NVPTX::BI__nvvm_atom_xchg_gen_l:
505 case NVPTX::BI__nvvm_atom_xchg_gen_ll:
506 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Xchg, E,
507 AtomicOrdering::Monotonic);
508
509 case NVPTX::BI__nvvm_atom_max_gen_i:
510 case NVPTX::BI__nvvm_atom_max_gen_l:
511 case NVPTX::BI__nvvm_atom_max_gen_ll:
512 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Max, E,
513 AtomicOrdering::Monotonic);
514
515 case NVPTX::BI__nvvm_atom_max_gen_ui:
516 case NVPTX::BI__nvvm_atom_max_gen_ul:
517 case NVPTX::BI__nvvm_atom_max_gen_ull:
518 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::UMax, E,
519 AtomicOrdering::Monotonic);
520
521 case NVPTX::BI__nvvm_atom_min_gen_i:
522 case NVPTX::BI__nvvm_atom_min_gen_l:
523 case NVPTX::BI__nvvm_atom_min_gen_ll:
524 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Min, E,
525 AtomicOrdering::Monotonic);
526
527 case NVPTX::BI__nvvm_atom_min_gen_ui:
528 case NVPTX::BI__nvvm_atom_min_gen_ul:
529 case NVPTX::BI__nvvm_atom_min_gen_ull:
530 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::UMin, E,
531 AtomicOrdering::Monotonic);
532
533 case NVPTX::BI__nvvm_atom_cas_gen_us:
534 case NVPTX::BI__nvvm_atom_cas_gen_i:
535 case NVPTX::BI__nvvm_atom_cas_gen_l:
536 case NVPTX::BI__nvvm_atom_cas_gen_ll:
537 // __nvvm_atom_cas_gen_* should return the old value rather than the
538 // success flag.
539 return MakeAtomicCmpXchgValue(*this, E, /*ReturnBool=*/false,
540 AtomicOrdering::Monotonic,
541 AtomicOrdering::Monotonic);
542
543 case NVPTX::BI__nvvm_atom_add_gen_f:
544 case NVPTX::BI__nvvm_atom_add_gen_d: {
545 Address DestAddr = EmitPointerWithAlignment(E->getArg(0));
546 Value *Val = EmitScalarExpr(E->getArg(1));
547
548 llvm::AtomicRMWInst *RMW = Builder.CreateAtomicRMW(
549 llvm::AtomicRMWInst::FAdd, DestAddr, Val, AtomicOrdering::Monotonic);
550
551 // CUDA's atomicAdd() is defined in terms of the native atom.add
552 // instruction, whose denormal behavior is fixed by the hardware and need
553 // not agree with the function's denormal mode. Without this the atomic is
554 // expanded into a CAS loop whenever the two disagree.
555 addAtomicIgnoreDenormalModeMetadata(*this, *RMW, /*AllowHalf=*/true);
556 return RMW;
557 }
558
559 case NVPTX::BI__nvvm_atom_inc_gen_ui:
560 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::UIncWrap, E,
561 AtomicOrdering::Monotonic);
562
563 case NVPTX::BI__nvvm_atom_dec_gen_ui:
564 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::UDecWrap, E,
565 AtomicOrdering::Monotonic);
566
567 case NVPTX::BI__nvvm_ldg_c:
568 case NVPTX::BI__nvvm_ldg_sc:
569 case NVPTX::BI__nvvm_ldg_c2:
570 case NVPTX::BI__nvvm_ldg_sc2:
571 case NVPTX::BI__nvvm_ldg_c4:
572 case NVPTX::BI__nvvm_ldg_sc4:
573 case NVPTX::BI__nvvm_ldg_s:
574 case NVPTX::BI__nvvm_ldg_s2:
575 case NVPTX::BI__nvvm_ldg_s4:
576 case NVPTX::BI__nvvm_ldg_i:
577 case NVPTX::BI__nvvm_ldg_i2:
578 case NVPTX::BI__nvvm_ldg_i4:
579 case NVPTX::BI__nvvm_ldg_l:
580 case NVPTX::BI__nvvm_ldg_l2:
581 case NVPTX::BI__nvvm_ldg_ll:
582 case NVPTX::BI__nvvm_ldg_ll2:
583 case NVPTX::BI__nvvm_ldg_uc:
584 case NVPTX::BI__nvvm_ldg_uc2:
585 case NVPTX::BI__nvvm_ldg_uc4:
586 case NVPTX::BI__nvvm_ldg_us:
587 case NVPTX::BI__nvvm_ldg_us2:
588 case NVPTX::BI__nvvm_ldg_us4:
589 case NVPTX::BI__nvvm_ldg_ui:
590 case NVPTX::BI__nvvm_ldg_ui2:
591 case NVPTX::BI__nvvm_ldg_ui4:
592 case NVPTX::BI__nvvm_ldg_ul:
593 case NVPTX::BI__nvvm_ldg_ul2:
594 case NVPTX::BI__nvvm_ldg_ull:
595 case NVPTX::BI__nvvm_ldg_ull2:
596 case NVPTX::BI__nvvm_ldg_f:
597 case NVPTX::BI__nvvm_ldg_f2:
598 case NVPTX::BI__nvvm_ldg_f4:
599 case NVPTX::BI__nvvm_ldg_d:
600 case NVPTX::BI__nvvm_ldg_d2:
601 // PTX Interoperability section 2.2: "For a vector with an even number of
602 // elements, its alignment is set to number of elements times the alignment
603 // of its member: n*alignof(t)."
604 return MakeLdg(*this, E);
605
606 case NVPTX::BI__nvvm_ldu_c:
607 case NVPTX::BI__nvvm_ldu_sc:
608 case NVPTX::BI__nvvm_ldu_c2:
609 case NVPTX::BI__nvvm_ldu_sc2:
610 case NVPTX::BI__nvvm_ldu_c4:
611 case NVPTX::BI__nvvm_ldu_sc4:
612 case NVPTX::BI__nvvm_ldu_s:
613 case NVPTX::BI__nvvm_ldu_s2:
614 case NVPTX::BI__nvvm_ldu_s4:
615 case NVPTX::BI__nvvm_ldu_i:
616 case NVPTX::BI__nvvm_ldu_i2:
617 case NVPTX::BI__nvvm_ldu_i4:
618 case NVPTX::BI__nvvm_ldu_l:
619 case NVPTX::BI__nvvm_ldu_l2:
620 case NVPTX::BI__nvvm_ldu_ll:
621 case NVPTX::BI__nvvm_ldu_ll2:
622 case NVPTX::BI__nvvm_ldu_uc:
623 case NVPTX::BI__nvvm_ldu_uc2:
624 case NVPTX::BI__nvvm_ldu_uc4:
625 case NVPTX::BI__nvvm_ldu_us:
626 case NVPTX::BI__nvvm_ldu_us2:
627 case NVPTX::BI__nvvm_ldu_us4:
628 case NVPTX::BI__nvvm_ldu_ui:
629 case NVPTX::BI__nvvm_ldu_ui2:
630 case NVPTX::BI__nvvm_ldu_ui4:
631 case NVPTX::BI__nvvm_ldu_ul:
632 case NVPTX::BI__nvvm_ldu_ul2:
633 case NVPTX::BI__nvvm_ldu_ull:
634 case NVPTX::BI__nvvm_ldu_ull2:
635 return MakeLdu(Intrinsic::nvvm_ldu_global_i, *this, E);
636 case NVPTX::BI__nvvm_ldu_f:
637 case NVPTX::BI__nvvm_ldu_f2:
638 case NVPTX::BI__nvvm_ldu_f4:
639 case NVPTX::BI__nvvm_ldu_d:
640 case NVPTX::BI__nvvm_ldu_d2:
641 return MakeLdu(Intrinsic::nvvm_ldu_global_f, *this, E);
642
643 case NVPTX::BI__nvvm_atom_cta_add_gen_i:
644 case NVPTX::BI__nvvm_atom_cta_add_gen_l:
645 case NVPTX::BI__nvvm_atom_cta_add_gen_ll:
646 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::Add,
647 llvm::AtomicScope::Workgroup);
648 case NVPTX::BI__nvvm_atom_sys_add_gen_i:
649 case NVPTX::BI__nvvm_atom_sys_add_gen_l:
650 case NVPTX::BI__nvvm_atom_sys_add_gen_ll:
651 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::Add,
652 llvm::AtomicScope::System);
653 case NVPTX::BI__nvvm_atom_cta_add_gen_f:
654 case NVPTX::BI__nvvm_atom_cta_add_gen_d:
655 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::FAdd,
656 llvm::AtomicScope::Workgroup);
657 case NVPTX::BI__nvvm_atom_sys_add_gen_f:
658 case NVPTX::BI__nvvm_atom_sys_add_gen_d:
659 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::FAdd,
660 llvm::AtomicScope::System);
661 case NVPTX::BI__nvvm_atom_cta_xchg_gen_i:
662 case NVPTX::BI__nvvm_atom_cta_xchg_gen_l:
663 case NVPTX::BI__nvvm_atom_cta_xchg_gen_ll:
664 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::Xchg,
665 llvm::AtomicScope::Workgroup);
666 case NVPTX::BI__nvvm_atom_sys_xchg_gen_i:
667 case NVPTX::BI__nvvm_atom_sys_xchg_gen_l:
668 case NVPTX::BI__nvvm_atom_sys_xchg_gen_ll:
669 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::Xchg,
670 llvm::AtomicScope::System);
671 case NVPTX::BI__nvvm_atom_cta_max_gen_i:
672 case NVPTX::BI__nvvm_atom_cta_max_gen_l:
673 case NVPTX::BI__nvvm_atom_cta_max_gen_ll:
674 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::Max,
675 llvm::AtomicScope::Workgroup);
676 case NVPTX::BI__nvvm_atom_cta_max_gen_ui:
677 case NVPTX::BI__nvvm_atom_cta_max_gen_ul:
678 case NVPTX::BI__nvvm_atom_cta_max_gen_ull:
679 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::UMax,
680 llvm::AtomicScope::Workgroup);
681 case NVPTX::BI__nvvm_atom_sys_max_gen_i:
682 case NVPTX::BI__nvvm_atom_sys_max_gen_l:
683 case NVPTX::BI__nvvm_atom_sys_max_gen_ll:
684 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::Max,
685 llvm::AtomicScope::System);
686 case NVPTX::BI__nvvm_atom_sys_max_gen_ui:
687 case NVPTX::BI__nvvm_atom_sys_max_gen_ul:
688 case NVPTX::BI__nvvm_atom_sys_max_gen_ull:
689 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::UMax,
690 llvm::AtomicScope::System);
691 case NVPTX::BI__nvvm_atom_cta_min_gen_i:
692 case NVPTX::BI__nvvm_atom_cta_min_gen_l:
693 case NVPTX::BI__nvvm_atom_cta_min_gen_ll:
694 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::Min,
695 llvm::AtomicScope::Workgroup);
696 case NVPTX::BI__nvvm_atom_cta_min_gen_ui:
697 case NVPTX::BI__nvvm_atom_cta_min_gen_ul:
698 case NVPTX::BI__nvvm_atom_cta_min_gen_ull:
699 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::UMin,
700 llvm::AtomicScope::Workgroup);
701 case NVPTX::BI__nvvm_atom_sys_min_gen_i:
702 case NVPTX::BI__nvvm_atom_sys_min_gen_l:
703 case NVPTX::BI__nvvm_atom_sys_min_gen_ll:
704 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::Min,
705 llvm::AtomicScope::System);
706 case NVPTX::BI__nvvm_atom_sys_min_gen_ui:
707 case NVPTX::BI__nvvm_atom_sys_min_gen_ul:
708 case NVPTX::BI__nvvm_atom_sys_min_gen_ull:
709 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::UMin,
710 llvm::AtomicScope::System);
711 case NVPTX::BI__nvvm_atom_cta_inc_gen_ui:
712 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::UIncWrap,
713 llvm::AtomicScope::Workgroup);
714 case NVPTX::BI__nvvm_atom_cta_dec_gen_ui:
715 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::UDecWrap,
716 llvm::AtomicScope::Workgroup);
717 case NVPTX::BI__nvvm_atom_sys_inc_gen_ui:
718 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::UIncWrap,
719 llvm::AtomicScope::System);
720 case NVPTX::BI__nvvm_atom_sys_dec_gen_ui:
721 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::UDecWrap,
722 llvm::AtomicScope::System);
723 case NVPTX::BI__nvvm_atom_cta_and_gen_i:
724 case NVPTX::BI__nvvm_atom_cta_and_gen_l:
725 case NVPTX::BI__nvvm_atom_cta_and_gen_ll:
726 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::And,
727 llvm::AtomicScope::Workgroup);
728 case NVPTX::BI__nvvm_atom_sys_and_gen_i:
729 case NVPTX::BI__nvvm_atom_sys_and_gen_l:
730 case NVPTX::BI__nvvm_atom_sys_and_gen_ll:
731 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::And,
732 llvm::AtomicScope::System);
733 case NVPTX::BI__nvvm_atom_cta_or_gen_i:
734 case NVPTX::BI__nvvm_atom_cta_or_gen_l:
735 case NVPTX::BI__nvvm_atom_cta_or_gen_ll:
736 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::Or,
737 llvm::AtomicScope::Workgroup);
738 case NVPTX::BI__nvvm_atom_sys_or_gen_i:
739 case NVPTX::BI__nvvm_atom_sys_or_gen_l:
740 case NVPTX::BI__nvvm_atom_sys_or_gen_ll:
741 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::Or,
742 llvm::AtomicScope::System);
743 case NVPTX::BI__nvvm_atom_cta_xor_gen_i:
744 case NVPTX::BI__nvvm_atom_cta_xor_gen_l:
745 case NVPTX::BI__nvvm_atom_cta_xor_gen_ll:
746 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::Xor,
747 llvm::AtomicScope::Workgroup);
748 case NVPTX::BI__nvvm_atom_sys_xor_gen_i:
749 case NVPTX::BI__nvvm_atom_sys_xor_gen_l:
750 case NVPTX::BI__nvvm_atom_sys_xor_gen_ll:
751 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::Xor,
752 llvm::AtomicScope::System);
753 case NVPTX::BI__nvvm_atom_cta_cas_gen_us:
754 case NVPTX::BI__nvvm_atom_cta_cas_gen_i:
755 case NVPTX::BI__nvvm_atom_cta_cas_gen_l:
756 case NVPTX::BI__nvvm_atom_cta_cas_gen_ll:
757 return MakeScopedAtomicCAS(*this, E, llvm::AtomicScope::Workgroup);
758 case NVPTX::BI__nvvm_atom_sys_cas_gen_us:
759 case NVPTX::BI__nvvm_atom_sys_cas_gen_i:
760 case NVPTX::BI__nvvm_atom_sys_cas_gen_l:
761 case NVPTX::BI__nvvm_atom_sys_cas_gen_ll:
762 return MakeScopedAtomicCAS(*this, E, llvm::AtomicScope::System);
763 case NVPTX::BI__nvvm_match_all_sync_i32p:
764 case NVPTX::BI__nvvm_match_all_sync_i64p: {
765 Value *Mask = EmitScalarExpr(E->getArg(0));
766 Value *Val = EmitScalarExpr(E->getArg(1));
767 Address PredOutPtr = EmitPointerWithAlignment(E->getArg(2));
768 Value *ResultPair = Builder.CreateCall(
769 CGM.getIntrinsic(BuiltinID == NVPTX::BI__nvvm_match_all_sync_i32p
770 ? Intrinsic::nvvm_match_all_sync_i32p
771 : Intrinsic::nvvm_match_all_sync_i64p),
772 {Mask, Val});
773 Value *Pred = Builder.CreateZExt(Builder.CreateExtractValue(ResultPair, 1),
774 PredOutPtr.getElementType());
775 Builder.CreateStore(Pred, PredOutPtr);
776 return Builder.CreateExtractValue(ResultPair, 0);
777 }
778
779 // FP MMA loads
780 case NVPTX::BI__hmma_m16n16k16_ld_a:
781 case NVPTX::BI__hmma_m16n16k16_ld_b:
782 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
783 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
784 case NVPTX::BI__hmma_m32n8k16_ld_a:
785 case NVPTX::BI__hmma_m32n8k16_ld_b:
786 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
787 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
788 case NVPTX::BI__hmma_m8n32k16_ld_a:
789 case NVPTX::BI__hmma_m8n32k16_ld_b:
790 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
791 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
792 // Integer MMA loads.
793 case NVPTX::BI__imma_m16n16k16_ld_a_s8:
794 case NVPTX::BI__imma_m16n16k16_ld_a_u8:
795 case NVPTX::BI__imma_m16n16k16_ld_b_s8:
796 case NVPTX::BI__imma_m16n16k16_ld_b_u8:
797 case NVPTX::BI__imma_m16n16k16_ld_c:
798 case NVPTX::BI__imma_m32n8k16_ld_a_s8:
799 case NVPTX::BI__imma_m32n8k16_ld_a_u8:
800 case NVPTX::BI__imma_m32n8k16_ld_b_s8:
801 case NVPTX::BI__imma_m32n8k16_ld_b_u8:
802 case NVPTX::BI__imma_m32n8k16_ld_c:
803 case NVPTX::BI__imma_m8n32k16_ld_a_s8:
804 case NVPTX::BI__imma_m8n32k16_ld_a_u8:
805 case NVPTX::BI__imma_m8n32k16_ld_b_s8:
806 case NVPTX::BI__imma_m8n32k16_ld_b_u8:
807 case NVPTX::BI__imma_m8n32k16_ld_c:
808 // Sub-integer MMA loads.
809 case NVPTX::BI__imma_m8n8k32_ld_a_s4:
810 case NVPTX::BI__imma_m8n8k32_ld_a_u4:
811 case NVPTX::BI__imma_m8n8k32_ld_b_s4:
812 case NVPTX::BI__imma_m8n8k32_ld_b_u4:
813 case NVPTX::BI__imma_m8n8k32_ld_c:
814 case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
815 case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
816 case NVPTX::BI__bmma_m8n8k128_ld_c:
817 // Double MMA loads.
818 case NVPTX::BI__dmma_m8n8k4_ld_a:
819 case NVPTX::BI__dmma_m8n8k4_ld_b:
820 case NVPTX::BI__dmma_m8n8k4_ld_c:
821 // Alternate float MMA loads.
822 case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
823 case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
824 case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
825 case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
826 case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
827 case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
828 case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
829 case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
830 case NVPTX::BI__mma_tf32_m16n16k8_ld_c: {
831 Address Dst = EmitPointerWithAlignment(E->getArg(0));
832 Value *Src = EmitScalarExpr(E->getArg(1));
833 Value *Ldm = EmitScalarExpr(E->getArg(2));
834 std::optional<llvm::APSInt> isColMajorArg =
835 E->getArg(3)->getIntegerConstantExpr(getContext());
836 if (!isColMajorArg)
837 return nullptr;
838 bool isColMajor = isColMajorArg->getSExtValue();
839 NVPTXMmaLdstInfo II = getNVPTXMmaLdstInfo(BuiltinID);
840 unsigned IID = isColMajor ? II.IID_col : II.IID_row;
841 if (IID == 0)
842 return nullptr;
843
844 Value *Result =
845 Builder.CreateCall(CGM.getIntrinsic(IID, Src->getType()), {Src, Ldm});
846
847 // Save returned values.
848 assert(II.NumResults);
849 if (II.NumResults == 1) {
850 Builder.CreateAlignedStore(Result, Dst.emitRawPointer(*this),
851 CharUnits::fromQuantity(4));
852 } else {
853 for (unsigned i = 0; i < II.NumResults; ++i) {
854 Builder.CreateAlignedStore(
855 Builder.CreateBitCast(Builder.CreateExtractValue(Result, i),
856 Dst.getElementType()),
857 Builder.CreateGEP(Dst.getElementType(), Dst.emitRawPointer(*this),
858 llvm::ConstantInt::get(IntTy, i)),
859 CharUnits::fromQuantity(4));
860 }
861 }
862 return Result;
863 }
864
865 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
866 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
867 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
868 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
869 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
870 case NVPTX::BI__hmma_m8n32k16_st_c_f32:
871 case NVPTX::BI__imma_m16n16k16_st_c_i32:
872 case NVPTX::BI__imma_m32n8k16_st_c_i32:
873 case NVPTX::BI__imma_m8n32k16_st_c_i32:
874 case NVPTX::BI__imma_m8n8k32_st_c_i32:
875 case NVPTX::BI__bmma_m8n8k128_st_c_i32:
876 case NVPTX::BI__dmma_m8n8k4_st_c_f64:
877 case NVPTX::BI__mma_m16n16k8_st_c_f32: {
878 Value *Dst = EmitScalarExpr(E->getArg(0));
879 Address Src = EmitPointerWithAlignment(E->getArg(1));
880 Value *Ldm = EmitScalarExpr(E->getArg(2));
881 std::optional<llvm::APSInt> isColMajorArg =
882 E->getArg(3)->getIntegerConstantExpr(getContext());
883 if (!isColMajorArg)
884 return nullptr;
885 bool isColMajor = isColMajorArg->getSExtValue();
886 NVPTXMmaLdstInfo II = getNVPTXMmaLdstInfo(BuiltinID);
887 unsigned IID = isColMajor ? II.IID_col : II.IID_row;
888 if (IID == 0)
889 return nullptr;
890 Function *Intrinsic =
891 CGM.getIntrinsic(IID, Dst->getType());
892 llvm::Type *ParamType = Intrinsic->getFunctionType()->getParamType(1);
893 SmallVector<Value *, 10> Values = {Dst};
894 for (unsigned i = 0; i < II.NumResults; ++i) {
895 Value *V = Builder.CreateAlignedLoad(
896 Src.getElementType(),
897 Builder.CreateGEP(Src.getElementType(), Src.emitRawPointer(*this),
898 llvm::ConstantInt::get(IntTy, i)),
899 CharUnits::fromQuantity(4));
900 Values.push_back(Builder.CreateBitCast(V, ParamType));
901 }
902 Values.push_back(Ldm);
903 Value *Result = Builder.CreateCall(Intrinsic, Values);
904 return Result;
905 }
906
907 // BI__hmma_m16n16k16_mma_<Dtype><CType>(d, a, b, c, layout, satf) -->
908 // Intrinsic::nvvm_wmma_m16n16k16_mma_sync<layout A,B><DType><CType><Satf>
909 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
910 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
911 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
912 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
913 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
914 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
915 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
916 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
917 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
918 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
919 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
920 case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
921 case NVPTX::BI__imma_m16n16k16_mma_s8:
922 case NVPTX::BI__imma_m16n16k16_mma_u8:
923 case NVPTX::BI__imma_m32n8k16_mma_s8:
924 case NVPTX::BI__imma_m32n8k16_mma_u8:
925 case NVPTX::BI__imma_m8n32k16_mma_s8:
926 case NVPTX::BI__imma_m8n32k16_mma_u8:
927 case NVPTX::BI__imma_m8n8k32_mma_s4:
928 case NVPTX::BI__imma_m8n8k32_mma_u4:
929 case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1:
930 case NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1:
931 case NVPTX::BI__dmma_m8n8k4_mma_f64:
932 case NVPTX::BI__mma_bf16_m16n16k16_mma_f32:
933 case NVPTX::BI__mma_bf16_m8n32k16_mma_f32:
934 case NVPTX::BI__mma_bf16_m32n8k16_mma_f32:
935 case NVPTX::BI__mma_tf32_m16n16k8_mma_f32: {
936 Address Dst = EmitPointerWithAlignment(E->getArg(0));
937 Address SrcA = EmitPointerWithAlignment(E->getArg(1));
938 Address SrcB = EmitPointerWithAlignment(E->getArg(2));
939 Address SrcC = EmitPointerWithAlignment(E->getArg(3));
940 std::optional<llvm::APSInt> LayoutArg =
941 E->getArg(4)->getIntegerConstantExpr(getContext());
942 if (!LayoutArg)
943 return nullptr;
944 int Layout = LayoutArg->getSExtValue();
945 if (Layout < 0 || Layout > 3)
946 return nullptr;
947 llvm::APSInt SatfArg;
948 if (BuiltinID == NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1 ||
949 BuiltinID == NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1)
950 SatfArg = 0; // .b1 does not have satf argument.
951 else if (std::optional<llvm::APSInt> OptSatfArg =
952 E->getArg(5)->getIntegerConstantExpr(getContext()))
953 SatfArg = *OptSatfArg;
954 else
955 return nullptr;
956 bool Satf = SatfArg.getSExtValue();
957 NVPTXMmaInfo MI = getNVPTXMmaInfo(BuiltinID);
958 unsigned IID = MI.getMMAIntrinsic(Layout, Satf);
959 if (IID == 0) // Unsupported combination of Layout/Satf.
960 return nullptr;
961
962 SmallVector<Value *, 24> Values;
963 Function *Intrinsic = CGM.getIntrinsic(IID);
964 llvm::Type *AType = Intrinsic->getFunctionType()->getParamType(0);
965 // Load A
966 for (unsigned i = 0; i < MI.NumEltsA; ++i) {
967 Value *V = Builder.CreateAlignedLoad(
968 SrcA.getElementType(),
969 Builder.CreateGEP(SrcA.getElementType(), SrcA.emitRawPointer(*this),
970 llvm::ConstantInt::get(IntTy, i)),
971 CharUnits::fromQuantity(4));
972 Values.push_back(Builder.CreateBitCast(V, AType));
973 }
974 // Load B
975 llvm::Type *BType = Intrinsic->getFunctionType()->getParamType(MI.NumEltsA);
976 for (unsigned i = 0; i < MI.NumEltsB; ++i) {
977 Value *V = Builder.CreateAlignedLoad(
978 SrcB.getElementType(),
979 Builder.CreateGEP(SrcB.getElementType(), SrcB.emitRawPointer(*this),
980 llvm::ConstantInt::get(IntTy, i)),
981 CharUnits::fromQuantity(4));
982 Values.push_back(Builder.CreateBitCast(V, BType));
983 }
984 // Load C
985 llvm::Type *CType =
986 Intrinsic->getFunctionType()->getParamType(MI.NumEltsA + MI.NumEltsB);
987 for (unsigned i = 0; i < MI.NumEltsC; ++i) {
988 Value *V = Builder.CreateAlignedLoad(
989 SrcC.getElementType(),
990 Builder.CreateGEP(SrcC.getElementType(), SrcC.emitRawPointer(*this),
991 llvm::ConstantInt::get(IntTy, i)),
992 CharUnits::fromQuantity(4));
993 Values.push_back(Builder.CreateBitCast(V, CType));
994 }
995 Value *Result = Builder.CreateCall(Intrinsic, Values);
996 llvm::Type *DType = Dst.getElementType();
997 for (unsigned i = 0; i < MI.NumEltsD; ++i)
998 Builder.CreateAlignedStore(
999 Builder.CreateBitCast(Builder.CreateExtractValue(Result, i), DType),
1000 Builder.CreateGEP(Dst.getElementType(), Dst.emitRawPointer(*this),
1001 llvm::ConstantInt::get(IntTy, i)),
1002 CharUnits::fromQuantity(4));
1003 return Result;
1004 }
1005 // The following builtins require half type support
1006 case NVPTX::BI__nvvm_ex2_approx_f16:
1007 return MakeHalfType(
1008 CGM.getIntrinsic(Intrinsic::nvvm_ex2_approx, Builder.getHalfTy()),
1009 BuiltinID, E, *this);
1010 case NVPTX::BI__nvvm_ex2_approx_f16x2:
1011 return MakeHalfType(
1012 CGM.getIntrinsic(Intrinsic::nvvm_ex2_approx,
1013 FixedVectorType::get(Builder.getHalfTy(), 2)),
1014 BuiltinID, E, *this);
1015 case NVPTX::BI__nvvm_ff2f16x2_rn:
1016 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rn, BuiltinID, E, *this);
1017 case NVPTX::BI__nvvm_ff2f16x2_rn_relu:
1018 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rn_relu, BuiltinID, E, *this);
1019 case NVPTX::BI__nvvm_ff2f16x2_rz:
1020 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rz, BuiltinID, E, *this);
1021 case NVPTX::BI__nvvm_ff2f16x2_rz_relu:
1022 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rz_relu, BuiltinID, E, *this);
1023
1024#define PZO_CVT(cvt) \
1025 case NVPTX::BI__nvvm_##cvt##_pzo: \
1026 return MakeHalfType(Intrinsic::nvvm_##cvt, BuiltinID, E, *this, \
1027 {Builder.getTrue()})
1028
1029 PZO_CVT(ff2f16x2_rn);
1030 PZO_CVT(ff2f16x2_rn_relu);
1031 PZO_CVT(ff2f16x2_rz);
1032 PZO_CVT(ff2f16x2_rz_relu);
1033 PZO_CVT(ff2f16x2_rn_satfinite);
1034 PZO_CVT(ff2f16x2_rn_relu_satfinite);
1035 PZO_CVT(ff2f16x2_rz_satfinite);
1036 PZO_CVT(ff2f16x2_rz_relu_satfinite);
1037 PZO_CVT(ff2bf16x2_rn);
1038 PZO_CVT(ff2bf16x2_rn_relu);
1039 PZO_CVT(ff2bf16x2_rz);
1040 PZO_CVT(ff2bf16x2_rz_relu);
1041 PZO_CVT(ff2bf16x2_rn_satfinite);
1042 PZO_CVT(ff2bf16x2_rn_relu_satfinite);
1043 PZO_CVT(ff2bf16x2_rz_satfinite);
1044 PZO_CVT(ff2bf16x2_rz_relu_satfinite);
1045 PZO_CVT(f2f16_rn);
1046 PZO_CVT(f2f16_rn_relu);
1047 PZO_CVT(f2f16_rz);
1048 PZO_CVT(f2f16_rz_relu);
1049 PZO_CVT(f2f16_rn_satfinite);
1050 PZO_CVT(f2f16_rn_relu_satfinite);
1051 PZO_CVT(f2f16_rz_satfinite);
1052 PZO_CVT(f2f16_rz_relu_satfinite);
1053 PZO_CVT(f2bf16_rn);
1054 PZO_CVT(f2bf16_rn_relu);
1055 PZO_CVT(f2bf16_rz);
1056 PZO_CVT(f2bf16_rz_relu);
1057 PZO_CVT(f2bf16_rn_satfinite);
1058 PZO_CVT(f2bf16_rn_relu_satfinite);
1059 PZO_CVT(f2bf16_rz_satfinite);
1060 PZO_CVT(f2bf16_rz_relu_satfinite);
1061
1062 PZO_CVT(ff_to_e4m3x2_rn);
1063 PZO_CVT(ff_to_e4m3x2_rn_relu);
1064 PZO_CVT(ff_to_e4m3x2_rz);
1065 PZO_CVT(ff_to_e4m3x2_rz_relu);
1066 PZO_CVT(ff_to_e5m2x2_rn);
1067 PZO_CVT(ff_to_e5m2x2_rn_relu);
1068 PZO_CVT(ff_to_e5m2x2_rz);
1069 PZO_CVT(ff_to_e5m2x2_rz_relu);
1070 PZO_CVT(f16x2_to_e4m3x2_rn);
1071 PZO_CVT(f16x2_to_e4m3x2_rn_relu);
1072 PZO_CVT(f16x2_to_e4m3x2_rz);
1073 PZO_CVT(f16x2_to_e4m3x2_rz_relu);
1074 PZO_CVT(f16x2_to_e5m2x2_rn);
1075 PZO_CVT(f16x2_to_e5m2x2_rn_relu);
1076 PZO_CVT(f16x2_to_e5m2x2_rz);
1077 PZO_CVT(f16x2_to_e5m2x2_rz_relu);
1078 PZO_CVT(bf16x2_to_e4m3x2_rn_satfinite);
1079 PZO_CVT(bf16x2_to_e4m3x2_rn_relu_satfinite);
1080 PZO_CVT(bf16x2_to_e4m3x2_rz_satfinite);
1081 PZO_CVT(bf16x2_to_e4m3x2_rz_relu_satfinite);
1082 PZO_CVT(bf16x2_to_e5m2x2_rn_satfinite);
1083 PZO_CVT(bf16x2_to_e5m2x2_rn_relu_satfinite);
1084 PZO_CVT(bf16x2_to_e5m2x2_rz_satfinite);
1085 PZO_CVT(bf16x2_to_e5m2x2_rz_relu_satfinite);
1086
1087 PZO_CVT(ff_to_e4m3x2_rn_scale_n1_ue8m0);
1088 PZO_CVT(ff_to_e4m3x2_rn_relu_scale_n1_ue8m0);
1089 PZO_CVT(ff_to_e4m3x2_rz_scale_n1_ue8m0);
1090 PZO_CVT(ff_to_e4m3x2_rz_relu_scale_n1_ue8m0);
1091 PZO_CVT(ff_to_e5m2x2_rn_scale_n1_ue8m0);
1092 PZO_CVT(ff_to_e5m2x2_rn_relu_scale_n1_ue8m0);
1093 PZO_CVT(ff_to_e5m2x2_rz_scale_n1_ue8m0);
1094 PZO_CVT(ff_to_e5m2x2_rz_relu_scale_n1_ue8m0);
1095 PZO_CVT(f16x2_to_e4m3x2_rn_scale_n1_ue8m0);
1096 PZO_CVT(f16x2_to_e4m3x2_rn_relu_scale_n1_ue8m0);
1097 PZO_CVT(f16x2_to_e4m3x2_rz_scale_n1_ue8m0);
1098 PZO_CVT(f16x2_to_e4m3x2_rz_relu_scale_n1_ue8m0);
1099 PZO_CVT(f16x2_to_e5m2x2_rn_scale_n1_ue8m0);
1100 PZO_CVT(f16x2_to_e5m2x2_rn_relu_scale_n1_ue8m0);
1101 PZO_CVT(f16x2_to_e5m2x2_rz_scale_n1_ue8m0);
1102 PZO_CVT(f16x2_to_e5m2x2_rz_relu_scale_n1_ue8m0);
1103 PZO_CVT(bf16x2_to_e4m3x2_rn_satfinite_scale_n1_ue8m0);
1104 PZO_CVT(bf16x2_to_e4m3x2_rn_relu_satfinite_scale_n1_ue8m0);
1105 PZO_CVT(bf16x2_to_e4m3x2_rz_satfinite_scale_n1_ue8m0);
1106 PZO_CVT(bf16x2_to_e4m3x2_rz_relu_satfinite_scale_n1_ue8m0);
1107 PZO_CVT(bf16x2_to_e5m2x2_rn_satfinite_scale_n1_ue8m0);
1108 PZO_CVT(bf16x2_to_e5m2x2_rn_relu_satfinite_scale_n1_ue8m0);
1109 PZO_CVT(bf16x2_to_e5m2x2_rz_satfinite_scale_n1_ue8m0);
1110 PZO_CVT(bf16x2_to_e5m2x2_rz_relu_satfinite_scale_n1_ue8m0);
1111
1112 PZO_CVT(ff_to_e2m3x2_rn_satfinite);
1113 PZO_CVT(ff_to_e2m3x2_rn_relu_satfinite);
1114 PZO_CVT(ff_to_e2m3x2_rz_satfinite);
1115 PZO_CVT(ff_to_e2m3x2_rz_relu_satfinite);
1116 PZO_CVT(ff_to_e3m2x2_rn_satfinite);
1117 PZO_CVT(ff_to_e3m2x2_rn_relu_satfinite);
1118 PZO_CVT(ff_to_e3m2x2_rz_satfinite);
1119 PZO_CVT(ff_to_e3m2x2_rz_relu_satfinite);
1120 PZO_CVT(f16x2_to_e2m3x2_rn_satfinite);
1121 PZO_CVT(f16x2_to_e2m3x2_rn_relu_satfinite);
1122 PZO_CVT(f16x2_to_e2m3x2_rz_satfinite);
1123 PZO_CVT(f16x2_to_e2m3x2_rz_relu_satfinite);
1124 PZO_CVT(f16x2_to_e3m2x2_rn_satfinite);
1125 PZO_CVT(f16x2_to_e3m2x2_rn_relu_satfinite);
1126 PZO_CVT(f16x2_to_e3m2x2_rz_satfinite);
1127 PZO_CVT(f16x2_to_e3m2x2_rz_relu_satfinite);
1128 PZO_CVT(bf16x2_to_e2m3x2_rn_satfinite);
1129 PZO_CVT(bf16x2_to_e2m3x2_rn_relu_satfinite);
1130 PZO_CVT(bf16x2_to_e2m3x2_rz_satfinite);
1131 PZO_CVT(bf16x2_to_e2m3x2_rz_relu_satfinite);
1132 PZO_CVT(bf16x2_to_e3m2x2_rn_satfinite);
1133 PZO_CVT(bf16x2_to_e3m2x2_rn_relu_satfinite);
1134 PZO_CVT(bf16x2_to_e3m2x2_rz_satfinite);
1135 PZO_CVT(bf16x2_to_e3m2x2_rz_relu_satfinite);
1136
1137 PZO_CVT(ff_to_e2m3x2_rn_satfinite_scale_n1_ue8m0);
1138 PZO_CVT(ff_to_e2m3x2_rn_relu_satfinite_scale_n1_ue8m0);
1139 PZO_CVT(ff_to_e2m3x2_rz_satfinite_scale_n1_ue8m0);
1140 PZO_CVT(ff_to_e2m3x2_rz_relu_satfinite_scale_n1_ue8m0);
1141 PZO_CVT(ff_to_e3m2x2_rn_satfinite_scale_n1_ue8m0);
1142 PZO_CVT(ff_to_e3m2x2_rn_relu_satfinite_scale_n1_ue8m0);
1143 PZO_CVT(ff_to_e3m2x2_rz_satfinite_scale_n1_ue8m0);
1144 PZO_CVT(ff_to_e3m2x2_rz_relu_satfinite_scale_n1_ue8m0);
1145 PZO_CVT(f16x2_to_e2m3x2_rn_satfinite_scale_n1_ue8m0);
1146 PZO_CVT(f16x2_to_e2m3x2_rn_relu_satfinite_scale_n1_ue8m0);
1147 PZO_CVT(f16x2_to_e2m3x2_rz_satfinite_scale_n1_ue8m0);
1148 PZO_CVT(f16x2_to_e2m3x2_rz_relu_satfinite_scale_n1_ue8m0);
1149 PZO_CVT(f16x2_to_e3m2x2_rn_satfinite_scale_n1_ue8m0);
1150 PZO_CVT(f16x2_to_e3m2x2_rn_relu_satfinite_scale_n1_ue8m0);
1151 PZO_CVT(f16x2_to_e3m2x2_rz_satfinite_scale_n1_ue8m0);
1152 PZO_CVT(f16x2_to_e3m2x2_rz_relu_satfinite_scale_n1_ue8m0);
1153 PZO_CVT(bf16x2_to_e2m3x2_rn_satfinite_scale_n1_ue8m0);
1154 PZO_CVT(bf16x2_to_e2m3x2_rn_relu_satfinite_scale_n1_ue8m0);
1155 PZO_CVT(bf16x2_to_e2m3x2_rz_satfinite_scale_n1_ue8m0);
1156 PZO_CVT(bf16x2_to_e2m3x2_rz_relu_satfinite_scale_n1_ue8m0);
1157 PZO_CVT(bf16x2_to_e3m2x2_rn_satfinite_scale_n1_ue8m0);
1158 PZO_CVT(bf16x2_to_e3m2x2_rn_relu_satfinite_scale_n1_ue8m0);
1159 PZO_CVT(bf16x2_to_e3m2x2_rz_satfinite_scale_n1_ue8m0);
1160 PZO_CVT(bf16x2_to_e3m2x2_rz_relu_satfinite_scale_n1_ue8m0);
1161
1162 PZO_CVT(ff_to_e2m1x2_rn_satfinite);
1163 PZO_CVT(ff_to_e2m1x2_rn_relu_satfinite);
1164 PZO_CVT(ff_to_e2m1x2_rz_satfinite);
1165 PZO_CVT(ff_to_e2m1x2_rz_relu_satfinite);
1166 PZO_CVT(f16x2_to_e2m1x2_rn_satfinite);
1167 PZO_CVT(f16x2_to_e2m1x2_rn_relu_satfinite);
1168 PZO_CVT(f16x2_to_e2m1x2_rz_satfinite);
1169 PZO_CVT(f16x2_to_e2m1x2_rz_relu_satfinite);
1170 PZO_CVT(bf16x2_to_e2m1x2_rn_satfinite);
1171 PZO_CVT(bf16x2_to_e2m1x2_rn_relu_satfinite);
1172 PZO_CVT(bf16x2_to_e2m1x2_rz_satfinite);
1173 PZO_CVT(bf16x2_to_e2m1x2_rz_relu_satfinite);
1174
1175 PZO_CVT(ff_to_e2m1x2_rn_satfinite_scale_n1_ue8m0);
1176 PZO_CVT(ff_to_e2m1x2_rn_relu_satfinite_scale_n1_ue8m0);
1177 PZO_CVT(ff_to_e2m1x2_rz_satfinite_scale_n1_ue8m0);
1178 PZO_CVT(ff_to_e2m1x2_rz_relu_satfinite_scale_n1_ue8m0);
1179 PZO_CVT(f16x2_to_e2m1x2_rn_satfinite_scale_n1_ue8m0);
1180 PZO_CVT(f16x2_to_e2m1x2_rn_relu_satfinite_scale_n1_ue8m0);
1181 PZO_CVT(f16x2_to_e2m1x2_rz_satfinite_scale_n1_ue8m0);
1182 PZO_CVT(f16x2_to_e2m1x2_rz_relu_satfinite_scale_n1_ue8m0);
1183 PZO_CVT(bf16x2_to_e2m1x2_rn_satfinite_scale_n1_ue8m0);
1184 PZO_CVT(bf16x2_to_e2m1x2_rn_relu_satfinite_scale_n1_ue8m0);
1185 PZO_CVT(bf16x2_to_e2m1x2_rz_satfinite_scale_n1_ue8m0);
1186 PZO_CVT(bf16x2_to_e2m1x2_rz_relu_satfinite_scale_n1_ue8m0);
1187
1188#undef PZO_CVT
1189
1190 case NVPTX::BI__nvvm_fma_rn_f16:
1191 return MakeHalfType(Intrinsic::nvvm_fma_rn_f16, BuiltinID, E, *this);
1192 case NVPTX::BI__nvvm_fma_rn_f16x2:
1193 return MakeHalfType(Intrinsic::nvvm_fma_rn_f16x2, BuiltinID, E, *this);
1194 case NVPTX::BI__nvvm_fma_rn_ftz_f16:
1195 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_f16, BuiltinID, E, *this);
1196 case NVPTX::BI__nvvm_fma_rn_ftz_f16x2:
1197 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_f16x2, BuiltinID, E, *this);
1198 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16:
1199 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_relu_f16, BuiltinID, E,
1200 *this);
1201 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16x2:
1202 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_relu_f16x2, BuiltinID, E,
1203 *this);
1204 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16:
1205 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_sat_f16, BuiltinID, E,
1206 *this);
1207 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16x2:
1208 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_sat_f16x2, BuiltinID, E,
1209 *this);
1210 case NVPTX::BI__nvvm_fma_rn_relu_f16:
1211 return MakeHalfType(Intrinsic::nvvm_fma_rn_relu_f16, BuiltinID, E, *this);
1212 case NVPTX::BI__nvvm_fma_rn_relu_f16x2:
1213 return MakeHalfType(Intrinsic::nvvm_fma_rn_relu_f16x2, BuiltinID, E, *this);
1214 case NVPTX::BI__nvvm_fma_rn_sat_f16:
1215 return MakeHalfType(Intrinsic::nvvm_fma_rn_sat_f16, BuiltinID, E, *this);
1216 case NVPTX::BI__nvvm_fma_rn_sat_f16x2:
1217 return MakeHalfType(Intrinsic::nvvm_fma_rn_sat_f16x2, BuiltinID, E, *this);
1218 case NVPTX::BI__nvvm_fma_rn_oob_f16:
1219 return MakeFMAOOB(Intrinsic::nvvm_fma_rn_oob, Builder.getHalfTy(), E,
1220 *this);
1221 case NVPTX::BI__nvvm_fma_rn_oob_f16x2:
1222 return MakeFMAOOB(Intrinsic::nvvm_fma_rn_oob,
1223 llvm::FixedVectorType::get(Builder.getHalfTy(), 2), E,
1224 *this);
1225 case NVPTX::BI__nvvm_fma_rn_oob_bf16:
1226 return MakeFMAOOB(Intrinsic::nvvm_fma_rn_oob, Builder.getBFloatTy(), E,
1227 *this);
1228 case NVPTX::BI__nvvm_fma_rn_oob_bf16x2:
1229 return MakeFMAOOB(Intrinsic::nvvm_fma_rn_oob,
1230 llvm::FixedVectorType::get(Builder.getBFloatTy(), 2), E,
1231 *this);
1232 case NVPTX::BI__nvvm_fma_rn_oob_relu_f16:
1233 return MakeFMAOOB(Intrinsic::nvvm_fma_rn_oob_relu, Builder.getHalfTy(), E,
1234 *this);
1235 case NVPTX::BI__nvvm_fma_rn_oob_relu_f16x2:
1236 return MakeFMAOOB(Intrinsic::nvvm_fma_rn_oob_relu,
1237 llvm::FixedVectorType::get(Builder.getHalfTy(), 2), E,
1238 *this);
1239 case NVPTX::BI__nvvm_fma_rn_oob_relu_bf16:
1240 return MakeFMAOOB(Intrinsic::nvvm_fma_rn_oob_relu, Builder.getBFloatTy(), E,
1241 *this);
1242 case NVPTX::BI__nvvm_fma_rn_oob_relu_bf16x2:
1243 return MakeFMAOOB(Intrinsic::nvvm_fma_rn_oob_relu,
1244 llvm::FixedVectorType::get(Builder.getBFloatTy(), 2), E,
1245 *this);
1246 case NVPTX::BI__nvvm_fmax_f16:
1247 return MakeHalfType(Intrinsic::nvvm_fmax_f16, BuiltinID, E, *this);
1248 case NVPTX::BI__nvvm_fmax_f16x2:
1249 return MakeHalfType(Intrinsic::nvvm_fmax_f16x2, BuiltinID, E, *this);
1250 case NVPTX::BI__nvvm_fmax_ftz_f16:
1251 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_f16, BuiltinID, E, *this);
1252 case NVPTX::BI__nvvm_fmax_ftz_f16x2:
1253 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_f16x2, BuiltinID, E, *this);
1254 case NVPTX::BI__nvvm_fmax_ftz_nan_f16:
1255 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_f16, BuiltinID, E, *this);
1256 case NVPTX::BI__nvvm_fmax_ftz_nan_f16x2:
1257 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_f16x2, BuiltinID, E,
1258 *this);
1259 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16:
1260 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f16, BuiltinID,
1261 E, *this);
1262 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16x2:
1263 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f16x2,
1264 BuiltinID, E, *this);
1265 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16:
1266 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_xorsign_abs_f16, BuiltinID, E,
1267 *this);
1268 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16x2:
1269 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_xorsign_abs_f16x2, BuiltinID,
1270 E, *this);
1271 case NVPTX::BI__nvvm_fmax_nan_f16:
1272 return MakeHalfType(Intrinsic::nvvm_fmax_nan_f16, BuiltinID, E, *this);
1273 case NVPTX::BI__nvvm_fmax_nan_f16x2:
1274 return MakeHalfType(Intrinsic::nvvm_fmax_nan_f16x2, BuiltinID, E, *this);
1275 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16:
1276 return MakeHalfType(Intrinsic::nvvm_fmax_nan_xorsign_abs_f16, BuiltinID, E,
1277 *this);
1278 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16x2:
1279 return MakeHalfType(Intrinsic::nvvm_fmax_nan_xorsign_abs_f16x2, BuiltinID,
1280 E, *this);
1281 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16:
1282 return MakeHalfType(Intrinsic::nvvm_fmax_xorsign_abs_f16, BuiltinID, E,
1283 *this);
1284 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16x2:
1285 return MakeHalfType(Intrinsic::nvvm_fmax_xorsign_abs_f16x2, BuiltinID, E,
1286 *this);
1287 case NVPTX::BI__nvvm_fmin_f16:
1288 return MakeHalfType(Intrinsic::nvvm_fmin_f16, BuiltinID, E, *this);
1289 case NVPTX::BI__nvvm_fmin_f16x2:
1290 return MakeHalfType(Intrinsic::nvvm_fmin_f16x2, BuiltinID, E, *this);
1291 case NVPTX::BI__nvvm_fmin_ftz_f16:
1292 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_f16, BuiltinID, E, *this);
1293 case NVPTX::BI__nvvm_fmin_ftz_f16x2:
1294 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_f16x2, BuiltinID, E, *this);
1295 case NVPTX::BI__nvvm_fmin_ftz_nan_f16:
1296 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_f16, BuiltinID, E, *this);
1297 case NVPTX::BI__nvvm_fmin_ftz_nan_f16x2:
1298 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_f16x2, BuiltinID, E,
1299 *this);
1300 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16:
1301 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f16, BuiltinID,
1302 E, *this);
1303 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16x2:
1304 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f16x2,
1305 BuiltinID, E, *this);
1306 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16:
1307 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_xorsign_abs_f16, BuiltinID, E,
1308 *this);
1309 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16x2:
1310 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_xorsign_abs_f16x2, BuiltinID,
1311 E, *this);
1312 case NVPTX::BI__nvvm_fmin_nan_f16:
1313 return MakeHalfType(Intrinsic::nvvm_fmin_nan_f16, BuiltinID, E, *this);
1314 case NVPTX::BI__nvvm_fmin_nan_f16x2:
1315 return MakeHalfType(Intrinsic::nvvm_fmin_nan_f16x2, BuiltinID, E, *this);
1316 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16:
1317 return MakeHalfType(Intrinsic::nvvm_fmin_nan_xorsign_abs_f16, BuiltinID, E,
1318 *this);
1319 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16x2:
1320 return MakeHalfType(Intrinsic::nvvm_fmin_nan_xorsign_abs_f16x2, BuiltinID,
1321 E, *this);
1322 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16:
1323 return MakeHalfType(Intrinsic::nvvm_fmin_xorsign_abs_f16, BuiltinID, E,
1324 *this);
1325 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16x2:
1326 return MakeHalfType(Intrinsic::nvvm_fmin_xorsign_abs_f16x2, BuiltinID, E,
1327 *this);
1328 case NVPTX::BI__nvvm_fabs_f:
1329 case NVPTX::BI__nvvm_abs_bf16:
1330 case NVPTX::BI__nvvm_abs_bf16x2:
1331 case NVPTX::BI__nvvm_fabs_f16:
1332 case NVPTX::BI__nvvm_fabs_f16x2:
1333 return Builder.CreateUnaryIntrinsic(Intrinsic::nvvm_fabs,
1334 EmitScalarExpr(E->getArg(0)));
1335 case NVPTX::BI__nvvm_fabs_ftz_f:
1336 case NVPTX::BI__nvvm_fabs_ftz_f16:
1337 case NVPTX::BI__nvvm_fabs_ftz_f16x2:
1338 return Builder.CreateUnaryIntrinsic(Intrinsic::nvvm_fabs_ftz,
1339 EmitScalarExpr(E->getArg(0)));
1340 case NVPTX::BI__nvvm_fabs_d:
1341 return Builder.CreateFAbs(EmitScalarExpr(E->getArg(0)));
1342 case NVPTX::BI__nvvm_ex2_approx_d:
1343 case NVPTX::BI__nvvm_ex2_approx_f:
1344 return Builder.CreateUnaryIntrinsic(Intrinsic::nvvm_ex2_approx,
1345 EmitScalarExpr(E->getArg(0)));
1346 case NVPTX::BI__nvvm_ex2_approx_ftz_f:
1347 return Builder.CreateUnaryIntrinsic(Intrinsic::nvvm_ex2_approx_ftz,
1348 EmitScalarExpr(E->getArg(0)));
1349 case NVPTX::BI__nvvm_add_rn_f:
1350 case NVPTX::BI__nvvm_add_rn_d:
1351 return MakeFPArith(Intrinsic::nvvm_fadd, APFloat::rmNearestTiesToEven,
1352 BuiltinID, E, *this);
1353 case NVPTX::BI__nvvm_add_rz_f:
1354 case NVPTX::BI__nvvm_add_rz_d:
1355 return MakeFPArith(Intrinsic::nvvm_fadd, APFloat::rmTowardZero, BuiltinID,
1356 E, *this);
1357 case NVPTX::BI__nvvm_add_rm_f:
1358 case NVPTX::BI__nvvm_add_rm_d:
1359 return MakeFPArith(Intrinsic::nvvm_fadd, APFloat::rmTowardNegative,
1360 BuiltinID, E, *this);
1361 case NVPTX::BI__nvvm_add_rp_f:
1362 case NVPTX::BI__nvvm_add_rp_d:
1363 return MakeFPArith(Intrinsic::nvvm_fadd, APFloat::rmTowardPositive,
1364 BuiltinID, E, *this);
1365 case NVPTX::BI__nvvm_add_rn_ftz_f:
1366 return MakeFPArith(Intrinsic::nvvm_fadd_ftz, APFloat::rmNearestTiesToEven,
1367 BuiltinID, E, *this);
1368 case NVPTX::BI__nvvm_add_rz_ftz_f:
1369 return MakeFPArith(Intrinsic::nvvm_fadd_ftz, APFloat::rmTowardZero,
1370 BuiltinID, E, *this);
1371 case NVPTX::BI__nvvm_add_rm_ftz_f:
1372 return MakeFPArith(Intrinsic::nvvm_fadd_ftz, APFloat::rmTowardNegative,
1373 BuiltinID, E, *this);
1374 case NVPTX::BI__nvvm_add_rp_ftz_f:
1375 return MakeFPArith(Intrinsic::nvvm_fadd_ftz, APFloat::rmTowardPositive,
1376 BuiltinID, E, *this);
1377 case NVPTX::BI__nvvm_add_rn_sat_f:
1378 case NVPTX::BI__nvvm_add_rn_sat_f16:
1379 case NVPTX::BI__nvvm_add_rn_sat_v2f16:
1380 return MakeFPArith(Intrinsic::nvvm_fadd_sat, APFloat::rmNearestTiesToEven,
1381 BuiltinID, E, *this);
1382 case NVPTX::BI__nvvm_add_rz_sat_f:
1383 return MakeFPArith(Intrinsic::nvvm_fadd_sat, APFloat::rmTowardZero,
1384 BuiltinID, E, *this);
1385 case NVPTX::BI__nvvm_add_rm_sat_f:
1386 return MakeFPArith(Intrinsic::nvvm_fadd_sat, APFloat::rmTowardNegative,
1387 BuiltinID, E, *this);
1388 case NVPTX::BI__nvvm_add_rp_sat_f:
1389 return MakeFPArith(Intrinsic::nvvm_fadd_sat, APFloat::rmTowardPositive,
1390 BuiltinID, E, *this);
1391 case NVPTX::BI__nvvm_add_rn_ftz_sat_f:
1392 case NVPTX::BI__nvvm_add_rn_ftz_sat_f16:
1393 case NVPTX::BI__nvvm_add_rn_ftz_sat_v2f16:
1394 return MakeFPArith(Intrinsic::nvvm_fadd_ftz_sat,
1395 APFloat::rmNearestTiesToEven, BuiltinID, E, *this);
1396 case NVPTX::BI__nvvm_add_rz_ftz_sat_f:
1397 return MakeFPArith(Intrinsic::nvvm_fadd_ftz_sat, APFloat::rmTowardZero,
1398 BuiltinID, E, *this);
1399 case NVPTX::BI__nvvm_add_rm_ftz_sat_f:
1400 return MakeFPArith(Intrinsic::nvvm_fadd_ftz_sat, APFloat::rmTowardNegative,
1401 BuiltinID, E, *this);
1402 case NVPTX::BI__nvvm_add_rp_ftz_sat_f:
1403 return MakeFPArith(Intrinsic::nvvm_fadd_ftz_sat, APFloat::rmTowardPositive,
1404 BuiltinID, E, *this);
1405 case NVPTX::BI__nvvm_mul_rn_f:
1406 case NVPTX::BI__nvvm_mul_rn_d:
1407 return MakeFPArith(Intrinsic::nvvm_fmul, APFloat::rmNearestTiesToEven,
1408 BuiltinID, E, *this);
1409 case NVPTX::BI__nvvm_mul_rz_f:
1410 case NVPTX::BI__nvvm_mul_rz_d:
1411 return MakeFPArith(Intrinsic::nvvm_fmul, APFloat::rmTowardZero, BuiltinID,
1412 E, *this);
1413 case NVPTX::BI__nvvm_mul_rm_f:
1414 case NVPTX::BI__nvvm_mul_rm_d:
1415 return MakeFPArith(Intrinsic::nvvm_fmul, APFloat::rmTowardNegative,
1416 BuiltinID, E, *this);
1417 case NVPTX::BI__nvvm_mul_rp_f:
1418 case NVPTX::BI__nvvm_mul_rp_d:
1419 return MakeFPArith(Intrinsic::nvvm_fmul, APFloat::rmTowardPositive,
1420 BuiltinID, E, *this);
1421 case NVPTX::BI__nvvm_mul_rn_ftz_f:
1422 return MakeFPArith(Intrinsic::nvvm_fmul_ftz, APFloat::rmNearestTiesToEven,
1423 BuiltinID, E, *this);
1424 case NVPTX::BI__nvvm_mul_rz_ftz_f:
1425 return MakeFPArith(Intrinsic::nvvm_fmul_ftz, APFloat::rmTowardZero,
1426 BuiltinID, E, *this);
1427 case NVPTX::BI__nvvm_mul_rm_ftz_f:
1428 return MakeFPArith(Intrinsic::nvvm_fmul_ftz, APFloat::rmTowardNegative,
1429 BuiltinID, E, *this);
1430 case NVPTX::BI__nvvm_mul_rp_ftz_f:
1431 return MakeFPArith(Intrinsic::nvvm_fmul_ftz, APFloat::rmTowardPositive,
1432 BuiltinID, E, *this);
1433 case NVPTX::BI__nvvm_mul_rn_sat_f16:
1434 case NVPTX::BI__nvvm_mul_rn_sat_v2f16:
1435 return MakeFPArith(Intrinsic::nvvm_fmul_sat, APFloat::rmNearestTiesToEven,
1436 BuiltinID, E, *this);
1437 case NVPTX::BI__nvvm_mul_rn_ftz_sat_f16:
1438 case NVPTX::BI__nvvm_mul_rn_ftz_sat_v2f16:
1439 return MakeFPArith(Intrinsic::nvvm_fmul_ftz_sat,
1440 APFloat::rmNearestTiesToEven, BuiltinID, E, *this);
1441 case NVPTX::BI__nvvm_ldg_h:
1442 case NVPTX::BI__nvvm_ldg_h2:
1443 return MakeLdg(*this, E);
1444 case NVPTX::BI__nvvm_ldu_h:
1445 case NVPTX::BI__nvvm_ldu_h2:
1446 return MakeLdu(Intrinsic::nvvm_ldu_global_f, *this, E);
1447 case NVPTX::BI__nvvm_cp_async_ca_shared_global_4:
1448 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_4,
1449 Intrinsic::nvvm_cp_async_ca_shared_global_4_s, *this, E,
1450 4);
1451 case NVPTX::BI__nvvm_cp_async_ca_shared_global_8:
1452 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_8,
1453 Intrinsic::nvvm_cp_async_ca_shared_global_8_s, *this, E,
1454 8);
1455 case NVPTX::BI__nvvm_cp_async_ca_shared_global_16:
1456 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_16,
1457 Intrinsic::nvvm_cp_async_ca_shared_global_16_s, *this, E,
1458 16);
1459 case NVPTX::BI__nvvm_cp_async_cg_shared_global_16:
1460 return MakeCpAsync(Intrinsic::nvvm_cp_async_cg_shared_global_16,
1461 Intrinsic::nvvm_cp_async_cg_shared_global_16_s, *this, E,
1462 16);
1463 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_x:
1464 return Builder.CreateCall(
1465 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_clusterid_x));
1466 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_y:
1467 return Builder.CreateCall(
1468 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_clusterid_y));
1469 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_z:
1470 return Builder.CreateCall(
1471 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_clusterid_z));
1472 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_w:
1473 return Builder.CreateCall(
1474 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_clusterid_w));
1475 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_x:
1476 return Builder.CreateCall(
1477 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_nclusterid_x));
1478 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_y:
1479 return Builder.CreateCall(
1480 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_nclusterid_y));
1481 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_z:
1482 return Builder.CreateCall(
1483 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_nclusterid_z));
1484 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_w:
1485 return Builder.CreateCall(
1486 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_nclusterid_w));
1487 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_x:
1488 return Builder.CreateCall(
1489 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_ctaid_x));
1490 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_y:
1491 return Builder.CreateCall(
1492 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_ctaid_y));
1493 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_z:
1494 return Builder.CreateCall(
1495 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_ctaid_z));
1496 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_w:
1497 return Builder.CreateCall(
1498 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_ctaid_w));
1499 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_x:
1500 return Builder.CreateCall(
1501 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_nctaid_x));
1502 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_y:
1503 return Builder.CreateCall(
1504 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_nctaid_y));
1505 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_z:
1506 return Builder.CreateCall(
1507 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_nctaid_z));
1508 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_w:
1509 return Builder.CreateCall(
1510 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_nctaid_w));
1511 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctarank:
1512 return Builder.CreateCall(
1513 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_ctarank));
1514 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctarank:
1515 return Builder.CreateCall(
1516 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_nctarank));
1517 case NVPTX::BI__nvvm_is_explicit_cluster:
1518 return Builder.CreateCall(
1519 CGM.getIntrinsic(Intrinsic::nvvm_is_explicit_cluster));
1520 case NVPTX::BI__nvvm_isspacep_shared_cluster:
1521 return Builder.CreateCall(
1522 CGM.getIntrinsic(Intrinsic::nvvm_isspacep_shared_cluster),
1523 EmitScalarExpr(E->getArg(0)));
1524 case NVPTX::BI__nvvm_mapa:
1525 return Builder.CreateCall(
1526 CGM.getIntrinsic(Intrinsic::nvvm_mapa),
1527 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
1528 case NVPTX::BI__nvvm_mapa_shared_cluster:
1529 return Builder.CreateCall(
1530 CGM.getIntrinsic(Intrinsic::nvvm_mapa_shared_cluster),
1531 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
1532 case NVPTX::BI__nvvm_getctarank:
1533 return Builder.CreateCall(
1534 CGM.getIntrinsic(Intrinsic::nvvm_getctarank),
1535 EmitScalarExpr(E->getArg(0)));
1536 case NVPTX::BI__nvvm_getctarank_shared_cluster:
1537 return Builder.CreateCall(
1538 CGM.getIntrinsic(Intrinsic::nvvm_getctarank_shared_cluster),
1539 EmitScalarExpr(E->getArg(0)));
1540 case NVPTX::BI__nvvm_barrier_cluster_arrive:
1541 return Builder.CreateCall(
1542 CGM.getIntrinsic(Intrinsic::nvvm_barrier_cluster_arrive));
1543 case NVPTX::BI__nvvm_barrier_cluster_arrive_relaxed:
1544 return Builder.CreateCall(
1545 CGM.getIntrinsic(Intrinsic::nvvm_barrier_cluster_arrive_relaxed));
1546 case NVPTX::BI__nvvm_barrier_cluster_wait:
1547 return Builder.CreateCall(
1548 CGM.getIntrinsic(Intrinsic::nvvm_barrier_cluster_wait));
1549 case NVPTX::BI__nvvm_fence_sc_cluster:
1550 return Builder.CreateCall(
1551 CGM.getIntrinsic(Intrinsic::nvvm_fence_sc_cluster));
1552 case NVPTX::BI__nvvm_bar_sync:
1553 return Builder.CreateCall(
1554 CGM.getIntrinsic(Intrinsic::nvvm_barrier_cta_sync_aligned_all),
1555 EmitScalarExpr(E->getArg(0)));
1556 case NVPTX::BI__syncthreads:
1557 return Builder.CreateCall(
1558 CGM.getIntrinsic(Intrinsic::nvvm_barrier_cta_sync_aligned_all),
1559 Builder.getInt32(0));
1560 case NVPTX::BI__nvvm_barrier_sync:
1561 return Builder.CreateCall(
1562 CGM.getIntrinsic(Intrinsic::nvvm_barrier_cta_sync_all),
1563 EmitScalarExpr(E->getArg(0)));
1564 case NVPTX::BI__nvvm_barrier_sync_cnt:
1565 return Builder.CreateCall(
1566 CGM.getIntrinsic(Intrinsic::nvvm_barrier_cta_sync_count),
1567 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
1568 case NVPTX::BI__nvvm_bar0_and:
1569 return Builder.CreateZExt(
1570 Builder.CreateIntrinsic(
1571 Intrinsic::nvvm_barrier_cta_red_and_aligned_all, {},
1572 {Builder.getInt32(0),
1573 Builder.CreateICmpNE(EmitScalarExpr(E->getArg(0)),
1574 Builder.getInt32(0))}),
1575 Builder.getInt32Ty());
1576 case NVPTX::BI__nvvm_bar0_or:
1577 return Builder.CreateZExt(
1578 Builder.CreateIntrinsic(
1579 Intrinsic::nvvm_barrier_cta_red_or_aligned_all, {},
1580 {Builder.getInt32(0),
1581 Builder.CreateICmpNE(EmitScalarExpr(E->getArg(0)),
1582 Builder.getInt32(0))}),
1583 Builder.getInt32Ty());
1584 case NVPTX::BI__nvvm_bar0_popc:
1585 return Builder.CreateIntrinsic(
1586 Intrinsic::nvvm_barrier_cta_red_popc_aligned_all, {},
1587 {Builder.getInt32(0), Builder.CreateICmpNE(EmitScalarExpr(E->getArg(0)),
1588 Builder.getInt32(0))});
1589 case NVPTX::BI__nvvm_mbarrier_init:
1590 case NVPTX::BI__nvvm_mbarrier_init_shared: {
1591 // The intrinsic is overloaded on the pointer, so the two builtins differ
1592 // only in the address space of their first argument.
1593 Value *Ptr = EmitScalarExpr(E->getArg(0));
1594 return Builder.CreateIntrinsic(
1595 Intrinsic::nvvm_mbarrier_init, {Ptr->getType()},
1596 {Ptr, EmitScalarExpr(E->getArg(1)), Builder.getInt32(0)});
1597 }
1598 case NVPTX::BI__nvvm_idp2a_lo_s_s:
1599 return MakeIdp2a(Intrinsic::nvvm_idp2a_s_s, /*IsHi=*/false, E, *this);
1600 case NVPTX::BI__nvvm_idp2a_lo_s_u:
1601 return MakeIdp2a(Intrinsic::nvvm_idp2a_s_u, /*IsHi=*/false, E, *this);
1602 case NVPTX::BI__nvvm_idp2a_lo_u_s:
1603 return MakeIdp2a(Intrinsic::nvvm_idp2a_u_s, /*IsHi=*/false, E, *this);
1604 case NVPTX::BI__nvvm_idp2a_lo_u_u:
1605 return MakeIdp2a(Intrinsic::nvvm_idp2a_u_u, /*IsHi=*/false, E, *this);
1606 case NVPTX::BI__nvvm_idp2a_hi_s_s:
1607 return MakeIdp2a(Intrinsic::nvvm_idp2a_s_s, /*IsHi=*/true, E, *this);
1608 case NVPTX::BI__nvvm_idp2a_hi_s_u:
1609 return MakeIdp2a(Intrinsic::nvvm_idp2a_s_u, /*IsHi=*/true, E, *this);
1610 case NVPTX::BI__nvvm_idp2a_hi_u_s:
1611 return MakeIdp2a(Intrinsic::nvvm_idp2a_u_s, /*IsHi=*/true, E, *this);
1612 case NVPTX::BI__nvvm_idp2a_hi_u_u:
1613 return MakeIdp2a(Intrinsic::nvvm_idp2a_u_u, /*IsHi=*/true, E, *this);
1614 default:
1615 return nullptr;
1616 }
1617}
V
#define V(N, I)
Definition ASTContext.h:3854
MakeAtomicCmpXchgValue
Value * MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E, bool ReturnBool, llvm::AtomicOrdering SuccessOrdering, llvm::AtomicOrdering FailureOrdering)
Utility to insert an atomic cmpxchg instruction.
Definition CGBuiltin.cpp:375
appendDefaultIntrinsicArgs
void appendDefaultIntrinsicArgs(SmallVectorImpl< llvm::Value * > &Args, llvm::Function *F)
Definition CGBuiltin.cpp:211
MakeBinaryAtomicValue
Value * MakeBinaryAtomicValue(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E, AtomicOrdering Ordering)
Utility to insert an atomic instruction based on Intrinsic::ID and the expression node.
Definition CGBuiltin.cpp:275
MMA_VARIANTS_B1_AND
#define MMA_VARIANTS_B1_AND(geom, type)
PZO_CVT
#define PZO_CVT(cvt)
MMA_INTR
#define MMA_INTR(geom_op_type, layout)
Definition NVPTX.cpp:32
MMA_VARIANTS
#define MMA_VARIANTS(geom, type)
MMA_SATF_VARIANTS
#define MMA_SATF_VARIANTS(geom, type)
MMA_LDST
#define MMA_LDST(n, geom_op_type)
Definition NVPTX.cpp:34
MMA_VARIANTS_B1_XOR
#define MMA_VARIANTS_B1_XOR(geom, type)
MMA_VARIANTS_I4
#define MMA_VARIANTS_I4(geom, type)
TargetBuiltins.h
Enumerates target-specific builtins in their own namespaces within namespace clang.
ASTContext::GE_None
@ GE_None
No error.
Definition ASTContext.h:2738
clang::ASTContext::GetBuiltinType
QualType GetBuiltinType(unsigned ID, GetBuiltinTypeError &Error, unsigned *IntegerConstantArgs=nullptr) const
Return the type for the specified builtin.
Definition ASTContext.cpp:13064
clang::CallExpr
CallExpr - Represents a function call (C99 6.5.2.2, C++ [expr.call]).
Definition Expr.h:3012
clang::CallExpr::getArg
Expr * getArg(unsigned Arg)
getArg - Return the specified argument.
Definition Expr.h:3216
clang::CallExpr::getNumArgs
unsigned getNumArgs() const
getNumArgs - Return the number of actual arguments to this call.
Definition Expr.h:3203
clang::CharUnits
This is an opaque type for sizes expressed in character units.
Definition CharUnits.h:38
clang::CharUnits::getAsAlign
llvm::Align getAsAlign() const
Returns Quantity as a valid llvm::Align, Beware llvm::Align assumes power of two 8-bit bytes.
Definition CharUnits.h:159
clang::CharUnits::fromQuantity
static CharUnits fromQuantity(QuantityType Quantity)
Construct a CharUnits quantity from a raw integer type.
Definition CharUnits.h:58
clang::CodeGen::Address
Like RawAddress, an abstract representation of an aligned address, but the pointer contained in this ...
Definition Address.h:128
clang::CodeGen::Address::emitRawPointer
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
Return the pointer contained in this class after authenticating it and adding offset to it if necessa...
Definition Address.h:253
clang::CodeGen::Address::getElementType
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Definition Address.h:209
clang::CodeGen::CGBuilderTy::CreateAtomicRMW
llvm::AtomicRMWInst * CreateAtomicRMW(llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val, llvm::AtomicOrdering Ordering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
Definition CGBuilder.h:192
clang::CodeGen::CGBuilderTy::CreateAtomicCmpXchg
llvm::AtomicCmpXchgInst * CreateAtomicCmpXchg(Address Addr, llvm::Value *Cmp, llvm::Value *New, llvm::AtomicOrdering SuccessOrdering, llvm::AtomicOrdering FailureOrdering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
Definition CGBuilder.h:181
clang::CodeGen::CGBuilderTy::CreateAlignedLoad
llvm::LoadInst * CreateAlignedLoad(llvm::Type *Ty, llvm::Value *Addr, CharUnits Align, const llvm::Twine &Name="")
Definition CGBuilder.h:140
clang::CodeGen::CGBuilderTy::CreateAddrSpaceCast
Address CreateAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
Definition CGBuilder.h:201
clang::CodeGen::CodeGenFunction
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
Definition CodeGenFunction.h:260
clang::CodeGen::CodeGenFunction::ConvertType
llvm::Type * ConvertType(QualType T)
Definition CodeGenFunction.cpp:229
clang::CodeGen::CodeGenFunction::getTarget
const TargetInfo & getTarget() const
Definition CodeGenFunction.h:2305
clang::CodeGen::CodeGenFunction::ConvertTypeForMem
llvm::Type * ConvertTypeForMem(QualType T)
Definition CodeGenFunction.cpp:225
clang::CodeGen::CodeGenFunction::EmitPointerWithAlignment
Address EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitPointerWithAlignment - Given an expression with a pointer type, emit the value and compute our be...
Definition CGExpr.cpp:1661
clang::CodeGen::CodeGenFunction::EmitScalarExpr
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
Definition CGExprScalar.cpp:6204
clang::CodeGen::CodeGenFunction::EmitNVPTXBuiltinExpr
llvm::Value * EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
Definition NVPTX.cpp:462
clang::CodeGen::CodeGenFunction::getLLVMContext
llvm::LLVMContext & getLLVMContext()
Definition CodeGenFunction.h:2306
clang::CodeGen::CodeGenModule::getContext
ASTContext & getContext() const
Definition CodeGenModule.h:897
clang::CodeGen::CodeGenModule::getNaturalPointeeTypeAlignment
CharUnits getNaturalPointeeTypeAlignment(QualType T, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
Definition CodeGenModule.cpp:8819
clang::CodeGen::CodeGenModule::getIntrinsic
llvm::Function * getIntrinsic(unsigned IID, ArrayRef< llvm::Type * > Tys={})
Definition CodeGenModule.cpp:7301
clang::Expr::getIntegerConstantExpr
std::optional< llvm::APSInt > getIntegerConstantExpr(const ASTContext &Ctx, bool AllowRelaxedEval=false) const
isIntegerConstantExpr - Return the value if this expression is a valid integer constant expression.
Definition ExprConstant.cpp:23447
clang::Expr::getType
QualType getType() const
Definition Expr.h:146
clang::QualType
A (possibly-)qualified type.
Definition TypeBase.h:938
clang::Scope
Scope - A scope is a transient data structure that is used while parsing the program.
Definition Scope.h:41
clang::TargetInfo::getTriple
const llvm::Triple & getTriple() const
Returns the target triple of the primary target.
Definition TargetInfo.h:1323
clang::Type::hasSignedIntegerRepresentation
bool hasSignedIntegerRepresentation() const
Determine whether this type has an signed integer representation of some sort, e.g....
Definition Type.cpp:2436
clang::Value::getType
QualType getType() const
Definition Value.cpp:238
clang::CodeGen::addAtomicIgnoreDenormalModeMetadata
void addAtomicIgnoreDenormalModeMetadata(CodeGenFunction &CGF, llvm::Instruction &AtomicInst, bool AllowHalf=false)
Attach !atomic.ignore.denormal.mode to AtomicInst, if it is an atomicrmw the metadata is meaningful f...
Definition TargetInfo.cpp:163
clang
Top level wrappers for InstallAPI frontend operations.
Definition CalledOnceCheck.h:17
clang::OpenACCClauseKind::DType
@ DType
'dtype' clause, an alias for 'device_type', stored separately for diagnostic purposes.
Definition OpenACCKinds.h:274
clang::ObjCSubstitutionContext::Result
@ Result
The result type of a method or function.
Definition TypeBase.h:906
llvm
Diagnostic wrappers for TextAPI types for error reporting.
Definition Dominators.h:30
Generated on for clang by doxygen 1.14.0