clang 24.0.0git
NVPTX.cpp
Go to the documentation of this file.
1//===-------- NVPTX.cpp - Emit LLVM Code for builtins ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Builtin calls as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGBuiltin.h"
14#include "clang/Basic/TargetBuiltins.h"
15#include "llvm/IR/IntrinsicsNVPTX.h"
16#include "llvm/TargetParser/AtomicScope.h"
17
18using namespace clang;
19using namespace CodeGen;
20using namespace llvm;
21
22namespace {
23// Helper classes for mapping MMA builtins to particular LLVM intrinsic variant.
24struct NVPTXMmaLdstInfo {
25 unsigned NumResults; // Number of elements to load/store
26 // Intrinsic IDs for row/col variants. 0 if particular layout is unsupported.
27 unsigned IID_col;
28 unsigned IID_row;
29};
30
31#define MMA_INTR(geom_op_type, layout) \
32 Intrinsic::nvvm_wmma_##geom_op_type##_##layout##_stride
33#define MMA_LDST(n, geom_op_type) \
34 { n, MMA_INTR(geom_op_type, col), MMA_INTR(geom_op_type, row) }
35
36static NVPTXMmaLdstInfo getNVPTXMmaLdstInfo(unsigned BuiltinID) {
37 switch (BuiltinID) {
38 // FP MMA loads
39 case NVPTX::BI__hmma_m16n16k16_ld_a:
40 return MMA_LDST(8, m16n16k16_load_a_f16);
41 case NVPTX::BI__hmma_m16n16k16_ld_b:
42 return MMA_LDST(8, m16n16k16_load_b_f16);
43 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
44 return MMA_LDST(4, m16n16k16_load_c_f16);
45 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
46 return MMA_LDST(8, m16n16k16_load_c_f32);
47 case NVPTX::BI__hmma_m32n8k16_ld_a:
48 return MMA_LDST(8, m32n8k16_load_a_f16);
49 case NVPTX::BI__hmma_m32n8k16_ld_b:
50 return MMA_LDST(8, m32n8k16_load_b_f16);
51 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
52 return MMA_LDST(4, m32n8k16_load_c_f16);
53 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
54 return MMA_LDST(8, m32n8k16_load_c_f32);
55 case NVPTX::BI__hmma_m8n32k16_ld_a:
56 return MMA_LDST(8, m8n32k16_load_a_f16);
57 case NVPTX::BI__hmma_m8n32k16_ld_b:
58 return MMA_LDST(8, m8n32k16_load_b_f16);
59 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
60 return MMA_LDST(4, m8n32k16_load_c_f16);
61 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
62 return MMA_LDST(8, m8n32k16_load_c_f32);
63
64 // Integer MMA loads
65 case NVPTX::BI__imma_m16n16k16_ld_a_s8:
66 return MMA_LDST(2, m16n16k16_load_a_s8);
67 case NVPTX::BI__imma_m16n16k16_ld_a_u8:
68 return MMA_LDST(2, m16n16k16_load_a_u8);
69 case NVPTX::BI__imma_m16n16k16_ld_b_s8:
70 return MMA_LDST(2, m16n16k16_load_b_s8);
71 case NVPTX::BI__imma_m16n16k16_ld_b_u8:
72 return MMA_LDST(2, m16n16k16_load_b_u8);
73 case NVPTX::BI__imma_m16n16k16_ld_c:
74 return MMA_LDST(8, m16n16k16_load_c_s32);
75 case NVPTX::BI__imma_m32n8k16_ld_a_s8:
76 return MMA_LDST(4, m32n8k16_load_a_s8);
77 case NVPTX::BI__imma_m32n8k16_ld_a_u8:
78 return MMA_LDST(4, m32n8k16_load_a_u8);
79 case NVPTX::BI__imma_m32n8k16_ld_b_s8:
80 return MMA_LDST(1, m32n8k16_load_b_s8);
81 case NVPTX::BI__imma_m32n8k16_ld_b_u8:
82 return MMA_LDST(1, m32n8k16_load_b_u8);
83 case NVPTX::BI__imma_m32n8k16_ld_c:
84 return MMA_LDST(8, m32n8k16_load_c_s32);
85 case NVPTX::BI__imma_m8n32k16_ld_a_s8:
86 return MMA_LDST(1, m8n32k16_load_a_s8);
87 case NVPTX::BI__imma_m8n32k16_ld_a_u8:
88 return MMA_LDST(1, m8n32k16_load_a_u8);
89 case NVPTX::BI__imma_m8n32k16_ld_b_s8:
90 return MMA_LDST(4, m8n32k16_load_b_s8);
91 case NVPTX::BI__imma_m8n32k16_ld_b_u8:
92 return MMA_LDST(4, m8n32k16_load_b_u8);
93 case NVPTX::BI__imma_m8n32k16_ld_c:
94 return MMA_LDST(8, m8n32k16_load_c_s32);
95
96 // Sub-integer MMA loads.
97 // Only row/col layout is supported by A/B fragments.
98 case NVPTX::BI__imma_m8n8k32_ld_a_s4:
99 return {1, 0, MMA_INTR(m8n8k32_load_a_s4, row)};
100 case NVPTX::BI__imma_m8n8k32_ld_a_u4:
101 return {1, 0, MMA_INTR(m8n8k32_load_a_u4, row)};
102 case NVPTX::BI__imma_m8n8k32_ld_b_s4:
103 return {1, MMA_INTR(m8n8k32_load_b_s4, col), 0};
104 case NVPTX::BI__imma_m8n8k32_ld_b_u4:
105 return {1, MMA_INTR(m8n8k32_load_b_u4, col), 0};
106 case NVPTX::BI__imma_m8n8k32_ld_c:
107 return MMA_LDST(2, m8n8k32_load_c_s32);
108 case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
109 return {1, 0, MMA_INTR(m8n8k128_load_a_b1, row)};
110 case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
111 return {1, MMA_INTR(m8n8k128_load_b_b1, col), 0};
112 case NVPTX::BI__bmma_m8n8k128_ld_c:
113 return MMA_LDST(2, m8n8k128_load_c_s32);
114
115 // Double MMA loads
116 case NVPTX::BI__dmma_m8n8k4_ld_a:
117 return MMA_LDST(1, m8n8k4_load_a_f64);
118 case NVPTX::BI__dmma_m8n8k4_ld_b:
119 return MMA_LDST(1, m8n8k4_load_b_f64);
120 case NVPTX::BI__dmma_m8n8k4_ld_c:
121 return MMA_LDST(2, m8n8k4_load_c_f64);
122
123 // Alternate float MMA loads
124 case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
125 return MMA_LDST(4, m16n16k16_load_a_bf16);
126 case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
127 return MMA_LDST(4, m16n16k16_load_b_bf16);
128 case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
129 return MMA_LDST(2, m8n32k16_load_a_bf16);
130 case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
131 return MMA_LDST(8, m8n32k16_load_b_bf16);
132 case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
133 return MMA_LDST(8, m32n8k16_load_a_bf16);
134 case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
135 return MMA_LDST(2, m32n8k16_load_b_bf16);
136 case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
137 return MMA_LDST(4, m16n16k8_load_a_tf32);
138 case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
139 return MMA_LDST(4, m16n16k8_load_b_tf32);
140 case NVPTX::BI__mma_tf32_m16n16k8_ld_c:
141 return MMA_LDST(8, m16n16k8_load_c_f32);
142
143 // NOTE: We need to follow inconsitent naming scheme used by NVCC. Unlike
144 // PTX and LLVM IR where stores always use fragment D, NVCC builtins always
145 // use fragment C for both loads and stores.
146 // FP MMA stores.
147 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
148 return MMA_LDST(4, m16n16k16_store_d_f16);
149 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
150 return MMA_LDST(8, m16n16k16_store_d_f32);
151 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
152 return MMA_LDST(4, m32n8k16_store_d_f16);
153 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
154 return MMA_LDST(8, m32n8k16_store_d_f32);
155 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
156 return MMA_LDST(4, m8n32k16_store_d_f16);
157 case NVPTX::BI__hmma_m8n32k16_st_c_f32:
158 return MMA_LDST(8, m8n32k16_store_d_f32);
159
160 // Integer and sub-integer MMA stores.
161 // Another naming quirk. Unlike other MMA builtins that use PTX types in the
162 // name, integer loads/stores use LLVM's i32.
163 case NVPTX::BI__imma_m16n16k16_st_c_i32:
164 return MMA_LDST(8, m16n16k16_store_d_s32);
165 case NVPTX::BI__imma_m32n8k16_st_c_i32:
166 return MMA_LDST(8, m32n8k16_store_d_s32);
167 case NVPTX::BI__imma_m8n32k16_st_c_i32:
168 return MMA_LDST(8, m8n32k16_store_d_s32);
169 case NVPTX::BI__imma_m8n8k32_st_c_i32:
170 return MMA_LDST(2, m8n8k32_store_d_s32);
171 case NVPTX::BI__bmma_m8n8k128_st_c_i32:
172 return MMA_LDST(2, m8n8k128_store_d_s32);
173
174 // Double MMA store
175 case NVPTX::BI__dmma_m8n8k4_st_c_f64:
176 return MMA_LDST(2, m8n8k4_store_d_f64);
177
178 // Alternate float MMA store
179 case NVPTX::BI__mma_m16n16k8_st_c_f32:
180 return MMA_LDST(8, m16n16k8_store_d_f32);
181
182 default:
183 llvm_unreachable("Unknown MMA builtin");
184 }
185}
186#undef MMA_LDST
187#undef MMA_INTR
188
189
190struct NVPTXMmaInfo {
191 unsigned NumEltsA;
192 unsigned NumEltsB;
193 unsigned NumEltsC;
194 unsigned NumEltsD;
195
196 // Variants are ordered by layout-A/layout-B/satf, where 'row' has priority
197 // over 'col' for layout. The index of non-satf variants is expected to match
198 // the undocumented layout constants used by CUDA's mma.hpp.
199 std::array<unsigned, 8> Variants;
200
201 unsigned getMMAIntrinsic(int Layout, bool Satf) {
202 unsigned Index = Layout + 4 * Satf;
203 if (Index >= Variants.size())
204 return 0;
205 return Variants[Index];
206 }
207};
208
209 // Returns an intrinsic that matches Layout and Satf for valid combinations of
210 // Layout and Satf, 0 otherwise.
211static NVPTXMmaInfo getNVPTXMmaInfo(unsigned BuiltinID) {
212 // clang-format off
213#define MMA_VARIANTS(geom, type) \
214 Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type, \
215 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type, \
216 Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type, \
217 Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type
218#define MMA_SATF_VARIANTS(geom, type) \
219 MMA_VARIANTS(geom, type), \
220 Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type##_satfinite, \
221 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type##_satfinite, \
222 Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type##_satfinite, \
223 Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type##_satfinite
224// Sub-integer MMA only supports row.col layout.
225#define MMA_VARIANTS_I4(geom, type) \
226 0, \
227 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type, \
228 0, \
229 0, \
230 0, \
231 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type##_satfinite, \
232 0, \
233 0
234// b1 MMA does not support .satfinite.
235#define MMA_VARIANTS_B1_XOR(geom, type) \
236 0, \
237 Intrinsic::nvvm_wmma_##geom##_mma_xor_popc_row_col_##type, \
238 0, \
239 0, \
240 0, \
241 0, \
242 0, \
243 0
244#define MMA_VARIANTS_B1_AND(geom, type) \
245 0, \
246 Intrinsic::nvvm_wmma_##geom##_mma_and_popc_row_col_##type, \
247 0, \
248 0, \
249 0, \
250 0, \
251 0, \
252 0
253 // clang-format on
254 switch (BuiltinID) {
255 // FP MMA
256 // Note that 'type' argument of MMA_SATF_VARIANTS uses D_C notation, while
257 // NumEltsN of return value are ordered as A,B,C,D.
258 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
259 return {8, 8, 4, 4, {{MMA_SATF_VARIANTS(m16n16k16, f16_f16)}}};
260 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
261 return {8, 8, 4, 8, {{MMA_SATF_VARIANTS(m16n16k16, f32_f16)}}};
262 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
263 return {8, 8, 8, 4, {{MMA_SATF_VARIANTS(m16n16k16, f16_f32)}}};
264 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
265 return {8, 8, 8, 8, {{MMA_SATF_VARIANTS(m16n16k16, f32_f32)}}};
266 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
267 return {8, 8, 4, 4, {{MMA_SATF_VARIANTS(m32n8k16, f16_f16)}}};
268 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
269 return {8, 8, 4, 8, {{MMA_SATF_VARIANTS(m32n8k16, f32_f16)}}};
270 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
271 return {8, 8, 8, 4, {{MMA_SATF_VARIANTS(m32n8k16, f16_f32)}}};
272 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
273 return {8, 8, 8, 8, {{MMA_SATF_VARIANTS(m32n8k16, f32_f32)}}};
274 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
275 return {8, 8, 4, 4, {{MMA_SATF_VARIANTS(m8n32k16, f16_f16)}}};
276 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
277 return {8, 8, 4, 8, {{MMA_SATF_VARIANTS(m8n32k16, f32_f16)}}};
278 case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
279 return {8, 8, 8, 4, {{MMA_SATF_VARIANTS(m8n32k16, f16_f32)}}};
280 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
281 return {8, 8, 8, 8, {{MMA_SATF_VARIANTS(m8n32k16, f32_f32)}}};
282
283 // Integer MMA
284 case NVPTX::BI__imma_m16n16k16_mma_s8:
285 return {2, 2, 8, 8, {{MMA_SATF_VARIANTS(m16n16k16, s8)}}};
286 case NVPTX::BI__imma_m16n16k16_mma_u8:
287 return {2, 2, 8, 8, {{MMA_SATF_VARIANTS(m16n16k16, u8)}}};
288 case NVPTX::BI__imma_m32n8k16_mma_s8:
289 return {4, 1, 8, 8, {{MMA_SATF_VARIANTS(m32n8k16, s8)}}};
290 case NVPTX::BI__imma_m32n8k16_mma_u8:
291 return {4, 1, 8, 8, {{MMA_SATF_VARIANTS(m32n8k16, u8)}}};
292 case NVPTX::BI__imma_m8n32k16_mma_s8:
293 return {1, 4, 8, 8, {{MMA_SATF_VARIANTS(m8n32k16, s8)}}};
294 case NVPTX::BI__imma_m8n32k16_mma_u8:
295 return {1, 4, 8, 8, {{MMA_SATF_VARIANTS(m8n32k16, u8)}}};
296
297 // Sub-integer MMA
298 case NVPTX::BI__imma_m8n8k32_mma_s4:
299 return {1, 1, 2, 2, {{MMA_VARIANTS_I4(m8n8k32, s4)}}};
300 case NVPTX::BI__imma_m8n8k32_mma_u4:
301 return {1, 1, 2, 2, {{MMA_VARIANTS_I4(m8n8k32, u4)}}};
302 case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1:
303 return {1, 1, 2, 2, {{MMA_VARIANTS_B1_XOR(m8n8k128, b1)}}};
304 case NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1:
305 return {1, 1, 2, 2, {{MMA_VARIANTS_B1_AND(m8n8k128, b1)}}};
306
307 // Double MMA
308 case NVPTX::BI__dmma_m8n8k4_mma_f64:
309 return {1, 1, 2, 2, {{MMA_VARIANTS(m8n8k4, f64)}}};
310
311 // Alternate FP MMA
312 case NVPTX::BI__mma_bf16_m16n16k16_mma_f32:
313 return {4, 4, 8, 8, {{MMA_VARIANTS(m16n16k16, bf16)}}};
314 case NVPTX::BI__mma_bf16_m8n32k16_mma_f32:
315 return {2, 8, 8, 8, {{MMA_VARIANTS(m8n32k16, bf16)}}};
316 case NVPTX::BI__mma_bf16_m32n8k16_mma_f32:
317 return {8, 2, 8, 8, {{MMA_VARIANTS(m32n8k16, bf16)}}};
318 case NVPTX::BI__mma_tf32_m16n16k8_mma_f32:
319 return {4, 4, 8, 8, {{MMA_VARIANTS(m16n16k8, tf32)}}};
320 default:
321 llvm_unreachable("Unexpected builtin ID.");
322 }
323#undef MMA_VARIANTS
324#undef MMA_SATF_VARIANTS
325#undef MMA_VARIANTS_I4
326#undef MMA_VARIANTS_B1_AND
327#undef MMA_VARIANTS_B1_XOR
328}
329
330static Value *MakeLdu(unsigned IntrinsicID, CodeGenFunction &CGF,
331 const CallExpr *E) {
332 Value *Ptr = CGF.EmitScalarExpr(E->getArg(0));
333 QualType ArgType = E->getArg(0)->getType();
334 clang::CharUnits Align = CGF.CGM.getNaturalPointeeTypeAlignment(ArgType);
335 llvm::Type *ElemTy = CGF.ConvertTypeForMem(ArgType->getPointeeType());
336 return CGF.Builder.CreateCall(
337 CGF.CGM.getIntrinsic(IntrinsicID, {ElemTy, Ptr->getType()}),
338 {Ptr, ConstantInt::get(CGF.Builder.getInt32Ty(), Align.getQuantity())});
339}
340
341static Value *MakeLdg(CodeGenFunction &CGF, const CallExpr *E) {
342 Value *Ptr = CGF.EmitScalarExpr(E->getArg(0));
343 QualType ArgType = E->getArg(0)->getType();
344 clang::CharUnits AlignV = CGF.CGM.getNaturalPointeeTypeAlignment(ArgType);
345 llvm::Type *ElemTy = CGF.ConvertTypeForMem(ArgType->getPointeeType());
346
347 // Use addrspace(1) for NVPTX ADDRESS_SPACE_GLOBAL
348 auto *ASC = CGF.Builder.CreateAddrSpaceCast(Ptr, CGF.Builder.getPtrTy(1));
349 auto *LD = CGF.Builder.CreateAlignedLoad(ElemTy, ASC, AlignV.getAsAlign());
350 MDNode *MD = MDNode::get(CGF.Builder.getContext(), {});
351 LD->setMetadata(LLVMContext::MD_invariant_load, MD);
352
353 return LD;
354}
355
356// `Scope` is AtomicScope::Workgroup for _cta builtins and AtomicScope::System
357// for _sys builtins.
358static Value *MakeScopedAtomicRMW(CodeGenFunction &CGF, const CallExpr *E,
359 llvm::AtomicRMWInst::BinOp Kind,
360 llvm::AtomicScope Scope) {
361 Address Ptr = CGF.EmitPointerWithAlignment(E->getArg(0));
362 Value *Val = CGF.EmitScalarExpr(E->getArg(1));
363 llvm::SyncScope::ID SSID = CGF.getLLVMContext().getOrInsertSyncScopeID(
364 *llvm::getAtomicScopeIRString(CGF.getTarget().getTriple(), Scope));
365 return CGF.Builder.CreateAtomicRMW(Kind, Ptr, Val,
366 llvm::AtomicOrdering::Monotonic, SSID);
367}
368
369// `Scope` is AtomicScope::Workgroup for _cta builtins and AtomicScope::System
370// for _sys builtins.
371static Value *MakeScopedAtomicCAS(CodeGenFunction &CGF, const CallExpr *E,
372 llvm::AtomicScope Scope) {
373 Address Ptr = CGF.EmitPointerWithAlignment(E->getArg(0));
374 Value *Cmp = CGF.EmitScalarExpr(E->getArg(1));
375 Value *New = CGF.EmitScalarExpr(E->getArg(2));
376 llvm::SyncScope::ID SSID = CGF.getLLVMContext().getOrInsertSyncScopeID(
377 *llvm::getAtomicScopeIRString(CGF.getTarget().getTriple(), Scope));
378 Value *Pair = CGF.Builder.CreateAtomicCmpXchg(
379 Ptr, Cmp, New, llvm::AtomicOrdering::Monotonic,
380 llvm::AtomicOrdering::Monotonic, SSID);
381 return CGF.Builder.CreateExtractValue(Pair, 0);
382}
383
384static Value *MakeCpAsync(unsigned IntrinsicID, unsigned IntrinsicIDS,
385 CodeGenFunction &CGF, const CallExpr *E,
386 int SrcSize) {
387 return E->getNumArgs() == 3
388 ? CGF.Builder.CreateCall(CGF.CGM.getIntrinsic(IntrinsicIDS),
389 {CGF.EmitScalarExpr(E->getArg(0)),
390 CGF.EmitScalarExpr(E->getArg(1)),
391 CGF.EmitScalarExpr(E->getArg(2))})
392 : CGF.Builder.CreateCall(CGF.CGM.getIntrinsic(IntrinsicID),
393 {CGF.EmitScalarExpr(E->getArg(0)),
394 CGF.EmitScalarExpr(E->getArg(1))});
395}
396
397static Value *MakeHalfType(Function *Intrinsic, unsigned BuiltinID,
398 const CallExpr *E, CodeGenFunction &CGF) {
399 SmallVector<Value *, 16> Args;
400 auto *FTy = Intrinsic->getFunctionType();
401 unsigned ICEArguments = 0;
402 ASTContext::GetBuiltinTypeError Error;
403 CGF.CGM.getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
404 assert(Error == ASTContext::GE_None && "Should not codegen an error");
405 for (unsigned i = 0, e = E->getNumArgs(); i != e; ++i) {
406 assert((ICEArguments & (1 << i)) == 0);
407 auto *ArgValue = CGF.EmitScalarExpr(E->getArg(i));
408 auto *PTy = FTy->getParamType(i);
409 if (PTy != ArgValue->getType())
410 ArgValue = CGF.Builder.CreateBitCast(ArgValue, PTy);
411 Args.push_back(ArgValue);
412 }
413
414 return CGF.Builder.CreateCall(Intrinsic, Args);
415}
416
417static Value *MakeHalfType(unsigned IntrinsicID, unsigned BuiltinID,
418 const CallExpr *E, CodeGenFunction &CGF) {
419 return MakeHalfType(CGF.CGM.getIntrinsic(IntrinsicID), BuiltinID, E, CGF);
420}
421
422static Value *MakeFMAOOB(unsigned IntrinsicID, llvm::Type *Ty,
423 const CallExpr *E, CodeGenFunction &CGF) {
424 return CGF.Builder.CreateCall(CGF.CGM.getIntrinsic(IntrinsicID, {Ty}),
425 {CGF.EmitScalarExpr(E->getArg(0)),
426 CGF.EmitScalarExpr(E->getArg(1)),
427 CGF.EmitScalarExpr(E->getArg(2))});
428}
429
430} // namespace
431
432Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
433 const CallExpr *E) {
434 switch (BuiltinID) {
435 case NVPTX::BI__nvvm_atom_add_gen_i:
436 case NVPTX::BI__nvvm_atom_add_gen_l:
437 case NVPTX::BI__nvvm_atom_add_gen_ll:
438 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Add, E,
439 AtomicOrdering::Monotonic);
440
441 case NVPTX::BI__nvvm_atom_sub_gen_i:
442 case NVPTX::BI__nvvm_atom_sub_gen_l:
443 case NVPTX::BI__nvvm_atom_sub_gen_ll:
444 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Sub, E,
445 AtomicOrdering::Monotonic);
446
447 case NVPTX::BI__nvvm_atom_and_gen_i:
448 case NVPTX::BI__nvvm_atom_and_gen_l:
449 case NVPTX::BI__nvvm_atom_and_gen_ll:
450 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::And, E,
451 AtomicOrdering::Monotonic);
452
453 case NVPTX::BI__nvvm_atom_or_gen_i:
454 case NVPTX::BI__nvvm_atom_or_gen_l:
455 case NVPTX::BI__nvvm_atom_or_gen_ll:
456 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Or, E,
457 AtomicOrdering::Monotonic);
458
459 case NVPTX::BI__nvvm_atom_xor_gen_i:
460 case NVPTX::BI__nvvm_atom_xor_gen_l:
461 case NVPTX::BI__nvvm_atom_xor_gen_ll:
462 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Xor, E,
463 AtomicOrdering::Monotonic);
464
465 case NVPTX::BI__nvvm_atom_xchg_gen_i:
466 case NVPTX::BI__nvvm_atom_xchg_gen_l:
467 case NVPTX::BI__nvvm_atom_xchg_gen_ll:
468 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Xchg, E,
469 AtomicOrdering::Monotonic);
470
471 case NVPTX::BI__nvvm_atom_max_gen_i:
472 case NVPTX::BI__nvvm_atom_max_gen_l:
473 case NVPTX::BI__nvvm_atom_max_gen_ll:
474 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Max, E,
475 AtomicOrdering::Monotonic);
476
477 case NVPTX::BI__nvvm_atom_max_gen_ui:
478 case NVPTX::BI__nvvm_atom_max_gen_ul:
479 case NVPTX::BI__nvvm_atom_max_gen_ull:
480 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::UMax, E,
481 AtomicOrdering::Monotonic);
482
483 case NVPTX::BI__nvvm_atom_min_gen_i:
484 case NVPTX::BI__nvvm_atom_min_gen_l:
485 case NVPTX::BI__nvvm_atom_min_gen_ll:
486 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Min, E,
487 AtomicOrdering::Monotonic);
488
489 case NVPTX::BI__nvvm_atom_min_gen_ui:
490 case NVPTX::BI__nvvm_atom_min_gen_ul:
491 case NVPTX::BI__nvvm_atom_min_gen_ull:
492 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::UMin, E,
493 AtomicOrdering::Monotonic);
494
495 case NVPTX::BI__nvvm_atom_cas_gen_us:
496 case NVPTX::BI__nvvm_atom_cas_gen_i:
497 case NVPTX::BI__nvvm_atom_cas_gen_l:
498 case NVPTX::BI__nvvm_atom_cas_gen_ll:
499 // __nvvm_atom_cas_gen_* should return the old value rather than the
500 // success flag.
501 return MakeAtomicCmpXchgValue(*this, E, /*ReturnBool=*/false,
502 AtomicOrdering::Monotonic,
503 AtomicOrdering::Monotonic);
504
505 case NVPTX::BI__nvvm_atom_add_gen_f:
506 case NVPTX::BI__nvvm_atom_add_gen_d: {
507 Address DestAddr = EmitPointerWithAlignment(E->getArg(0));
508 Value *Val = EmitScalarExpr(E->getArg(1));
509
510 return Builder.CreateAtomicRMW(llvm::AtomicRMWInst::FAdd, DestAddr, Val,
511 AtomicOrdering::Monotonic);
512 }
513
514 case NVPTX::BI__nvvm_atom_inc_gen_ui:
515 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::UIncWrap, E,
516 AtomicOrdering::Monotonic);
517
518 case NVPTX::BI__nvvm_atom_dec_gen_ui:
519 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::UDecWrap, E,
520 AtomicOrdering::Monotonic);
521
522 case NVPTX::BI__nvvm_ldg_c:
523 case NVPTX::BI__nvvm_ldg_sc:
524 case NVPTX::BI__nvvm_ldg_c2:
525 case NVPTX::BI__nvvm_ldg_sc2:
526 case NVPTX::BI__nvvm_ldg_c4:
527 case NVPTX::BI__nvvm_ldg_sc4:
528 case NVPTX::BI__nvvm_ldg_s:
529 case NVPTX::BI__nvvm_ldg_s2:
530 case NVPTX::BI__nvvm_ldg_s4:
531 case NVPTX::BI__nvvm_ldg_i:
532 case NVPTX::BI__nvvm_ldg_i2:
533 case NVPTX::BI__nvvm_ldg_i4:
534 case NVPTX::BI__nvvm_ldg_l:
535 case NVPTX::BI__nvvm_ldg_l2:
536 case NVPTX::BI__nvvm_ldg_ll:
537 case NVPTX::BI__nvvm_ldg_ll2:
538 case NVPTX::BI__nvvm_ldg_uc:
539 case NVPTX::BI__nvvm_ldg_uc2:
540 case NVPTX::BI__nvvm_ldg_uc4:
541 case NVPTX::BI__nvvm_ldg_us:
542 case NVPTX::BI__nvvm_ldg_us2:
543 case NVPTX::BI__nvvm_ldg_us4:
544 case NVPTX::BI__nvvm_ldg_ui:
545 case NVPTX::BI__nvvm_ldg_ui2:
546 case NVPTX::BI__nvvm_ldg_ui4:
547 case NVPTX::BI__nvvm_ldg_ul:
548 case NVPTX::BI__nvvm_ldg_ul2:
549 case NVPTX::BI__nvvm_ldg_ull:
550 case NVPTX::BI__nvvm_ldg_ull2:
551 case NVPTX::BI__nvvm_ldg_f:
552 case NVPTX::BI__nvvm_ldg_f2:
553 case NVPTX::BI__nvvm_ldg_f4:
554 case NVPTX::BI__nvvm_ldg_d:
555 case NVPTX::BI__nvvm_ldg_d2:
556 // PTX Interoperability section 2.2: "For a vector with an even number of
557 // elements, its alignment is set to number of elements times the alignment
558 // of its member: n*alignof(t)."
559 return MakeLdg(*this, E);
560
561 case NVPTX::BI__nvvm_ldu_c:
562 case NVPTX::BI__nvvm_ldu_sc:
563 case NVPTX::BI__nvvm_ldu_c2:
564 case NVPTX::BI__nvvm_ldu_sc2:
565 case NVPTX::BI__nvvm_ldu_c4:
566 case NVPTX::BI__nvvm_ldu_sc4:
567 case NVPTX::BI__nvvm_ldu_s:
568 case NVPTX::BI__nvvm_ldu_s2:
569 case NVPTX::BI__nvvm_ldu_s4:
570 case NVPTX::BI__nvvm_ldu_i:
571 case NVPTX::BI__nvvm_ldu_i2:
572 case NVPTX::BI__nvvm_ldu_i4:
573 case NVPTX::BI__nvvm_ldu_l:
574 case NVPTX::BI__nvvm_ldu_l2:
575 case NVPTX::BI__nvvm_ldu_ll:
576 case NVPTX::BI__nvvm_ldu_ll2:
577 case NVPTX::BI__nvvm_ldu_uc:
578 case NVPTX::BI__nvvm_ldu_uc2:
579 case NVPTX::BI__nvvm_ldu_uc4:
580 case NVPTX::BI__nvvm_ldu_us:
581 case NVPTX::BI__nvvm_ldu_us2:
582 case NVPTX::BI__nvvm_ldu_us4:
583 case NVPTX::BI__nvvm_ldu_ui:
584 case NVPTX::BI__nvvm_ldu_ui2:
585 case NVPTX::BI__nvvm_ldu_ui4:
586 case NVPTX::BI__nvvm_ldu_ul:
587 case NVPTX::BI__nvvm_ldu_ul2:
588 case NVPTX::BI__nvvm_ldu_ull:
589 case NVPTX::BI__nvvm_ldu_ull2:
590 return MakeLdu(Intrinsic::nvvm_ldu_global_i, *this, E);
591 case NVPTX::BI__nvvm_ldu_f:
592 case NVPTX::BI__nvvm_ldu_f2:
593 case NVPTX::BI__nvvm_ldu_f4:
594 case NVPTX::BI__nvvm_ldu_d:
595 case NVPTX::BI__nvvm_ldu_d2:
596 return MakeLdu(Intrinsic::nvvm_ldu_global_f, *this, E);
597
598 case NVPTX::BI__nvvm_atom_cta_add_gen_i:
599 case NVPTX::BI__nvvm_atom_cta_add_gen_l:
600 case NVPTX::BI__nvvm_atom_cta_add_gen_ll:
601 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::Add,
602 llvm::AtomicScope::Workgroup);
603 case NVPTX::BI__nvvm_atom_sys_add_gen_i:
604 case NVPTX::BI__nvvm_atom_sys_add_gen_l:
605 case NVPTX::BI__nvvm_atom_sys_add_gen_ll:
606 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::Add,
607 llvm::AtomicScope::System);
608 case NVPTX::BI__nvvm_atom_cta_add_gen_f:
609 case NVPTX::BI__nvvm_atom_cta_add_gen_d:
610 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::FAdd,
611 llvm::AtomicScope::Workgroup);
612 case NVPTX::BI__nvvm_atom_sys_add_gen_f:
613 case NVPTX::BI__nvvm_atom_sys_add_gen_d:
614 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::FAdd,
615 llvm::AtomicScope::System);
616 case NVPTX::BI__nvvm_atom_cta_xchg_gen_i:
617 case NVPTX::BI__nvvm_atom_cta_xchg_gen_l:
618 case NVPTX::BI__nvvm_atom_cta_xchg_gen_ll:
619 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::Xchg,
620 llvm::AtomicScope::Workgroup);
621 case NVPTX::BI__nvvm_atom_sys_xchg_gen_i:
622 case NVPTX::BI__nvvm_atom_sys_xchg_gen_l:
623 case NVPTX::BI__nvvm_atom_sys_xchg_gen_ll:
624 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::Xchg,
625 llvm::AtomicScope::System);
626 case NVPTX::BI__nvvm_atom_cta_max_gen_i:
627 case NVPTX::BI__nvvm_atom_cta_max_gen_l:
628 case NVPTX::BI__nvvm_atom_cta_max_gen_ll:
629 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::Max,
630 llvm::AtomicScope::Workgroup);
631 case NVPTX::BI__nvvm_atom_cta_max_gen_ui:
632 case NVPTX::BI__nvvm_atom_cta_max_gen_ul:
633 case NVPTX::BI__nvvm_atom_cta_max_gen_ull:
634 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::UMax,
635 llvm::AtomicScope::Workgroup);
636 case NVPTX::BI__nvvm_atom_sys_max_gen_i:
637 case NVPTX::BI__nvvm_atom_sys_max_gen_l:
638 case NVPTX::BI__nvvm_atom_sys_max_gen_ll:
639 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::Max,
640 llvm::AtomicScope::System);
641 case NVPTX::BI__nvvm_atom_sys_max_gen_ui:
642 case NVPTX::BI__nvvm_atom_sys_max_gen_ul:
643 case NVPTX::BI__nvvm_atom_sys_max_gen_ull:
644 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::UMax,
645 llvm::AtomicScope::System);
646 case NVPTX::BI__nvvm_atom_cta_min_gen_i:
647 case NVPTX::BI__nvvm_atom_cta_min_gen_l:
648 case NVPTX::BI__nvvm_atom_cta_min_gen_ll:
649 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::Min,
650 llvm::AtomicScope::Workgroup);
651 case NVPTX::BI__nvvm_atom_cta_min_gen_ui:
652 case NVPTX::BI__nvvm_atom_cta_min_gen_ul:
653 case NVPTX::BI__nvvm_atom_cta_min_gen_ull:
654 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::UMin,
655 llvm::AtomicScope::Workgroup);
656 case NVPTX::BI__nvvm_atom_sys_min_gen_i:
657 case NVPTX::BI__nvvm_atom_sys_min_gen_l:
658 case NVPTX::BI__nvvm_atom_sys_min_gen_ll:
659 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::Min,
660 llvm::AtomicScope::System);
661 case NVPTX::BI__nvvm_atom_sys_min_gen_ui:
662 case NVPTX::BI__nvvm_atom_sys_min_gen_ul:
663 case NVPTX::BI__nvvm_atom_sys_min_gen_ull:
664 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::UMin,
665 llvm::AtomicScope::System);
666 case NVPTX::BI__nvvm_atom_cta_inc_gen_ui:
667 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::UIncWrap,
668 llvm::AtomicScope::Workgroup);
669 case NVPTX::BI__nvvm_atom_cta_dec_gen_ui:
670 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::UDecWrap,
671 llvm::AtomicScope::Workgroup);
672 case NVPTX::BI__nvvm_atom_sys_inc_gen_ui:
673 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::UIncWrap,
674 llvm::AtomicScope::System);
675 case NVPTX::BI__nvvm_atom_sys_dec_gen_ui:
676 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::UDecWrap,
677 llvm::AtomicScope::System);
678 case NVPTX::BI__nvvm_atom_cta_and_gen_i:
679 case NVPTX::BI__nvvm_atom_cta_and_gen_l:
680 case NVPTX::BI__nvvm_atom_cta_and_gen_ll:
681 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::And,
682 llvm::AtomicScope::Workgroup);
683 case NVPTX::BI__nvvm_atom_sys_and_gen_i:
684 case NVPTX::BI__nvvm_atom_sys_and_gen_l:
685 case NVPTX::BI__nvvm_atom_sys_and_gen_ll:
686 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::And,
687 llvm::AtomicScope::System);
688 case NVPTX::BI__nvvm_atom_cta_or_gen_i:
689 case NVPTX::BI__nvvm_atom_cta_or_gen_l:
690 case NVPTX::BI__nvvm_atom_cta_or_gen_ll:
691 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::Or,
692 llvm::AtomicScope::Workgroup);
693 case NVPTX::BI__nvvm_atom_sys_or_gen_i:
694 case NVPTX::BI__nvvm_atom_sys_or_gen_l:
695 case NVPTX::BI__nvvm_atom_sys_or_gen_ll:
696 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::Or,
697 llvm::AtomicScope::System);
698 case NVPTX::BI__nvvm_atom_cta_xor_gen_i:
699 case NVPTX::BI__nvvm_atom_cta_xor_gen_l:
700 case NVPTX::BI__nvvm_atom_cta_xor_gen_ll:
701 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::Xor,
702 llvm::AtomicScope::Workgroup);
703 case NVPTX::BI__nvvm_atom_sys_xor_gen_i:
704 case NVPTX::BI__nvvm_atom_sys_xor_gen_l:
705 case NVPTX::BI__nvvm_atom_sys_xor_gen_ll:
706 return MakeScopedAtomicRMW(*this, E, llvm::AtomicRMWInst::Xor,
707 llvm::AtomicScope::System);
708 case NVPTX::BI__nvvm_atom_cta_cas_gen_us:
709 case NVPTX::BI__nvvm_atom_cta_cas_gen_i:
710 case NVPTX::BI__nvvm_atom_cta_cas_gen_l:
711 case NVPTX::BI__nvvm_atom_cta_cas_gen_ll:
712 return MakeScopedAtomicCAS(*this, E, llvm::AtomicScope::Workgroup);
713 case NVPTX::BI__nvvm_atom_sys_cas_gen_us:
714 case NVPTX::BI__nvvm_atom_sys_cas_gen_i:
715 case NVPTX::BI__nvvm_atom_sys_cas_gen_l:
716 case NVPTX::BI__nvvm_atom_sys_cas_gen_ll:
717 return MakeScopedAtomicCAS(*this, E, llvm::AtomicScope::System);
718 case NVPTX::BI__nvvm_match_all_sync_i32p:
719 case NVPTX::BI__nvvm_match_all_sync_i64p: {
720 Value *Mask = EmitScalarExpr(E->getArg(0));
721 Value *Val = EmitScalarExpr(E->getArg(1));
722 Address PredOutPtr = EmitPointerWithAlignment(E->getArg(2));
723 Value *ResultPair = Builder.CreateCall(
724 CGM.getIntrinsic(BuiltinID == NVPTX::BI__nvvm_match_all_sync_i32p
725 ? Intrinsic::nvvm_match_all_sync_i32p
726 : Intrinsic::nvvm_match_all_sync_i64p),
727 {Mask, Val});
728 Value *Pred = Builder.CreateZExt(Builder.CreateExtractValue(ResultPair, 1),
729 PredOutPtr.getElementType());
730 Builder.CreateStore(Pred, PredOutPtr);
731 return Builder.CreateExtractValue(ResultPair, 0);
732 }
733
734 // FP MMA loads
735 case NVPTX::BI__hmma_m16n16k16_ld_a:
736 case NVPTX::BI__hmma_m16n16k16_ld_b:
737 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
738 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
739 case NVPTX::BI__hmma_m32n8k16_ld_a:
740 case NVPTX::BI__hmma_m32n8k16_ld_b:
741 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
742 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
743 case NVPTX::BI__hmma_m8n32k16_ld_a:
744 case NVPTX::BI__hmma_m8n32k16_ld_b:
745 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
746 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
747 // Integer MMA loads.
748 case NVPTX::BI__imma_m16n16k16_ld_a_s8:
749 case NVPTX::BI__imma_m16n16k16_ld_a_u8:
750 case NVPTX::BI__imma_m16n16k16_ld_b_s8:
751 case NVPTX::BI__imma_m16n16k16_ld_b_u8:
752 case NVPTX::BI__imma_m16n16k16_ld_c:
753 case NVPTX::BI__imma_m32n8k16_ld_a_s8:
754 case NVPTX::BI__imma_m32n8k16_ld_a_u8:
755 case NVPTX::BI__imma_m32n8k16_ld_b_s8:
756 case NVPTX::BI__imma_m32n8k16_ld_b_u8:
757 case NVPTX::BI__imma_m32n8k16_ld_c:
758 case NVPTX::BI__imma_m8n32k16_ld_a_s8:
759 case NVPTX::BI__imma_m8n32k16_ld_a_u8:
760 case NVPTX::BI__imma_m8n32k16_ld_b_s8:
761 case NVPTX::BI__imma_m8n32k16_ld_b_u8:
762 case NVPTX::BI__imma_m8n32k16_ld_c:
763 // Sub-integer MMA loads.
764 case NVPTX::BI__imma_m8n8k32_ld_a_s4:
765 case NVPTX::BI__imma_m8n8k32_ld_a_u4:
766 case NVPTX::BI__imma_m8n8k32_ld_b_s4:
767 case NVPTX::BI__imma_m8n8k32_ld_b_u4:
768 case NVPTX::BI__imma_m8n8k32_ld_c:
769 case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
770 case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
771 case NVPTX::BI__bmma_m8n8k128_ld_c:
772 // Double MMA loads.
773 case NVPTX::BI__dmma_m8n8k4_ld_a:
774 case NVPTX::BI__dmma_m8n8k4_ld_b:
775 case NVPTX::BI__dmma_m8n8k4_ld_c:
776 // Alternate float MMA loads.
777 case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
778 case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
779 case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
780 case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
781 case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
782 case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
783 case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
784 case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
785 case NVPTX::BI__mma_tf32_m16n16k8_ld_c: {
786 Address Dst = EmitPointerWithAlignment(E->getArg(0));
787 Value *Src = EmitScalarExpr(E->getArg(1));
788 Value *Ldm = EmitScalarExpr(E->getArg(2));
789 std::optional<llvm::APSInt> isColMajorArg =
790 E->getArg(3)->getIntegerConstantExpr(getContext());
791 if (!isColMajorArg)
792 return nullptr;
793 bool isColMajor = isColMajorArg->getSExtValue();
794 NVPTXMmaLdstInfo II = getNVPTXMmaLdstInfo(BuiltinID);
795 unsigned IID = isColMajor ? II.IID_col : II.IID_row;
796 if (IID == 0)
797 return nullptr;
798
799 Value *Result =
800 Builder.CreateCall(CGM.getIntrinsic(IID, Src->getType()), {Src, Ldm});
801
802 // Save returned values.
803 assert(II.NumResults);
804 if (II.NumResults == 1) {
805 Builder.CreateAlignedStore(Result, Dst.emitRawPointer(*this),
806 CharUnits::fromQuantity(4));
807 } else {
808 for (unsigned i = 0; i < II.NumResults; ++i) {
809 Builder.CreateAlignedStore(
810 Builder.CreateBitCast(Builder.CreateExtractValue(Result, i),
811 Dst.getElementType()),
812 Builder.CreateGEP(Dst.getElementType(), Dst.emitRawPointer(*this),
813 llvm::ConstantInt::get(IntTy, i)),
814 CharUnits::fromQuantity(4));
815 }
816 }
817 return Result;
818 }
819
820 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
821 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
822 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
823 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
824 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
825 case NVPTX::BI__hmma_m8n32k16_st_c_f32:
826 case NVPTX::BI__imma_m16n16k16_st_c_i32:
827 case NVPTX::BI__imma_m32n8k16_st_c_i32:
828 case NVPTX::BI__imma_m8n32k16_st_c_i32:
829 case NVPTX::BI__imma_m8n8k32_st_c_i32:
830 case NVPTX::BI__bmma_m8n8k128_st_c_i32:
831 case NVPTX::BI__dmma_m8n8k4_st_c_f64:
832 case NVPTX::BI__mma_m16n16k8_st_c_f32: {
833 Value *Dst = EmitScalarExpr(E->getArg(0));
834 Address Src = EmitPointerWithAlignment(E->getArg(1));
835 Value *Ldm = EmitScalarExpr(E->getArg(2));
836 std::optional<llvm::APSInt> isColMajorArg =
837 E->getArg(3)->getIntegerConstantExpr(getContext());
838 if (!isColMajorArg)
839 return nullptr;
840 bool isColMajor = isColMajorArg->getSExtValue();
841 NVPTXMmaLdstInfo II = getNVPTXMmaLdstInfo(BuiltinID);
842 unsigned IID = isColMajor ? II.IID_col : II.IID_row;
843 if (IID == 0)
844 return nullptr;
845 Function *Intrinsic =
846 CGM.getIntrinsic(IID, Dst->getType());
847 llvm::Type *ParamType = Intrinsic->getFunctionType()->getParamType(1);
848 SmallVector<Value *, 10> Values = {Dst};
849 for (unsigned i = 0; i < II.NumResults; ++i) {
850 Value *V = Builder.CreateAlignedLoad(
851 Src.getElementType(),
852 Builder.CreateGEP(Src.getElementType(), Src.emitRawPointer(*this),
853 llvm::ConstantInt::get(IntTy, i)),
854 CharUnits::fromQuantity(4));
855 Values.push_back(Builder.CreateBitCast(V, ParamType));
856 }
857 Values.push_back(Ldm);
858 Value *Result = Builder.CreateCall(Intrinsic, Values);
859 return Result;
860 }
861
862 // BI__hmma_m16n16k16_mma_<Dtype><CType>(d, a, b, c, layout, satf) -->
863 // Intrinsic::nvvm_wmma_m16n16k16_mma_sync<layout A,B><DType><CType><Satf>
864 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
865 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
866 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
867 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
868 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
869 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
870 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
871 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
872 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
873 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
874 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
875 case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
876 case NVPTX::BI__imma_m16n16k16_mma_s8:
877 case NVPTX::BI__imma_m16n16k16_mma_u8:
878 case NVPTX::BI__imma_m32n8k16_mma_s8:
879 case NVPTX::BI__imma_m32n8k16_mma_u8:
880 case NVPTX::BI__imma_m8n32k16_mma_s8:
881 case NVPTX::BI__imma_m8n32k16_mma_u8:
882 case NVPTX::BI__imma_m8n8k32_mma_s4:
883 case NVPTX::BI__imma_m8n8k32_mma_u4:
884 case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1:
885 case NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1:
886 case NVPTX::BI__dmma_m8n8k4_mma_f64:
887 case NVPTX::BI__mma_bf16_m16n16k16_mma_f32:
888 case NVPTX::BI__mma_bf16_m8n32k16_mma_f32:
889 case NVPTX::BI__mma_bf16_m32n8k16_mma_f32:
890 case NVPTX::BI__mma_tf32_m16n16k8_mma_f32: {
891 Address Dst = EmitPointerWithAlignment(E->getArg(0));
892 Address SrcA = EmitPointerWithAlignment(E->getArg(1));
893 Address SrcB = EmitPointerWithAlignment(E->getArg(2));
894 Address SrcC = EmitPointerWithAlignment(E->getArg(3));
895 std::optional<llvm::APSInt> LayoutArg =
896 E->getArg(4)->getIntegerConstantExpr(getContext());
897 if (!LayoutArg)
898 return nullptr;
899 int Layout = LayoutArg->getSExtValue();
900 if (Layout < 0 || Layout > 3)
901 return nullptr;
902 llvm::APSInt SatfArg;
903 if (BuiltinID == NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1 ||
904 BuiltinID == NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1)
905 SatfArg = 0; // .b1 does not have satf argument.
906 else if (std::optional<llvm::APSInt> OptSatfArg =
907 E->getArg(5)->getIntegerConstantExpr(getContext()))
908 SatfArg = *OptSatfArg;
909 else
910 return nullptr;
911 bool Satf = SatfArg.getSExtValue();
912 NVPTXMmaInfo MI = getNVPTXMmaInfo(BuiltinID);
913 unsigned IID = MI.getMMAIntrinsic(Layout, Satf);
914 if (IID == 0) // Unsupported combination of Layout/Satf.
915 return nullptr;
916
917 SmallVector<Value *, 24> Values;
918 Function *Intrinsic = CGM.getIntrinsic(IID);
919 llvm::Type *AType = Intrinsic->getFunctionType()->getParamType(0);
920 // Load A
921 for (unsigned i = 0; i < MI.NumEltsA; ++i) {
922 Value *V = Builder.CreateAlignedLoad(
923 SrcA.getElementType(),
924 Builder.CreateGEP(SrcA.getElementType(), SrcA.emitRawPointer(*this),
925 llvm::ConstantInt::get(IntTy, i)),
926 CharUnits::fromQuantity(4));
927 Values.push_back(Builder.CreateBitCast(V, AType));
928 }
929 // Load B
930 llvm::Type *BType = Intrinsic->getFunctionType()->getParamType(MI.NumEltsA);
931 for (unsigned i = 0; i < MI.NumEltsB; ++i) {
932 Value *V = Builder.CreateAlignedLoad(
933 SrcB.getElementType(),
934 Builder.CreateGEP(SrcB.getElementType(), SrcB.emitRawPointer(*this),
935 llvm::ConstantInt::get(IntTy, i)),
936 CharUnits::fromQuantity(4));
937 Values.push_back(Builder.CreateBitCast(V, BType));
938 }
939 // Load C
940 llvm::Type *CType =
941 Intrinsic->getFunctionType()->getParamType(MI.NumEltsA + MI.NumEltsB);
942 for (unsigned i = 0; i < MI.NumEltsC; ++i) {
943 Value *V = Builder.CreateAlignedLoad(
944 SrcC.getElementType(),
945 Builder.CreateGEP(SrcC.getElementType(), SrcC.emitRawPointer(*this),
946 llvm::ConstantInt::get(IntTy, i)),
947 CharUnits::fromQuantity(4));
948 Values.push_back(Builder.CreateBitCast(V, CType));
949 }
950 Value *Result = Builder.CreateCall(Intrinsic, Values);
951 llvm::Type *DType = Dst.getElementType();
952 for (unsigned i = 0; i < MI.NumEltsD; ++i)
953 Builder.CreateAlignedStore(
954 Builder.CreateBitCast(Builder.CreateExtractValue(Result, i), DType),
955 Builder.CreateGEP(Dst.getElementType(), Dst.emitRawPointer(*this),
956 llvm::ConstantInt::get(IntTy, i)),
957 CharUnits::fromQuantity(4));
958 return Result;
959 }
960 // The following builtins require half type support
961 case NVPTX::BI__nvvm_ex2_approx_f16:
962 return MakeHalfType(
963 CGM.getIntrinsic(Intrinsic::nvvm_ex2_approx, Builder.getHalfTy()),
964 BuiltinID, E, *this);
965 case NVPTX::BI__nvvm_ex2_approx_f16x2:
966 return MakeHalfType(
967 CGM.getIntrinsic(Intrinsic::nvvm_ex2_approx,
968 FixedVectorType::get(Builder.getHalfTy(), 2)),
969 BuiltinID, E, *this);
970 case NVPTX::BI__nvvm_ff2f16x2_rn:
971 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rn, BuiltinID, E, *this);
972 case NVPTX::BI__nvvm_ff2f16x2_rn_relu:
973 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rn_relu, BuiltinID, E, *this);
974 case NVPTX::BI__nvvm_ff2f16x2_rz:
975 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rz, BuiltinID, E, *this);
976 case NVPTX::BI__nvvm_ff2f16x2_rz_relu:
977 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rz_relu, BuiltinID, E, *this);
978 case NVPTX::BI__nvvm_fma_rn_f16:
979 return MakeHalfType(Intrinsic::nvvm_fma_rn_f16, BuiltinID, E, *this);
980 case NVPTX::BI__nvvm_fma_rn_f16x2:
981 return MakeHalfType(Intrinsic::nvvm_fma_rn_f16x2, BuiltinID, E, *this);
982 case NVPTX::BI__nvvm_fma_rn_ftz_f16:
983 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_f16, BuiltinID, E, *this);
984 case NVPTX::BI__nvvm_fma_rn_ftz_f16x2:
985 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_f16x2, BuiltinID, E, *this);
986 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16:
987 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_relu_f16, BuiltinID, E,
988 *this);
989 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16x2:
990 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_relu_f16x2, BuiltinID, E,
991 *this);
992 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16:
993 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_sat_f16, BuiltinID, E,
994 *this);
995 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16x2:
996 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_sat_f16x2, BuiltinID, E,
997 *this);
998 case NVPTX::BI__nvvm_fma_rn_relu_f16:
999 return MakeHalfType(Intrinsic::nvvm_fma_rn_relu_f16, BuiltinID, E, *this);
1000 case NVPTX::BI__nvvm_fma_rn_relu_f16x2:
1001 return MakeHalfType(Intrinsic::nvvm_fma_rn_relu_f16x2, BuiltinID, E, *this);
1002 case NVPTX::BI__nvvm_fma_rn_sat_f16:
1003 return MakeHalfType(Intrinsic::nvvm_fma_rn_sat_f16, BuiltinID, E, *this);
1004 case NVPTX::BI__nvvm_fma_rn_sat_f16x2:
1005 return MakeHalfType(Intrinsic::nvvm_fma_rn_sat_f16x2, BuiltinID, E, *this);
1006 case NVPTX::BI__nvvm_fma_rn_oob_f16:
1007 return MakeFMAOOB(Intrinsic::nvvm_fma_rn_oob, Builder.getHalfTy(), E,
1008 *this);
1009 case NVPTX::BI__nvvm_fma_rn_oob_f16x2:
1010 return MakeFMAOOB(Intrinsic::nvvm_fma_rn_oob,
1011 llvm::FixedVectorType::get(Builder.getHalfTy(), 2), E,
1012 *this);
1013 case NVPTX::BI__nvvm_fma_rn_oob_bf16:
1014 return MakeFMAOOB(Intrinsic::nvvm_fma_rn_oob, Builder.getBFloatTy(), E,
1015 *this);
1016 case NVPTX::BI__nvvm_fma_rn_oob_bf16x2:
1017 return MakeFMAOOB(Intrinsic::nvvm_fma_rn_oob,
1018 llvm::FixedVectorType::get(Builder.getBFloatTy(), 2), E,
1019 *this);
1020 case NVPTX::BI__nvvm_fma_rn_oob_relu_f16:
1021 return MakeFMAOOB(Intrinsic::nvvm_fma_rn_oob_relu, Builder.getHalfTy(), E,
1022 *this);
1023 case NVPTX::BI__nvvm_fma_rn_oob_relu_f16x2:
1024 return MakeFMAOOB(Intrinsic::nvvm_fma_rn_oob_relu,
1025 llvm::FixedVectorType::get(Builder.getHalfTy(), 2), E,
1026 *this);
1027 case NVPTX::BI__nvvm_fma_rn_oob_relu_bf16:
1028 return MakeFMAOOB(Intrinsic::nvvm_fma_rn_oob_relu, Builder.getBFloatTy(), E,
1029 *this);
1030 case NVPTX::BI__nvvm_fma_rn_oob_relu_bf16x2:
1031 return MakeFMAOOB(Intrinsic::nvvm_fma_rn_oob_relu,
1032 llvm::FixedVectorType::get(Builder.getBFloatTy(), 2), E,
1033 *this);
1034 case NVPTX::BI__nvvm_fmax_f16:
1035 return MakeHalfType(Intrinsic::nvvm_fmax_f16, BuiltinID, E, *this);
1036 case NVPTX::BI__nvvm_fmax_f16x2:
1037 return MakeHalfType(Intrinsic::nvvm_fmax_f16x2, BuiltinID, E, *this);
1038 case NVPTX::BI__nvvm_fmax_ftz_f16:
1039 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_f16, BuiltinID, E, *this);
1040 case NVPTX::BI__nvvm_fmax_ftz_f16x2:
1041 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_f16x2, BuiltinID, E, *this);
1042 case NVPTX::BI__nvvm_fmax_ftz_nan_f16:
1043 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_f16, BuiltinID, E, *this);
1044 case NVPTX::BI__nvvm_fmax_ftz_nan_f16x2:
1045 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_f16x2, BuiltinID, E,
1046 *this);
1047 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16:
1048 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f16, BuiltinID,
1049 E, *this);
1050 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16x2:
1051 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f16x2,
1052 BuiltinID, E, *this);
1053 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16:
1054 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_xorsign_abs_f16, BuiltinID, E,
1055 *this);
1056 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16x2:
1057 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_xorsign_abs_f16x2, BuiltinID,
1058 E, *this);
1059 case NVPTX::BI__nvvm_fmax_nan_f16:
1060 return MakeHalfType(Intrinsic::nvvm_fmax_nan_f16, BuiltinID, E, *this);
1061 case NVPTX::BI__nvvm_fmax_nan_f16x2:
1062 return MakeHalfType(Intrinsic::nvvm_fmax_nan_f16x2, BuiltinID, E, *this);
1063 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16:
1064 return MakeHalfType(Intrinsic::nvvm_fmax_nan_xorsign_abs_f16, BuiltinID, E,
1065 *this);
1066 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16x2:
1067 return MakeHalfType(Intrinsic::nvvm_fmax_nan_xorsign_abs_f16x2, BuiltinID,
1068 E, *this);
1069 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16:
1070 return MakeHalfType(Intrinsic::nvvm_fmax_xorsign_abs_f16, BuiltinID, E,
1071 *this);
1072 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16x2:
1073 return MakeHalfType(Intrinsic::nvvm_fmax_xorsign_abs_f16x2, BuiltinID, E,
1074 *this);
1075 case NVPTX::BI__nvvm_fmin_f16:
1076 return MakeHalfType(Intrinsic::nvvm_fmin_f16, BuiltinID, E, *this);
1077 case NVPTX::BI__nvvm_fmin_f16x2:
1078 return MakeHalfType(Intrinsic::nvvm_fmin_f16x2, BuiltinID, E, *this);
1079 case NVPTX::BI__nvvm_fmin_ftz_f16:
1080 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_f16, BuiltinID, E, *this);
1081 case NVPTX::BI__nvvm_fmin_ftz_f16x2:
1082 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_f16x2, BuiltinID, E, *this);
1083 case NVPTX::BI__nvvm_fmin_ftz_nan_f16:
1084 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_f16, BuiltinID, E, *this);
1085 case NVPTX::BI__nvvm_fmin_ftz_nan_f16x2:
1086 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_f16x2, BuiltinID, E,
1087 *this);
1088 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16:
1089 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f16, BuiltinID,
1090 E, *this);
1091 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16x2:
1092 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f16x2,
1093 BuiltinID, E, *this);
1094 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16:
1095 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_xorsign_abs_f16, BuiltinID, E,
1096 *this);
1097 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16x2:
1098 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_xorsign_abs_f16x2, BuiltinID,
1099 E, *this);
1100 case NVPTX::BI__nvvm_fmin_nan_f16:
1101 return MakeHalfType(Intrinsic::nvvm_fmin_nan_f16, BuiltinID, E, *this);
1102 case NVPTX::BI__nvvm_fmin_nan_f16x2:
1103 return MakeHalfType(Intrinsic::nvvm_fmin_nan_f16x2, BuiltinID, E, *this);
1104 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16:
1105 return MakeHalfType(Intrinsic::nvvm_fmin_nan_xorsign_abs_f16, BuiltinID, E,
1106 *this);
1107 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16x2:
1108 return MakeHalfType(Intrinsic::nvvm_fmin_nan_xorsign_abs_f16x2, BuiltinID,
1109 E, *this);
1110 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16:
1111 return MakeHalfType(Intrinsic::nvvm_fmin_xorsign_abs_f16, BuiltinID, E,
1112 *this);
1113 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16x2:
1114 return MakeHalfType(Intrinsic::nvvm_fmin_xorsign_abs_f16x2, BuiltinID, E,
1115 *this);
1116 case NVPTX::BI__nvvm_fabs_f:
1117 case NVPTX::BI__nvvm_abs_bf16:
1118 case NVPTX::BI__nvvm_abs_bf16x2:
1119 case NVPTX::BI__nvvm_fabs_f16:
1120 case NVPTX::BI__nvvm_fabs_f16x2:
1121 return Builder.CreateUnaryIntrinsic(Intrinsic::nvvm_fabs,
1122 EmitScalarExpr(E->getArg(0)));
1123 case NVPTX::BI__nvvm_fabs_ftz_f:
1124 case NVPTX::BI__nvvm_fabs_ftz_f16:
1125 case NVPTX::BI__nvvm_fabs_ftz_f16x2:
1126 return Builder.CreateUnaryIntrinsic(Intrinsic::nvvm_fabs_ftz,
1127 EmitScalarExpr(E->getArg(0)));
1128 case NVPTX::BI__nvvm_fabs_d:
1129 return Builder.CreateFAbs(EmitScalarExpr(E->getArg(0)));
1130 case NVPTX::BI__nvvm_ex2_approx_d:
1131 case NVPTX::BI__nvvm_ex2_approx_f:
1132 return Builder.CreateUnaryIntrinsic(Intrinsic::nvvm_ex2_approx,
1133 EmitScalarExpr(E->getArg(0)));
1134 case NVPTX::BI__nvvm_ex2_approx_ftz_f:
1135 return Builder.CreateUnaryIntrinsic(Intrinsic::nvvm_ex2_approx_ftz,
1136 EmitScalarExpr(E->getArg(0)));
1137 case NVPTX::BI__nvvm_ldg_h:
1138 case NVPTX::BI__nvvm_ldg_h2:
1139 return MakeLdg(*this, E);
1140 case NVPTX::BI__nvvm_ldu_h:
1141 case NVPTX::BI__nvvm_ldu_h2:
1142 return MakeLdu(Intrinsic::nvvm_ldu_global_f, *this, E);
1143 case NVPTX::BI__nvvm_cp_async_ca_shared_global_4:
1144 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_4,
1145 Intrinsic::nvvm_cp_async_ca_shared_global_4_s, *this, E,
1146 4);
1147 case NVPTX::BI__nvvm_cp_async_ca_shared_global_8:
1148 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_8,
1149 Intrinsic::nvvm_cp_async_ca_shared_global_8_s, *this, E,
1150 8);
1151 case NVPTX::BI__nvvm_cp_async_ca_shared_global_16:
1152 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_16,
1153 Intrinsic::nvvm_cp_async_ca_shared_global_16_s, *this, E,
1154 16);
1155 case NVPTX::BI__nvvm_cp_async_cg_shared_global_16:
1156 return MakeCpAsync(Intrinsic::nvvm_cp_async_cg_shared_global_16,
1157 Intrinsic::nvvm_cp_async_cg_shared_global_16_s, *this, E,
1158 16);
1159 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_x:
1160 return Builder.CreateCall(
1161 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_clusterid_x));
1162 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_y:
1163 return Builder.CreateCall(
1164 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_clusterid_y));
1165 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_z:
1166 return Builder.CreateCall(
1167 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_clusterid_z));
1168 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_w:
1169 return Builder.CreateCall(
1170 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_clusterid_w));
1171 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_x:
1172 return Builder.CreateCall(
1173 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_nclusterid_x));
1174 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_y:
1175 return Builder.CreateCall(
1176 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_nclusterid_y));
1177 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_z:
1178 return Builder.CreateCall(
1179 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_nclusterid_z));
1180 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_w:
1181 return Builder.CreateCall(
1182 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_nclusterid_w));
1183 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_x:
1184 return Builder.CreateCall(
1185 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_ctaid_x));
1186 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_y:
1187 return Builder.CreateCall(
1188 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_ctaid_y));
1189 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_z:
1190 return Builder.CreateCall(
1191 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_ctaid_z));
1192 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_w:
1193 return Builder.CreateCall(
1194 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_ctaid_w));
1195 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_x:
1196 return Builder.CreateCall(
1197 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_nctaid_x));
1198 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_y:
1199 return Builder.CreateCall(
1200 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_nctaid_y));
1201 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_z:
1202 return Builder.CreateCall(
1203 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_nctaid_z));
1204 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_w:
1205 return Builder.CreateCall(
1206 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_nctaid_w));
1207 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctarank:
1208 return Builder.CreateCall(
1209 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_ctarank));
1210 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctarank:
1211 return Builder.CreateCall(
1212 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_nctarank));
1213 case NVPTX::BI__nvvm_is_explicit_cluster:
1214 return Builder.CreateCall(
1215 CGM.getIntrinsic(Intrinsic::nvvm_is_explicit_cluster));
1216 case NVPTX::BI__nvvm_isspacep_shared_cluster:
1217 return Builder.CreateCall(
1218 CGM.getIntrinsic(Intrinsic::nvvm_isspacep_shared_cluster),
1219 EmitScalarExpr(E->getArg(0)));
1220 case NVPTX::BI__nvvm_mapa:
1221 return Builder.CreateCall(
1222 CGM.getIntrinsic(Intrinsic::nvvm_mapa),
1223 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
1224 case NVPTX::BI__nvvm_mapa_shared_cluster:
1225 return Builder.CreateCall(
1226 CGM.getIntrinsic(Intrinsic::nvvm_mapa_shared_cluster),
1227 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
1228 case NVPTX::BI__nvvm_getctarank:
1229 return Builder.CreateCall(
1230 CGM.getIntrinsic(Intrinsic::nvvm_getctarank),
1231 EmitScalarExpr(E->getArg(0)));
1232 case NVPTX::BI__nvvm_getctarank_shared_cluster:
1233 return Builder.CreateCall(
1234 CGM.getIntrinsic(Intrinsic::nvvm_getctarank_shared_cluster),
1235 EmitScalarExpr(E->getArg(0)));
1236 case NVPTX::BI__nvvm_barrier_cluster_arrive:
1237 return Builder.CreateCall(
1238 CGM.getIntrinsic(Intrinsic::nvvm_barrier_cluster_arrive));
1239 case NVPTX::BI__nvvm_barrier_cluster_arrive_relaxed:
1240 return Builder.CreateCall(
1241 CGM.getIntrinsic(Intrinsic::nvvm_barrier_cluster_arrive_relaxed));
1242 case NVPTX::BI__nvvm_barrier_cluster_wait:
1243 return Builder.CreateCall(
1244 CGM.getIntrinsic(Intrinsic::nvvm_barrier_cluster_wait));
1245 case NVPTX::BI__nvvm_fence_sc_cluster:
1246 return Builder.CreateCall(
1247 CGM.getIntrinsic(Intrinsic::nvvm_fence_sc_cluster));
1248 case NVPTX::BI__nvvm_bar_sync:
1249 return Builder.CreateCall(
1250 CGM.getIntrinsic(Intrinsic::nvvm_barrier_cta_sync_aligned_all),
1251 EmitScalarExpr(E->getArg(0)));
1252 case NVPTX::BI__syncthreads:
1253 return Builder.CreateCall(
1254 CGM.getIntrinsic(Intrinsic::nvvm_barrier_cta_sync_aligned_all),
1255 Builder.getInt32(0));
1256 case NVPTX::BI__nvvm_barrier_sync:
1257 return Builder.CreateCall(
1258 CGM.getIntrinsic(Intrinsic::nvvm_barrier_cta_sync_all),
1259 EmitScalarExpr(E->getArg(0)));
1260 case NVPTX::BI__nvvm_barrier_sync_cnt:
1261 return Builder.CreateCall(
1262 CGM.getIntrinsic(Intrinsic::nvvm_barrier_cta_sync_count),
1263 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
1264 case NVPTX::BI__nvvm_bar0_and:
1265 return Builder.CreateZExt(
1266 Builder.CreateIntrinsic(
1267 Intrinsic::nvvm_barrier_cta_red_and_aligned_all, {},
1268 {Builder.getInt32(0),
1269 Builder.CreateICmpNE(EmitScalarExpr(E->getArg(0)),
1270 Builder.getInt32(0))}),
1271 Builder.getInt32Ty());
1272 case NVPTX::BI__nvvm_bar0_or:
1273 return Builder.CreateZExt(
1274 Builder.CreateIntrinsic(
1275 Intrinsic::nvvm_barrier_cta_red_or_aligned_all, {},
1276 {Builder.getInt32(0),
1277 Builder.CreateICmpNE(EmitScalarExpr(E->getArg(0)),
1278 Builder.getInt32(0))}),
1279 Builder.getInt32Ty());
1280 case NVPTX::BI__nvvm_bar0_popc:
1281 return Builder.CreateIntrinsic(
1282 Intrinsic::nvvm_barrier_cta_red_popc_aligned_all, {},
1283 {Builder.getInt32(0), Builder.CreateICmpNE(EmitScalarExpr(E->getArg(0)),
1284 Builder.getInt32(0))});
1285 default:
1286 return nullptr;
1287 }
1288}
V
#define V(N, I)
Definition ASTContext.h:3756
MakeAtomicCmpXchgValue
Value * MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E, bool ReturnBool, llvm::AtomicOrdering SuccessOrdering, llvm::AtomicOrdering FailureOrdering)
Utility to insert an atomic cmpxchg instruction.
Definition CGBuiltin.cpp:399
MakeBinaryAtomicValue
Value * MakeBinaryAtomicValue(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E, AtomicOrdering Ordering)
Utility to insert an atomic instruction based on Intrinsic::ID and the expression node.
Definition CGBuiltin.cpp:299
MMA_VARIANTS_B1_AND
#define MMA_VARIANTS_B1_AND(geom, type)
MMA_INTR
#define MMA_INTR(geom_op_type, layout)
Definition NVPTX.cpp:31
MMA_VARIANTS
#define MMA_VARIANTS(geom, type)
MMA_SATF_VARIANTS
#define MMA_SATF_VARIANTS(geom, type)
MMA_LDST
#define MMA_LDST(n, geom_op_type)
Definition NVPTX.cpp:33
MMA_VARIANTS_B1_XOR
#define MMA_VARIANTS_B1_XOR(geom, type)
MMA_VARIANTS_I4
#define MMA_VARIANTS_I4(geom, type)
TargetBuiltins.h
Enumerates target-specific builtins in their own namespaces within namespace clang.
ASTContext::GE_None
@ GE_None
No error.
Definition ASTContext.h:2659
clang::ASTContext::GetBuiltinType
QualType GetBuiltinType(unsigned ID, GetBuiltinTypeError &Error, unsigned *IntegerConstantArgs=nullptr) const
Return the type for the specified builtin.
Definition ASTContext.cpp:13025
clang::CallExpr
CallExpr - Represents a function call (C99 6.5.2.2, C++ [expr.call]).
Definition Expr.h:2949
clang::CallExpr::getArg
Expr * getArg(unsigned Arg)
getArg - Return the specified argument.
Definition Expr.h:3153
clang::CallExpr::getNumArgs
unsigned getNumArgs() const
getNumArgs - Return the number of actual arguments to this call.
Definition Expr.h:3140
clang::CharUnits
CharUnits - This is an opaque type for sizes expressed in character units.
Definition CharUnits.h:38
clang::CharUnits::getAsAlign
llvm::Align getAsAlign() const
getAsAlign - Returns Quantity as a valid llvm::Align, Beware llvm::Align assumes power of two 8-bit b...
Definition CharUnits.h:189
clang::CharUnits::fromQuantity
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
Definition CharUnits.h:63
clang::CodeGen::Address
Like RawAddress, an abstract representation of an aligned address, but the pointer contained in this ...
Definition Address.h:128
clang::CodeGen::Address::emitRawPointer
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
Return the pointer contained in this class after authenticating it and adding offset to it if necessa...
Definition Address.h:253
clang::CodeGen::Address::getElementType
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Definition Address.h:209
clang::CodeGen::CGBuilderTy::CreateAtomicRMW
llvm::AtomicRMWInst * CreateAtomicRMW(llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val, llvm::AtomicOrdering Ordering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
Definition CGBuilder.h:190
clang::CodeGen::CGBuilderTy::CreateAtomicCmpXchg
llvm::AtomicCmpXchgInst * CreateAtomicCmpXchg(Address Addr, llvm::Value *Cmp, llvm::Value *New, llvm::AtomicOrdering SuccessOrdering, llvm::AtomicOrdering FailureOrdering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
Definition CGBuilder.h:179
clang::CodeGen::CGBuilderTy::CreateAlignedLoad
llvm::LoadInst * CreateAlignedLoad(llvm::Type *Ty, llvm::Value *Addr, CharUnits Align, const llvm::Twine &Name="")
Definition CGBuilder.h:138
clang::CodeGen::CGBuilderTy::CreateAddrSpaceCast
Address CreateAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
Definition CGBuilder.h:199
clang::CodeGen::CodeGenFunction
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
Definition CodeGenFunction.h:258
clang::CodeGen::CodeGenFunction::getTarget
const TargetInfo & getTarget() const
Definition CodeGenFunction.h:2238
clang::CodeGen::CodeGenFunction::ConvertTypeForMem
llvm::Type * ConvertTypeForMem(QualType T)
Definition CodeGenFunction.cpp:241
clang::CodeGen::CodeGenFunction::EmitPointerWithAlignment
Address EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitPointerWithAlignment - Given an expression with a pointer type, emit the value and compute our be...
Definition CGExpr.cpp:1621
clang::CodeGen::CodeGenFunction::EmitScalarExpr
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
Definition CGExprScalar.cpp:6195
clang::CodeGen::CodeGenFunction::EmitNVPTXBuiltinExpr
llvm::Value * EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
Definition NVPTX.cpp:432
clang::CodeGen::CodeGenFunction::getLLVMContext
llvm::LLVMContext & getLLVMContext()
Definition CodeGenFunction.h:2239
clang::CodeGen::CodeGenModule::getContext
ASTContext & getContext() const
Definition CodeGenModule.h:899
clang::CodeGen::CodeGenModule::getNaturalPointeeTypeAlignment
CharUnits getNaturalPointeeTypeAlignment(QualType T, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
Definition CodeGenModule.cpp:8737
clang::CodeGen::CodeGenModule::getIntrinsic
llvm::Function * getIntrinsic(unsigned IID, ArrayRef< llvm::Type * > Tys={})
Definition CodeGenModule.cpp:7252
clang::Expr::getIntegerConstantExpr
std::optional< llvm::APSInt > getIntegerConstantExpr(const ASTContext &Ctx) const
isIntegerConstantExpr - Return the value if this expression is a valid integer constant expression.
Definition ExprConstant.cpp:22758
clang::Expr::getType
QualType getType() const
Definition Expr.h:144
clang::QualType
A (possibly-)qualified type.
Definition TypeBase.h:938
clang::Scope
Scope - A scope is a transient data structure that is used while parsing the program.
Definition Scope.h:41
clang::TargetInfo::getTriple
const llvm::Triple & getTriple() const
Returns the target triple of the primary target.
Definition TargetInfo.h:1327
clang::Value::getType
QualType getType() const
Definition Value.cpp:238
clang
The JSON file list parser is used to communicate input to InstallAPI.
Definition CalledOnceCheck.h:17
clang::OpenACCClauseKind::DType
@ DType
'dtype' clause, an alias for 'device_type', stored separately for diagnostic purposes.
Definition OpenACCKinds.h:274
clang::ObjCSubstitutionContext::Result
@ Result
The result type of a method or function.
Definition TypeBase.h:906
llvm
Diagnostic wrappers for TextAPI types for error reporting.
Definition Dominators.h:30
Generated on for clang by doxygen 1.14.0