clang 23.0.0git
CIRGenBuiltinAArch64.cpp
Go to the documentation of this file.
1//===---- CIRGenBuiltinAArch64.cpp - Emit CIR for AArch64 builtins --------===//
2// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
3// See https://llvm.org/LICENSE.txt for license information.
4// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5//
6//===----------------------------------------------------------------------===//
7//
8// This contains code to emit ARM64 Builtin calls as CIR or a function call
9// to be later resolved.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CIRGenBuilder.h"
14#include "CIRGenFunction.h"
15#include "clang/Basic/TargetBuiltins.h"
16#include "clang/CIR/MissingFeatures.h"
17
18// TODO(cir): once all builtins are covered, decide whether we still
19// need to use LLVM intrinsics or if there's a better approach to follow. Right
20// now the intrinsics are reused to make it convenient to encode all thousands
21// of them and passing down to LLVM lowering.
22#include "llvm/IR/Intrinsics.h"
23#include "llvm/IR/IntrinsicsAArch64.h"
24
25#include "mlir/IR/Value.h"
26#include "clang/AST/GlobalDecl.h"
27#include "clang/Basic/Builtins.h"
28
29using namespace clang;
30using namespace clang::CIRGen;
31using namespace llvm;
32
33// Generate vscale * scalingFactor
34static mlir::Value genVscaleTimesFactor(mlir::Location loc,
35 CIRGenBuilderTy builder,
36 mlir::Type cirTy,
37 int32_t scalingFactor) {
38 mlir::Value vscale = builder.emitIntrinsicCallOp(loc, "vscale", cirTy);
39 return builder.createNUWAMul(loc, vscale,
40 builder.getUInt64(scalingFactor, loc));
41}
42
43//===----------------------------------------------------------------------===//
44// Intrinsics maps
45//
46// Maps that help automate code-generation.
47//
48// TODO(cir): Share this code with ARM.cpp
49//===----------------------------------------------------------------------===//
50enum {
51 AddRetType = (1 << 0),
52 Add1ArgType = (1 << 1),
53 Add2ArgTypes = (1 << 2),
54
55 VectorizeRetType = (1 << 3),
56 VectorizeArgTypes = (1 << 4),
57
58 InventFloatType = (1 << 5),
59 UnsignedAlts = (1 << 6),
60
61 Use64BitVectors = (1 << 7),
62 Use128BitVectors = (1 << 8),
63
64 Vectorize1ArgType = Add1ArgType | VectorizeArgTypes,
65 VectorRet = AddRetType | VectorizeRetType,
66 VectorRetGetArgs01 =
67 AddRetType | Add2ArgTypes | VectorizeRetType | VectorizeArgTypes,
68 FpCmpzModifiers =
69 AddRetType | VectorizeRetType | Add1ArgType | InventFloatType
70};
71
72namespace {
73struct ARMVectorIntrinsicInfo {
74 const char *nameHint;
75 unsigned builtinID;
76 unsigned llvmIntrinsic;
77 unsigned altLLVMIntrinsic;
78 uint64_t typeModifier;
79
80 bool operator<(unsigned rhsBuiltinID) const {
81 return builtinID < rhsBuiltinID;
82 }
83 bool operator<(const ARMVectorIntrinsicInfo &te) const {
84 return builtinID < te.builtinID;
85 }
86};
87} // end anonymous namespace
88
89#define NEONMAP0(NameBase) \
90 {#NameBase, NEON::BI__builtin_neon_##NameBase, 0, 0, 0}
91
92#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
93 {#NameBase, NEON::BI__builtin_neon_##NameBase, Intrinsic::LLVMIntrinsic, 0, \
94 TypeModifier}
95
96#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier) \
97 {#NameBase, NEON::BI__builtin_neon_##NameBase, Intrinsic::LLVMIntrinsic, \
98 Intrinsic::AltLLVMIntrinsic, TypeModifier}
99
100static const ARMVectorIntrinsicInfo AArch64SIMDIntrinsicMap[] = {
101 NEONMAP0(splat_lane_v),
102 NEONMAP0(splat_laneq_v),
103 NEONMAP0(splatq_lane_v),
104 NEONMAP0(splatq_laneq_v),
105 NEONMAP1(vabs_v, aarch64_neon_abs, 0),
106 NEONMAP1(vabsq_v, aarch64_neon_abs, 0),
107 NEONMAP0(vadd_v),
108 NEONMAP0(vaddhn_v),
109 NEONMAP0(vaddq_p128),
110 NEONMAP0(vaddq_v),
111 NEONMAP1(vaesdq_u8, aarch64_crypto_aesd, 0),
112 NEONMAP1(vaeseq_u8, aarch64_crypto_aese, 0),
113 NEONMAP1(vaesimcq_u8, aarch64_crypto_aesimc, 0),
114 NEONMAP1(vaesmcq_u8, aarch64_crypto_aesmc, 0),
115 NEONMAP2(vbcaxq_s16, aarch64_crypto_bcaxu, aarch64_crypto_bcaxs,
116 Add1ArgType | UnsignedAlts),
117 NEONMAP2(vbcaxq_s32, aarch64_crypto_bcaxu, aarch64_crypto_bcaxs,
118 Add1ArgType | UnsignedAlts),
119 NEONMAP2(vbcaxq_s64, aarch64_crypto_bcaxu, aarch64_crypto_bcaxs,
120 Add1ArgType | UnsignedAlts),
121 NEONMAP2(vbcaxq_s8, aarch64_crypto_bcaxu, aarch64_crypto_bcaxs,
122 Add1ArgType | UnsignedAlts),
123 NEONMAP2(vbcaxq_u16, aarch64_crypto_bcaxu, aarch64_crypto_bcaxs,
124 Add1ArgType | UnsignedAlts),
125 NEONMAP2(vbcaxq_u32, aarch64_crypto_bcaxu, aarch64_crypto_bcaxs,
126 Add1ArgType | UnsignedAlts),
127 NEONMAP2(vbcaxq_u64, aarch64_crypto_bcaxu, aarch64_crypto_bcaxs,
128 Add1ArgType | UnsignedAlts),
129 NEONMAP2(vbcaxq_u8, aarch64_crypto_bcaxu, aarch64_crypto_bcaxs,
130 Add1ArgType | UnsignedAlts),
131 NEONMAP1(vbfdot_f32, aarch64_neon_bfdot, 0),
132 NEONMAP1(vbfdotq_f32, aarch64_neon_bfdot, 0),
133 NEONMAP1(vbfmlalbq_f32, aarch64_neon_bfmlalb, 0),
134 NEONMAP1(vbfmlaltq_f32, aarch64_neon_bfmlalt, 0),
135 NEONMAP1(vbfmmlaq_f32, aarch64_neon_bfmmla, 0),
136 NEONMAP1(vcadd_rot270_f16, aarch64_neon_vcadd_rot270, Add1ArgType),
137 NEONMAP1(vcadd_rot270_f32, aarch64_neon_vcadd_rot270, Add1ArgType),
138 NEONMAP1(vcadd_rot90_f16, aarch64_neon_vcadd_rot90, Add1ArgType),
139 NEONMAP1(vcadd_rot90_f32, aarch64_neon_vcadd_rot90, Add1ArgType),
140 NEONMAP1(vcaddq_rot270_f16, aarch64_neon_vcadd_rot270, Add1ArgType),
141 NEONMAP1(vcaddq_rot270_f32, aarch64_neon_vcadd_rot270, Add1ArgType),
142 NEONMAP1(vcaddq_rot270_f64, aarch64_neon_vcadd_rot270, Add1ArgType),
143 NEONMAP1(vcaddq_rot90_f16, aarch64_neon_vcadd_rot90, Add1ArgType),
144 NEONMAP1(vcaddq_rot90_f32, aarch64_neon_vcadd_rot90, Add1ArgType),
145 NEONMAP1(vcaddq_rot90_f64, aarch64_neon_vcadd_rot90, Add1ArgType),
146 NEONMAP1(vcage_v, aarch64_neon_facge, 0),
147 NEONMAP1(vcageq_v, aarch64_neon_facge, 0),
148 NEONMAP1(vcagt_v, aarch64_neon_facgt, 0),
149 NEONMAP1(vcagtq_v, aarch64_neon_facgt, 0),
150 NEONMAP1(vcale_v, aarch64_neon_facge, 0),
151 NEONMAP1(vcaleq_v, aarch64_neon_facge, 0),
152 NEONMAP1(vcalt_v, aarch64_neon_facgt, 0),
153 NEONMAP1(vcaltq_v, aarch64_neon_facgt, 0),
154 NEONMAP0(vceqz_v),
155 NEONMAP0(vceqzq_v),
156 NEONMAP0(vcgez_v),
157 NEONMAP0(vcgezq_v),
158 NEONMAP0(vcgtz_v),
159 NEONMAP0(vcgtzq_v),
160 NEONMAP0(vclez_v),
161 NEONMAP0(vclezq_v),
162 NEONMAP1(vcls_v, aarch64_neon_cls, Add1ArgType),
163 NEONMAP1(vclsq_v, aarch64_neon_cls, Add1ArgType),
164 NEONMAP0(vcltz_v),
165 NEONMAP0(vcltzq_v),
166 NEONMAP1(vclz_v, ctlz, Add1ArgType),
167 NEONMAP1(vclzq_v, ctlz, Add1ArgType),
168 NEONMAP1(vcmla_f16, aarch64_neon_vcmla_rot0, Add1ArgType),
169 NEONMAP1(vcmla_f32, aarch64_neon_vcmla_rot0, Add1ArgType),
170 NEONMAP1(vcmla_rot180_f16, aarch64_neon_vcmla_rot180, Add1ArgType),
171 NEONMAP1(vcmla_rot180_f32, aarch64_neon_vcmla_rot180, Add1ArgType),
172 NEONMAP1(vcmla_rot270_f16, aarch64_neon_vcmla_rot270, Add1ArgType),
173 NEONMAP1(vcmla_rot270_f32, aarch64_neon_vcmla_rot270, Add1ArgType),
174 NEONMAP1(vcmla_rot90_f16, aarch64_neon_vcmla_rot90, Add1ArgType),
175 NEONMAP1(vcmla_rot90_f32, aarch64_neon_vcmla_rot90, Add1ArgType),
176 NEONMAP1(vcmlaq_f16, aarch64_neon_vcmla_rot0, Add1ArgType),
177 NEONMAP1(vcmlaq_f32, aarch64_neon_vcmla_rot0, Add1ArgType),
178 NEONMAP1(vcmlaq_f64, aarch64_neon_vcmla_rot0, Add1ArgType),
179 NEONMAP1(vcmlaq_rot180_f16, aarch64_neon_vcmla_rot180, Add1ArgType),
180 NEONMAP1(vcmlaq_rot180_f32, aarch64_neon_vcmla_rot180, Add1ArgType),
181 NEONMAP1(vcmlaq_rot180_f64, aarch64_neon_vcmla_rot180, Add1ArgType),
182 NEONMAP1(vcmlaq_rot270_f16, aarch64_neon_vcmla_rot270, Add1ArgType),
183 NEONMAP1(vcmlaq_rot270_f32, aarch64_neon_vcmla_rot270, Add1ArgType),
184 NEONMAP1(vcmlaq_rot270_f64, aarch64_neon_vcmla_rot270, Add1ArgType),
185 NEONMAP1(vcmlaq_rot90_f16, aarch64_neon_vcmla_rot90, Add1ArgType),
186 NEONMAP1(vcmlaq_rot90_f32, aarch64_neon_vcmla_rot90, Add1ArgType),
187 NEONMAP1(vcmlaq_rot90_f64, aarch64_neon_vcmla_rot90, Add1ArgType),
188 NEONMAP1(vcnt_v, ctpop, Add1ArgType),
189 NEONMAP1(vcntq_v, ctpop, Add1ArgType),
190 NEONMAP1(vcvt_f16_f32, aarch64_neon_vcvtfp2hf, 0),
191 NEONMAP0(vcvt_f16_s16),
192 NEONMAP0(vcvt_f16_u16),
193 NEONMAP1(vcvt_f32_f16, aarch64_neon_vcvthf2fp, 0),
194 NEONMAP0(vcvt_f32_v),
195 NEONMAP1(vcvt_n_f16_s16, aarch64_neon_vcvtfxs2fp, 0),
196 NEONMAP1(vcvt_n_f16_u16, aarch64_neon_vcvtfxu2fp, 0),
197 NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
198 NEONMAP2(vcvt_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
199 NEONMAP1(vcvt_n_s16_f16, aarch64_neon_vcvtfp2fxs, 0),
200 NEONMAP1(vcvt_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
201 NEONMAP1(vcvt_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
202 NEONMAP1(vcvt_n_u16_f16, aarch64_neon_vcvtfp2fxu, 0),
203 NEONMAP1(vcvt_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
204 NEONMAP1(vcvt_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
205 NEONMAP0(vcvtq_f16_s16),
206 NEONMAP0(vcvtq_f16_u16),
207 NEONMAP0(vcvtq_f32_v),
208 NEONMAP0(vcvtq_high_bf16_f32),
209 NEONMAP0(vcvtq_low_bf16_f32),
210 NEONMAP1(vcvtq_n_f16_s16, aarch64_neon_vcvtfxs2fp, 0),
211 NEONMAP1(vcvtq_n_f16_u16, aarch64_neon_vcvtfxu2fp, 0),
212 NEONMAP2(vcvtq_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp,
213 0),
214 NEONMAP2(vcvtq_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp,
215 0),
216 NEONMAP1(vcvtq_n_s16_f16, aarch64_neon_vcvtfp2fxs, 0),
217 NEONMAP1(vcvtq_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
218 NEONMAP1(vcvtq_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
219 NEONMAP1(vcvtq_n_u16_f16, aarch64_neon_vcvtfp2fxu, 0),
220 NEONMAP1(vcvtq_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
221 NEONMAP1(vcvtq_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
222 NEONMAP1(vcvtx_f32_v, aarch64_neon_fcvtxn, AddRetType | Add1ArgType),
223 NEONMAP1(vdot_s32, aarch64_neon_sdot, 0),
224 NEONMAP1(vdot_u32, aarch64_neon_udot, 0),
225 NEONMAP1(vdotq_s32, aarch64_neon_sdot, 0),
226 NEONMAP1(vdotq_u32, aarch64_neon_udot, 0),
227 NEONMAP2(veor3q_s16, aarch64_crypto_eor3u, aarch64_crypto_eor3s,
228 Add1ArgType | UnsignedAlts),
229 NEONMAP2(veor3q_s32, aarch64_crypto_eor3u, aarch64_crypto_eor3s,
230 Add1ArgType | UnsignedAlts),
231 NEONMAP2(veor3q_s64, aarch64_crypto_eor3u, aarch64_crypto_eor3s,
232 Add1ArgType | UnsignedAlts),
233 NEONMAP2(veor3q_s8, aarch64_crypto_eor3u, aarch64_crypto_eor3s,
234 Add1ArgType | UnsignedAlts),
235 NEONMAP2(veor3q_u16, aarch64_crypto_eor3u, aarch64_crypto_eor3s,
236 Add1ArgType | UnsignedAlts),
237 NEONMAP2(veor3q_u32, aarch64_crypto_eor3u, aarch64_crypto_eor3s,
238 Add1ArgType | UnsignedAlts),
239 NEONMAP2(veor3q_u64, aarch64_crypto_eor3u, aarch64_crypto_eor3s,
240 Add1ArgType | UnsignedAlts),
241 NEONMAP2(veor3q_u8, aarch64_crypto_eor3u, aarch64_crypto_eor3s,
242 Add1ArgType | UnsignedAlts),
243 NEONMAP0(vext_v),
244 NEONMAP0(vextq_v),
245 NEONMAP0(vfma_v),
246 NEONMAP0(vfmaq_v),
247 NEONMAP1(vfmlal_high_f16, aarch64_neon_fmlal2, 0),
248 NEONMAP1(vfmlal_low_f16, aarch64_neon_fmlal, 0),
249 NEONMAP1(vfmlalq_high_f16, aarch64_neon_fmlal2, 0),
250 NEONMAP1(vfmlalq_low_f16, aarch64_neon_fmlal, 0),
251 NEONMAP1(vfmlsl_high_f16, aarch64_neon_fmlsl2, 0),
252 NEONMAP1(vfmlsl_low_f16, aarch64_neon_fmlsl, 0),
253 NEONMAP1(vfmlslq_high_f16, aarch64_neon_fmlsl2, 0),
254 NEONMAP1(vfmlslq_low_f16, aarch64_neon_fmlsl, 0),
255 NEONMAP2(vhadd_v, aarch64_neon_uhadd, aarch64_neon_shadd,
256 Add1ArgType | UnsignedAlts),
257 NEONMAP2(vhaddq_v, aarch64_neon_uhadd, aarch64_neon_shadd,
258 Add1ArgType | UnsignedAlts),
259 NEONMAP2(vhsub_v, aarch64_neon_uhsub, aarch64_neon_shsub,
260 Add1ArgType | UnsignedAlts),
261 NEONMAP2(vhsubq_v, aarch64_neon_uhsub, aarch64_neon_shsub,
262 Add1ArgType | UnsignedAlts),
263 NEONMAP1(vld1_x2_v, aarch64_neon_ld1x2, 0),
264 NEONMAP1(vld1_x3_v, aarch64_neon_ld1x3, 0),
265 NEONMAP1(vld1_x4_v, aarch64_neon_ld1x4, 0),
266 NEONMAP1(vld1q_x2_v, aarch64_neon_ld1x2, 0),
267 NEONMAP1(vld1q_x3_v, aarch64_neon_ld1x3, 0),
268 NEONMAP1(vld1q_x4_v, aarch64_neon_ld1x4, 0),
269 NEONMAP1(vmmlaq_s32, aarch64_neon_smmla, 0),
270 NEONMAP1(vmmlaq_u32, aarch64_neon_ummla, 0),
271 NEONMAP0(vmovl_v),
272 NEONMAP0(vmovn_v),
273 NEONMAP1(vmul_v, aarch64_neon_pmul, Add1ArgType),
274 NEONMAP1(vmulq_v, aarch64_neon_pmul, Add1ArgType),
275 NEONMAP1(vpadd_v, aarch64_neon_addp, Add1ArgType),
276 NEONMAP2(vpaddl_v, aarch64_neon_uaddlp, aarch64_neon_saddlp, UnsignedAlts),
277 NEONMAP2(vpaddlq_v, aarch64_neon_uaddlp, aarch64_neon_saddlp, UnsignedAlts),
278 NEONMAP1(vpaddq_v, aarch64_neon_addp, Add1ArgType),
279 NEONMAP1(vqabs_v, aarch64_neon_sqabs, Add1ArgType),
280 NEONMAP1(vqabsq_v, aarch64_neon_sqabs, Add1ArgType),
281 NEONMAP2(vqadd_v, aarch64_neon_uqadd, aarch64_neon_sqadd,
282 Add1ArgType | UnsignedAlts),
283 NEONMAP2(vqaddq_v, aarch64_neon_uqadd, aarch64_neon_sqadd,
284 Add1ArgType | UnsignedAlts),
285 NEONMAP2(vqdmlal_v, aarch64_neon_sqdmull, aarch64_neon_sqadd, 0),
286 NEONMAP2(vqdmlsl_v, aarch64_neon_sqdmull, aarch64_neon_sqsub, 0),
287 NEONMAP1(vqdmulh_lane_v, aarch64_neon_sqdmulh_lane, 0),
288 NEONMAP1(vqdmulh_laneq_v, aarch64_neon_sqdmulh_laneq, 0),
289 NEONMAP1(vqdmulh_v, aarch64_neon_sqdmulh, Add1ArgType),
290 NEONMAP1(vqdmulhq_lane_v, aarch64_neon_sqdmulh_lane, 0),
291 NEONMAP1(vqdmulhq_laneq_v, aarch64_neon_sqdmulh_laneq, 0),
292 NEONMAP1(vqdmulhq_v, aarch64_neon_sqdmulh, Add1ArgType),
293 NEONMAP1(vqdmull_v, aarch64_neon_sqdmull, Add1ArgType),
294 NEONMAP2(vqmovn_v, aarch64_neon_uqxtn, aarch64_neon_sqxtn,
295 Add1ArgType | UnsignedAlts),
296 NEONMAP1(vqmovun_v, aarch64_neon_sqxtun, Add1ArgType),
297 NEONMAP1(vqneg_v, aarch64_neon_sqneg, Add1ArgType),
298 NEONMAP1(vqnegq_v, aarch64_neon_sqneg, Add1ArgType),
299 NEONMAP1(vqrdmlah_s16, aarch64_neon_sqrdmlah, Add1ArgType),
300 NEONMAP1(vqrdmlah_s32, aarch64_neon_sqrdmlah, Add1ArgType),
301 NEONMAP1(vqrdmlahq_s16, aarch64_neon_sqrdmlah, Add1ArgType),
302 NEONMAP1(vqrdmlahq_s32, aarch64_neon_sqrdmlah, Add1ArgType),
303 NEONMAP1(vqrdmlsh_s16, aarch64_neon_sqrdmlsh, Add1ArgType),
304 NEONMAP1(vqrdmlsh_s32, aarch64_neon_sqrdmlsh, Add1ArgType),
305 NEONMAP1(vqrdmlshq_s16, aarch64_neon_sqrdmlsh, Add1ArgType),
306 NEONMAP1(vqrdmlshq_s32, aarch64_neon_sqrdmlsh, Add1ArgType),
307 NEONMAP1(vqrdmulh_lane_v, aarch64_neon_sqrdmulh_lane, 0),
308 NEONMAP1(vqrdmulh_laneq_v, aarch64_neon_sqrdmulh_laneq, 0),
309 NEONMAP1(vqrdmulh_v, aarch64_neon_sqrdmulh, Add1ArgType),
310 NEONMAP1(vqrdmulhq_lane_v, aarch64_neon_sqrdmulh_lane, 0),
311 NEONMAP1(vqrdmulhq_laneq_v, aarch64_neon_sqrdmulh_laneq, 0),
312 NEONMAP1(vqrdmulhq_v, aarch64_neon_sqrdmulh, Add1ArgType),
313 NEONMAP2(vqrshl_v, aarch64_neon_uqrshl, aarch64_neon_sqrshl,
314 Add1ArgType | UnsignedAlts),
315 NEONMAP2(vqrshlq_v, aarch64_neon_uqrshl, aarch64_neon_sqrshl,
316 Add1ArgType | UnsignedAlts),
317 NEONMAP2(vqshl_n_v, aarch64_neon_uqshl, aarch64_neon_sqshl, UnsignedAlts),
318 NEONMAP2(vqshl_v, aarch64_neon_uqshl, aarch64_neon_sqshl,
319 Add1ArgType | UnsignedAlts),
320 NEONMAP2(vqshlq_n_v, aarch64_neon_uqshl, aarch64_neon_sqshl, UnsignedAlts),
321 NEONMAP2(vqshlq_v, aarch64_neon_uqshl, aarch64_neon_sqshl,
322 Add1ArgType | UnsignedAlts),
323 NEONMAP1(vqshlu_n_v, aarch64_neon_sqshlu, 0),
324 NEONMAP1(vqshluq_n_v, aarch64_neon_sqshlu, 0),
325 NEONMAP2(vqsub_v, aarch64_neon_uqsub, aarch64_neon_sqsub,
326 Add1ArgType | UnsignedAlts),
327 NEONMAP2(vqsubq_v, aarch64_neon_uqsub, aarch64_neon_sqsub,
328 Add1ArgType | UnsignedAlts),
329 NEONMAP1(vraddhn_v, aarch64_neon_raddhn, Add1ArgType),
330 NEONMAP1(vrax1q_u64, aarch64_crypto_rax1, 0),
331 NEONMAP2(vrecpe_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
332 NEONMAP2(vrecpeq_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
333 NEONMAP1(vrecps_v, aarch64_neon_frecps, Add1ArgType),
334 NEONMAP1(vrecpsq_v, aarch64_neon_frecps, Add1ArgType),
335 NEONMAP2(vrhadd_v, aarch64_neon_urhadd, aarch64_neon_srhadd,
336 Add1ArgType | UnsignedAlts),
337 NEONMAP2(vrhaddq_v, aarch64_neon_urhadd, aarch64_neon_srhadd,
338 Add1ArgType | UnsignedAlts),
339 NEONMAP1(vrnd32x_f32, aarch64_neon_frint32x, Add1ArgType),
340 NEONMAP1(vrnd32x_f64, aarch64_neon_frint32x, Add1ArgType),
341 NEONMAP1(vrnd32xq_f32, aarch64_neon_frint32x, Add1ArgType),
342 NEONMAP1(vrnd32xq_f64, aarch64_neon_frint32x, Add1ArgType),
343 NEONMAP1(vrnd32z_f32, aarch64_neon_frint32z, Add1ArgType),
344 NEONMAP1(vrnd32z_f64, aarch64_neon_frint32z, Add1ArgType),
345 NEONMAP1(vrnd32zq_f32, aarch64_neon_frint32z, Add1ArgType),
346 NEONMAP1(vrnd32zq_f64, aarch64_neon_frint32z, Add1ArgType),
347 NEONMAP1(vrnd64x_f32, aarch64_neon_frint64x, Add1ArgType),
348 NEONMAP1(vrnd64x_f64, aarch64_neon_frint64x, Add1ArgType),
349 NEONMAP1(vrnd64xq_f32, aarch64_neon_frint64x, Add1ArgType),
350 NEONMAP1(vrnd64xq_f64, aarch64_neon_frint64x, Add1ArgType),
351 NEONMAP1(vrnd64z_f32, aarch64_neon_frint64z, Add1ArgType),
352 NEONMAP1(vrnd64z_f64, aarch64_neon_frint64z, Add1ArgType),
353 NEONMAP1(vrnd64zq_f32, aarch64_neon_frint64z, Add1ArgType),
354 NEONMAP1(vrnd64zq_f64, aarch64_neon_frint64z, Add1ArgType),
355 NEONMAP0(vrndi_v),
356 NEONMAP0(vrndiq_v),
357 NEONMAP2(vrshl_v, aarch64_neon_urshl, aarch64_neon_srshl,
358 Add1ArgType | UnsignedAlts),
359 NEONMAP2(vrshlq_v, aarch64_neon_urshl, aarch64_neon_srshl,
360 Add1ArgType | UnsignedAlts),
361 NEONMAP2(vrshr_n_v, aarch64_neon_urshl, aarch64_neon_srshl, UnsignedAlts),
362 NEONMAP2(vrshrq_n_v, aarch64_neon_urshl, aarch64_neon_srshl, UnsignedAlts),
363 NEONMAP2(vrsqrte_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
364 NEONMAP2(vrsqrteq_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
365 NEONMAP1(vrsqrts_v, aarch64_neon_frsqrts, Add1ArgType),
366 NEONMAP1(vrsqrtsq_v, aarch64_neon_frsqrts, Add1ArgType),
367 NEONMAP1(vrsubhn_v, aarch64_neon_rsubhn, Add1ArgType),
368 NEONMAP1(vsha1su0q_u32, aarch64_crypto_sha1su0, 0),
369 NEONMAP1(vsha1su1q_u32, aarch64_crypto_sha1su1, 0),
370 NEONMAP1(vsha256h2q_u32, aarch64_crypto_sha256h2, 0),
371 NEONMAP1(vsha256hq_u32, aarch64_crypto_sha256h, 0),
372 NEONMAP1(vsha256su0q_u32, aarch64_crypto_sha256su0, 0),
373 NEONMAP1(vsha256su1q_u32, aarch64_crypto_sha256su1, 0),
374 NEONMAP1(vsha512h2q_u64, aarch64_crypto_sha512h2, 0),
375 NEONMAP1(vsha512hq_u64, aarch64_crypto_sha512h, 0),
376 NEONMAP1(vsha512su0q_u64, aarch64_crypto_sha512su0, 0),
377 NEONMAP1(vsha512su1q_u64, aarch64_crypto_sha512su1, 0),
378 NEONMAP0(vshl_n_v),
379 NEONMAP2(vshl_v, aarch64_neon_ushl, aarch64_neon_sshl,
380 Add1ArgType | UnsignedAlts),
381 NEONMAP0(vshll_n_v),
382 NEONMAP0(vshlq_n_v),
383 NEONMAP2(vshlq_v, aarch64_neon_ushl, aarch64_neon_sshl,
384 Add1ArgType | UnsignedAlts),
385 NEONMAP0(vshr_n_v),
386 NEONMAP0(vshrn_n_v),
387 NEONMAP0(vshrq_n_v),
388 NEONMAP1(vsm3partw1q_u32, aarch64_crypto_sm3partw1, 0),
389 NEONMAP1(vsm3partw2q_u32, aarch64_crypto_sm3partw2, 0),
390 NEONMAP1(vsm3ss1q_u32, aarch64_crypto_sm3ss1, 0),
391 NEONMAP1(vsm3tt1aq_u32, aarch64_crypto_sm3tt1a, 0),
392 NEONMAP1(vsm3tt1bq_u32, aarch64_crypto_sm3tt1b, 0),
393 NEONMAP1(vsm3tt2aq_u32, aarch64_crypto_sm3tt2a, 0),
394 NEONMAP1(vsm3tt2bq_u32, aarch64_crypto_sm3tt2b, 0),
395 NEONMAP1(vsm4ekeyq_u32, aarch64_crypto_sm4ekey, 0),
396 NEONMAP1(vsm4eq_u32, aarch64_crypto_sm4e, 0),
397 NEONMAP1(vst1_x2_v, aarch64_neon_st1x2, 0),
398 NEONMAP1(vst1_x3_v, aarch64_neon_st1x3, 0),
399 NEONMAP1(vst1_x4_v, aarch64_neon_st1x4, 0),
400 NEONMAP1(vst1q_x2_v, aarch64_neon_st1x2, 0),
401 NEONMAP1(vst1q_x3_v, aarch64_neon_st1x3, 0),
402 NEONMAP1(vst1q_x4_v, aarch64_neon_st1x4, 0),
403 NEONMAP0(vsubhn_v),
404 NEONMAP0(vtst_v),
405 NEONMAP0(vtstq_v),
406 NEONMAP1(vusdot_s32, aarch64_neon_usdot, 0),
407 NEONMAP1(vusdotq_s32, aarch64_neon_usdot, 0),
408 NEONMAP1(vusmmlaq_s32, aarch64_neon_usmmla, 0),
409 NEONMAP1(vxarq_u64, aarch64_crypto_xar, 0),
410};
411
412#define SVEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
413 {#NameBase, SVE::BI__builtin_sve_##NameBase, Intrinsic::LLVMIntrinsic, 0, \
414 TypeModifier}
415
416#define SVEMAP2(NameBase, TypeModifier) \
417 {#NameBase, SVE::BI__builtin_sve_##NameBase, 0, 0, TypeModifier}
418static const ARMVectorIntrinsicInfo aarch64SVEIntrinsicMap[] = {
419#define GET_SVE_LLVM_INTRINSIC_MAP
420#include "clang/Basic/arm_sve_builtin_cg.inc"
421#undef GET_SVE_LLVM_INTRINSIC_MAP
422};
423
424static bool aarch64SIMDIntrinsicsProvenSorted = false;
425static bool aarch64SVEIntrinsicsProvenSorted = false;
426
427// Check if Builtin `builtinId` is present in `intrinsicMap`. If yes, returns
428// the corresponding info struct.
429static const ARMVectorIntrinsicInfo *
430findARMVectorIntrinsicInMap(ArrayRef<ARMVectorIntrinsicInfo> intrinsicMap,
431 unsigned builtinID, bool &mapProvenSorted) {
432
433#ifndef NDEBUG
434 if (!mapProvenSorted) {
435 assert(llvm::is_sorted(intrinsicMap));
436 mapProvenSorted = true;
437 }
438#endif
439
440 const ARMVectorIntrinsicInfo *info =
441 llvm::lower_bound(intrinsicMap, builtinID);
442
443 if (info != intrinsicMap.end() && info->builtinID == builtinID)
444 return info;
445
446 return nullptr;
447}
448
449//===----------------------------------------------------------------------===//
450// Emit-helpers
451//===----------------------------------------------------------------------===//
452static mlir::Value
453emitAArch64CompareBuiltinExpr(CIRGenFunction &cgf, CIRGenBuilderTy &builder,
454 mlir::Location loc, mlir::Value src,
455 mlir::Type retTy, const cir::CmpOpKind kind) {
456
457 bool scalarCmp = !isa<cir::VectorType>(src.getType());
458 if (!scalarCmp) {
459 assert(!cast<cir::VectorType>(retTy).getIsScalable() &&
460 "This is only intended for fixed-width vectors");
461 // Vector types are cast to i8 vectors. Recover original type.
462 src = builder.createBitcast(src, retTy);
463 }
464
465 mlir::Value zero = builder.getNullValue(src.getType(), loc);
466
467 if (!scalarCmp)
468 return builder.createVecCompare(loc, kind, src, zero);
469
470 // For scalars, cast !cir.bool to !cir.int<s, 1> so that the compare
471 // result is sign- rather zero-extended when casting to the output
472 // retType.
473 mlir::Value cmp = builder.createCast(
474 loc, cir::CastKind::bool_to_int,
475 builder.createCompare(loc, kind, src, zero), builder.getSIntNTy(1));
476
477 return builder.createCast(loc, cir::CastKind::integral, cmp, retTy);
478}
479
480// TODO(cir): Remove `loc` from the list of arguments once all NYIs are gone.
481static cir::VectorType getNeonType(CIRGenFunction *cgf, NeonTypeFlags typeFlags,
482 mlir::Location loc,
483 bool hasLegalHalfType = true,
484 bool v1Ty = false,
485 bool allowBFloatArgsAndRet = true) {
486 int isQuad = typeFlags.isQuad();
487 switch (typeFlags.getEltType()) {
488 case NeonTypeFlags::Int8:
489 case NeonTypeFlags::Poly8:
490 return cir::VectorType::get(typeFlags.isUnsigned() ? cgf->uInt8Ty
491 : cgf->sInt8Ty,
492 v1Ty ? 1 : (8 << isQuad));
493 case NeonTypeFlags::MFloat8:
494 cgf->getCIRGenModule().errorNYI(loc, std::string("NEON type: MFloat8"));
495 [[fallthrough]];
496 case NeonTypeFlags::Int16:
497 case NeonTypeFlags::Poly16:
498 return cir::VectorType::get(typeFlags.isUnsigned() ? cgf->uInt16Ty
499 : cgf->sInt16Ty,
500 v1Ty ? 1 : (4 << isQuad));
501 case NeonTypeFlags::BFloat16:
502 if (allowBFloatArgsAndRet)
503 cgf->getCIRGenModule().errorNYI(loc, std::string("NEON type: BFloat16"));
504 else
505 cgf->getCIRGenModule().errorNYI(loc, std::string("NEON type: BFloat16"));
506 [[fallthrough]];
507 case NeonTypeFlags::Float16:
508 if (hasLegalHalfType)
509 cgf->getCIRGenModule().errorNYI(loc, std::string("NEON type: Float16"));
510 else
511 cgf->getCIRGenModule().errorNYI(loc, std::string("NEON type: Float16"));
512 [[fallthrough]];
513 case NeonTypeFlags::Int32:
514 return cir::VectorType::get(typeFlags.isUnsigned() ? cgf->uInt32Ty
515 : cgf->sInt32Ty,
516 v1Ty ? 1 : (2 << isQuad));
517 case NeonTypeFlags::Int64:
518 case NeonTypeFlags::Poly64:
519 return cir::VectorType::get(typeFlags.isUnsigned() ? cgf->uInt64Ty
520 : cgf->sInt64Ty,
521 v1Ty ? 1 : (1 << isQuad));
522 case NeonTypeFlags::Poly128:
523 // FIXME: i128 and f128 doesn't get fully support in Clang and llvm.
524 // There is a lot of i128 and f128 API missing.
525 // so we use v16i8 to represent poly128 and get pattern matched.
526 cgf->getCIRGenModule().errorNYI(loc, std::string("NEON type: Poly128"));
527 [[fallthrough]];
528 case NeonTypeFlags::Float32:
529 return cir::VectorType::get(cgf->getCIRGenModule().floatTy,
530 v1Ty ? 1 : (2 << isQuad));
531 case NeonTypeFlags::Float64:
532 return cir::VectorType::get(cgf->getCIRGenModule().doubleTy,
533 v1Ty ? 1 : (1 << isQuad));
534 }
535 llvm_unreachable("Unknown vector element type!");
536}
537
538static mlir::Value emitCommonNeonBuiltinExpr(
539 CIRGenFunction &cgf, unsigned builtinID, unsigned llvmIntrinsic,
540 unsigned altLLVMIntrinsic, const char *nameHint, unsigned modifier,
541 const CallExpr *expr, llvm::SmallVectorImpl<mlir::Value> &ops) {
542
543 mlir::Location loc = cgf.getLoc(expr->getExprLoc());
544 clang::ASTContext &ctx = cgf.getContext();
545
546 // Extract the trailing immediate argument that encodes the type discriminator
547 // for this overloaded intrinsic.
548 // TODO: Move to the parent code that takes care of argument processing.
549 const clang::Expr *arg = expr->getArg(expr->getNumArgs() - 1);
550 std::optional<llvm::APSInt> neonTypeConst = arg->getIntegerConstantExpr(ctx);
551 if (!neonTypeConst)
552 return nullptr;
553
554 // Determine the type of this overloaded NEON intrinsic.
555 NeonTypeFlags neonType(neonTypeConst->getZExtValue());
556 const bool hasLegalHalfType = cgf.getTarget().hasFastHalfType();
557
558 // The value of allowBFloatArgsAndRet is true for AArch64, but it should
559 // come from ABI info.
560 const bool allowBFloatArgsAndRet = false;
561 // FIXME
562 // getTargetHooks().getABIInfo().allowBFloatArgsAndRet();
563
564 cir::VectorType vTy = getNeonType(&cgf, neonType, loc, hasLegalHalfType,
565 false, allowBFloatArgsAndRet);
566 mlir::Type ty = vTy;
567 if (!ty)
568 return nullptr;
569
570 switch (builtinID) {
571 case NEON::BI__builtin_neon_splat_lane_v:
572 case NEON::BI__builtin_neon_splat_laneq_v:
573 case NEON::BI__builtin_neon_splatq_lane_v:
574 case NEON::BI__builtin_neon_splatq_laneq_v:
575 case NEON::BI__builtin_neon_vpadd_v:
576 case NEON::BI__builtin_neon_vpaddq_v:
577 case NEON::BI__builtin_neon_vabs_v:
578 case NEON::BI__builtin_neon_vabsq_v:
579 case NEON::BI__builtin_neon_vadd_v:
580 case NEON::BI__builtin_neon_vaddq_v:
581 case NEON::BI__builtin_neon_vaddhn_v:
582 case NEON::BI__builtin_neon_vcale_v:
583 case NEON::BI__builtin_neon_vcaleq_v:
584 case NEON::BI__builtin_neon_vcalt_v:
585 case NEON::BI__builtin_neon_vcaltq_v:
586 case NEON::BI__builtin_neon_vcage_v:
587 case NEON::BI__builtin_neon_vcageq_v:
588 case NEON::BI__builtin_neon_vcagt_v:
589 case NEON::BI__builtin_neon_vcagtq_v:
590 cgf.cgm.errorNYI(expr->getSourceRange(),
591 std::string("unimplemented AArch64 builtin call: ") +
592 ctx.BuiltinInfo.getName(builtinID));
593 return mlir::Value{};
594 case NEON::BI__builtin_neon_vceqz_v:
595 case NEON::BI__builtin_neon_vceqzq_v:
596 return emitAArch64CompareBuiltinExpr(cgf, cgf.getBuilder(), loc, ops[0],
597 vTy, cir::CmpOpKind::eq);
598 case NEON::BI__builtin_neon_vcgez_v:
599 case NEON::BI__builtin_neon_vcgezq_v:
600 case NEON::BI__builtin_neon_vclez_v:
601 case NEON::BI__builtin_neon_vclezq_v:
602 case NEON::BI__builtin_neon_vcgtz_v:
603 case NEON::BI__builtin_neon_vcgtzq_v:
604 case NEON::BI__builtin_neon_vcltz_v:
605 case NEON::BI__builtin_neon_vcltzq_v:
606 case NEON::BI__builtin_neon_vclz_v:
607 case NEON::BI__builtin_neon_vclzq_v:
608 case NEON::BI__builtin_neon_vcvt_f32_v:
609 case NEON::BI__builtin_neon_vcvtq_f32_v:
610 case NEON::BI__builtin_neon_vcvt_f16_s16:
611 case NEON::BI__builtin_neon_vcvt_f16_u16:
612 case NEON::BI__builtin_neon_vcvtq_f16_s16:
613 case NEON::BI__builtin_neon_vcvtq_f16_u16:
614 case NEON::BI__builtin_neon_vcvt_n_f16_s16:
615 case NEON::BI__builtin_neon_vcvt_n_f16_u16:
616 case NEON::BI__builtin_neon_vcvtq_n_f16_s16:
617 case NEON::BI__builtin_neon_vcvtq_n_f16_u16:
618 case NEON::BI__builtin_neon_vcvt_n_f32_v:
619 case NEON::BI__builtin_neon_vcvt_n_f64_v:
620 case NEON::BI__builtin_neon_vcvtq_n_f32_v:
621 case NEON::BI__builtin_neon_vcvtq_n_f64_v:
622 case NEON::BI__builtin_neon_vcvt_n_s16_f16:
623 case NEON::BI__builtin_neon_vcvt_n_s32_v:
624 case NEON::BI__builtin_neon_vcvt_n_u16_f16:
625 case NEON::BI__builtin_neon_vcvt_n_u32_v:
626 case NEON::BI__builtin_neon_vcvt_n_s64_v:
627 case NEON::BI__builtin_neon_vcvt_n_u64_v:
628 case NEON::BI__builtin_neon_vcvtq_n_s16_f16:
629 case NEON::BI__builtin_neon_vcvtq_n_s32_v:
630 case NEON::BI__builtin_neon_vcvtq_n_u16_f16:
631 case NEON::BI__builtin_neon_vcvtq_n_u32_v:
632 case NEON::BI__builtin_neon_vcvtq_n_s64_v:
633 case NEON::BI__builtin_neon_vcvtq_n_u64_v:
634 case NEON::BI__builtin_neon_vcvt_s32_v:
635 case NEON::BI__builtin_neon_vcvt_u32_v:
636 case NEON::BI__builtin_neon_vcvt_s64_v:
637 case NEON::BI__builtin_neon_vcvt_u64_v:
638 case NEON::BI__builtin_neon_vcvt_s16_f16:
639 case NEON::BI__builtin_neon_vcvt_u16_f16:
640 case NEON::BI__builtin_neon_vcvtq_s32_v:
641 case NEON::BI__builtin_neon_vcvtq_u32_v:
642 case NEON::BI__builtin_neon_vcvtq_s64_v:
643 case NEON::BI__builtin_neon_vcvtq_u64_v:
644 case NEON::BI__builtin_neon_vcvtq_s16_f16:
645 case NEON::BI__builtin_neon_vcvtq_u16_f16:
646 case NEON::BI__builtin_neon_vcvta_s16_f16:
647 case NEON::BI__builtin_neon_vcvta_s32_v:
648 case NEON::BI__builtin_neon_vcvta_s64_v:
649 case NEON::BI__builtin_neon_vcvta_u16_f16:
650 case NEON::BI__builtin_neon_vcvta_u32_v:
651 case NEON::BI__builtin_neon_vcvta_u64_v:
652 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
653 case NEON::BI__builtin_neon_vcvtaq_s32_v:
654 case NEON::BI__builtin_neon_vcvtaq_s64_v:
655 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
656 case NEON::BI__builtin_neon_vcvtaq_u32_v:
657 case NEON::BI__builtin_neon_vcvtaq_u64_v:
658 case NEON::BI__builtin_neon_vcvtn_s16_f16:
659 case NEON::BI__builtin_neon_vcvtn_s32_v:
660 case NEON::BI__builtin_neon_vcvtn_s64_v:
661 case NEON::BI__builtin_neon_vcvtn_u16_f16:
662 case NEON::BI__builtin_neon_vcvtn_u32_v:
663 case NEON::BI__builtin_neon_vcvtn_u64_v:
664 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
665 case NEON::BI__builtin_neon_vcvtnq_s32_v:
666 case NEON::BI__builtin_neon_vcvtnq_s64_v:
667 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
668 case NEON::BI__builtin_neon_vcvtnq_u32_v:
669 case NEON::BI__builtin_neon_vcvtnq_u64_v:
670 case NEON::BI__builtin_neon_vcvtp_s16_f16:
671 case NEON::BI__builtin_neon_vcvtp_s32_v:
672 case NEON::BI__builtin_neon_vcvtp_s64_v:
673 case NEON::BI__builtin_neon_vcvtp_u16_f16:
674 case NEON::BI__builtin_neon_vcvtp_u32_v:
675 case NEON::BI__builtin_neon_vcvtp_u64_v:
676 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
677 case NEON::BI__builtin_neon_vcvtpq_s32_v:
678 case NEON::BI__builtin_neon_vcvtpq_s64_v:
679 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
680 case NEON::BI__builtin_neon_vcvtpq_u32_v:
681 case NEON::BI__builtin_neon_vcvtpq_u64_v:
682 case NEON::BI__builtin_neon_vcvtm_s16_f16:
683 case NEON::BI__builtin_neon_vcvtm_s32_v:
684 case NEON::BI__builtin_neon_vcvtm_s64_v:
685 case NEON::BI__builtin_neon_vcvtm_u16_f16:
686 case NEON::BI__builtin_neon_vcvtm_u32_v:
687 case NEON::BI__builtin_neon_vcvtm_u64_v:
688 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
689 case NEON::BI__builtin_neon_vcvtmq_s32_v:
690 case NEON::BI__builtin_neon_vcvtmq_s64_v:
691 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
692 case NEON::BI__builtin_neon_vcvtmq_u32_v:
693 case NEON::BI__builtin_neon_vcvtmq_u64_v:
694 case NEON::BI__builtin_neon_vcvtx_f32_v:
695 case NEON::BI__builtin_neon_vext_v:
696 case NEON::BI__builtin_neon_vextq_v:
697 case NEON::BI__builtin_neon_vfma_v:
698 case NEON::BI__builtin_neon_vfmaq_v:
699 case NEON::BI__builtin_neon_vld1_v:
700 case NEON::BI__builtin_neon_vld1q_v:
701 case NEON::BI__builtin_neon_vld1_x2_v:
702 case NEON::BI__builtin_neon_vld1q_x2_v:
703 case NEON::BI__builtin_neon_vld1_x3_v:
704 case NEON::BI__builtin_neon_vld1q_x3_v:
705 case NEON::BI__builtin_neon_vld1_x4_v:
706 case NEON::BI__builtin_neon_vld1q_x4_v:
707 case NEON::BI__builtin_neon_vld2_v:
708 case NEON::BI__builtin_neon_vld2q_v:
709 case NEON::BI__builtin_neon_vld3_v:
710 case NEON::BI__builtin_neon_vld3q_v:
711 case NEON::BI__builtin_neon_vld4_v:
712 case NEON::BI__builtin_neon_vld4q_v:
713 case NEON::BI__builtin_neon_vld2_dup_v:
714 case NEON::BI__builtin_neon_vld2q_dup_v:
715 case NEON::BI__builtin_neon_vld3_dup_v:
716 case NEON::BI__builtin_neon_vld3q_dup_v:
717 case NEON::BI__builtin_neon_vld4_dup_v:
718 case NEON::BI__builtin_neon_vld4q_dup_v:
719 case NEON::BI__builtin_neon_vld1_dup_v:
720 case NEON::BI__builtin_neon_vld1q_dup_v:
721 case NEON::BI__builtin_neon_vld2_lane_v:
722 case NEON::BI__builtin_neon_vld2q_lane_v:
723 case NEON::BI__builtin_neon_vld3_lane_v:
724 case NEON::BI__builtin_neon_vld3q_lane_v:
725 case NEON::BI__builtin_neon_vld4_lane_v:
726 case NEON::BI__builtin_neon_vld4q_lane_v:
727 case NEON::BI__builtin_neon_vmovl_v:
728 case NEON::BI__builtin_neon_vmovn_v:
729 case NEON::BI__builtin_neon_vmull_v:
730 case NEON::BI__builtin_neon_vpadal_v:
731 case NEON::BI__builtin_neon_vpadalq_v:
732 case NEON::BI__builtin_neon_vpaddl_v:
733 case NEON::BI__builtin_neon_vpaddlq_v:
734 case NEON::BI__builtin_neon_vqdmlal_v:
735 case NEON::BI__builtin_neon_vqdmlsl_v:
736 case NEON::BI__builtin_neon_vqdmulhq_lane_v:
737 case NEON::BI__builtin_neon_vqdmulh_lane_v:
738 case NEON::BI__builtin_neon_vqrdmulhq_lane_v:
739 case NEON::BI__builtin_neon_vqrdmulh_lane_v:
740 case NEON::BI__builtin_neon_vqdmulhq_laneq_v:
741 case NEON::BI__builtin_neon_vqdmulh_laneq_v:
742 case NEON::BI__builtin_neon_vqrdmulhq_laneq_v:
743 case NEON::BI__builtin_neon_vqrdmulh_laneq_v:
744 case NEON::BI__builtin_neon_vqshl_n_v:
745 case NEON::BI__builtin_neon_vqshlq_n_v:
746 case NEON::BI__builtin_neon_vqshlu_n_v:
747 case NEON::BI__builtin_neon_vqshluq_n_v:
748 case NEON::BI__builtin_neon_vrecpe_v:
749 case NEON::BI__builtin_neon_vrecpeq_v:
750 case NEON::BI__builtin_neon_vrsqrte_v:
751 case NEON::BI__builtin_neon_vrsqrteq_v:
752 case NEON::BI__builtin_neon_vrndi_v:
753 case NEON::BI__builtin_neon_vrndiq_v:
754 case NEON::BI__builtin_neon_vrshr_n_v:
755 case NEON::BI__builtin_neon_vrshrq_n_v:
756 case NEON::BI__builtin_neon_vsha512hq_u64:
757 case NEON::BI__builtin_neon_vsha512h2q_u64:
758 case NEON::BI__builtin_neon_vsha512su0q_u64:
759 case NEON::BI__builtin_neon_vsha512su1q_u64:
760 case NEON::BI__builtin_neon_vshl_n_v:
761 case NEON::BI__builtin_neon_vshlq_n_v:
762 case NEON::BI__builtin_neon_vshll_n_v:
763 case NEON::BI__builtin_neon_vshrn_n_v:
764 case NEON::BI__builtin_neon_vshr_n_v:
765 case NEON::BI__builtin_neon_vshrq_n_v:
766 case NEON::BI__builtin_neon_vst1_v:
767 case NEON::BI__builtin_neon_vst1q_v:
768 case NEON::BI__builtin_neon_vst2_v:
769 case NEON::BI__builtin_neon_vst2q_v:
770 case NEON::BI__builtin_neon_vst3_v:
771 case NEON::BI__builtin_neon_vst3q_v:
772 case NEON::BI__builtin_neon_vst4_v:
773 case NEON::BI__builtin_neon_vst4q_v:
774 case NEON::BI__builtin_neon_vst2_lane_v:
775 case NEON::BI__builtin_neon_vst2q_lane_v:
776 case NEON::BI__builtin_neon_vst3_lane_v:
777 case NEON::BI__builtin_neon_vst3q_lane_v:
778 case NEON::BI__builtin_neon_vst4_lane_v:
779 case NEON::BI__builtin_neon_vst4q_lane_v:
780 case NEON::BI__builtin_neon_vsm3partw1q_u32:
781 case NEON::BI__builtin_neon_vsm3partw2q_u32:
782 case NEON::BI__builtin_neon_vsm3ss1q_u32:
783 case NEON::BI__builtin_neon_vsm4ekeyq_u32:
784 case NEON::BI__builtin_neon_vsm4eq_u32:
785 case NEON::BI__builtin_neon_vsm3tt1aq_u32:
786 case NEON::BI__builtin_neon_vsm3tt1bq_u32:
787 case NEON::BI__builtin_neon_vsm3tt2aq_u32:
788 case NEON::BI__builtin_neon_vsm3tt2bq_u32:
789 case NEON::BI__builtin_neon_vst1_x2_v:
790 case NEON::BI__builtin_neon_vst1q_x2_v:
791 case NEON::BI__builtin_neon_vst1_x3_v:
792 case NEON::BI__builtin_neon_vst1q_x3_v:
793 case NEON::BI__builtin_neon_vst1_x4_v:
794 case NEON::BI__builtin_neon_vst1q_x4_v:
795 case NEON::BI__builtin_neon_vsubhn_v:
796 case NEON::BI__builtin_neon_vtrn_v:
797 case NEON::BI__builtin_neon_vtrnq_v:
798 case NEON::BI__builtin_neon_vtst_v:
799 case NEON::BI__builtin_neon_vtstq_v:
800 case NEON::BI__builtin_neon_vuzp_v:
801 case NEON::BI__builtin_neon_vuzpq_v:
802 case NEON::BI__builtin_neon_vxarq_u64:
803 case NEON::BI__builtin_neon_vzip_v:
804 case NEON::BI__builtin_neon_vzipq_v:
805 case NEON::BI__builtin_neon_vdot_s32:
806 case NEON::BI__builtin_neon_vdot_u32:
807 case NEON::BI__builtin_neon_vdotq_s32:
808 case NEON::BI__builtin_neon_vdotq_u32:
809 case NEON::BI__builtin_neon_vfmlal_low_f16:
810 case NEON::BI__builtin_neon_vfmlalq_low_f16:
811 case NEON::BI__builtin_neon_vfmlsl_low_f16:
812 case NEON::BI__builtin_neon_vfmlslq_low_f16:
813 case NEON::BI__builtin_neon_vfmlal_high_f16:
814 case NEON::BI__builtin_neon_vfmlalq_high_f16:
815 case NEON::BI__builtin_neon_vfmlsl_high_f16:
816 case NEON::BI__builtin_neon_vfmlslq_high_f16:
817 case NEON::BI__builtin_neon_vmmlaq_s32:
818 case NEON::BI__builtin_neon_vmmlaq_u32:
819 case NEON::BI__builtin_neon_vusmmlaq_s32:
820 case NEON::BI__builtin_neon_vusdot_s32:
821 case NEON::BI__builtin_neon_vusdotq_s32:
822 case NEON::BI__builtin_neon_vbfdot_f32:
823 case NEON::BI__builtin_neon_vbfdotq_f32:
824 case NEON::BI__builtin_neon___a32_vcvt_bf16_f32:
825 default:
826 cgf.cgm.errorNYI(expr->getSourceRange(),
827 std::string("unimplemented AArch64 builtin call: ") +
828 ctx.BuiltinInfo.getName(builtinID));
829 return mlir::Value{};
830
831 cgf.cgm.errorNYI(expr->getSourceRange(),
832 std::string("unimplemented AArch64 builtin call: ") +
833 ctx.BuiltinInfo.getName(builtinID));
834 return mlir::Value{};
835 }
836}
837
838// Emit an intrinsic where all operands are of the same type as the result.
839// Depending on mode, this may be a constrained floating-point intrinsic.
840static mlir::Value
841emitCallMaybeConstrainedBuiltin(CIRGenBuilderTy &builder, mlir::Location loc,
842 StringRef intrName, mlir::Type retTy,
843 llvm::SmallVector<mlir::Value> &ops) {
844 assert(!cir::MissingFeatures::emitConstrainedFPCall());
845
846 return builder.emitIntrinsicCallOp(loc, intrName, retTy, ops);
847}
848
849bool CIRGenFunction::getAArch64SVEProcessedOperands(
850 unsigned builtinID, const CallExpr *expr, SmallVectorImpl<mlir::Value> &ops,
851 SVETypeFlags typeFlags) {
852 // Find out if any arguments are required to be integer constant expressions.
853 unsigned iceArguments = 0;
854 ASTContext::GetBuiltinTypeError error;
855 getContext().GetBuiltinType(builtinID, error, &iceArguments);
856 assert(error == ASTContext::GE_None && "Should not codegen an error");
857
858 for (unsigned i = 0, e = expr->getNumArgs(); i != e; i++) {
859 bool isIce = iceArguments & (1 << i);
860 mlir::Value arg = emitScalarExpr(expr->getArg(i));
861
862 if (isIce) {
863 cgm.errorNYI(expr->getSourceRange(),
864 std::string("unimplemented AArch64 builtin call: ") +
865 getContext().BuiltinInfo.getName(builtinID));
866 }
867
868 // FIXME: Handle types like svint16x2_t, which are currently incorrectly
869 // converted to i32. These should be treated as structs and unpacked.
870
871 ops.push_back(arg);
872 }
873 return true;
874}
875
876static llvm::StringRef getLLVMIntrNameNoPrefix(llvm::Intrinsic::ID intrID) {
877 llvm::StringRef llvmIntrName = llvm::Intrinsic::getBaseName(intrID);
878 assert(llvmIntrName.starts_with("llvm.") && "Not an LLVM intrinsic!");
879 return llvmIntrName.drop_front(/*strlen("llvm.")=*/5);
880}
881
882// Reinterpret the input predicate so that it can be used to correctly isolate
883// the elements of the specified datatype.
884mlir::Value CIRGenFunction::emitSVEPredicateCast(mlir::Value pred,
885 unsigned minNumElts,
886 mlir::Location loc) {
887
888 // TODO: Handle "aarch64.svcount" once we get round to supporting SME.
889
890 auto retTy = cir::VectorType::get(builder.getUIntNTy(1), minNumElts,
891 /*is_scalable=*/true);
892 if (pred.getType() == retTy)
893 return pred;
894
895 llvm::Intrinsic::ID intID;
896 switch (minNumElts) {
897 default:
898 llvm_unreachable("unsupported element count!");
899 case 1:
900 case 2:
901 case 4:
902 case 8:
903 intID = Intrinsic::aarch64_sve_convert_from_svbool;
904 break;
905 case 16:
906 intID = Intrinsic::aarch64_sve_convert_to_svbool;
907 break;
908 }
909
910 llvm::StringRef llvmIntrName = getLLVMIntrNameNoPrefix(intID);
911 auto call = builder.emitIntrinsicCallOp(loc, llvmIntrName, retTy,
912 mlir::ValueRange{pred});
913 assert(call.getType() == retTy && "Unexpected return type!");
914 return call;
915}
916
917//===----------------------------------------------------------------------===//
918// SVE helpers
919//===----------------------------------------------------------------------===//
920// Get the minimum number of elements in an SVE vector for the given element
921// type. The actual number of elements in the vector would be an integer (power
922// of two) multiple of this value.
923static unsigned getSVEMinEltCount(clang::SVETypeFlags::EltType sveType) {
924 switch (sveType) {
925 default:
926 llvm_unreachable("Invalid SVETypeFlag!");
927
928 case SVETypeFlags::EltTyInt8:
929 return 16;
930 case SVETypeFlags::EltTyInt16:
931 return 8;
932 case SVETypeFlags::EltTyInt32:
933 return 4;
934 case SVETypeFlags::EltTyInt64:
935 return 2;
936
937 case SVETypeFlags::EltTyMFloat8:
938 return 16;
939 case SVETypeFlags::EltTyFloat16:
940 case SVETypeFlags::EltTyBFloat16:
941 return 8;
942 case SVETypeFlags::EltTyFloat32:
943 return 4;
944 case SVETypeFlags::EltTyFloat64:
945 return 2;
946
947 case SVETypeFlags::EltTyBool8:
948 return 16;
949 case SVETypeFlags::EltTyBool16:
950 return 8;
951 case SVETypeFlags::EltTyBool32:
952 return 4;
953 case SVETypeFlags::EltTyBool64:
954 return 2;
955 }
956}
957
958// TODO(cir): Share with OGCG
959constexpr unsigned sveBitsPerBlock = 128;
960
961static cir::VectorType getSVEVectorForElementType(CIRGenModule &cgm,
962 mlir::Type eltTy) {
963 unsigned numElts =
964 sveBitsPerBlock / cgm.getDataLayout().getTypeSizeInBits(eltTy);
965 return cir::VectorType::get(eltTy, numElts, /*is_scalable=*/true);
966}
967
968//===----------------------------------------------------------------------===//
969// NEON helpers
970//===----------------------------------------------------------------------===//
971/// Return true if BuiltinID is an overloaded Neon intrinsic with an extra
972/// argument that specifies the vector type. The additional argument is meant
973/// for Sema checking (see `CheckNeonBuiltinFunctionCall`) and this function
974/// should be kept consistent with the logic in Sema.
975/// TODO: Make this return false for SISD builtins.
976/// TODO(cir): Share this with ARM.cpp
977static bool hasExtraNeonArgument(unsigned builtinID) {
978 // Required by the headers included below, but not in this particular
979 // function.
980 [[maybe_unused]] int PtrArgNum = -1;
981 [[maybe_unused]] bool HasConstPtr = false;
982
983 // The mask encodes the type. We don't care about the actual value. Instead,
984 // we just check whether its been set.
985 uint64_t mask = 0;
986 switch (builtinID) {
987#define GET_NEON_OVERLOAD_CHECK
988#include "clang/Basic/arm_fp16.inc"
989#include "clang/Basic/arm_neon.inc"
990#undef GET_NEON_OVERLOAD_CHECK
991 // Non-neon builtins for controling VFP that take extra argument for
992 // discriminating the type.
993 case ARM::BI__builtin_arm_vcvtr_f:
994 case ARM::BI__builtin_arm_vcvtr_d:
995 mask = 1;
996 }
997 switch (builtinID) {
998 default:
999 break;
1000 }
1001
1002 return mask != 0;
1003}
1004
1005// TODO(cir): Remove `cgm` from the list of arguments once all NYI(s) are gone.
1006template <typename Operation>
1007static mlir::Value
1008emitNeonCallToOp(CIRGenModule &cgm, CIRGenBuilderTy &builder,
1009 llvm::SmallVector<mlir::Type> argTypes,
1010 llvm::SmallVectorImpl<mlir::Value> &args,
1011 std::optional<llvm::StringRef> intrinsicName,
1012 mlir::Type funcResTy, mlir::Location loc,
1013 bool isConstrainedFPIntrinsic = false, unsigned shift = 0,
1014 bool rightshift = false) {
1015 // TODO(cir): Consider removing the following unreachable when we have
1016 // emitConstrainedFPCall feature implemented
1017 assert(!cir::MissingFeatures::emitConstrainedFPCall());
1018 if (isConstrainedFPIntrinsic)
1019 cgm.errorNYI(loc, std::string("constrained FP intrinsic"));
1020
1021 for (unsigned j = 0; j < argTypes.size(); ++j) {
1022 if (isConstrainedFPIntrinsic) {
1023 assert(!cir::MissingFeatures::emitConstrainedFPCall());
1024 }
1025 if (shift > 0 && shift == j) {
1026 cgm.errorNYI(loc, std::string("intrinsic requiring a shift Op"));
1027 } else {
1028 args[j] = builder.createBitcast(args[j], argTypes[j]);
1029 }
1030 }
1031 if (isConstrainedFPIntrinsic) {
1032 assert(!cir::MissingFeatures::emitConstrainedFPCall());
1033 return nullptr;
1034 }
1035 if constexpr (std::is_same_v<Operation, cir::LLVMIntrinsicCallOp>) {
1036 return Operation::create(builder, loc,
1037 builder.getStringAttr(intrinsicName.value()),
1038 funcResTy, args)
1039 .getResult();
1040 } else {
1041 return Operation::create(builder, loc, funcResTy, args).getResult();
1042 }
1043}
1044
1045// TODO(cir): Remove `cgm` from the list of arguments once all NYI(s) are gone.
1046static mlir::Value emitNeonCall(CIRGenModule &cgm, CIRGenBuilderTy &builder,
1047 llvm::SmallVector<mlir::Type> argTypes,
1048 llvm::SmallVectorImpl<mlir::Value> &args,
1049 llvm::StringRef intrinsicName,
1050 mlir::Type funcResTy, mlir::Location loc,
1051 bool isConstrainedFPIntrinsic = false,
1052 unsigned shift = 0, bool rightshift = false) {
1053 return emitNeonCallToOp<cir::LLVMIntrinsicCallOp>(
1054 cgm, builder, std::move(argTypes), args, intrinsicName, funcResTy, loc,
1055 isConstrainedFPIntrinsic, shift, rightshift);
1056}
1057
1058std::optional<mlir::Value>
1059CIRGenFunction::emitAArch64SVEBuiltinExpr(unsigned builtinID,
1060 const CallExpr *expr) {
1061 mlir::Type ty = convertType(expr->getType());
1062
1063 if (builtinID >= SVE::BI__builtin_sve_reinterpret_s8_s8 &&
1064 builtinID <= SVE::BI__builtin_sve_reinterpret_f64_f64_x4) {
1065 cgm.errorNYI(expr->getSourceRange(),
1066 std::string("unimplemented AArch64 builtin call: ") +
1067 getContext().BuiltinInfo.getName(builtinID));
1068 return mlir::Value{};
1069 }
1070
1071 assert(!cir::MissingFeatures::aarch64SVEIntrinsics());
1072
1073 auto *builtinIntrInfo = findARMVectorIntrinsicInMap(
1074 aarch64SVEIntrinsicMap, builtinID, aarch64SVEIntrinsicsProvenSorted);
1075
1076 // The operands of the builtin call
1077 llvm::SmallVector<mlir::Value> ops;
1078
1079 SVETypeFlags typeFlags(builtinIntrInfo->typeModifier);
1080 if (!CIRGenFunction::getAArch64SVEProcessedOperands(builtinID, expr, ops,
1081 typeFlags))
1082 return mlir::Value{};
1083
1084 if (typeFlags.isLoad() || typeFlags.isStore() || typeFlags.isGatherLoad() ||
1085 typeFlags.isScatterStore() || typeFlags.isPrefetch() ||
1086 typeFlags.isGatherPrefetch() || typeFlags.isStructLoad() ||
1087 typeFlags.isStructStore() || typeFlags.isTupleSet() ||
1088 typeFlags.isTupleGet() || typeFlags.isTupleCreate() ||
1089 typeFlags.isUndef())
1090 cgm.errorNYI(expr->getSourceRange(),
1091 std::string("unimplemented AArch64 builtin call: ") +
1092 getContext().BuiltinInfo.getName(builtinID));
1093
1094 mlir::Location loc = getLoc(expr->getExprLoc());
1095
1096 // Handle built-ins for which there is a corresponding LLVM Intrinsic.
1097 // -------------------------------------------------------------------
1098 if (builtinIntrInfo->llvmIntrinsic != 0) {
1099 // Emit set FPMR for intrinsics that require it.
1100 if (typeFlags.setsFPMR())
1101 cgm.errorNYI(expr->getSourceRange(),
1102 std::string("unimplemented AArch64 builtin call: ") +
1103 getContext().BuiltinInfo.getName(builtinID));
1104
1105 // Zero-ing predication
1106 if (typeFlags.getMergeType() == SVETypeFlags::MergeZeroExp) {
1107 auto null = builder.getNullValue(convertType(expr->getType()),
1108 getLoc(expr->getExprLoc()));
1109 ops.insert(ops.begin(), null);
1110 }
1111
1112 if (typeFlags.getMergeType() == SVETypeFlags::MergeAnyExp)
1113 ops.insert(ops.begin(),
1114 builder.getConstant(loc, cir::UndefAttr::get(ty)));
1115
1116 // Some ACLE builtins leave out the argument to specify the predicate
1117 // pattern, which is expected to be expanded to an SV_ALL pattern.
1118 if (typeFlags.isAppendSVALL())
1119 cgm.errorNYI(expr->getSourceRange(),
1120 std::string("unimplemented AArch64 builtin call: ") +
1121 getContext().BuiltinInfo.getName(builtinID));
1122 if (typeFlags.isInsertOp1SVALL())
1123 cgm.errorNYI(expr->getSourceRange(),
1124 std::string("unimplemented AArch64 builtin call: ") +
1125 getContext().BuiltinInfo.getName(builtinID));
1126
1127 // Predicates must match the main datatype.
1128 for (mlir::Value &op : ops)
1129 if (auto predTy = dyn_cast<cir::VectorType>(op.getType()))
1130 if (auto cirInt = dyn_cast<cir::IntType>(predTy.getElementType()))
1131 if (cirInt.getWidth() == 1)
1132 op = emitSVEPredicateCast(
1133 op, getSVEMinEltCount(typeFlags.getEltType()), loc);
1134
1135 // Splat scalar operand to vector (intrinsics with _n infix)
1136 if (typeFlags.hasSplatOperand()) {
1137 unsigned opNo = typeFlags.getSplatOperand();
1138 ops[opNo] = cir::VecSplatOp::create(
1139 builder, loc, getSVEVectorForElementType(cgm, ops[opNo].getType()),
1140 ops[opNo]);
1141 }
1142
1143 if (typeFlags.isReverseCompare())
1144 cgm.errorNYI(expr->getSourceRange(),
1145 std::string("unimplemented AArch64 builtin call: ") +
1146 getContext().BuiltinInfo.getName(builtinID));
1147 if (typeFlags.isReverseUSDOT())
1148 cgm.errorNYI(expr->getSourceRange(),
1149 std::string("unimplemented AArch64 builtin call: ") +
1150 getContext().BuiltinInfo.getName(builtinID));
1151 if (typeFlags.isReverseMergeAnyBinOp() &&
1152 typeFlags.getMergeType() == SVETypeFlags::MergeAny)
1153 cgm.errorNYI(expr->getSourceRange(),
1154 std::string("unimplemented AArch64 builtin call: ") +
1155 getContext().BuiltinInfo.getName(builtinID));
1156 if (typeFlags.isReverseMergeAnyAccOp() &&
1157 typeFlags.getMergeType() == SVETypeFlags::MergeAny)
1158 cgm.errorNYI(expr->getSourceRange(),
1159 std::string("unimplemented AArch64 builtin call: ") +
1160 getContext().BuiltinInfo.getName(builtinID));
1161
1162 // Predicated intrinsics with _z suffix.
1163 if (typeFlags.getMergeType() == SVETypeFlags::MergeZero) {
1164 cgm.errorNYI(expr->getSourceRange(),
1165 std::string("unimplemented AArch64 builtin call: ") +
1166 getContext().BuiltinInfo.getName(builtinID));
1167 }
1168
1169 llvm::StringRef llvmIntrName = getLLVMIntrNameNoPrefix(
1170 static_cast<llvm::Intrinsic::ID>(builtinIntrInfo->llvmIntrinsic));
1171 auto retTy = convertType(expr->getType());
1172
1173 auto call = builder.emitIntrinsicCallOp(loc, llvmIntrName, retTy,
1174 mlir::ValueRange{ops});
1175 if (call.getType() == retTy)
1176 return call;
1177
1178 // Predicate results must be converted to svbool_t.
1179 if (isa<mlir::VectorType>(retTy) &&
1180 cast<mlir::VectorType>(retTy).isScalable())
1181 cgm.errorNYI(expr->getSourceRange(),
1182 std::string("unimplemented AArch64 builtin call: ") +
1183 getContext().BuiltinInfo.getName(builtinID));
1184 // TODO Handle struct types, e.g. svint8x2_t (update the converter first).
1185
1186 llvm_unreachable("unsupported element count!");
1187 }
1188
1189 // Handle the remaining built-ins.
1190 // -------------------------------
1191 switch (builtinID) {
1192 default:
1193 return std::nullopt;
1194
1195 case SVE::BI__builtin_sve_svreinterpret_b:
1196 case SVE::BI__builtin_sve_svreinterpret_c:
1197 case SVE::BI__builtin_sve_svpsel_lane_b8:
1198 case SVE::BI__builtin_sve_svpsel_lane_b16:
1199 case SVE::BI__builtin_sve_svpsel_lane_b32:
1200 case SVE::BI__builtin_sve_svpsel_lane_b64:
1201 case SVE::BI__builtin_sve_svpsel_lane_c8:
1202 case SVE::BI__builtin_sve_svpsel_lane_c16:
1203 case SVE::BI__builtin_sve_svpsel_lane_c32:
1204 case SVE::BI__builtin_sve_svpsel_lane_c64:
1205 case SVE::BI__builtin_sve_svmov_b_z:
1206 case SVE::BI__builtin_sve_svnot_b_z:
1207 case SVE::BI__builtin_sve_svmovlb_u16:
1208 case SVE::BI__builtin_sve_svmovlb_u32:
1209 case SVE::BI__builtin_sve_svmovlb_u64:
1210 case SVE::BI__builtin_sve_svmovlb_s16:
1211 case SVE::BI__builtin_sve_svmovlb_s32:
1212 case SVE::BI__builtin_sve_svmovlb_s64:
1213 case SVE::BI__builtin_sve_svmovlt_u16:
1214 case SVE::BI__builtin_sve_svmovlt_u32:
1215 case SVE::BI__builtin_sve_svmovlt_u64:
1216 case SVE::BI__builtin_sve_svmovlt_s16:
1217 case SVE::BI__builtin_sve_svmovlt_s32:
1218 case SVE::BI__builtin_sve_svmovlt_s64:
1219 case SVE::BI__builtin_sve_svpmullt_u16:
1220 case SVE::BI__builtin_sve_svpmullt_u64:
1221 case SVE::BI__builtin_sve_svpmullt_n_u16:
1222 case SVE::BI__builtin_sve_svpmullt_n_u64:
1223 case SVE::BI__builtin_sve_svpmullb_u16:
1224 case SVE::BI__builtin_sve_svpmullb_u64:
1225 case SVE::BI__builtin_sve_svpmullb_n_u16:
1226 case SVE::BI__builtin_sve_svpmullb_n_u64:
1227
1228 case SVE::BI__builtin_sve_svdup_n_b8:
1229 case SVE::BI__builtin_sve_svdup_n_b16:
1230 case SVE::BI__builtin_sve_svdup_n_b32:
1231 case SVE::BI__builtin_sve_svdup_n_b64:
1232
1233 case SVE::BI__builtin_sve_svdupq_n_b8:
1234 case SVE::BI__builtin_sve_svdupq_n_b16:
1235 case SVE::BI__builtin_sve_svdupq_n_b32:
1236 case SVE::BI__builtin_sve_svdupq_n_b64:
1237 case SVE::BI__builtin_sve_svdupq_n_u8:
1238 case SVE::BI__builtin_sve_svdupq_n_s8:
1239 case SVE::BI__builtin_sve_svdupq_n_u64:
1240 case SVE::BI__builtin_sve_svdupq_n_f64:
1241 case SVE::BI__builtin_sve_svdupq_n_s64:
1242 case SVE::BI__builtin_sve_svdupq_n_u16:
1243 case SVE::BI__builtin_sve_svdupq_n_f16:
1244 case SVE::BI__builtin_sve_svdupq_n_bf16:
1245 case SVE::BI__builtin_sve_svdupq_n_s16:
1246 case SVE::BI__builtin_sve_svdupq_n_u32:
1247 case SVE::BI__builtin_sve_svdupq_n_f32:
1248 case SVE::BI__builtin_sve_svdupq_n_s32:
1249 case SVE::BI__builtin_sve_svpfalse_b:
1250 case SVE::BI__builtin_sve_svpfalse_c:
1251 cgm.errorNYI(expr->getSourceRange(),
1252 std::string("unimplemented AArch64 builtin call: ") +
1253 getContext().BuiltinInfo.getName(builtinID));
1254 return mlir::Value{};
1255
1256 case SVE::BI__builtin_sve_svlen_u8:
1257 case SVE::BI__builtin_sve_svlen_s8:
1258 return genVscaleTimesFactor(loc, builder, convertType(expr->getType()), 16);
1259
1260 case SVE::BI__builtin_sve_svlen_u16:
1261 case SVE::BI__builtin_sve_svlen_s16:
1262 case SVE::BI__builtin_sve_svlen_f16:
1263 case SVE::BI__builtin_sve_svlen_bf16:
1264 return genVscaleTimesFactor(loc, builder, convertType(expr->getType()), 8);
1265
1266 case SVE::BI__builtin_sve_svlen_u32:
1267 case SVE::BI__builtin_sve_svlen_s32:
1268 case SVE::BI__builtin_sve_svlen_f32:
1269 return genVscaleTimesFactor(loc, builder, convertType(expr->getType()), 4);
1270
1271 case SVE::BI__builtin_sve_svlen_u64:
1272 case SVE::BI__builtin_sve_svlen_s64:
1273 case SVE::BI__builtin_sve_svlen_f64:
1274 return genVscaleTimesFactor(loc, builder, convertType(expr->getType()), 2);
1275
1276 case SVE::BI__builtin_sve_svtbl2_u8:
1277 case SVE::BI__builtin_sve_svtbl2_s8:
1278 case SVE::BI__builtin_sve_svtbl2_u16:
1279 case SVE::BI__builtin_sve_svtbl2_s16:
1280 case SVE::BI__builtin_sve_svtbl2_u32:
1281 case SVE::BI__builtin_sve_svtbl2_s32:
1282 case SVE::BI__builtin_sve_svtbl2_u64:
1283 case SVE::BI__builtin_sve_svtbl2_s64:
1284 case SVE::BI__builtin_sve_svtbl2_f16:
1285 case SVE::BI__builtin_sve_svtbl2_bf16:
1286 case SVE::BI__builtin_sve_svtbl2_f32:
1287 case SVE::BI__builtin_sve_svtbl2_f64:
1288 case SVE::BI__builtin_sve_svset_neonq_s8:
1289 case SVE::BI__builtin_sve_svset_neonq_s16:
1290 case SVE::BI__builtin_sve_svset_neonq_s32:
1291 case SVE::BI__builtin_sve_svset_neonq_s64:
1292 case SVE::BI__builtin_sve_svset_neonq_u8:
1293 case SVE::BI__builtin_sve_svset_neonq_u16:
1294 case SVE::BI__builtin_sve_svset_neonq_u32:
1295 case SVE::BI__builtin_sve_svset_neonq_u64:
1296 case SVE::BI__builtin_sve_svset_neonq_f16:
1297 case SVE::BI__builtin_sve_svset_neonq_f32:
1298 case SVE::BI__builtin_sve_svset_neonq_f64:
1299 case SVE::BI__builtin_sve_svset_neonq_bf16:
1300 case SVE::BI__builtin_sve_svget_neonq_s8:
1301 case SVE::BI__builtin_sve_svget_neonq_s16:
1302 case SVE::BI__builtin_sve_svget_neonq_s32:
1303 case SVE::BI__builtin_sve_svget_neonq_s64:
1304 case SVE::BI__builtin_sve_svget_neonq_u8:
1305 case SVE::BI__builtin_sve_svget_neonq_u16:
1306 case SVE::BI__builtin_sve_svget_neonq_u32:
1307 case SVE::BI__builtin_sve_svget_neonq_u64:
1308 case SVE::BI__builtin_sve_svget_neonq_f16:
1309 case SVE::BI__builtin_sve_svget_neonq_f32:
1310 case SVE::BI__builtin_sve_svget_neonq_f64:
1311 case SVE::BI__builtin_sve_svget_neonq_bf16:
1312 case SVE::BI__builtin_sve_svdup_neonq_s8:
1313 case SVE::BI__builtin_sve_svdup_neonq_s16:
1314 case SVE::BI__builtin_sve_svdup_neonq_s32:
1315 case SVE::BI__builtin_sve_svdup_neonq_s64:
1316 case SVE::BI__builtin_sve_svdup_neonq_u8:
1317 case SVE::BI__builtin_sve_svdup_neonq_u16:
1318 case SVE::BI__builtin_sve_svdup_neonq_u32:
1319 case SVE::BI__builtin_sve_svdup_neonq_u64:
1320 case SVE::BI__builtin_sve_svdup_neonq_f16:
1321 case SVE::BI__builtin_sve_svdup_neonq_f32:
1322 case SVE::BI__builtin_sve_svdup_neonq_f64:
1323 case SVE::BI__builtin_sve_svdup_neonq_bf16:
1324 cgm.errorNYI(expr->getSourceRange(),
1325 std::string("unimplemented AArch64 builtin call: ") +
1326 getContext().BuiltinInfo.getName(builtinID));
1327 return mlir::Value{};
1328 }
1329
1330 // Unreachable: All cases in the switch above return.
1331}
1332
1333std::optional<mlir::Value>
1334CIRGenFunction::emitAArch64SMEBuiltinExpr(unsigned builtinID,
1335 const CallExpr *expr) {
1336 assert(!cir::MissingFeatures::aarch64SMEIntrinsics());
1337
1338 cgm.errorNYI(expr->getSourceRange(),
1339 std::string("unimplemented AArch64 builtin call: ") +
1340 getContext().BuiltinInfo.getName(builtinID));
1341 return mlir::Value{};
1342}
1343
1344// Some intrinsics are equivalent for codegen.
1345static const std::pair<unsigned, unsigned> neonEquivalentIntrinsicMap[] = {
1346 {
1347 NEON::BI__builtin_neon_splat_lane_bf16,
1348 NEON::BI__builtin_neon_splat_lane_v,
1349 },
1350 {
1351 NEON::BI__builtin_neon_splat_laneq_bf16,
1352 NEON::BI__builtin_neon_splat_laneq_v,
1353 },
1354 {
1355 NEON::BI__builtin_neon_splatq_lane_bf16,
1356 NEON::BI__builtin_neon_splatq_lane_v,
1357 },
1358 {
1359 NEON::BI__builtin_neon_splatq_laneq_bf16,
1360 NEON::BI__builtin_neon_splatq_laneq_v,
1361 },
1362 {
1363 NEON::BI__builtin_neon_vabd_f16,
1364 NEON::BI__builtin_neon_vabd_v,
1365 },
1366 {
1367 NEON::BI__builtin_neon_vabdq_f16,
1368 NEON::BI__builtin_neon_vabdq_v,
1369 },
1370 {
1371 NEON::BI__builtin_neon_vabs_f16,
1372 NEON::BI__builtin_neon_vabs_v,
1373 },
1374 {
1375 NEON::BI__builtin_neon_vabsq_f16,
1376 NEON::BI__builtin_neon_vabsq_v,
1377 },
1378 {
1379 NEON::BI__builtin_neon_vcage_f16,
1380 NEON::BI__builtin_neon_vcage_v,
1381 },
1382 {
1383 NEON::BI__builtin_neon_vcageq_f16,
1384 NEON::BI__builtin_neon_vcageq_v,
1385 },
1386 {
1387 NEON::BI__builtin_neon_vcagt_f16,
1388 NEON::BI__builtin_neon_vcagt_v,
1389 },
1390 {
1391 NEON::BI__builtin_neon_vcagtq_f16,
1392 NEON::BI__builtin_neon_vcagtq_v,
1393 },
1394 {
1395 NEON::BI__builtin_neon_vcale_f16,
1396 NEON::BI__builtin_neon_vcale_v,
1397 },
1398 {
1399 NEON::BI__builtin_neon_vcaleq_f16,
1400 NEON::BI__builtin_neon_vcaleq_v,
1401 },
1402 {
1403 NEON::BI__builtin_neon_vcalt_f16,
1404 NEON::BI__builtin_neon_vcalt_v,
1405 },
1406 {
1407 NEON::BI__builtin_neon_vcaltq_f16,
1408 NEON::BI__builtin_neon_vcaltq_v,
1409 },
1410 {
1411 NEON::BI__builtin_neon_vceqz_f16,
1412 NEON::BI__builtin_neon_vceqz_v,
1413 },
1414 {
1415 NEON::BI__builtin_neon_vceqzq_f16,
1416 NEON::BI__builtin_neon_vceqzq_v,
1417 },
1418 {
1419 NEON::BI__builtin_neon_vcgez_f16,
1420 NEON::BI__builtin_neon_vcgez_v,
1421 },
1422 {
1423 NEON::BI__builtin_neon_vcgezq_f16,
1424 NEON::BI__builtin_neon_vcgezq_v,
1425 },
1426 {
1427 NEON::BI__builtin_neon_vcgtz_f16,
1428 NEON::BI__builtin_neon_vcgtz_v,
1429 },
1430 {
1431 NEON::BI__builtin_neon_vcgtzq_f16,
1432 NEON::BI__builtin_neon_vcgtzq_v,
1433 },
1434 {
1435 NEON::BI__builtin_neon_vclez_f16,
1436 NEON::BI__builtin_neon_vclez_v,
1437 },
1438 {
1439 NEON::BI__builtin_neon_vclezq_f16,
1440 NEON::BI__builtin_neon_vclezq_v,
1441 },
1442 {
1443 NEON::BI__builtin_neon_vcltz_f16,
1444 NEON::BI__builtin_neon_vcltz_v,
1445 },
1446 {
1447 NEON::BI__builtin_neon_vcltzq_f16,
1448 NEON::BI__builtin_neon_vcltzq_v,
1449 },
1450 {
1451 NEON::BI__builtin_neon_vfma_f16,
1452 NEON::BI__builtin_neon_vfma_v,
1453 },
1454 {
1455 NEON::BI__builtin_neon_vfma_lane_f16,
1456 NEON::BI__builtin_neon_vfma_lane_v,
1457 },
1458 {
1459 NEON::BI__builtin_neon_vfma_laneq_f16,
1460 NEON::BI__builtin_neon_vfma_laneq_v,
1461 },
1462 {
1463 NEON::BI__builtin_neon_vfmaq_f16,
1464 NEON::BI__builtin_neon_vfmaq_v,
1465 },
1466 {
1467 NEON::BI__builtin_neon_vfmaq_lane_f16,
1468 NEON::BI__builtin_neon_vfmaq_lane_v,
1469 },
1470 {
1471 NEON::BI__builtin_neon_vfmaq_laneq_f16,
1472 NEON::BI__builtin_neon_vfmaq_laneq_v,
1473 },
1474 {NEON::BI__builtin_neon_vld1_bf16_x2, NEON::BI__builtin_neon_vld1_x2_v},
1475 {NEON::BI__builtin_neon_vld1_bf16_x3, NEON::BI__builtin_neon_vld1_x3_v},
1476 {NEON::BI__builtin_neon_vld1_bf16_x4, NEON::BI__builtin_neon_vld1_x4_v},
1477 {NEON::BI__builtin_neon_vld1_bf16, NEON::BI__builtin_neon_vld1_v},
1478 {NEON::BI__builtin_neon_vld1_dup_bf16, NEON::BI__builtin_neon_vld1_dup_v},
1479 {NEON::BI__builtin_neon_vld1_lane_bf16, NEON::BI__builtin_neon_vld1_lane_v},
1480 {NEON::BI__builtin_neon_vld1q_bf16_x2, NEON::BI__builtin_neon_vld1q_x2_v},
1481 {NEON::BI__builtin_neon_vld1q_bf16_x3, NEON::BI__builtin_neon_vld1q_x3_v},
1482 {NEON::BI__builtin_neon_vld1q_bf16_x4, NEON::BI__builtin_neon_vld1q_x4_v},
1483 {NEON::BI__builtin_neon_vld1q_bf16, NEON::BI__builtin_neon_vld1q_v},
1484 {NEON::BI__builtin_neon_vld1q_dup_bf16, NEON::BI__builtin_neon_vld1q_dup_v},
1485 {NEON::BI__builtin_neon_vld1q_lane_bf16,
1486 NEON::BI__builtin_neon_vld1q_lane_v},
1487 {NEON::BI__builtin_neon_vld2_bf16, NEON::BI__builtin_neon_vld2_v},
1488 {NEON::BI__builtin_neon_vld2_dup_bf16, NEON::BI__builtin_neon_vld2_dup_v},
1489 {NEON::BI__builtin_neon_vld2_lane_bf16, NEON::BI__builtin_neon_vld2_lane_v},
1490 {NEON::BI__builtin_neon_vld2q_bf16, NEON::BI__builtin_neon_vld2q_v},
1491 {NEON::BI__builtin_neon_vld2q_dup_bf16, NEON::BI__builtin_neon_vld2q_dup_v},
1492 {NEON::BI__builtin_neon_vld2q_lane_bf16,
1493 NEON::BI__builtin_neon_vld2q_lane_v},
1494 {NEON::BI__builtin_neon_vld3_bf16, NEON::BI__builtin_neon_vld3_v},
1495 {NEON::BI__builtin_neon_vld3_dup_bf16, NEON::BI__builtin_neon_vld3_dup_v},
1496 {NEON::BI__builtin_neon_vld3_lane_bf16, NEON::BI__builtin_neon_vld3_lane_v},
1497 {NEON::BI__builtin_neon_vld3q_bf16, NEON::BI__builtin_neon_vld3q_v},
1498 {NEON::BI__builtin_neon_vld3q_dup_bf16, NEON::BI__builtin_neon_vld3q_dup_v},
1499 {NEON::BI__builtin_neon_vld3q_lane_bf16,
1500 NEON::BI__builtin_neon_vld3q_lane_v},
1501 {NEON::BI__builtin_neon_vld4_bf16, NEON::BI__builtin_neon_vld4_v},
1502 {NEON::BI__builtin_neon_vld4_dup_bf16, NEON::BI__builtin_neon_vld4_dup_v},
1503 {NEON::BI__builtin_neon_vld4_lane_bf16, NEON::BI__builtin_neon_vld4_lane_v},
1504 {NEON::BI__builtin_neon_vld4q_bf16, NEON::BI__builtin_neon_vld4q_v},
1505 {NEON::BI__builtin_neon_vld4q_dup_bf16, NEON::BI__builtin_neon_vld4q_dup_v},
1506 {NEON::BI__builtin_neon_vld4q_lane_bf16,
1507 NEON::BI__builtin_neon_vld4q_lane_v},
1508 {
1509 NEON::BI__builtin_neon_vmax_f16,
1510 NEON::BI__builtin_neon_vmax_v,
1511 },
1512 {
1513 NEON::BI__builtin_neon_vmaxnm_f16,
1514 NEON::BI__builtin_neon_vmaxnm_v,
1515 },
1516 {
1517 NEON::BI__builtin_neon_vmaxnmq_f16,
1518 NEON::BI__builtin_neon_vmaxnmq_v,
1519 },
1520 {
1521 NEON::BI__builtin_neon_vmaxq_f16,
1522 NEON::BI__builtin_neon_vmaxq_v,
1523 },
1524 {
1525 NEON::BI__builtin_neon_vmin_f16,
1526 NEON::BI__builtin_neon_vmin_v,
1527 },
1528 {
1529 NEON::BI__builtin_neon_vminnm_f16,
1530 NEON::BI__builtin_neon_vminnm_v,
1531 },
1532 {
1533 NEON::BI__builtin_neon_vminnmq_f16,
1534 NEON::BI__builtin_neon_vminnmq_v,
1535 },
1536 {
1537 NEON::BI__builtin_neon_vminq_f16,
1538 NEON::BI__builtin_neon_vminq_v,
1539 },
1540 {
1541 NEON::BI__builtin_neon_vmulx_f16,
1542 NEON::BI__builtin_neon_vmulx_v,
1543 },
1544 {
1545 NEON::BI__builtin_neon_vmulxq_f16,
1546 NEON::BI__builtin_neon_vmulxq_v,
1547 },
1548 {
1549 NEON::BI__builtin_neon_vpadd_f16,
1550 NEON::BI__builtin_neon_vpadd_v,
1551 },
1552 {
1553 NEON::BI__builtin_neon_vpaddq_f16,
1554 NEON::BI__builtin_neon_vpaddq_v,
1555 },
1556 {
1557 NEON::BI__builtin_neon_vpmax_f16,
1558 NEON::BI__builtin_neon_vpmax_v,
1559 },
1560 {
1561 NEON::BI__builtin_neon_vpmaxnm_f16,
1562 NEON::BI__builtin_neon_vpmaxnm_v,
1563 },
1564 {
1565 NEON::BI__builtin_neon_vpmaxnmq_f16,
1566 NEON::BI__builtin_neon_vpmaxnmq_v,
1567 },
1568 {
1569 NEON::BI__builtin_neon_vpmaxq_f16,
1570 NEON::BI__builtin_neon_vpmaxq_v,
1571 },
1572 {
1573 NEON::BI__builtin_neon_vpmin_f16,
1574 NEON::BI__builtin_neon_vpmin_v,
1575 },
1576 {
1577 NEON::BI__builtin_neon_vpminnm_f16,
1578 NEON::BI__builtin_neon_vpminnm_v,
1579 },
1580 {
1581 NEON::BI__builtin_neon_vpminnmq_f16,
1582 NEON::BI__builtin_neon_vpminnmq_v,
1583 },
1584 {
1585 NEON::BI__builtin_neon_vpminq_f16,
1586 NEON::BI__builtin_neon_vpminq_v,
1587 },
1588 {
1589 NEON::BI__builtin_neon_vrecpe_f16,
1590 NEON::BI__builtin_neon_vrecpe_v,
1591 },
1592 {
1593 NEON::BI__builtin_neon_vrecpeq_f16,
1594 NEON::BI__builtin_neon_vrecpeq_v,
1595 },
1596 {
1597 NEON::BI__builtin_neon_vrecps_f16,
1598 NEON::BI__builtin_neon_vrecps_v,
1599 },
1600 {
1601 NEON::BI__builtin_neon_vrecpsq_f16,
1602 NEON::BI__builtin_neon_vrecpsq_v,
1603 },
1604 {
1605 NEON::BI__builtin_neon_vrnd_f16,
1606 NEON::BI__builtin_neon_vrnd_v,
1607 },
1608 {
1609 NEON::BI__builtin_neon_vrnda_f16,
1610 NEON::BI__builtin_neon_vrnda_v,
1611 },
1612 {
1613 NEON::BI__builtin_neon_vrndaq_f16,
1614 NEON::BI__builtin_neon_vrndaq_v,
1615 },
1616 {
1617 NEON::BI__builtin_neon_vrndi_f16,
1618 NEON::BI__builtin_neon_vrndi_v,
1619 },
1620 {
1621 NEON::BI__builtin_neon_vrndiq_f16,
1622 NEON::BI__builtin_neon_vrndiq_v,
1623 },
1624 {
1625 NEON::BI__builtin_neon_vrndm_f16,
1626 NEON::BI__builtin_neon_vrndm_v,
1627 },
1628 {
1629 NEON::BI__builtin_neon_vrndmq_f16,
1630 NEON::BI__builtin_neon_vrndmq_v,
1631 },
1632 {
1633 NEON::BI__builtin_neon_vrndn_f16,
1634 NEON::BI__builtin_neon_vrndn_v,
1635 },
1636 {
1637 NEON::BI__builtin_neon_vrndnq_f16,
1638 NEON::BI__builtin_neon_vrndnq_v,
1639 },
1640 {
1641 NEON::BI__builtin_neon_vrndp_f16,
1642 NEON::BI__builtin_neon_vrndp_v,
1643 },
1644 {
1645 NEON::BI__builtin_neon_vrndpq_f16,
1646 NEON::BI__builtin_neon_vrndpq_v,
1647 },
1648 {
1649 NEON::BI__builtin_neon_vrndq_f16,
1650 NEON::BI__builtin_neon_vrndq_v,
1651 },
1652 {
1653 NEON::BI__builtin_neon_vrndx_f16,
1654 NEON::BI__builtin_neon_vrndx_v,
1655 },
1656 {
1657 NEON::BI__builtin_neon_vrndxq_f16,
1658 NEON::BI__builtin_neon_vrndxq_v,
1659 },
1660 {
1661 NEON::BI__builtin_neon_vrsqrte_f16,
1662 NEON::BI__builtin_neon_vrsqrte_v,
1663 },
1664 {
1665 NEON::BI__builtin_neon_vrsqrteq_f16,
1666 NEON::BI__builtin_neon_vrsqrteq_v,
1667 },
1668 {
1669 NEON::BI__builtin_neon_vrsqrts_f16,
1670 NEON::BI__builtin_neon_vrsqrts_v,
1671 },
1672 {
1673 NEON::BI__builtin_neon_vrsqrtsq_f16,
1674 NEON::BI__builtin_neon_vrsqrtsq_v,
1675 },
1676 {
1677 NEON::BI__builtin_neon_vsqrt_f16,
1678 NEON::BI__builtin_neon_vsqrt_v,
1679 },
1680 {
1681 NEON::BI__builtin_neon_vsqrtq_f16,
1682 NEON::BI__builtin_neon_vsqrtq_v,
1683 },
1684 {NEON::BI__builtin_neon_vst1_bf16_x2, NEON::BI__builtin_neon_vst1_x2_v},
1685 {NEON::BI__builtin_neon_vst1_bf16_x3, NEON::BI__builtin_neon_vst1_x3_v},
1686 {NEON::BI__builtin_neon_vst1_bf16_x4, NEON::BI__builtin_neon_vst1_x4_v},
1687 {NEON::BI__builtin_neon_vst1_bf16, NEON::BI__builtin_neon_vst1_v},
1688 {NEON::BI__builtin_neon_vst1_lane_bf16, NEON::BI__builtin_neon_vst1_lane_v},
1689 {NEON::BI__builtin_neon_vst1q_bf16_x2, NEON::BI__builtin_neon_vst1q_x2_v},
1690 {NEON::BI__builtin_neon_vst1q_bf16_x3, NEON::BI__builtin_neon_vst1q_x3_v},
1691 {NEON::BI__builtin_neon_vst1q_bf16_x4, NEON::BI__builtin_neon_vst1q_x4_v},
1692 {NEON::BI__builtin_neon_vst1q_bf16, NEON::BI__builtin_neon_vst1q_v},
1693 {NEON::BI__builtin_neon_vst1q_lane_bf16,
1694 NEON::BI__builtin_neon_vst1q_lane_v},
1695 {NEON::BI__builtin_neon_vst2_bf16, NEON::BI__builtin_neon_vst2_v},
1696 {NEON::BI__builtin_neon_vst2_lane_bf16, NEON::BI__builtin_neon_vst2_lane_v},
1697 {NEON::BI__builtin_neon_vst2q_bf16, NEON::BI__builtin_neon_vst2q_v},
1698 {NEON::BI__builtin_neon_vst2q_lane_bf16,
1699 NEON::BI__builtin_neon_vst2q_lane_v},
1700 {NEON::BI__builtin_neon_vst3_bf16, NEON::BI__builtin_neon_vst3_v},
1701 {NEON::BI__builtin_neon_vst3_lane_bf16, NEON::BI__builtin_neon_vst3_lane_v},
1702 {NEON::BI__builtin_neon_vst3q_bf16, NEON::BI__builtin_neon_vst3q_v},
1703 {NEON::BI__builtin_neon_vst3q_lane_bf16,
1704 NEON::BI__builtin_neon_vst3q_lane_v},
1705 {NEON::BI__builtin_neon_vst4_bf16, NEON::BI__builtin_neon_vst4_v},
1706 {NEON::BI__builtin_neon_vst4_lane_bf16, NEON::BI__builtin_neon_vst4_lane_v},
1707 {NEON::BI__builtin_neon_vst4q_bf16, NEON::BI__builtin_neon_vst4q_v},
1708 {NEON::BI__builtin_neon_vst4q_lane_bf16,
1709 NEON::BI__builtin_neon_vst4q_lane_v},
1710 // The mangling rules cause us to have one ID for each type for
1711 // vldap1(q)_lane and vstl1(q)_lane, but codegen is equivalent for all of
1712 // them. Choose an arbitrary one to be handled as tha canonical variation.
1713 {NEON::BI__builtin_neon_vldap1_lane_u64,
1714 NEON::BI__builtin_neon_vldap1_lane_s64},
1715 {NEON::BI__builtin_neon_vldap1_lane_f64,
1716 NEON::BI__builtin_neon_vldap1_lane_s64},
1717 {NEON::BI__builtin_neon_vldap1_lane_p64,
1718 NEON::BI__builtin_neon_vldap1_lane_s64},
1719 {NEON::BI__builtin_neon_vldap1q_lane_u64,
1720 NEON::BI__builtin_neon_vldap1q_lane_s64},
1721 {NEON::BI__builtin_neon_vldap1q_lane_f64,
1722 NEON::BI__builtin_neon_vldap1q_lane_s64},
1723 {NEON::BI__builtin_neon_vldap1q_lane_p64,
1724 NEON::BI__builtin_neon_vldap1q_lane_s64},
1725 {NEON::BI__builtin_neon_vstl1_lane_u64,
1726 NEON::BI__builtin_neon_vstl1_lane_s64},
1727 {NEON::BI__builtin_neon_vstl1_lane_f64,
1728 NEON::BI__builtin_neon_vstl1_lane_s64},
1729 {NEON::BI__builtin_neon_vstl1_lane_p64,
1730 NEON::BI__builtin_neon_vstl1_lane_s64},
1731 {NEON::BI__builtin_neon_vstl1q_lane_u64,
1732 NEON::BI__builtin_neon_vstl1q_lane_s64},
1733 {NEON::BI__builtin_neon_vstl1q_lane_f64,
1734 NEON::BI__builtin_neon_vstl1q_lane_s64},
1735 {NEON::BI__builtin_neon_vstl1q_lane_p64,
1736 NEON::BI__builtin_neon_vstl1q_lane_s64},
1737};
1738
1739std::optional<mlir::Value>
1740CIRGenFunction::emitAArch64BuiltinExpr(unsigned builtinID, const CallExpr *expr,
1741 ReturnValueSlot returnValue,
1742 llvm::Triple::ArchType arch) {
1743 if (builtinID >= clang::AArch64::FirstSVEBuiltin &&
1744 builtinID <= clang::AArch64::LastSVEBuiltin)
1745 return emitAArch64SVEBuiltinExpr(builtinID, expr);
1746
1747 if (builtinID >= clang::AArch64::FirstSMEBuiltin &&
1748 builtinID <= clang::AArch64::LastSMEBuiltin)
1749 return emitAArch64SMEBuiltinExpr(builtinID, expr);
1750
1751 if (builtinID == Builtin::BI__builtin_cpu_supports) {
1752 cgm.errorNYI(expr->getSourceRange(),
1753 std::string("unimplemented AArch64 builtin call: ") +
1754 getContext().BuiltinInfo.getName(builtinID));
1755 return mlir::Value{};
1756 }
1757
1758 switch (builtinID) {
1759 default:
1760 break;
1761 case clang::AArch64::BI__builtin_arm_nop:
1762 case clang::AArch64::BI__builtin_arm_yield:
1763 case clang::AArch64::BI__yield:
1764 case clang::AArch64::BI__builtin_arm_wfe:
1765 case clang::AArch64::BI__wfe:
1766 case clang::AArch64::BI__builtin_arm_wfi:
1767 case clang::AArch64::BI__wfi:
1768 case clang::AArch64::BI__builtin_arm_sev:
1769 case clang::AArch64::BI__sev:
1770 case clang::AArch64::BI__builtin_arm_sevl:
1771 case clang::AArch64::BI__sevl:
1772 cgm.errorNYI(expr->getSourceRange(),
1773 std::string("unimplemented AArch64 builtin call: ") +
1774 getContext().BuiltinInfo.getName(builtinID));
1775 return mlir::Value{};
1776 }
1777
1778 if (builtinID == clang::AArch64::BI__builtin_arm_trap) {
1779 cgm.errorNYI(expr->getSourceRange(),
1780 std::string("unimplemented AArch64 builtin call: ") +
1781 getContext().BuiltinInfo.getName(builtinID));
1782 return mlir::Value{};
1783 }
1784
1785 if (builtinID == clang::AArch64::BI__builtin_arm_get_sme_state) {
1786 cgm.errorNYI(expr->getSourceRange(),
1787 std::string("unimplemented AArch64 builtin call: ") +
1788 getContext().BuiltinInfo.getName(builtinID));
1789 return mlir::Value{};
1790 }
1791
1792 if (builtinID == clang::AArch64::BI__builtin_arm_rbit) {
1793 cgm.errorNYI(expr->getSourceRange(),
1794 std::string("unimplemented AArch64 builtin call: ") +
1795 getContext().BuiltinInfo.getName(builtinID));
1796 return mlir::Value{};
1797 }
1798 if (builtinID == clang::AArch64::BI__builtin_arm_rbit64) {
1799 cgm.errorNYI(expr->getSourceRange(),
1800 std::string("unimplemented AArch64 builtin call: ") +
1801 getContext().BuiltinInfo.getName(builtinID));
1802 return mlir::Value{};
1803 }
1804
1805 if (builtinID == clang::AArch64::BI__builtin_arm_clz ||
1806 builtinID == clang::AArch64::BI__builtin_arm_clz64) {
1807 cgm.errorNYI(expr->getSourceRange(),
1808 std::string("unimplemented AArch64 builtin call: ") +
1809 getContext().BuiltinInfo.getName(builtinID));
1810 return mlir::Value{};
1811 }
1812
1813 if (builtinID == clang::AArch64::BI__builtin_arm_cls) {
1814 cgm.errorNYI(expr->getSourceRange(),
1815 std::string("unimplemented AArch64 builtin call: ") +
1816 getContext().BuiltinInfo.getName(builtinID));
1817 return mlir::Value{};
1818 }
1819 if (builtinID == clang::AArch64::BI__builtin_arm_cls64) {
1820 cgm.errorNYI(expr->getSourceRange(),
1821 std::string("unimplemented AArch64 builtin call: ") +
1822 getContext().BuiltinInfo.getName(builtinID));
1823 return mlir::Value{};
1824 }
1825
1826 if (builtinID == clang::AArch64::BI__builtin_arm_rint32zf ||
1827 builtinID == clang::AArch64::BI__builtin_arm_rint32z) {
1828 cgm.errorNYI(expr->getSourceRange(),
1829 std::string("unimplemented AArch64 builtin call: ") +
1830 getContext().BuiltinInfo.getName(builtinID));
1831 return mlir::Value{};
1832 }
1833
1834 if (builtinID == clang::AArch64::BI__builtin_arm_rint64zf ||
1835 builtinID == clang::AArch64::BI__builtin_arm_rint64z) {
1836 cgm.errorNYI(expr->getSourceRange(),
1837 std::string("unimplemented AArch64 builtin call: ") +
1838 getContext().BuiltinInfo.getName(builtinID));
1839 return mlir::Value{};
1840 }
1841
1842 if (builtinID == clang::AArch64::BI__builtin_arm_rint32xf ||
1843 builtinID == clang::AArch64::BI__builtin_arm_rint32x) {
1844 cgm.errorNYI(expr->getSourceRange(),
1845 std::string("unimplemented AArch64 builtin call: ") +
1846 getContext().BuiltinInfo.getName(builtinID));
1847 return mlir::Value{};
1848 }
1849
1850 if (builtinID == clang::AArch64::BI__builtin_arm_rint64xf ||
1851 builtinID == clang::AArch64::BI__builtin_arm_rint64x) {
1852 cgm.errorNYI(expr->getSourceRange(),
1853 std::string("unimplemented AArch64 builtin call: ") +
1854 getContext().BuiltinInfo.getName(builtinID));
1855 return mlir::Value{};
1856 }
1857
1858 if (builtinID == clang::AArch64::BI__builtin_arm_jcvt) {
1859 cgm.errorNYI(expr->getSourceRange(),
1860 std::string("unimplemented AArch64 builtin call: ") +
1861 getContext().BuiltinInfo.getName(builtinID));
1862 return mlir::Value{};
1863 }
1864
1865 if (builtinID == clang::AArch64::BI__builtin_arm_ld64b ||
1866 builtinID == clang::AArch64::BI__builtin_arm_st64b ||
1867 builtinID == clang::AArch64::BI__builtin_arm_st64bv ||
1868 builtinID == clang::AArch64::BI__builtin_arm_st64bv0) {
1869 cgm.errorNYI(expr->getSourceRange(),
1870 std::string("unimplemented AArch64 builtin call: ") +
1871 getContext().BuiltinInfo.getName(builtinID));
1872 return mlir::Value{};
1873 }
1874
1875 if (builtinID == clang::AArch64::BI__builtin_arm_atomic_store_with_stshh) {
1876 cgm.errorNYI(expr->getSourceRange(),
1877 std::string("unimplemented AArch64 builtin call: ") +
1878 getContext().BuiltinInfo.getName(builtinID));
1879 return mlir::Value{};
1880 }
1881
1882 if (builtinID == clang::AArch64::BI__builtin_arm_rndr ||
1883 builtinID == clang::AArch64::BI__builtin_arm_rndrrs) {
1884 cgm.errorNYI(expr->getSourceRange(),
1885 std::string("unimplemented AArch64 builtin call: ") +
1886 getContext().BuiltinInfo.getName(builtinID));
1887 return mlir::Value{};
1888 }
1889
1890 if (builtinID == clang::AArch64::BI__clear_cache) {
1891 cgm.errorNYI(expr->getSourceRange(),
1892 std::string("unimplemented AArch64 builtin call: ") +
1893 getContext().BuiltinInfo.getName(builtinID));
1894 return mlir::Value{};
1895 }
1896
1897 if ((builtinID == clang::AArch64::BI__builtin_arm_ldrex ||
1898 builtinID == clang::AArch64::BI__builtin_arm_ldaex) &&
1899 getContext().getTypeSize(expr->getType()) == 128) {
1900 cgm.errorNYI(expr->getSourceRange(),
1901 std::string("unimplemented AArch64 builtin call: ") +
1902 getContext().BuiltinInfo.getName(builtinID));
1903 return mlir::Value{};
1904 }
1905 if (builtinID == clang::AArch64::BI__builtin_arm_ldrex ||
1906 builtinID == clang::AArch64::BI__builtin_arm_ldaex) {
1907 cgm.errorNYI(expr->getSourceRange(),
1908 std::string("unimplemented AArch64 builtin call: ") +
1909 getContext().BuiltinInfo.getName(builtinID));
1910 return mlir::Value{};
1911 }
1912
1913 if ((builtinID == clang::AArch64::BI__builtin_arm_strex ||
1914 builtinID == clang::AArch64::BI__builtin_arm_stlex) &&
1915 getContext().getTypeSize(expr->getArg(0)->getType()) == 128) {
1916 cgm.errorNYI(expr->getSourceRange(),
1917 std::string("unimplemented AArch64 builtin call: ") +
1918 getContext().BuiltinInfo.getName(builtinID));
1919 return mlir::Value{};
1920 }
1921
1922 if (builtinID == clang::AArch64::BI__builtin_arm_strex ||
1923 builtinID == clang::AArch64::BI__builtin_arm_stlex) {
1924 cgm.errorNYI(expr->getSourceRange(),
1925 std::string("unimplemented AArch64 builtin call: ") +
1926 getContext().BuiltinInfo.getName(builtinID));
1927 return mlir::Value{};
1928 }
1929
1930 if (builtinID == clang::AArch64::BI__getReg) {
1931 cgm.errorNYI(expr->getSourceRange(),
1932 std::string("unimplemented AArch64 builtin call: ") +
1933 getContext().BuiltinInfo.getName(builtinID));
1934 return mlir::Value{};
1935 }
1936
1937 if (builtinID == clang::AArch64::BI__break) {
1938 cgm.errorNYI(expr->getSourceRange(),
1939 std::string("unimplemented AArch64 builtin call: ") +
1940 getContext().BuiltinInfo.getName(builtinID));
1941 return mlir::Value{};
1942 }
1943
1944 if (builtinID == clang::AArch64::BI__builtin_arm_clrex) {
1945 cgm.errorNYI(expr->getSourceRange(),
1946 std::string("unimplemented AArch64 builtin call: ") +
1947 getContext().BuiltinInfo.getName(builtinID));
1948 return mlir::Value{};
1949 }
1950
1951 if (builtinID == clang::AArch64::BI_ReadWriteBarrier) {
1952 cgm.errorNYI(expr->getSourceRange(),
1953 std::string("unimplemented AArch64 builtin call: ") +
1954 getContext().BuiltinInfo.getName(builtinID));
1955 return mlir::Value{};
1956 }
1957
1958 // CRC32
1959 Intrinsic::ID crcIntrinsicID = Intrinsic::not_intrinsic;
1960 switch (builtinID) {
1961 case clang::AArch64::BI__builtin_arm_crc32b:
1962 crcIntrinsicID = Intrinsic::aarch64_crc32b;
1963 break;
1964 case clang::AArch64::BI__builtin_arm_crc32cb:
1965 crcIntrinsicID = Intrinsic::aarch64_crc32cb;
1966 break;
1967 case clang::AArch64::BI__builtin_arm_crc32h:
1968 crcIntrinsicID = Intrinsic::aarch64_crc32h;
1969 break;
1970 case clang::AArch64::BI__builtin_arm_crc32ch:
1971 crcIntrinsicID = Intrinsic::aarch64_crc32ch;
1972 break;
1973 case clang::AArch64::BI__builtin_arm_crc32w:
1974 crcIntrinsicID = Intrinsic::aarch64_crc32w;
1975 break;
1976 case clang::AArch64::BI__builtin_arm_crc32cw:
1977 crcIntrinsicID = Intrinsic::aarch64_crc32cw;
1978 break;
1979 case clang::AArch64::BI__builtin_arm_crc32d:
1980 crcIntrinsicID = Intrinsic::aarch64_crc32x;
1981 break;
1982 case clang::AArch64::BI__builtin_arm_crc32cd:
1983 crcIntrinsicID = Intrinsic::aarch64_crc32cx;
1984 break;
1985 }
1986
1987 if (crcIntrinsicID != Intrinsic::not_intrinsic) {
1988 cgm.errorNYI(expr->getSourceRange(),
1989 std::string("unimplemented AArch64 builtin call: ") +
1990 getContext().BuiltinInfo.getName(builtinID));
1991 return mlir::Value{};
1992 }
1993
1994 // Memory Operations (MOPS)
1995 if (builtinID == AArch64::BI__builtin_arm_mops_memset_tag) {
1996 cgm.errorNYI(expr->getSourceRange(),
1997 std::string("unimplemented AArch64 builtin call: ") +
1998 getContext().BuiltinInfo.getName(builtinID));
1999 return mlir::Value{};
2000 }
2001
2002 // Memory Tagging Extensions (MTE) Intrinsics
2003 Intrinsic::ID mteIntrinsicID = Intrinsic::not_intrinsic;
2004 switch (builtinID) {
2005 case clang::AArch64::BI__builtin_arm_irg:
2006 mteIntrinsicID = Intrinsic::aarch64_irg;
2007 break;
2008 case clang::AArch64::BI__builtin_arm_addg:
2009 mteIntrinsicID = Intrinsic::aarch64_addg;
2010 break;
2011 case clang::AArch64::BI__builtin_arm_gmi:
2012 mteIntrinsicID = Intrinsic::aarch64_gmi;
2013 break;
2014 case clang::AArch64::BI__builtin_arm_ldg:
2015 mteIntrinsicID = Intrinsic::aarch64_ldg;
2016 break;
2017 case clang::AArch64::BI__builtin_arm_stg:
2018 mteIntrinsicID = Intrinsic::aarch64_stg;
2019 break;
2020 case clang::AArch64::BI__builtin_arm_subp:
2021 mteIntrinsicID = Intrinsic::aarch64_subp;
2022 break;
2023 }
2024
2025 if (mteIntrinsicID != Intrinsic::not_intrinsic) {
2026 cgm.errorNYI(expr->getSourceRange(),
2027 std::string("unimplemented AArch64 builtin call: ") +
2028 getContext().BuiltinInfo.getName(builtinID));
2029 return mlir::Value{};
2030 }
2031
2032 if (builtinID == clang::AArch64::BI__builtin_arm_rsr ||
2033 builtinID == clang::AArch64::BI__builtin_arm_rsr64 ||
2034 builtinID == clang::AArch64::BI__builtin_arm_rsr128 ||
2035 builtinID == clang::AArch64::BI__builtin_arm_rsrp ||
2036 builtinID == clang::AArch64::BI__builtin_arm_wsr ||
2037 builtinID == clang::AArch64::BI__builtin_arm_wsr64 ||
2038 builtinID == clang::AArch64::BI__builtin_arm_wsr128 ||
2039 builtinID == clang::AArch64::BI__builtin_arm_wsrp) {
2040 cgm.errorNYI(expr->getSourceRange(),
2041 std::string("unimplemented AArch64 builtin call: ") +
2042 getContext().BuiltinInfo.getName(builtinID));
2043 return mlir::Value{};
2044 }
2045
2046 if (builtinID == clang::AArch64::BI_ReadStatusReg ||
2047 builtinID == clang::AArch64::BI_WriteStatusReg ||
2048 builtinID == clang::AArch64::BI__sys) {
2049 cgm.errorNYI(expr->getSourceRange(),
2050 std::string("unimplemented AArch64 builtin call: ") +
2051 getContext().BuiltinInfo.getName(builtinID));
2052 return mlir::Value{};
2053 }
2054
2055 if (builtinID == clang::AArch64::BI_AddressOfReturnAddress) {
2056 cgm.errorNYI(expr->getSourceRange(),
2057 std::string("unimplemented AArch64 builtin call: ") +
2058 getContext().BuiltinInfo.getName(builtinID));
2059 return mlir::Value{};
2060 }
2061
2062 if (builtinID == clang::AArch64::BI__builtin_sponentry) {
2063 cgm.errorNYI(expr->getSourceRange(),
2064 std::string("unimplemented AArch64 builtin call: ") +
2065 getContext().BuiltinInfo.getName(builtinID));
2066 return mlir::Value{};
2067 }
2068
2069 if (builtinID == clang::AArch64::BI__mulh ||
2070 builtinID == clang::AArch64::BI__umulh) {
2071 cgm.errorNYI(expr->getSourceRange(),
2072 std::string("unimplemented AArch64 builtin call: ") +
2073 getContext().BuiltinInfo.getName(builtinID));
2074 return mlir::Value{};
2075 }
2076
2077 if (builtinID == AArch64::BI__writex18byte ||
2078 builtinID == AArch64::BI__writex18word ||
2079 builtinID == AArch64::BI__writex18dword ||
2080 builtinID == AArch64::BI__writex18qword) {
2081 cgm.errorNYI(expr->getSourceRange(),
2082 std::string("unimplemented AArch64 builtin call: ") +
2083 getContext().BuiltinInfo.getName(builtinID));
2084 return mlir::Value{};
2085 }
2086
2087 if (builtinID == AArch64::BI__readx18byte ||
2088 builtinID == AArch64::BI__readx18word ||
2089 builtinID == AArch64::BI__readx18dword ||
2090 builtinID == AArch64::BI__readx18qword) {
2091 cgm.errorNYI(expr->getSourceRange(),
2092 std::string("unimplemented AArch64 builtin call: ") +
2093 getContext().BuiltinInfo.getName(builtinID));
2094 return mlir::Value{};
2095 }
2096
2097 if (builtinID == AArch64::BI__addx18byte ||
2098 builtinID == AArch64::BI__addx18word ||
2099 builtinID == AArch64::BI__addx18dword ||
2100 builtinID == AArch64::BI__addx18qword ||
2101 builtinID == AArch64::BI__incx18byte ||
2102 builtinID == AArch64::BI__incx18word ||
2103 builtinID == AArch64::BI__incx18dword ||
2104 builtinID == AArch64::BI__incx18qword) {
2105 cgm.errorNYI(expr->getSourceRange(),
2106 std::string("unimplemented AArch64 builtin call: ") +
2107 getContext().BuiltinInfo.getName(builtinID));
2108 return mlir::Value{};
2109 }
2110
2111 if (builtinID == AArch64::BI_CopyDoubleFromInt64 ||
2112 builtinID == AArch64::BI_CopyFloatFromInt32 ||
2113 builtinID == AArch64::BI_CopyInt32FromFloat ||
2114 builtinID == AArch64::BI_CopyInt64FromDouble) {
2115 cgm.errorNYI(expr->getSourceRange(),
2116 std::string("unimplemented AArch64 builtin call: ") +
2117 getContext().BuiltinInfo.getName(builtinID));
2118 return mlir::Value{};
2119 }
2120
2121 if (builtinID == AArch64::BI_CountLeadingOnes ||
2122 builtinID == AArch64::BI_CountLeadingOnes64 ||
2123 builtinID == AArch64::BI_CountLeadingZeros ||
2124 builtinID == AArch64::BI_CountLeadingZeros64) {
2125 cgm.errorNYI(expr->getSourceRange(),
2126 std::string("unimplemented AArch64 builtin call: ") +
2127 getContext().BuiltinInfo.getName(builtinID));
2128 return mlir::Value{};
2129 }
2130
2131 if (builtinID == AArch64::BI_CountLeadingSigns ||
2132 builtinID == AArch64::BI_CountLeadingSigns64) {
2133 cgm.errorNYI(expr->getSourceRange(),
2134 std::string("unimplemented AArch64 builtin call: ") +
2135 getContext().BuiltinInfo.getName(builtinID));
2136 return mlir::Value{};
2137 }
2138
2139 if (builtinID == AArch64::BI_CountOneBits ||
2140 builtinID == AArch64::BI_CountOneBits64) {
2141 cgm.errorNYI(expr->getSourceRange(),
2142 std::string("unimplemented AArch64 builtin call: ") +
2143 getContext().BuiltinInfo.getName(builtinID));
2144 return mlir::Value{};
2145 }
2146
2147 if (builtinID == AArch64::BI__prefetch) {
2148 cgm.errorNYI(expr->getSourceRange(),
2149 std::string("unimplemented AArch64 builtin call: ") +
2150 getContext().BuiltinInfo.getName(builtinID));
2151 return mlir::Value{};
2152 }
2153
2154 if (builtinID == AArch64::BI__hlt) {
2155 cgm.errorNYI(expr->getSourceRange(),
2156 std::string("unimplemented AArch64 builtin call: ") +
2157 getContext().BuiltinInfo.getName(builtinID));
2158 return mlir::Value{};
2159 }
2160
2161 if (builtinID == NEON::BI__builtin_neon_vcvth_bf16_f32) {
2162 cgm.errorNYI(expr->getSourceRange(),
2163 std::string("unimplemented AArch64 builtin call: ") +
2164 getContext().BuiltinInfo.getName(builtinID));
2165 return mlir::Value{};
2166 }
2167
2168 // Handle MSVC intrinsics before argument evaluation to prevent double
2169 // evaluation.
2170 assert(!cir::MissingFeatures::msvcBuiltins());
2171
2172 // Some intrinsics are equivalent - if they are use the base intrinsic ID.
2173 auto it = llvm::find_if(neonEquivalentIntrinsicMap, [builtinID](auto &p) {
2174 return p.first == builtinID;
2175 });
2176 if (it != end(neonEquivalentIntrinsicMap))
2177 builtinID = it->second;
2178
2179 // Find out if any arguments are required to be integer constant
2180 // expressions.
2181 assert(!cir::MissingFeatures::handleBuiltinICEArguments());
2182 unsigned iceArguments = 0;
2183 ASTContext::GetBuiltinTypeError error;
2184 getContext().GetBuiltinType(builtinID, error, &iceArguments);
2185 assert(error == ASTContext::GE_None && "Should not codegen an error");
2186 llvm::SmallVector<mlir::Value> ops;
2187
2188 // Skip extra arguments used to discriminate vector types and that are
2189 // intended for Sema checking.
2190 bool hasExtraArg = hasExtraNeonArgument(builtinID);
2191 unsigned numArgs = expr->getNumArgs() - (hasExtraArg ? 1 : 0);
2192 for (unsigned i = 0, e = numArgs; i != e; i++) {
2193 if (i == 0) {
2194 switch (builtinID) {
2195 case NEON::BI__builtin_neon_vld1_v:
2196 case NEON::BI__builtin_neon_vld1q_v:
2197 case NEON::BI__builtin_neon_vld1_dup_v:
2198 case NEON::BI__builtin_neon_vld1q_dup_v:
2199 case NEON::BI__builtin_neon_vld1_lane_v:
2200 case NEON::BI__builtin_neon_vld1q_lane_v:
2201 case NEON::BI__builtin_neon_vst1_v:
2202 case NEON::BI__builtin_neon_vst1q_v:
2203 case NEON::BI__builtin_neon_vst1_lane_v:
2204 case NEON::BI__builtin_neon_vst1q_lane_v:
2205 case NEON::BI__builtin_neon_vldap1_lane_s64:
2206 case NEON::BI__builtin_neon_vldap1q_lane_s64:
2207 case NEON::BI__builtin_neon_vstl1_lane_s64:
2208 case NEON::BI__builtin_neon_vstl1q_lane_s64:
2209 // Get the alignment for the argument in addition to the value;
2210 // we'll use it later.
2211 cgm.errorNYI(
2212 expr->getSourceRange(),
2213 std::string("unimplemented AArch64 builtin argument handling ") +
2214 getContext().BuiltinInfo.getName(builtinID));
2215 }
2216 }
2217 ops.push_back(
2218 emitScalarOrConstFoldImmArg(iceArguments, i, expr->getArg(i)));
2219 }
2220
2221 assert(!cir::MissingFeatures::neonSISDIntrinsics());
2222
2223 // Not all intrinsics handled by the common case work for AArch64 yet, so only
2224 // defer to common code if it's been added to our special map.
2225 assert(!cir::MissingFeatures::aarch64SIMDIntrinsics());
2226
2227 assert(!cir::MissingFeatures::aarch64TblBuiltinExpr());
2228
2229 const Expr *arg = expr->getArg(expr->getNumArgs() - 1);
2230 NeonTypeFlags type(0);
2231 // A trailing constant integer is used for discriminating overloaded builtin
2232 // calls. Use it to determine the type of this overloaded NEON intrinsic.
2233 if (std::optional<llvm::APSInt> result =
2234 arg->getIntegerConstantExpr(getContext()))
2235 type = NeonTypeFlags(result->getZExtValue());
2236
2237 bool usgn = type.isUnsigned();
2238
2239 mlir::Location loc = getLoc(expr->getExprLoc());
2240
2241 // Not all intrinsics handled by the common case work for AArch64 yet, so only
2242 // defer to common code if it's been added to our special map.
2243 const ARMVectorIntrinsicInfo *builtin;
2244 builtin = findARMVectorIntrinsicInMap(AArch64SIMDIntrinsicMap, builtinID,
2245 aarch64SIMDIntrinsicsProvenSorted);
2246
2247 if (builtin)
2248 return emitCommonNeonBuiltinExpr(
2249 *this, builtin->builtinID, builtin->llvmIntrinsic,
2250 builtin->altLLVMIntrinsic, builtin->nameHint, builtin->typeModifier,
2251 expr, ops);
2252
2253 // Handle non-overloaded intrinsics first.
2254 switch (builtinID) {
2255 default:
2256 break;
2257 case NEON::BI__builtin_neon_vabsh_f16: {
2258 return cir::FAbsOp::create(builder, loc, ops);
2259 }
2260 case NEON::BI__builtin_neon_vaddq_p128:
2261 case NEON::BI__builtin_neon_vldrq_p128:
2262 case NEON::BI__builtin_neon_vstrq_p128:
2263 case NEON::BI__builtin_neon_vcvts_f32_u32:
2264 case NEON::BI__builtin_neon_vcvtd_f64_u64:
2265 case NEON::BI__builtin_neon_vcvts_f32_s32:
2266 case NEON::BI__builtin_neon_vcvtd_f64_s64:
2267 case NEON::BI__builtin_neon_vcvth_f16_u16:
2268 case NEON::BI__builtin_neon_vcvth_f16_u32:
2269 case NEON::BI__builtin_neon_vcvth_f16_u64:
2270 case NEON::BI__builtin_neon_vcvth_f16_s16:
2271 case NEON::BI__builtin_neon_vcvth_f16_s32:
2272 case NEON::BI__builtin_neon_vcvth_f16_s64:
2273 case NEON::BI__builtin_neon_vcvtah_u16_f16:
2274 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
2275 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
2276 case NEON::BI__builtin_neon_vcvtph_u16_f16:
2277 case NEON::BI__builtin_neon_vcvth_u16_f16:
2278 case NEON::BI__builtin_neon_vcvtah_s16_f16:
2279 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
2280 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
2281 case NEON::BI__builtin_neon_vcvtph_s16_f16:
2282 case NEON::BI__builtin_neon_vcvth_s16_f16:
2283 case NEON::BI__builtin_neon_vcaleh_f16:
2284 case NEON::BI__builtin_neon_vcalth_f16:
2285 case NEON::BI__builtin_neon_vcageh_f16:
2286 case NEON::BI__builtin_neon_vcagth_f16:
2287 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
2288 case NEON::BI__builtin_neon_vcvth_n_u16_f16:
2289 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
2290 case NEON::BI__builtin_neon_vcvth_n_f16_u16:
2291 case NEON::BI__builtin_neon_vpaddd_s64:
2292 case NEON::BI__builtin_neon_vpaddd_f64:
2293 case NEON::BI__builtin_neon_vpadds_f32:
2294 cgm.errorNYI(expr->getSourceRange(),
2295 std::string("unimplemented AArch64 builtin call: ") +
2296 getContext().BuiltinInfo.getName(builtinID));
2297 return mlir::Value{};
2298 case NEON::BI__builtin_neon_vceqzd_s64:
2299 return emitAArch64CompareBuiltinExpr(
2300 *this, builder, loc, ops[0],
2301 convertType(expr->getCallReturnType(getContext())), cir::CmpOpKind::eq);
2302 case NEON::BI__builtin_neon_vceqzd_f64:
2303 case NEON::BI__builtin_neon_vceqzs_f32:
2304 case NEON::BI__builtin_neon_vceqzh_f16:
2305 case NEON::BI__builtin_neon_vcgezd_s64:
2306 case NEON::BI__builtin_neon_vcgezd_f64:
2307 case NEON::BI__builtin_neon_vcgezs_f32:
2308 case NEON::BI__builtin_neon_vcgezh_f16:
2309 case NEON::BI__builtin_neon_vclezd_s64:
2310 case NEON::BI__builtin_neon_vclezd_f64:
2311 case NEON::BI__builtin_neon_vclezs_f32:
2312 case NEON::BI__builtin_neon_vclezh_f16:
2313 case NEON::BI__builtin_neon_vcgtzd_s64:
2314 case NEON::BI__builtin_neon_vcgtzd_f64:
2315 case NEON::BI__builtin_neon_vcgtzs_f32:
2316 case NEON::BI__builtin_neon_vcgtzh_f16:
2317 case NEON::BI__builtin_neon_vcltzd_s64:
2318 case NEON::BI__builtin_neon_vcltzd_f64:
2319 case NEON::BI__builtin_neon_vcltzs_f32:
2320 case NEON::BI__builtin_neon_vcltzh_f16:
2321 case NEON::BI__builtin_neon_vceqzd_u64:
2322 case NEON::BI__builtin_neon_vceqd_f64:
2323 case NEON::BI__builtin_neon_vcled_f64:
2324 case NEON::BI__builtin_neon_vcltd_f64:
2325 case NEON::BI__builtin_neon_vcged_f64:
2326 case NEON::BI__builtin_neon_vcgtd_f64:
2327 case NEON::BI__builtin_neon_vceqs_f32:
2328 case NEON::BI__builtin_neon_vcles_f32:
2329 case NEON::BI__builtin_neon_vclts_f32:
2330 case NEON::BI__builtin_neon_vcges_f32:
2331 case NEON::BI__builtin_neon_vcgts_f32:
2332 case NEON::BI__builtin_neon_vceqh_f16:
2333 case NEON::BI__builtin_neon_vcleh_f16:
2334 case NEON::BI__builtin_neon_vclth_f16:
2335 case NEON::BI__builtin_neon_vcgeh_f16:
2336 case NEON::BI__builtin_neon_vcgth_f16:
2337 case NEON::BI__builtin_neon_vceqd_s64:
2338 case NEON::BI__builtin_neon_vceqd_u64:
2339 case NEON::BI__builtin_neon_vcgtd_s64:
2340 case NEON::BI__builtin_neon_vcgtd_u64:
2341 case NEON::BI__builtin_neon_vcltd_s64:
2342 case NEON::BI__builtin_neon_vcltd_u64:
2343 case NEON::BI__builtin_neon_vcged_u64:
2344 case NEON::BI__builtin_neon_vcged_s64:
2345 case NEON::BI__builtin_neon_vcled_u64:
2346 case NEON::BI__builtin_neon_vcled_s64:
2347 cgm.errorNYI(expr->getSourceRange(),
2348 std::string("unimplemented AArch64 builtin call: ") +
2349 getContext().BuiltinInfo.getName(builtinID));
2350 return mlir::Value{};
2351 case NEON::BI__builtin_neon_vnegd_s64: {
2352 return builder.createNeg(ops[0]);
2353 }
2354 case NEON::BI__builtin_neon_vnegh_f16: {
2355 return builder.createFNeg(ops[0]);
2356 }
2357 case NEON::BI__builtin_neon_vtstd_s64:
2358 case NEON::BI__builtin_neon_vtstd_u64:
2359 case NEON::BI__builtin_neon_vset_lane_i8:
2360 case NEON::BI__builtin_neon_vset_lane_i16:
2361 case NEON::BI__builtin_neon_vset_lane_i32:
2362 case NEON::BI__builtin_neon_vset_lane_i64:
2363 case NEON::BI__builtin_neon_vset_lane_bf16:
2364 case NEON::BI__builtin_neon_vset_lane_f32:
2365 case NEON::BI__builtin_neon_vsetq_lane_i8:
2366 case NEON::BI__builtin_neon_vsetq_lane_i16:
2367 case NEON::BI__builtin_neon_vsetq_lane_i32:
2368 case NEON::BI__builtin_neon_vsetq_lane_i64:
2369 case NEON::BI__builtin_neon_vsetq_lane_bf16:
2370 case NEON::BI__builtin_neon_vsetq_lane_f32:
2371 case NEON::BI__builtin_neon_vset_lane_f64:
2372 case NEON::BI__builtin_neon_vset_lane_mf8:
2373 case NEON::BI__builtin_neon_vsetq_lane_mf8:
2374 case NEON::BI__builtin_neon_vsetq_lane_f64:
2375 case NEON::BI__builtin_neon_vget_lane_i8:
2376 case NEON::BI__builtin_neon_vdupb_lane_i8:
2377 case NEON::BI__builtin_neon_vgetq_lane_i8:
2378 case NEON::BI__builtin_neon_vdupb_laneq_i8:
2379 case NEON::BI__builtin_neon_vget_lane_mf8:
2380 case NEON::BI__builtin_neon_vdupb_lane_mf8:
2381 case NEON::BI__builtin_neon_vgetq_lane_mf8:
2382 case NEON::BI__builtin_neon_vdupb_laneq_mf8:
2383 case NEON::BI__builtin_neon_vget_lane_i16:
2384 case NEON::BI__builtin_neon_vduph_lane_i16:
2385 case NEON::BI__builtin_neon_vgetq_lane_i16:
2386 case NEON::BI__builtin_neon_vduph_laneq_i16:
2387 case NEON::BI__builtin_neon_vget_lane_i32:
2388 case NEON::BI__builtin_neon_vdups_lane_i32:
2389 case NEON::BI__builtin_neon_vdups_lane_f32:
2390 case NEON::BI__builtin_neon_vgetq_lane_i32:
2391 case NEON::BI__builtin_neon_vdups_laneq_i32:
2392 case NEON::BI__builtin_neon_vget_lane_i64:
2393 case NEON::BI__builtin_neon_vdupd_lane_i64:
2394 case NEON::BI__builtin_neon_vdupd_lane_f64:
2395 case NEON::BI__builtin_neon_vgetq_lane_i64:
2396 case NEON::BI__builtin_neon_vdupd_laneq_i64:
2397 case NEON::BI__builtin_neon_vget_lane_f32:
2398 case NEON::BI__builtin_neon_vget_lane_f64:
2399 case NEON::BI__builtin_neon_vgetq_lane_f32:
2400 case NEON::BI__builtin_neon_vdups_laneq_f32:
2401 case NEON::BI__builtin_neon_vgetq_lane_f64:
2402 case NEON::BI__builtin_neon_vdupd_laneq_f64:
2403 case NEON::BI__builtin_neon_vaddh_f16:
2404 case NEON::BI__builtin_neon_vsubh_f16:
2405 case NEON::BI__builtin_neon_vmulh_f16:
2406 case NEON::BI__builtin_neon_vdivh_f16:
2407 cgm.errorNYI(expr->getSourceRange(),
2408 std::string("unimplemented AArch64 builtin call: ") +
2409 getContext().BuiltinInfo.getName(builtinID));
2410 return mlir::Value{};
2411 case NEON::BI__builtin_neon_vfmah_f16:
2412 // NEON intrinsic puts accumulator first, unlike the LLVM fma.
2413 std::rotate(ops.begin(), ops.begin() + 1, ops.end());
2414 return emitCallMaybeConstrainedBuiltin(builder, loc, "fma",
2415 convertType(expr->getType()), ops);
2416 break;
2417 case NEON::BI__builtin_neon_vfmsh_f16:
2418 // NEON intrinsic puts accumulator first, unlike the LLVM fma.
2419 std::rotate(ops.begin(), ops.begin() + 1, ops.end());
2420 ops[0] = builder.createFNeg(ops[0]);
2421 return emitCallMaybeConstrainedBuiltin(builder, loc, "fma",
2422 convertType(expr->getType()), ops);
2423 case NEON::BI__builtin_neon_vaddd_s64:
2424 case NEON::BI__builtin_neon_vaddd_u64:
2425 case NEON::BI__builtin_neon_vsubd_s64:
2426 case NEON::BI__builtin_neon_vsubd_u64:
2427 case NEON::BI__builtin_neon_vqdmlalh_s16:
2428 case NEON::BI__builtin_neon_vqdmlslh_s16:
2429 case NEON::BI__builtin_neon_vqshlud_n_s64:
2430 case NEON::BI__builtin_neon_vqshld_n_u64:
2431 case NEON::BI__builtin_neon_vqshld_n_s64:
2432 case NEON::BI__builtin_neon_vrshrd_n_u64:
2433 case NEON::BI__builtin_neon_vrshrd_n_s64:
2434 case NEON::BI__builtin_neon_vrsrad_n_u64:
2435 case NEON::BI__builtin_neon_vrsrad_n_s64:
2436 case NEON::BI__builtin_neon_vshld_n_s64:
2437 case NEON::BI__builtin_neon_vshld_n_u64:
2438 case NEON::BI__builtin_neon_vshrd_n_s64:
2439 case NEON::BI__builtin_neon_vshrd_n_u64:
2440 case NEON::BI__builtin_neon_vsrad_n_s64:
2441 case NEON::BI__builtin_neon_vsrad_n_u64:
2442 case NEON::BI__builtin_neon_vqdmlalh_lane_s16:
2443 case NEON::BI__builtin_neon_vqdmlalh_laneq_s16:
2444 case NEON::BI__builtin_neon_vqdmlslh_lane_s16:
2445 case NEON::BI__builtin_neon_vqdmlslh_laneq_s16:
2446 case NEON::BI__builtin_neon_vqdmlals_s32:
2447 case NEON::BI__builtin_neon_vqdmlsls_s32:
2448 case NEON::BI__builtin_neon_vqdmlals_lane_s32:
2449 case NEON::BI__builtin_neon_vqdmlals_laneq_s32:
2450 case NEON::BI__builtin_neon_vqdmlsls_lane_s32:
2451 case NEON::BI__builtin_neon_vqdmlsls_laneq_s32:
2452 case NEON::BI__builtin_neon_vget_lane_bf16:
2453 case NEON::BI__builtin_neon_vduph_lane_bf16:
2454 case NEON::BI__builtin_neon_vduph_lane_f16:
2455 case NEON::BI__builtin_neon_vgetq_lane_bf16:
2456 case NEON::BI__builtin_neon_vduph_laneq_bf16:
2457 case NEON::BI__builtin_neon_vduph_laneq_f16:
2458 case NEON::BI__builtin_neon_vcvt_bf16_f32:
2459 case NEON::BI__builtin_neon_vcvtq_low_bf16_f32:
2460 case NEON::BI__builtin_neon_vcvtq_high_bf16_f32:
2461 case clang::AArch64::BI_InterlockedAdd:
2462 case clang::AArch64::BI_InterlockedAdd_acq:
2463 case clang::AArch64::BI_InterlockedAdd_rel:
2464 case clang::AArch64::BI_InterlockedAdd_nf:
2465 case clang::AArch64::BI_InterlockedAdd64:
2466 case clang::AArch64::BI_InterlockedAdd64_acq:
2467 case clang::AArch64::BI_InterlockedAdd64_rel:
2468 case clang::AArch64::BI_InterlockedAdd64_nf:
2469 cgm.errorNYI(expr->getSourceRange(),
2470 std::string("unimplemented AArch64 builtin call: ") +
2471 getContext().BuiltinInfo.getName(builtinID));
2472 return mlir::Value{};
2473 }
2474
2475 cir::VectorType ty = getNeonType(this, type, loc);
2476 if (!ty)
2477 return nullptr;
2478
2479 llvm::StringRef intrName;
2480
2481 switch (builtinID) {
2482 default:
2483 return std::nullopt;
2484 case NEON::BI__builtin_neon_vbsl_v:
2485 case NEON::BI__builtin_neon_vbslq_v:
2486 case NEON::BI__builtin_neon_vfma_lane_v:
2487 case NEON::BI__builtin_neon_vfmaq_lane_v:
2488 case NEON::BI__builtin_neon_vfma_laneq_v:
2489 case NEON::BI__builtin_neon_vfmaq_laneq_v:
2490 case NEON::BI__builtin_neon_vfmah_lane_f16:
2491 case NEON::BI__builtin_neon_vfmas_lane_f32:
2492 case NEON::BI__builtin_neon_vfmah_laneq_f16:
2493 case NEON::BI__builtin_neon_vfmas_laneq_f32:
2494 case NEON::BI__builtin_neon_vfmad_lane_f64:
2495 case NEON::BI__builtin_neon_vfmad_laneq_f64:
2496 case NEON::BI__builtin_neon_vmull_v:
2497 case NEON::BI__builtin_neon_vmax_v:
2498 case NEON::BI__builtin_neon_vmaxq_v:
2499 case NEON::BI__builtin_neon_vmaxh_f16:
2500 case NEON::BI__builtin_neon_vmin_v:
2501 case NEON::BI__builtin_neon_vminq_v:
2502 case NEON::BI__builtin_neon_vminh_f16:
2503 cgm.errorNYI(expr->getSourceRange(),
2504 std::string("unimplemented AArch64 builtin call: ") +
2505 getContext().BuiltinInfo.getName(builtinID));
2506 return mlir::Value{};
2507 case NEON::BI__builtin_neon_vabd_v:
2508 case NEON::BI__builtin_neon_vabdq_v:
2509 intrName = usgn ? "aarch64.neon.uabd" : "aarch64.neon.sabd";
2510 if (cir::isFPOrVectorOfFPType(ty))
2511 intrName = "aarch64.neon.fabd";
2512 return emitNeonCall(cgm, builder, {ty, ty}, ops, intrName, ty, loc);
2513 case NEON::BI__builtin_neon_vpadal_v:
2514 case NEON::BI__builtin_neon_vpadalq_v:
2515 case NEON::BI__builtin_neon_vpmin_v:
2516 case NEON::BI__builtin_neon_vpminq_v:
2517 case NEON::BI__builtin_neon_vpmax_v:
2518 case NEON::BI__builtin_neon_vpmaxq_v:
2519 case NEON::BI__builtin_neon_vminnm_v:
2520 case NEON::BI__builtin_neon_vminnmq_v:
2521 case NEON::BI__builtin_neon_vminnmh_f16:
2522 case NEON::BI__builtin_neon_vmaxnm_v:
2523 case NEON::BI__builtin_neon_vmaxnmq_v:
2524 case NEON::BI__builtin_neon_vmaxnmh_f16:
2525 case NEON::BI__builtin_neon_vrecpss_f32:
2526 case NEON::BI__builtin_neon_vrecpsd_f64:
2527 case NEON::BI__builtin_neon_vrecpsh_f16:
2528 case NEON::BI__builtin_neon_vqshrun_n_v:
2529 case NEON::BI__builtin_neon_vqrshrun_n_v:
2530 case NEON::BI__builtin_neon_vqshrn_n_v:
2531 case NEON::BI__builtin_neon_vrshrn_n_v:
2532 case NEON::BI__builtin_neon_vqrshrn_n_v:
2533 case NEON::BI__builtin_neon_vrndah_f16:
2534 case NEON::BI__builtin_neon_vrnda_v:
2535 case NEON::BI__builtin_neon_vrndaq_v:
2536 case NEON::BI__builtin_neon_vrndih_f16:
2537 case NEON::BI__builtin_neon_vrndmh_f16:
2538 case NEON::BI__builtin_neon_vrndm_v:
2539 case NEON::BI__builtin_neon_vrndmq_v:
2540 case NEON::BI__builtin_neon_vrndnh_f16:
2541 case NEON::BI__builtin_neon_vrndn_v:
2542 case NEON::BI__builtin_neon_vrndnq_v:
2543 case NEON::BI__builtin_neon_vrndns_f32:
2544 case NEON::BI__builtin_neon_vrndph_f16:
2545 case NEON::BI__builtin_neon_vrndp_v:
2546 case NEON::BI__builtin_neon_vrndpq_v:
2547 case NEON::BI__builtin_neon_vrndxh_f16:
2548 case NEON::BI__builtin_neon_vrndx_v:
2549 case NEON::BI__builtin_neon_vrndxq_v:
2550 case NEON::BI__builtin_neon_vrndh_f16:
2551 case NEON::BI__builtin_neon_vrnd32x_f32:
2552 case NEON::BI__builtin_neon_vrnd32xq_f32:
2553 case NEON::BI__builtin_neon_vrnd32x_f64:
2554 case NEON::BI__builtin_neon_vrnd32xq_f64:
2555 case NEON::BI__builtin_neon_vrnd32z_f32:
2556 case NEON::BI__builtin_neon_vrnd32zq_f32:
2557 case NEON::BI__builtin_neon_vrnd32z_f64:
2558 case NEON::BI__builtin_neon_vrnd32zq_f64:
2559 case NEON::BI__builtin_neon_vrnd64x_f32:
2560 case NEON::BI__builtin_neon_vrnd64xq_f32:
2561 case NEON::BI__builtin_neon_vrnd64x_f64:
2562 case NEON::BI__builtin_neon_vrnd64xq_f64:
2563 case NEON::BI__builtin_neon_vrnd64z_f32:
2564 case NEON::BI__builtin_neon_vrnd64zq_f32:
2565 case NEON::BI__builtin_neon_vrnd64z_f64:
2566 case NEON::BI__builtin_neon_vrnd64zq_f64:
2567 case NEON::BI__builtin_neon_vrnd_v:
2568 case NEON::BI__builtin_neon_vrndq_v:
2569 case NEON::BI__builtin_neon_vcvt_f64_v:
2570 case NEON::BI__builtin_neon_vcvtq_f64_v:
2571 case NEON::BI__builtin_neon_vcvt_f64_f32:
2572 case NEON::BI__builtin_neon_vcvt_f32_f64:
2573 case NEON::BI__builtin_neon_vcvt_s32_v:
2574 case NEON::BI__builtin_neon_vcvt_u32_v:
2575 case NEON::BI__builtin_neon_vcvt_s64_v:
2576 case NEON::BI__builtin_neon_vcvt_u64_v:
2577 case NEON::BI__builtin_neon_vcvt_s16_f16:
2578 case NEON::BI__builtin_neon_vcvt_u16_f16:
2579 case NEON::BI__builtin_neon_vcvtq_s32_v:
2580 case NEON::BI__builtin_neon_vcvtq_u32_v:
2581 case NEON::BI__builtin_neon_vcvtq_s64_v:
2582 case NEON::BI__builtin_neon_vcvtq_u64_v:
2583 case NEON::BI__builtin_neon_vcvtq_s16_f16:
2584 case NEON::BI__builtin_neon_vcvtq_u16_f16:
2585 case NEON::BI__builtin_neon_vcvta_s16_f16:
2586 case NEON::BI__builtin_neon_vcvta_u16_f16:
2587 case NEON::BI__builtin_neon_vcvta_s32_v:
2588 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
2589 case NEON::BI__builtin_neon_vcvtaq_s32_v:
2590 case NEON::BI__builtin_neon_vcvta_u32_v:
2591 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
2592 case NEON::BI__builtin_neon_vcvtaq_u32_v:
2593 case NEON::BI__builtin_neon_vcvta_s64_v:
2594 case NEON::BI__builtin_neon_vcvtaq_s64_v:
2595 case NEON::BI__builtin_neon_vcvta_u64_v:
2596 case NEON::BI__builtin_neon_vcvtaq_u64_v:
2597 case NEON::BI__builtin_neon_vcvtm_s16_f16:
2598 case NEON::BI__builtin_neon_vcvtm_s32_v:
2599 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
2600 case NEON::BI__builtin_neon_vcvtmq_s32_v:
2601 case NEON::BI__builtin_neon_vcvtm_u16_f16:
2602 case NEON::BI__builtin_neon_vcvtm_u32_v:
2603 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
2604 case NEON::BI__builtin_neon_vcvtmq_u32_v:
2605 case NEON::BI__builtin_neon_vcvtm_s64_v:
2606 case NEON::BI__builtin_neon_vcvtmq_s64_v:
2607 case NEON::BI__builtin_neon_vcvtm_u64_v:
2608 case NEON::BI__builtin_neon_vcvtmq_u64_v:
2609 case NEON::BI__builtin_neon_vcvtn_s16_f16:
2610 case NEON::BI__builtin_neon_vcvtn_s32_v:
2611 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
2612 case NEON::BI__builtin_neon_vcvtnq_s32_v:
2613 case NEON::BI__builtin_neon_vcvtn_u16_f16:
2614 case NEON::BI__builtin_neon_vcvtn_u32_v:
2615 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
2616 case NEON::BI__builtin_neon_vcvtnq_u32_v:
2617 case NEON::BI__builtin_neon_vcvtn_s64_v:
2618 case NEON::BI__builtin_neon_vcvtnq_s64_v:
2619 case NEON::BI__builtin_neon_vcvtn_u64_v:
2620 case NEON::BI__builtin_neon_vcvtnq_u64_v:
2621 case NEON::BI__builtin_neon_vcvtp_s16_f16:
2622 case NEON::BI__builtin_neon_vcvtp_s32_v:
2623 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
2624 case NEON::BI__builtin_neon_vcvtpq_s32_v:
2625 case NEON::BI__builtin_neon_vcvtp_u16_f16:
2626 case NEON::BI__builtin_neon_vcvtp_u32_v:
2627 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
2628 case NEON::BI__builtin_neon_vcvtpq_u32_v:
2629 case NEON::BI__builtin_neon_vcvtp_s64_v:
2630 case NEON::BI__builtin_neon_vcvtpq_s64_v:
2631 case NEON::BI__builtin_neon_vcvtp_u64_v:
2632 case NEON::BI__builtin_neon_vcvtpq_u64_v:
2633 case NEON::BI__builtin_neon_vmulx_v:
2634 case NEON::BI__builtin_neon_vmulxq_v:
2635 case NEON::BI__builtin_neon_vmulxh_lane_f16:
2636 case NEON::BI__builtin_neon_vmulxh_laneq_f16:
2637 case NEON::BI__builtin_neon_vmul_lane_v:
2638 case NEON::BI__builtin_neon_vmul_laneq_v:
2639 case NEON::BI__builtin_neon_vpmaxnm_v:
2640 case NEON::BI__builtin_neon_vpmaxnmq_v:
2641 case NEON::BI__builtin_neon_vpminnm_v:
2642 case NEON::BI__builtin_neon_vpminnmq_v:
2643 case NEON::BI__builtin_neon_vsqrth_f16:
2644 case NEON::BI__builtin_neon_vsqrt_v:
2645 case NEON::BI__builtin_neon_vsqrtq_v:
2646 case NEON::BI__builtin_neon_vrbit_v:
2647 case NEON::BI__builtin_neon_vrbitq_v:
2648 case NEON::BI__builtin_neon_vmaxv_f16:
2649 case NEON::BI__builtin_neon_vmaxvq_f16:
2650 case NEON::BI__builtin_neon_vminv_f16:
2651 case NEON::BI__builtin_neon_vminvq_f16:
2652 case NEON::BI__builtin_neon_vmaxnmv_f16:
2653 case NEON::BI__builtin_neon_vmaxnmvq_f16:
2654 case NEON::BI__builtin_neon_vminnmv_f16:
2655 case NEON::BI__builtin_neon_vminnmvq_f16:
2656 case NEON::BI__builtin_neon_vmul_n_f64:
2657 case NEON::BI__builtin_neon_vaddlv_u8:
2658 case NEON::BI__builtin_neon_vaddlv_u16:
2659 case NEON::BI__builtin_neon_vaddlvq_u8:
2660 case NEON::BI__builtin_neon_vaddlvq_u16:
2661 case NEON::BI__builtin_neon_vaddlv_s8:
2662 case NEON::BI__builtin_neon_vaddlv_s16:
2663 case NEON::BI__builtin_neon_vaddlvq_s8:
2664 case NEON::BI__builtin_neon_vaddlvq_s16:
2665 case NEON::BI__builtin_neon_vsri_n_v:
2666 case NEON::BI__builtin_neon_vsriq_n_v:
2667 case NEON::BI__builtin_neon_vsli_n_v:
2668 case NEON::BI__builtin_neon_vsliq_n_v:
2669 case NEON::BI__builtin_neon_vsra_n_v:
2670 case NEON::BI__builtin_neon_vsraq_n_v:
2671 case NEON::BI__builtin_neon_vrsra_n_v:
2672 case NEON::BI__builtin_neon_vrsraq_n_v:
2673 case NEON::BI__builtin_neon_vld1_v:
2674 case NEON::BI__builtin_neon_vld1q_v:
2675 case NEON::BI__builtin_neon_vst1_v:
2676 case NEON::BI__builtin_neon_vst1q_v:
2677 case NEON::BI__builtin_neon_vld1_lane_v:
2678 case NEON::BI__builtin_neon_vld1q_lane_v:
2679 case NEON::BI__builtin_neon_vldap1_lane_s64:
2680 case NEON::BI__builtin_neon_vldap1q_lane_s64:
2681 case NEON::BI__builtin_neon_vld1_dup_v:
2682 case NEON::BI__builtin_neon_vld1q_dup_v:
2683 case NEON::BI__builtin_neon_vst1_lane_v:
2684 case NEON::BI__builtin_neon_vst1q_lane_v:
2685 case NEON::BI__builtin_neon_vstl1_lane_s64:
2686 case NEON::BI__builtin_neon_vstl1q_lane_s64:
2687 case NEON::BI__builtin_neon_vld2_v:
2688 case NEON::BI__builtin_neon_vld2q_v:
2689 case NEON::BI__builtin_neon_vld3_v:
2690 case NEON::BI__builtin_neon_vld3q_v:
2691 case NEON::BI__builtin_neon_vld4_v:
2692 case NEON::BI__builtin_neon_vld4q_v:
2693 case NEON::BI__builtin_neon_vld2_dup_v:
2694 case NEON::BI__builtin_neon_vld2q_dup_v:
2695 case NEON::BI__builtin_neon_vld3_dup_v:
2696 case NEON::BI__builtin_neon_vld3q_dup_v:
2697 case NEON::BI__builtin_neon_vld4_dup_v:
2698 case NEON::BI__builtin_neon_vld4q_dup_v:
2699 case NEON::BI__builtin_neon_vld2_lane_v:
2700 case NEON::BI__builtin_neon_vld2q_lane_v:
2701 case NEON::BI__builtin_neon_vld3_lane_v:
2702 case NEON::BI__builtin_neon_vld3q_lane_v:
2703 case NEON::BI__builtin_neon_vld4_lane_v:
2704 case NEON::BI__builtin_neon_vld4q_lane_v:
2705 case NEON::BI__builtin_neon_vst2_v:
2706 case NEON::BI__builtin_neon_vst2q_v:
2707 case NEON::BI__builtin_neon_vst2_lane_v:
2708 case NEON::BI__builtin_neon_vst2q_lane_v:
2709 case NEON::BI__builtin_neon_vst3_v:
2710 case NEON::BI__builtin_neon_vst3q_v:
2711 case NEON::BI__builtin_neon_vst3_lane_v:
2712 case NEON::BI__builtin_neon_vst3q_lane_v:
2713 case NEON::BI__builtin_neon_vst4_v:
2714 case NEON::BI__builtin_neon_vst4q_v:
2715 case NEON::BI__builtin_neon_vst4_lane_v:
2716 case NEON::BI__builtin_neon_vst4q_lane_v:
2717 case NEON::BI__builtin_neon_vtrn_v:
2718 case NEON::BI__builtin_neon_vtrnq_v:
2719 case NEON::BI__builtin_neon_vuzp_v:
2720 case NEON::BI__builtin_neon_vuzpq_v:
2721 case NEON::BI__builtin_neon_vzip_v:
2722 case NEON::BI__builtin_neon_vzipq_v:
2723 case NEON::BI__builtin_neon_vqtbl1q_v:
2724 case NEON::BI__builtin_neon_vqtbl2q_v:
2725 case NEON::BI__builtin_neon_vqtbl3q_v:
2726 case NEON::BI__builtin_neon_vqtbl4q_v:
2727 case NEON::BI__builtin_neon_vqtbx1q_v:
2728 case NEON::BI__builtin_neon_vqtbx2q_v:
2729 case NEON::BI__builtin_neon_vqtbx3q_v:
2730 case NEON::BI__builtin_neon_vqtbx4q_v:
2731 case NEON::BI__builtin_neon_vsqadd_v:
2732 case NEON::BI__builtin_neon_vsqaddq_v:
2733 case NEON::BI__builtin_neon_vuqadd_v:
2734 case NEON::BI__builtin_neon_vuqaddq_v:
2735 case NEON::BI__builtin_neon_vluti2_laneq_mf8:
2736 case NEON::BI__builtin_neon_vluti2_laneq_bf16:
2737 case NEON::BI__builtin_neon_vluti2_laneq_f16:
2738 case NEON::BI__builtin_neon_vluti2_laneq_p16:
2739 case NEON::BI__builtin_neon_vluti2_laneq_p8:
2740 case NEON::BI__builtin_neon_vluti2_laneq_s16:
2741 case NEON::BI__builtin_neon_vluti2_laneq_s8:
2742 case NEON::BI__builtin_neon_vluti2_laneq_u16:
2743 case NEON::BI__builtin_neon_vluti2_laneq_u8:
2744 case NEON::BI__builtin_neon_vluti2q_laneq_mf8:
2745 case NEON::BI__builtin_neon_vluti2q_laneq_bf16:
2746 case NEON::BI__builtin_neon_vluti2q_laneq_f16:
2747 case NEON::BI__builtin_neon_vluti2q_laneq_p16:
2748 case NEON::BI__builtin_neon_vluti2q_laneq_p8:
2749 case NEON::BI__builtin_neon_vluti2q_laneq_s16:
2750 case NEON::BI__builtin_neon_vluti2q_laneq_s8:
2751 case NEON::BI__builtin_neon_vluti2q_laneq_u16:
2752 case NEON::BI__builtin_neon_vluti2q_laneq_u8:
2753 case NEON::BI__builtin_neon_vluti2_lane_mf8:
2754 case NEON::BI__builtin_neon_vluti2_lane_bf16:
2755 case NEON::BI__builtin_neon_vluti2_lane_f16:
2756 case NEON::BI__builtin_neon_vluti2_lane_p16:
2757 case NEON::BI__builtin_neon_vluti2_lane_p8:
2758 case NEON::BI__builtin_neon_vluti2_lane_s16:
2759 case NEON::BI__builtin_neon_vluti2_lane_s8:
2760 case NEON::BI__builtin_neon_vluti2_lane_u16:
2761 case NEON::BI__builtin_neon_vluti2_lane_u8:
2762 case NEON::BI__builtin_neon_vluti2q_lane_mf8:
2763 case NEON::BI__builtin_neon_vluti2q_lane_bf16:
2764 case NEON::BI__builtin_neon_vluti2q_lane_f16:
2765 case NEON::BI__builtin_neon_vluti2q_lane_p16:
2766 case NEON::BI__builtin_neon_vluti2q_lane_p8:
2767 case NEON::BI__builtin_neon_vluti2q_lane_s16:
2768 case NEON::BI__builtin_neon_vluti2q_lane_s8:
2769 case NEON::BI__builtin_neon_vluti2q_lane_u16:
2770 case NEON::BI__builtin_neon_vluti2q_lane_u8:
2771 case NEON::BI__builtin_neon_vluti4q_lane_mf8:
2772 case NEON::BI__builtin_neon_vluti4q_lane_p8:
2773 case NEON::BI__builtin_neon_vluti4q_lane_s8:
2774 case NEON::BI__builtin_neon_vluti4q_lane_u8:
2775 case NEON::BI__builtin_neon_vluti4q_laneq_mf8:
2776 case NEON::BI__builtin_neon_vluti4q_laneq_p8:
2777 case NEON::BI__builtin_neon_vluti4q_laneq_s8:
2778 case NEON::BI__builtin_neon_vluti4q_laneq_u8:
2779 case NEON::BI__builtin_neon_vluti4q_lane_bf16_x2:
2780 case NEON::BI__builtin_neon_vluti4q_lane_f16_x2:
2781 case NEON::BI__builtin_neon_vluti4q_lane_p16_x2:
2782 case NEON::BI__builtin_neon_vluti4q_lane_s16_x2:
2783 case NEON::BI__builtin_neon_vluti4q_lane_u16_x2:
2784 case NEON::BI__builtin_neon_vluti4q_laneq_bf16_x2:
2785 case NEON::BI__builtin_neon_vluti4q_laneq_f16_x2:
2786 case NEON::BI__builtin_neon_vluti4q_laneq_p16_x2:
2787 case NEON::BI__builtin_neon_vluti4q_laneq_s16_x2:
2788 case NEON::BI__builtin_neon_vluti4q_laneq_u16_x2:
2789 case NEON::BI__builtin_neon_vmmlaq_f16_mf8_fpm:
2790 case NEON::BI__builtin_neon_vmmlaq_f32_mf8_fpm:
2791 case NEON::BI__builtin_neon_vcvt1_low_bf16_mf8_fpm:
2792 case NEON::BI__builtin_neon_vcvt1_bf16_mf8_fpm:
2793 case NEON::BI__builtin_neon_vcvt1_high_bf16_mf8_fpm:
2794 case NEON::BI__builtin_neon_vcvt2_low_bf16_mf8_fpm:
2795 case NEON::BI__builtin_neon_vcvt2_bf16_mf8_fpm:
2796 case NEON::BI__builtin_neon_vcvt2_high_bf16_mf8_fpm:
2797 case NEON::BI__builtin_neon_vcvt1_low_f16_mf8_fpm:
2798 case NEON::BI__builtin_neon_vcvt1_f16_mf8_fpm:
2799 case NEON::BI__builtin_neon_vcvt1_high_f16_mf8_fpm:
2800 case NEON::BI__builtin_neon_vcvt2_low_f16_mf8_fpm:
2801 case NEON::BI__builtin_neon_vcvt2_f16_mf8_fpm:
2802 case NEON::BI__builtin_neon_vcvt2_high_f16_mf8_fpm:
2803 case NEON::BI__builtin_neon_vcvt_mf8_f32_fpm:
2804 case NEON::BI__builtin_neon_vcvt_mf8_f16_fpm:
2805 case NEON::BI__builtin_neon_vcvtq_mf8_f16_fpm:
2806 case NEON::BI__builtin_neon_vcvt_high_mf8_f32_fpm:
2807 case NEON::BI__builtin_neon_vdot_f16_mf8_fpm:
2808 case NEON::BI__builtin_neon_vdotq_f16_mf8_fpm:
2809 case NEON::BI__builtin_neon_vdot_lane_f16_mf8_fpm:
2810 case NEON::BI__builtin_neon_vdotq_lane_f16_mf8_fpm:
2811 case NEON::BI__builtin_neon_vdot_laneq_f16_mf8_fpm:
2812 case NEON::BI__builtin_neon_vdotq_laneq_f16_mf8_fpm:
2813 case NEON::BI__builtin_neon_vdot_f32_mf8_fpm:
2814 case NEON::BI__builtin_neon_vdotq_f32_mf8_fpm:
2815 case NEON::BI__builtin_neon_vdot_lane_f32_mf8_fpm:
2816 case NEON::BI__builtin_neon_vdotq_lane_f32_mf8_fpm:
2817 case NEON::BI__builtin_neon_vdot_laneq_f32_mf8_fpm:
2818 case NEON::BI__builtin_neon_vdotq_laneq_f32_mf8_fpm:
2819 case NEON::BI__builtin_neon_vmlalbq_f16_mf8_fpm:
2820 case NEON::BI__builtin_neon_vmlaltq_f16_mf8_fpm:
2821 case NEON::BI__builtin_neon_vmlallbbq_f32_mf8_fpm:
2822 case NEON::BI__builtin_neon_vmlallbtq_f32_mf8_fpm:
2823 case NEON::BI__builtin_neon_vmlalltbq_f32_mf8_fpm:
2824 case NEON::BI__builtin_neon_vmlallttq_f32_mf8_fpm:
2825 case NEON::BI__builtin_neon_vmlalbq_lane_f16_mf8_fpm:
2826 case NEON::BI__builtin_neon_vmlalbq_laneq_f16_mf8_fpm:
2827 case NEON::BI__builtin_neon_vmlaltq_lane_f16_mf8_fpm:
2828 case NEON::BI__builtin_neon_vmlaltq_laneq_f16_mf8_fpm:
2829 case NEON::BI__builtin_neon_vmlallbbq_lane_f32_mf8_fpm:
2830 case NEON::BI__builtin_neon_vmlallbbq_laneq_f32_mf8_fpm:
2831 case NEON::BI__builtin_neon_vmlallbtq_lane_f32_mf8_fpm:
2832 case NEON::BI__builtin_neon_vmlallbtq_laneq_f32_mf8_fpm:
2833 case NEON::BI__builtin_neon_vmlalltbq_lane_f32_mf8_fpm:
2834 case NEON::BI__builtin_neon_vmlalltbq_laneq_f32_mf8_fpm:
2835 case NEON::BI__builtin_neon_vmlallttq_lane_f32_mf8_fpm:
2836 case NEON::BI__builtin_neon_vmlallttq_laneq_f32_mf8_fpm:
2837 case NEON::BI__builtin_neon_vamin_f16:
2838 case NEON::BI__builtin_neon_vaminq_f16:
2839 case NEON::BI__builtin_neon_vamin_f32:
2840 case NEON::BI__builtin_neon_vaminq_f32:
2841 case NEON::BI__builtin_neon_vaminq_f64:
2842 case NEON::BI__builtin_neon_vamax_f16:
2843 case NEON::BI__builtin_neon_vamaxq_f16:
2844 case NEON::BI__builtin_neon_vamax_f32:
2845 case NEON::BI__builtin_neon_vamaxq_f32:
2846 case NEON::BI__builtin_neon_vamaxq_f64:
2847 case NEON::BI__builtin_neon_vscale_f16:
2848 case NEON::BI__builtin_neon_vscaleq_f16:
2849 case NEON::BI__builtin_neon_vscale_f32:
2850 case NEON::BI__builtin_neon_vscaleq_f32:
2851 case NEON::BI__builtin_neon_vscaleq_f64:
2852 cgm.errorNYI(expr->getSourceRange(),
2853 std::string("unimplemented AArch64 builtin call: ") +
2854 getContext().BuiltinInfo.getName(builtinID));
2855 return mlir::Value{};
2856 }
2857
2858 // Unreachable: All cases in the switch above return.
2859}
Builtins.h
Defines enum values for all the target-independent builtin functions.
NEONMAP2
#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier)
Definition CIRGenBuiltinAArch64.cpp:96
hasExtraNeonArgument
static bool hasExtraNeonArgument(unsigned builtinID)
Return true if BuiltinID is an overloaded Neon intrinsic with an extra argument that specifies the ve...
Definition CIRGenBuiltinAArch64.cpp:977
aarch64SVEIntrinsicsProvenSorted
static bool aarch64SVEIntrinsicsProvenSorted
Definition CIRGenBuiltinAArch64.cpp:425
neonEquivalentIntrinsicMap
static const std::pair< unsigned, unsigned > neonEquivalentIntrinsicMap[]
Definition CIRGenBuiltinAArch64.cpp:1345
emitNeonCallToOp
static mlir::Value emitNeonCallToOp(CIRGenModule &cgm, CIRGenBuilderTy &builder, llvm::SmallVector< mlir::Type > argTypes, llvm::SmallVectorImpl< mlir::Value > &args, std::optional< llvm::StringRef > intrinsicName, mlir::Type funcResTy, mlir::Location loc, bool isConstrainedFPIntrinsic=false, unsigned shift=0, bool rightshift=false)
Definition CIRGenBuiltinAArch64.cpp:1008
aarch64SVEIntrinsicMap
static const ARMVectorIntrinsicInfo aarch64SVEIntrinsicMap[]
Definition CIRGenBuiltinAArch64.cpp:418
getSVEVectorForElementType
static cir::VectorType getSVEVectorForElementType(CIRGenModule &cgm, mlir::Type eltTy)
Definition CIRGenBuiltinAArch64.cpp:961
getSVEMinEltCount
static unsigned getSVEMinEltCount(clang::SVETypeFlags::EltType sveType)
Definition CIRGenBuiltinAArch64.cpp:923
NEONMAP1
#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier)
Definition CIRGenBuiltinAArch64.cpp:92
genVscaleTimesFactor
static mlir::Value genVscaleTimesFactor(mlir::Location loc, CIRGenBuilderTy builder, mlir::Type cirTy, int32_t scalingFactor)
Definition CIRGenBuiltinAArch64.cpp:34
getLLVMIntrNameNoPrefix
static llvm::StringRef getLLVMIntrNameNoPrefix(llvm::Intrinsic::ID intrID)
Definition CIRGenBuiltinAArch64.cpp:876
emitCallMaybeConstrainedBuiltin
static mlir::Value emitCallMaybeConstrainedBuiltin(CIRGenBuilderTy &builder, mlir::Location loc, StringRef intrName, mlir::Type retTy, llvm::SmallVector< mlir::Value > &ops)
Definition CIRGenBuiltinAArch64.cpp:841
emitCommonNeonBuiltinExpr
static mlir::Value emitCommonNeonBuiltinExpr(CIRGenFunction &cgf, unsigned builtinID, unsigned llvmIntrinsic, unsigned altLLVMIntrinsic, const char *nameHint, unsigned modifier, const CallExpr *expr, llvm::SmallVectorImpl< mlir::Value > &ops)
Definition CIRGenBuiltinAArch64.cpp:538
NEONMAP0
#define NEONMAP0(NameBase)
Definition CIRGenBuiltinAArch64.cpp:89
emitNeonCall
static mlir::Value emitNeonCall(CIRGenModule &cgm, CIRGenBuilderTy &builder, llvm::SmallVector< mlir::Type > argTypes, llvm::SmallVectorImpl< mlir::Value > &args, llvm::StringRef intrinsicName, mlir::Type funcResTy, mlir::Location loc, bool isConstrainedFPIntrinsic=false, unsigned shift=0, bool rightshift=false)
Definition CIRGenBuiltinAArch64.cpp:1046
Vectorize1ArgType
@ Vectorize1ArgType
Definition CIRGenBuiltinAArch64.cpp:64
FpCmpzModifiers
@ FpCmpzModifiers
Definition CIRGenBuiltinAArch64.cpp:68
Use64BitVectors
@ Use64BitVectors
Definition CIRGenBuiltinAArch64.cpp:61
VectorizeArgTypes
@ VectorizeArgTypes
Definition CIRGenBuiltinAArch64.cpp:56
VectorRetGetArgs01
@ VectorRetGetArgs01
Definition CIRGenBuiltinAArch64.cpp:66
InventFloatType
@ InventFloatType
Definition CIRGenBuiltinAArch64.cpp:58
VectorizeRetType
@ VectorizeRetType
Definition CIRGenBuiltinAArch64.cpp:55
Use128BitVectors
@ Use128BitVectors
Definition CIRGenBuiltinAArch64.cpp:62
findARMVectorIntrinsicInMap
static const ARMVectorIntrinsicInfo * findARMVectorIntrinsicInMap(ArrayRef< ARMVectorIntrinsicInfo > intrinsicMap, unsigned builtinID, bool &mapProvenSorted)
Definition CIRGenBuiltinAArch64.cpp:430
aarch64SIMDIntrinsicsProvenSorted
static bool aarch64SIMDIntrinsicsProvenSorted
Definition CIRGenBuiltinAArch64.cpp:424
sveBitsPerBlock
constexpr unsigned sveBitsPerBlock
Definition CIRGenBuiltinAArch64.cpp:959
AArch64SIMDIntrinsicMap
static const ARMVectorIntrinsicInfo AArch64SIMDIntrinsicMap[]
Definition CIRGenBuiltinAArch64.cpp:100
emitAArch64CompareBuiltinExpr
static mlir::Value emitAArch64CompareBuiltinExpr(CIRGenFunction &cgf, CIRGenBuilderTy &builder, mlir::Location loc, mlir::Value src, mlir::Type retTy, const cir::CmpOpKind kind)
Definition CIRGenBuiltinAArch64.cpp:453
getNeonType
static cir::VectorType getNeonType(CIRGenFunction *cgf, NeonTypeFlags typeFlags, mlir::Location loc, bool hasLegalHalfType=true, bool v1Ty=false, bool allowBFloatArgsAndRet=true)
Definition CIRGenBuiltinAArch64.cpp:481
getType
TokenType getType() const
Returns the token's type, e.g.
Definition FormatToken.h:984
kind
*collection of selector each with an associated kind and an ordered *collection of selectors A selector has a kind
Definition OpenMPClause.h:9709
TargetBuiltins.h
Enumerates target-specific builtins in their own namespaces within namespace clang.
cir::CIRBaseBuilderTy::getNullValue
cir::ConstantOp getNullValue(mlir::Type ty, mlir::Location loc)
Definition CIRBaseBuilder.h:97
cir::CIRBaseBuilderTy::createCast
mlir::Value createCast(mlir::Location loc, cir::CastKind kind, mlir::Value src, mlir::Type newTy)
Definition CIRBaseBuilder.h:470
cir::CIRBaseBuilderTy::createNUWAMul
mlir::Value createNUWAMul(mlir::Location loc, mlir::Value lhs, mlir::Value rhs)
Definition CIRBaseBuilder.h:603
cir::CIRBaseBuilderTy::createVecCompare
cir::VecCmpOp createVecCompare(mlir::Location loc, cir::CmpOpKind kind, mlir::Value lhs, mlir::Value rhs)
Definition CIRBaseBuilder.h:667
cir::CIRBaseBuilderTy::createBitcast
mlir::Value createBitcast(mlir::Value src, mlir::Type newTy)
Definition CIRBaseBuilder.h:504
cir::CIRBaseBuilderTy::createCompare
cir::CmpOp createCompare(mlir::Location loc, cir::CmpOpKind kind, mlir::Value lhs, mlir::Value rhs)
Definition CIRBaseBuilder.h:662
cir::CIRDataLayout::getTypeSizeInBits
llvm::TypeSize getTypeSizeInBits(mlir::Type ty) const
Definition CIRDataLayout.cpp:46
clang::ASTContext
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition ASTContext.h:226
clang::ASTContext::BuiltinInfo
Builtin::Context & BuiltinInfo
Definition ASTContext.h:799
clang::ASTContext::GetBuiltinType
QualType GetBuiltinType(unsigned ID, GetBuiltinTypeError &Error, unsigned *IntegerConstantArgs=nullptr) const
Return the type for the specified builtin.
Definition ASTContext.cpp:12893
clang::ASTContext::GE_None
@ GE_None
No error.
Definition ASTContext.h:2617
clang::Builtin::Context::getName
std::string getName(unsigned ID) const
Return the identifier name for the specified builtin, e.g.
Definition Builtins.cpp:80
clang::CIRGen::CIRGenBuilderTy::getUInt64
cir::ConstantOp getUInt64(uint64_t c, mlir::Location loc)
Definition CIRGenBuilder.h:409
clang::CIRGen::CIRGenBuilderTy::emitIntrinsicCallOp
mlir::Value emitIntrinsicCallOp(mlir::Location loc, const llvm::StringRef str, const mlir::Type &resTy, Operands &&...op)
Definition CIRGenBuilder.h:789
clang::CIRGen::CIRGenBuilderTy::getSIntNTy
cir::IntType getSIntNTy(int n)
Definition CIRGenBuilder.h:330
clang::CIRGen::CIRGenFunction::convertType
mlir::Type convertType(clang::QualType t)
Definition CIRGenFunction.cpp:102
clang::CIRGen::CIRGenFunction::getTarget
const TargetInfo & getTarget() const
Definition CIRGenFunction.h:233
clang::CIRGen::CIRGenFunction::getLoc
mlir::Location getLoc(clang::SourceLocation srcLoc)
Helpers to convert Clang's SourceLocation to a MLIR Location.
Definition CIRGenFunction.cpp:106
clang::CIRGen::CIRGenFunction::emitSVEPredicateCast
mlir::Value emitSVEPredicateCast(mlir::Value pred, unsigned minNumElts, mlir::Location loc)
Definition CIRGenBuiltinAArch64.cpp:884
clang::CIRGen::CIRGenFunction::getAArch64SVEProcessedOperands
bool getAArch64SVEProcessedOperands(unsigned builtinID, const CallExpr *expr, SmallVectorImpl< mlir::Value > &ops, clang::SVETypeFlags typeFlags)
Definition CIRGenBuiltinAArch64.cpp:849
clang::CIRGen::CIRGenFunction::returnValue
Address returnValue
The temporary alloca to hold the return value.
Definition CIRGenFunction.h:90
clang::CIRGen::CIRGenFunction::emitAArch64BuiltinExpr
std::optional< mlir::Value > emitAArch64BuiltinExpr(unsigned builtinID, const CallExpr *expr, ReturnValueSlot returnValue, llvm::Triple::ArchType arch)
Definition CIRGenBuiltinAArch64.cpp:1740
clang::CIRGen::CIRGenFunction::emitAArch64SMEBuiltinExpr
std::optional< mlir::Value > emitAArch64SMEBuiltinExpr(unsigned builtinID, const CallExpr *expr)
Definition CIRGenBuiltinAArch64.cpp:1334
clang::CIRGen::CIRGenFunction::emitScalarExpr
mlir::Value emitScalarExpr(const clang::Expr *e, bool ignoreResultAssign=false)
Emit the computation of the specified expression of scalar type.
Definition CIRGenExprScalar.cpp:1664
clang::CIRGen::CIRGenFunction::getBuilder
CIRGenBuilderTy & getBuilder()
Definition CIRGenFunction.h:145
clang::CIRGen::CIRGenFunction::getContext
clang::ASTContext & getContext() const
Definition CIRGenFunction.h:143
clang::CIRGen::CIRGenFunction::emitAArch64SVEBuiltinExpr
std::optional< mlir::Value > emitAArch64SVEBuiltinExpr(unsigned builtinID, const CallExpr *expr)
Definition CIRGenBuiltinAArch64.cpp:1059
clang::CIRGen::CIRGenFunction::emitScalarOrConstFoldImmArg
mlir::Value emitScalarOrConstFoldImmArg(unsigned iceArguments, unsigned idx, const Expr *argExpr)
Definition CIRGenBuiltin.cpp:2379
clang::CIRGen::CIRGenModule
This class organizes the cross-function state that is used while generating CIR code.
Definition CIRGenModule.h:57
clang::CIRGen::CIRGenModule::errorNYI
DiagnosticBuilder errorNYI(SourceLocation, llvm::StringRef)
Helpers to emit "not yet implemented" error diagnostics.
Definition CIRGenModule.cpp:2893
clang::CIRGen::CIRGenModule::getDataLayout
const cir::CIRDataLayout getDataLayout() const
Definition CIRGenModule.h:138
clang::CIRGen::ReturnValueSlot
Contains the address where the return value of a function can be stored, and whether the address is v...
Definition CIRGenCall.h:260
clang::CallExpr
CallExpr - Represents a function call (C99 6.5.2.2, C++ [expr.call]).
Definition Expr.h:2946
clang::Expr
This represents one expression.
Definition Expr.h:112
clang::NeonTypeFlags
Flags to identify the types for overloaded Neon builtins.
Definition TargetBuiltins.h:253
clang::NeonTypeFlags::getEltType
EltType getEltType() const
Definition TargetBuiltins.h:286
clang::SVETypeFlags
Flags to identify the types for overloaded SVE builtins.
Definition TargetBuiltins.h:330
clang::SVETypeFlags::isReverseUSDOT
bool isReverseUSDOT() const
Definition TargetBuiltins.h:409
clang::SVETypeFlags::isGatherLoad
bool isGatherLoad() const
Definition TargetBuiltins.h:389
clang::SVETypeFlags::getEltType
EltType getEltType() const
Definition TargetBuiltins.h:367
clang::SVETypeFlags::isPrefetch
bool isPrefetch() const
Definition TargetBuiltins.h:404
clang::SVETypeFlags::isTupleSet
bool isTupleSet() const
Definition TargetBuiltins.h:415
clang::SVETypeFlags::isReverseMergeAnyAccOp
bool isReverseMergeAnyAccOp() const
Definition TargetBuiltins.h:411
clang::SVETypeFlags::isTupleGet
bool isTupleGet() const
Definition TargetBuiltins.h:414
clang::SVETypeFlags::isInsertOp1SVALL
bool isInsertOp1SVALL() const
Definition TargetBuiltins.h:407
clang::SVETypeFlags::isAppendSVALL
bool isAppendSVALL() const
Definition TargetBuiltins.h:406
clang::SVETypeFlags::isReverseMergeAnyBinOp
bool isReverseMergeAnyBinOp() const
Definition TargetBuiltins.h:410
clang::SVETypeFlags::isStructStore
bool isStructStore() const
Definition TargetBuiltins.h:392
clang::SVETypeFlags::isTupleCreate
bool isTupleCreate() const
Definition TargetBuiltins.h:413
clang::SVETypeFlags::isGatherPrefetch
bool isGatherPrefetch() const
Definition TargetBuiltins.h:408
clang::SVETypeFlags::hasSplatOperand
bool hasSplatOperand() const
Definition TargetBuiltins.h:383
clang::SVETypeFlags::getMergeType
MergeType getMergeType() const
Definition TargetBuiltins.h:375
clang::SVETypeFlags::isStructLoad
bool isStructLoad() const
Definition TargetBuiltins.h:391
clang::SVETypeFlags::getSplatOperand
unsigned getSplatOperand() const
Definition TargetBuiltins.h:379
clang::SVETypeFlags::isScatterStore
bool isScatterStore() const
Definition TargetBuiltins.h:390
clang::SVETypeFlags::isReverseCompare
bool isReverseCompare() const
Definition TargetBuiltins.h:405
clang::TargetInfo::hasFastHalfType
virtual bool hasFastHalfType() const
Determine whether the target has fast native support for operations on half types.
Definition TargetInfo.h:711
clang::ast_matchers::type
const internal::VariadicAllOfMatcher< Type > type
Matches Types in the clang AST.
Definition ASTMatchersInternal.cpp:809
clang::ast_matchers::expr
const internal::VariadicDynCastAllOfMatcher< Stmt, Expr > expr
Matches expressions.
Definition ASTMatchersInternal.cpp:937
clang
The JSON file list parser is used to communicate input to InstallAPI.
Definition CalledOnceCheck.h:17
clang::isa
bool isa(CodeGen::Address addr)
Definition Address.h:330
clang::operator<
bool operator<(DeclarationName LHS, DeclarationName RHS)
Ordering on two declaration names.
Definition DeclarationName.h:578
clang::cast
U cast(CodeGen::Address addr)
Definition Address.h:327
hlsl::uint64_t
unsigned long uint64_t
Definition hlsl_basic_types.h:42
llvm
Diagnostic wrappers for TextAPI types for error reporting.
Definition Dominators.h:30
cir::MissingFeatures::msvcBuiltins
static bool msvcBuiltins()
Definition MissingFeatures.h:320
cir::MissingFeatures::neonSISDIntrinsics
static bool neonSISDIntrinsics()
Definition MissingFeatures.h:314
cir::MissingFeatures::handleBuiltinICEArguments
static bool handleBuiltinICEArguments()
Definition MissingFeatures.h:291
cir::MissingFeatures::aarch64SIMDIntrinsics
static bool aarch64SIMDIntrinsics()
Definition MissingFeatures.h:197
cir::MissingFeatures::aarch64SVEIntrinsics
static bool aarch64SVEIntrinsics()
Definition MissingFeatures.h:199
cir::MissingFeatures::emitConstrainedFPCall
static bool emitConstrainedFPCall()
Definition MissingFeatures.h:268
cir::MissingFeatures::aarch64SMEIntrinsics
static bool aarch64SMEIntrinsics()
Definition MissingFeatures.h:198
cir::MissingFeatures::aarch64TblBuiltinExpr
static bool aarch64TblBuiltinExpr()
Definition MissingFeatures.h:200
Generated on for clang by doxygen 1.14.0