clang 22.0.0git
CIRGenBuiltinAArch64.cpp
Go to the documentation of this file.
1//===---- CIRGenBuiltinAArch64.cpp - Emit CIR for AArch64 builtins --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit ARM64 Builtin calls as CIR or a function call
10// to be later resolved.
11//
12//===----------------------------------------------------------------------===//
13
14#include "CIRGenBuilder.h"
15#include "CIRGenFunction.h"
16#include "clang/CIR/MissingFeatures.h"
17
18// TODO(cir): once all builtins are covered, decide whether we still
19// need to use LLVM intrinsics or if there's a better approach to follow. Right
20// now the intrinsics are reused to make it convenient to encode all thousands
21// of them and passing down to LLVM lowering.
22#include "llvm/IR/Intrinsics.h"
23#include "llvm/IR/IntrinsicsAArch64.h"
24
25#include "mlir/IR/Value.h"
26#include "clang/AST/GlobalDecl.h"
27#include "clang/Basic/Builtins.h"
28#include "clang/Basic/TargetBuiltins.h"
29
30using namespace clang;
31using namespace clang::CIRGen;
32using namespace llvm;
33
34template <typename... Operands>
35static mlir::Value emitIntrinsicCallOp(CIRGenBuilderTy &builder,
36 mlir::Location loc, const StringRef str,
37 const mlir::Type &resTy,
38 Operands &&...op) {
39 return cir::LLVMIntrinsicCallOp::create(builder, loc,
40 builder.getStringAttr(str), resTy,
41 std::forward<Operands>(op)...)
42 .getResult();
43}
44
45// Generate vscale * scalingFactor
46static mlir::Value genVscaleTimesFactor(mlir::Location loc,
47 CIRGenBuilderTy builder,
48 mlir::Type cirTy,
49 int32_t scalingFactor) {
50 mlir::Value vscale = emitIntrinsicCallOp(builder, loc, "vscale", cirTy);
51 return builder.createNUWAMul(loc, vscale,
52 builder.getUInt64(scalingFactor, loc));
53}
54
55std::optional<mlir::Value>
56CIRGenFunction::emitAArch64SVEBuiltinExpr(unsigned builtinID,
57 const CallExpr *expr) {
58 if (builtinID >= SVE::BI__builtin_sve_reinterpret_s8_s8 &&
59 builtinID <= SVE::BI__builtin_sve_reinterpret_f64_f64_x4) {
60 cgm.errorNYI(expr->getSourceRange(),
61 std::string("unimplemented AArch64 builtin call: ") +
62 getContext().BuiltinInfo.getName(builtinID));
63 return mlir::Value{};
64 }
65
66 assert(!cir::MissingFeatures::aarch64SVEIntrinsics());
67
68 mlir::Location loc = getLoc(expr->getExprLoc());
69
70 switch (builtinID) {
71 default:
72 return std::nullopt;
73
74 case SVE::BI__builtin_sve_svreinterpret_b:
75 case SVE::BI__builtin_sve_svreinterpret_c:
76 case SVE::BI__builtin_sve_svpsel_lane_b8:
77 case SVE::BI__builtin_sve_svpsel_lane_b16:
78 case SVE::BI__builtin_sve_svpsel_lane_b32:
79 case SVE::BI__builtin_sve_svpsel_lane_b64:
80 case SVE::BI__builtin_sve_svpsel_lane_c8:
81 case SVE::BI__builtin_sve_svpsel_lane_c16:
82 case SVE::BI__builtin_sve_svpsel_lane_c32:
83 case SVE::BI__builtin_sve_svpsel_lane_c64:
84 case SVE::BI__builtin_sve_svmov_b_z:
85 case SVE::BI__builtin_sve_svnot_b_z:
86 case SVE::BI__builtin_sve_svmovlb_u16:
87 case SVE::BI__builtin_sve_svmovlb_u32:
88 case SVE::BI__builtin_sve_svmovlb_u64:
89 case SVE::BI__builtin_sve_svmovlb_s16:
90 case SVE::BI__builtin_sve_svmovlb_s32:
91 case SVE::BI__builtin_sve_svmovlb_s64:
92 case SVE::BI__builtin_sve_svmovlt_u16:
93 case SVE::BI__builtin_sve_svmovlt_u32:
94 case SVE::BI__builtin_sve_svmovlt_u64:
95 case SVE::BI__builtin_sve_svmovlt_s16:
96 case SVE::BI__builtin_sve_svmovlt_s32:
97 case SVE::BI__builtin_sve_svmovlt_s64:
98 case SVE::BI__builtin_sve_svpmullt_u16:
99 case SVE::BI__builtin_sve_svpmullt_u64:
100 case SVE::BI__builtin_sve_svpmullt_n_u16:
101 case SVE::BI__builtin_sve_svpmullt_n_u64:
102 case SVE::BI__builtin_sve_svpmullb_u16:
103 case SVE::BI__builtin_sve_svpmullb_u64:
104 case SVE::BI__builtin_sve_svpmullb_n_u16:
105 case SVE::BI__builtin_sve_svpmullb_n_u64:
106 case SVE::BI__builtin_sve_svdup_n_b8:
107 case SVE::BI__builtin_sve_svdup_n_b16:
108 case SVE::BI__builtin_sve_svdup_n_b32:
109 case SVE::BI__builtin_sve_svdup_n_b64:
110 case SVE::BI__builtin_sve_svdupq_n_b8:
111 case SVE::BI__builtin_sve_svdupq_n_b16:
112 case SVE::BI__builtin_sve_svdupq_n_b32:
113 case SVE::BI__builtin_sve_svdupq_n_b64:
114 case SVE::BI__builtin_sve_svdupq_n_u8:
115 case SVE::BI__builtin_sve_svdupq_n_s8:
116 case SVE::BI__builtin_sve_svdupq_n_u64:
117 case SVE::BI__builtin_sve_svdupq_n_f64:
118 case SVE::BI__builtin_sve_svdupq_n_s64:
119 case SVE::BI__builtin_sve_svdupq_n_u16:
120 case SVE::BI__builtin_sve_svdupq_n_f16:
121 case SVE::BI__builtin_sve_svdupq_n_bf16:
122 case SVE::BI__builtin_sve_svdupq_n_s16:
123 case SVE::BI__builtin_sve_svdupq_n_u32:
124 case SVE::BI__builtin_sve_svdupq_n_f32:
125 case SVE::BI__builtin_sve_svdupq_n_s32:
126 case SVE::BI__builtin_sve_svpfalse_b:
127 case SVE::BI__builtin_sve_svpfalse_c:
128 cgm.errorNYI(expr->getSourceRange(),
129 std::string("unimplemented AArch64 builtin call: ") +
130 getContext().BuiltinInfo.getName(builtinID));
131 return mlir::Value{};
132 case SVE::BI__builtin_sve_svlen_u8:
133 case SVE::BI__builtin_sve_svlen_s8:
134 return genVscaleTimesFactor(loc, builder, convertType(expr->getType()), 16);
135 case SVE::BI__builtin_sve_svlen_u16:
136 case SVE::BI__builtin_sve_svlen_s16:
137 case SVE::BI__builtin_sve_svlen_f16:
138 case SVE::BI__builtin_sve_svlen_bf16:
139 return genVscaleTimesFactor(loc, builder, convertType(expr->getType()), 8);
140 case SVE::BI__builtin_sve_svlen_u32:
141 case SVE::BI__builtin_sve_svlen_s32:
142 case SVE::BI__builtin_sve_svlen_f32:
143 return genVscaleTimesFactor(loc, builder, convertType(expr->getType()), 4);
144 case SVE::BI__builtin_sve_svlen_u64:
145 case SVE::BI__builtin_sve_svlen_s64:
146 case SVE::BI__builtin_sve_svlen_f64:
147 return genVscaleTimesFactor(loc, builder, convertType(expr->getType()), 2);
148 case SVE::BI__builtin_sve_svtbl2_u8:
149 case SVE::BI__builtin_sve_svtbl2_s8:
150 case SVE::BI__builtin_sve_svtbl2_u16:
151 case SVE::BI__builtin_sve_svtbl2_s16:
152 case SVE::BI__builtin_sve_svtbl2_u32:
153 case SVE::BI__builtin_sve_svtbl2_s32:
154 case SVE::BI__builtin_sve_svtbl2_u64:
155 case SVE::BI__builtin_sve_svtbl2_s64:
156 case SVE::BI__builtin_sve_svtbl2_f16:
157 case SVE::BI__builtin_sve_svtbl2_bf16:
158 case SVE::BI__builtin_sve_svtbl2_f32:
159 case SVE::BI__builtin_sve_svtbl2_f64:
160 case SVE::BI__builtin_sve_svset_neonq_s8:
161 case SVE::BI__builtin_sve_svset_neonq_s16:
162 case SVE::BI__builtin_sve_svset_neonq_s32:
163 case SVE::BI__builtin_sve_svset_neonq_s64:
164 case SVE::BI__builtin_sve_svset_neonq_u8:
165 case SVE::BI__builtin_sve_svset_neonq_u16:
166 case SVE::BI__builtin_sve_svset_neonq_u32:
167 case SVE::BI__builtin_sve_svset_neonq_u64:
168 case SVE::BI__builtin_sve_svset_neonq_f16:
169 case SVE::BI__builtin_sve_svset_neonq_f32:
170 case SVE::BI__builtin_sve_svset_neonq_f64:
171 case SVE::BI__builtin_sve_svset_neonq_bf16:
172 case SVE::BI__builtin_sve_svget_neonq_s8:
173 case SVE::BI__builtin_sve_svget_neonq_s16:
174 case SVE::BI__builtin_sve_svget_neonq_s32:
175 case SVE::BI__builtin_sve_svget_neonq_s64:
176 case SVE::BI__builtin_sve_svget_neonq_u8:
177 case SVE::BI__builtin_sve_svget_neonq_u16:
178 case SVE::BI__builtin_sve_svget_neonq_u32:
179 case SVE::BI__builtin_sve_svget_neonq_u64:
180 case SVE::BI__builtin_sve_svget_neonq_f16:
181 case SVE::BI__builtin_sve_svget_neonq_f32:
182 case SVE::BI__builtin_sve_svget_neonq_f64:
183 case SVE::BI__builtin_sve_svget_neonq_bf16:
184 case SVE::BI__builtin_sve_svdup_neonq_s8:
185 case SVE::BI__builtin_sve_svdup_neonq_s16:
186 case SVE::BI__builtin_sve_svdup_neonq_s32:
187 case SVE::BI__builtin_sve_svdup_neonq_s64:
188 case SVE::BI__builtin_sve_svdup_neonq_u8:
189 case SVE::BI__builtin_sve_svdup_neonq_u16:
190 case SVE::BI__builtin_sve_svdup_neonq_u32:
191 case SVE::BI__builtin_sve_svdup_neonq_u64:
192 case SVE::BI__builtin_sve_svdup_neonq_f16:
193 case SVE::BI__builtin_sve_svdup_neonq_f32:
194 case SVE::BI__builtin_sve_svdup_neonq_f64:
195 case SVE::BI__builtin_sve_svdup_neonq_bf16:
196 cgm.errorNYI(expr->getSourceRange(),
197 std::string("unimplemented AArch64 builtin call: ") +
198 getContext().BuiltinInfo.getName(builtinID));
199 return mlir::Value{};
200 }
201
202 // Unreachable: All cases in the switch above return.
203}
204
205std::optional<mlir::Value>
206CIRGenFunction::emitAArch64SMEBuiltinExpr(unsigned builtinID,
207 const CallExpr *expr) {
208 assert(!cir::MissingFeatures::aarch64SMEIntrinsics());
209
210 cgm.errorNYI(expr->getSourceRange(),
211 std::string("unimplemented AArch64 builtin call: ") +
212 getContext().BuiltinInfo.getName(builtinID));
213 return mlir::Value{};
214}
215
216// Some intrinsics are equivalent for codegen.
217static const std::pair<unsigned, unsigned> neonEquivalentIntrinsicMap[] = {
218 {
219 NEON::BI__builtin_neon_splat_lane_bf16,
220 NEON::BI__builtin_neon_splat_lane_v,
221 },
222 {
223 NEON::BI__builtin_neon_splat_laneq_bf16,
224 NEON::BI__builtin_neon_splat_laneq_v,
225 },
226 {
227 NEON::BI__builtin_neon_splatq_lane_bf16,
228 NEON::BI__builtin_neon_splatq_lane_v,
229 },
230 {
231 NEON::BI__builtin_neon_splatq_laneq_bf16,
232 NEON::BI__builtin_neon_splatq_laneq_v,
233 },
234 {
235 NEON::BI__builtin_neon_vabd_f16,
236 NEON::BI__builtin_neon_vabd_v,
237 },
238 {
239 NEON::BI__builtin_neon_vabdq_f16,
240 NEON::BI__builtin_neon_vabdq_v,
241 },
242 {
243 NEON::BI__builtin_neon_vabs_f16,
244 NEON::BI__builtin_neon_vabs_v,
245 },
246 {
247 NEON::BI__builtin_neon_vabsq_f16,
248 NEON::BI__builtin_neon_vabsq_v,
249 },
250 {
251 NEON::BI__builtin_neon_vcage_f16,
252 NEON::BI__builtin_neon_vcage_v,
253 },
254 {
255 NEON::BI__builtin_neon_vcageq_f16,
256 NEON::BI__builtin_neon_vcageq_v,
257 },
258 {
259 NEON::BI__builtin_neon_vcagt_f16,
260 NEON::BI__builtin_neon_vcagt_v,
261 },
262 {
263 NEON::BI__builtin_neon_vcagtq_f16,
264 NEON::BI__builtin_neon_vcagtq_v,
265 },
266 {
267 NEON::BI__builtin_neon_vcale_f16,
268 NEON::BI__builtin_neon_vcale_v,
269 },
270 {
271 NEON::BI__builtin_neon_vcaleq_f16,
272 NEON::BI__builtin_neon_vcaleq_v,
273 },
274 {
275 NEON::BI__builtin_neon_vcalt_f16,
276 NEON::BI__builtin_neon_vcalt_v,
277 },
278 {
279 NEON::BI__builtin_neon_vcaltq_f16,
280 NEON::BI__builtin_neon_vcaltq_v,
281 },
282 {
283 NEON::BI__builtin_neon_vceqz_f16,
284 NEON::BI__builtin_neon_vceqz_v,
285 },
286 {
287 NEON::BI__builtin_neon_vceqzq_f16,
288 NEON::BI__builtin_neon_vceqzq_v,
289 },
290 {
291 NEON::BI__builtin_neon_vcgez_f16,
292 NEON::BI__builtin_neon_vcgez_v,
293 },
294 {
295 NEON::BI__builtin_neon_vcgezq_f16,
296 NEON::BI__builtin_neon_vcgezq_v,
297 },
298 {
299 NEON::BI__builtin_neon_vcgtz_f16,
300 NEON::BI__builtin_neon_vcgtz_v,
301 },
302 {
303 NEON::BI__builtin_neon_vcgtzq_f16,
304 NEON::BI__builtin_neon_vcgtzq_v,
305 },
306 {
307 NEON::BI__builtin_neon_vclez_f16,
308 NEON::BI__builtin_neon_vclez_v,
309 },
310 {
311 NEON::BI__builtin_neon_vclezq_f16,
312 NEON::BI__builtin_neon_vclezq_v,
313 },
314 {
315 NEON::BI__builtin_neon_vcltz_f16,
316 NEON::BI__builtin_neon_vcltz_v,
317 },
318 {
319 NEON::BI__builtin_neon_vcltzq_f16,
320 NEON::BI__builtin_neon_vcltzq_v,
321 },
322 {
323 NEON::BI__builtin_neon_vfma_f16,
324 NEON::BI__builtin_neon_vfma_v,
325 },
326 {
327 NEON::BI__builtin_neon_vfma_lane_f16,
328 NEON::BI__builtin_neon_vfma_lane_v,
329 },
330 {
331 NEON::BI__builtin_neon_vfma_laneq_f16,
332 NEON::BI__builtin_neon_vfma_laneq_v,
333 },
334 {
335 NEON::BI__builtin_neon_vfmaq_f16,
336 NEON::BI__builtin_neon_vfmaq_v,
337 },
338 {
339 NEON::BI__builtin_neon_vfmaq_lane_f16,
340 NEON::BI__builtin_neon_vfmaq_lane_v,
341 },
342 {
343 NEON::BI__builtin_neon_vfmaq_laneq_f16,
344 NEON::BI__builtin_neon_vfmaq_laneq_v,
345 },
346 {NEON::BI__builtin_neon_vld1_bf16_x2, NEON::BI__builtin_neon_vld1_x2_v},
347 {NEON::BI__builtin_neon_vld1_bf16_x3, NEON::BI__builtin_neon_vld1_x3_v},
348 {NEON::BI__builtin_neon_vld1_bf16_x4, NEON::BI__builtin_neon_vld1_x4_v},
349 {NEON::BI__builtin_neon_vld1_bf16, NEON::BI__builtin_neon_vld1_v},
350 {NEON::BI__builtin_neon_vld1_dup_bf16, NEON::BI__builtin_neon_vld1_dup_v},
351 {NEON::BI__builtin_neon_vld1_lane_bf16, NEON::BI__builtin_neon_vld1_lane_v},
352 {NEON::BI__builtin_neon_vld1q_bf16_x2, NEON::BI__builtin_neon_vld1q_x2_v},
353 {NEON::BI__builtin_neon_vld1q_bf16_x3, NEON::BI__builtin_neon_vld1q_x3_v},
354 {NEON::BI__builtin_neon_vld1q_bf16_x4, NEON::BI__builtin_neon_vld1q_x4_v},
355 {NEON::BI__builtin_neon_vld1q_bf16, NEON::BI__builtin_neon_vld1q_v},
356 {NEON::BI__builtin_neon_vld1q_dup_bf16, NEON::BI__builtin_neon_vld1q_dup_v},
357 {NEON::BI__builtin_neon_vld1q_lane_bf16,
358 NEON::BI__builtin_neon_vld1q_lane_v},
359 {NEON::BI__builtin_neon_vld2_bf16, NEON::BI__builtin_neon_vld2_v},
360 {NEON::BI__builtin_neon_vld2_dup_bf16, NEON::BI__builtin_neon_vld2_dup_v},
361 {NEON::BI__builtin_neon_vld2_lane_bf16, NEON::BI__builtin_neon_vld2_lane_v},
362 {NEON::BI__builtin_neon_vld2q_bf16, NEON::BI__builtin_neon_vld2q_v},
363 {NEON::BI__builtin_neon_vld2q_dup_bf16, NEON::BI__builtin_neon_vld2q_dup_v},
364 {NEON::BI__builtin_neon_vld2q_lane_bf16,
365 NEON::BI__builtin_neon_vld2q_lane_v},
366 {NEON::BI__builtin_neon_vld3_bf16, NEON::BI__builtin_neon_vld3_v},
367 {NEON::BI__builtin_neon_vld3_dup_bf16, NEON::BI__builtin_neon_vld3_dup_v},
368 {NEON::BI__builtin_neon_vld3_lane_bf16, NEON::BI__builtin_neon_vld3_lane_v},
369 {NEON::BI__builtin_neon_vld3q_bf16, NEON::BI__builtin_neon_vld3q_v},
370 {NEON::BI__builtin_neon_vld3q_dup_bf16, NEON::BI__builtin_neon_vld3q_dup_v},
371 {NEON::BI__builtin_neon_vld3q_lane_bf16,
372 NEON::BI__builtin_neon_vld3q_lane_v},
373 {NEON::BI__builtin_neon_vld4_bf16, NEON::BI__builtin_neon_vld4_v},
374 {NEON::BI__builtin_neon_vld4_dup_bf16, NEON::BI__builtin_neon_vld4_dup_v},
375 {NEON::BI__builtin_neon_vld4_lane_bf16, NEON::BI__builtin_neon_vld4_lane_v},
376 {NEON::BI__builtin_neon_vld4q_bf16, NEON::BI__builtin_neon_vld4q_v},
377 {NEON::BI__builtin_neon_vld4q_dup_bf16, NEON::BI__builtin_neon_vld4q_dup_v},
378 {NEON::BI__builtin_neon_vld4q_lane_bf16,
379 NEON::BI__builtin_neon_vld4q_lane_v},
380 {
381 NEON::BI__builtin_neon_vmax_f16,
382 NEON::BI__builtin_neon_vmax_v,
383 },
384 {
385 NEON::BI__builtin_neon_vmaxnm_f16,
386 NEON::BI__builtin_neon_vmaxnm_v,
387 },
388 {
389 NEON::BI__builtin_neon_vmaxnmq_f16,
390 NEON::BI__builtin_neon_vmaxnmq_v,
391 },
392 {
393 NEON::BI__builtin_neon_vmaxq_f16,
394 NEON::BI__builtin_neon_vmaxq_v,
395 },
396 {
397 NEON::BI__builtin_neon_vmin_f16,
398 NEON::BI__builtin_neon_vmin_v,
399 },
400 {
401 NEON::BI__builtin_neon_vminnm_f16,
402 NEON::BI__builtin_neon_vminnm_v,
403 },
404 {
405 NEON::BI__builtin_neon_vminnmq_f16,
406 NEON::BI__builtin_neon_vminnmq_v,
407 },
408 {
409 NEON::BI__builtin_neon_vminq_f16,
410 NEON::BI__builtin_neon_vminq_v,
411 },
412 {
413 NEON::BI__builtin_neon_vmulx_f16,
414 NEON::BI__builtin_neon_vmulx_v,
415 },
416 {
417 NEON::BI__builtin_neon_vmulxq_f16,
418 NEON::BI__builtin_neon_vmulxq_v,
419 },
420 {
421 NEON::BI__builtin_neon_vpadd_f16,
422 NEON::BI__builtin_neon_vpadd_v,
423 },
424 {
425 NEON::BI__builtin_neon_vpaddq_f16,
426 NEON::BI__builtin_neon_vpaddq_v,
427 },
428 {
429 NEON::BI__builtin_neon_vpmax_f16,
430 NEON::BI__builtin_neon_vpmax_v,
431 },
432 {
433 NEON::BI__builtin_neon_vpmaxnm_f16,
434 NEON::BI__builtin_neon_vpmaxnm_v,
435 },
436 {
437 NEON::BI__builtin_neon_vpmaxnmq_f16,
438 NEON::BI__builtin_neon_vpmaxnmq_v,
439 },
440 {
441 NEON::BI__builtin_neon_vpmaxq_f16,
442 NEON::BI__builtin_neon_vpmaxq_v,
443 },
444 {
445 NEON::BI__builtin_neon_vpmin_f16,
446 NEON::BI__builtin_neon_vpmin_v,
447 },
448 {
449 NEON::BI__builtin_neon_vpminnm_f16,
450 NEON::BI__builtin_neon_vpminnm_v,
451 },
452 {
453 NEON::BI__builtin_neon_vpminnmq_f16,
454 NEON::BI__builtin_neon_vpminnmq_v,
455 },
456 {
457 NEON::BI__builtin_neon_vpminq_f16,
458 NEON::BI__builtin_neon_vpminq_v,
459 },
460 {
461 NEON::BI__builtin_neon_vrecpe_f16,
462 NEON::BI__builtin_neon_vrecpe_v,
463 },
464 {
465 NEON::BI__builtin_neon_vrecpeq_f16,
466 NEON::BI__builtin_neon_vrecpeq_v,
467 },
468 {
469 NEON::BI__builtin_neon_vrecps_f16,
470 NEON::BI__builtin_neon_vrecps_v,
471 },
472 {
473 NEON::BI__builtin_neon_vrecpsq_f16,
474 NEON::BI__builtin_neon_vrecpsq_v,
475 },
476 {
477 NEON::BI__builtin_neon_vrnd_f16,
478 NEON::BI__builtin_neon_vrnd_v,
479 },
480 {
481 NEON::BI__builtin_neon_vrnda_f16,
482 NEON::BI__builtin_neon_vrnda_v,
483 },
484 {
485 NEON::BI__builtin_neon_vrndaq_f16,
486 NEON::BI__builtin_neon_vrndaq_v,
487 },
488 {
489 NEON::BI__builtin_neon_vrndi_f16,
490 NEON::BI__builtin_neon_vrndi_v,
491 },
492 {
493 NEON::BI__builtin_neon_vrndiq_f16,
494 NEON::BI__builtin_neon_vrndiq_v,
495 },
496 {
497 NEON::BI__builtin_neon_vrndm_f16,
498 NEON::BI__builtin_neon_vrndm_v,
499 },
500 {
501 NEON::BI__builtin_neon_vrndmq_f16,
502 NEON::BI__builtin_neon_vrndmq_v,
503 },
504 {
505 NEON::BI__builtin_neon_vrndn_f16,
506 NEON::BI__builtin_neon_vrndn_v,
507 },
508 {
509 NEON::BI__builtin_neon_vrndnq_f16,
510 NEON::BI__builtin_neon_vrndnq_v,
511 },
512 {
513 NEON::BI__builtin_neon_vrndp_f16,
514 NEON::BI__builtin_neon_vrndp_v,
515 },
516 {
517 NEON::BI__builtin_neon_vrndpq_f16,
518 NEON::BI__builtin_neon_vrndpq_v,
519 },
520 {
521 NEON::BI__builtin_neon_vrndq_f16,
522 NEON::BI__builtin_neon_vrndq_v,
523 },
524 {
525 NEON::BI__builtin_neon_vrndx_f16,
526 NEON::BI__builtin_neon_vrndx_v,
527 },
528 {
529 NEON::BI__builtin_neon_vrndxq_f16,
530 NEON::BI__builtin_neon_vrndxq_v,
531 },
532 {
533 NEON::BI__builtin_neon_vrsqrte_f16,
534 NEON::BI__builtin_neon_vrsqrte_v,
535 },
536 {
537 NEON::BI__builtin_neon_vrsqrteq_f16,
538 NEON::BI__builtin_neon_vrsqrteq_v,
539 },
540 {
541 NEON::BI__builtin_neon_vrsqrts_f16,
542 NEON::BI__builtin_neon_vrsqrts_v,
543 },
544 {
545 NEON::BI__builtin_neon_vrsqrtsq_f16,
546 NEON::BI__builtin_neon_vrsqrtsq_v,
547 },
548 {
549 NEON::BI__builtin_neon_vsqrt_f16,
550 NEON::BI__builtin_neon_vsqrt_v,
551 },
552 {
553 NEON::BI__builtin_neon_vsqrtq_f16,
554 NEON::BI__builtin_neon_vsqrtq_v,
555 },
556 {NEON::BI__builtin_neon_vst1_bf16_x2, NEON::BI__builtin_neon_vst1_x2_v},
557 {NEON::BI__builtin_neon_vst1_bf16_x3, NEON::BI__builtin_neon_vst1_x3_v},
558 {NEON::BI__builtin_neon_vst1_bf16_x4, NEON::BI__builtin_neon_vst1_x4_v},
559 {NEON::BI__builtin_neon_vst1_bf16, NEON::BI__builtin_neon_vst1_v},
560 {NEON::BI__builtin_neon_vst1_lane_bf16, NEON::BI__builtin_neon_vst1_lane_v},
561 {NEON::BI__builtin_neon_vst1q_bf16_x2, NEON::BI__builtin_neon_vst1q_x2_v},
562 {NEON::BI__builtin_neon_vst1q_bf16_x3, NEON::BI__builtin_neon_vst1q_x3_v},
563 {NEON::BI__builtin_neon_vst1q_bf16_x4, NEON::BI__builtin_neon_vst1q_x4_v},
564 {NEON::BI__builtin_neon_vst1q_bf16, NEON::BI__builtin_neon_vst1q_v},
565 {NEON::BI__builtin_neon_vst1q_lane_bf16,
566 NEON::BI__builtin_neon_vst1q_lane_v},
567 {NEON::BI__builtin_neon_vst2_bf16, NEON::BI__builtin_neon_vst2_v},
568 {NEON::BI__builtin_neon_vst2_lane_bf16, NEON::BI__builtin_neon_vst2_lane_v},
569 {NEON::BI__builtin_neon_vst2q_bf16, NEON::BI__builtin_neon_vst2q_v},
570 {NEON::BI__builtin_neon_vst2q_lane_bf16,
571 NEON::BI__builtin_neon_vst2q_lane_v},
572 {NEON::BI__builtin_neon_vst3_bf16, NEON::BI__builtin_neon_vst3_v},
573 {NEON::BI__builtin_neon_vst3_lane_bf16, NEON::BI__builtin_neon_vst3_lane_v},
574 {NEON::BI__builtin_neon_vst3q_bf16, NEON::BI__builtin_neon_vst3q_v},
575 {NEON::BI__builtin_neon_vst3q_lane_bf16,
576 NEON::BI__builtin_neon_vst3q_lane_v},
577 {NEON::BI__builtin_neon_vst4_bf16, NEON::BI__builtin_neon_vst4_v},
578 {NEON::BI__builtin_neon_vst4_lane_bf16, NEON::BI__builtin_neon_vst4_lane_v},
579 {NEON::BI__builtin_neon_vst4q_bf16, NEON::BI__builtin_neon_vst4q_v},
580 {NEON::BI__builtin_neon_vst4q_lane_bf16,
581 NEON::BI__builtin_neon_vst4q_lane_v},
582 // The mangling rules cause us to have one ID for each type for
583 // vldap1(q)_lane and vstl1(q)_lane, but codegen is equivalent for all of
584 // them. Choose an arbitrary one to be handled as tha canonical variation.
585 {NEON::BI__builtin_neon_vldap1_lane_u64,
586 NEON::BI__builtin_neon_vldap1_lane_s64},
587 {NEON::BI__builtin_neon_vldap1_lane_f64,
588 NEON::BI__builtin_neon_vldap1_lane_s64},
589 {NEON::BI__builtin_neon_vldap1_lane_p64,
590 NEON::BI__builtin_neon_vldap1_lane_s64},
591 {NEON::BI__builtin_neon_vldap1q_lane_u64,
592 NEON::BI__builtin_neon_vldap1q_lane_s64},
593 {NEON::BI__builtin_neon_vldap1q_lane_f64,
594 NEON::BI__builtin_neon_vldap1q_lane_s64},
595 {NEON::BI__builtin_neon_vldap1q_lane_p64,
596 NEON::BI__builtin_neon_vldap1q_lane_s64},
597 {NEON::BI__builtin_neon_vstl1_lane_u64,
598 NEON::BI__builtin_neon_vstl1_lane_s64},
599 {NEON::BI__builtin_neon_vstl1_lane_f64,
600 NEON::BI__builtin_neon_vstl1_lane_s64},
601 {NEON::BI__builtin_neon_vstl1_lane_p64,
602 NEON::BI__builtin_neon_vstl1_lane_s64},
603 {NEON::BI__builtin_neon_vstl1q_lane_u64,
604 NEON::BI__builtin_neon_vstl1q_lane_s64},
605 {NEON::BI__builtin_neon_vstl1q_lane_f64,
606 NEON::BI__builtin_neon_vstl1q_lane_s64},
607 {NEON::BI__builtin_neon_vstl1q_lane_p64,
608 NEON::BI__builtin_neon_vstl1q_lane_s64},
609};
610
611std::optional<mlir::Value>
612CIRGenFunction::emitAArch64BuiltinExpr(unsigned builtinID, const CallExpr *expr,
613 ReturnValueSlot returnValue,
614 llvm::Triple::ArchType arch) {
615 if (builtinID >= clang::AArch64::FirstSVEBuiltin &&
616 builtinID <= clang::AArch64::LastSVEBuiltin)
617 return emitAArch64SVEBuiltinExpr(builtinID, expr);
618
619 if (builtinID >= clang::AArch64::FirstSMEBuiltin &&
620 builtinID <= clang::AArch64::LastSMEBuiltin)
621 return emitAArch64SMEBuiltinExpr(builtinID, expr);
622
623 if (builtinID == Builtin::BI__builtin_cpu_supports) {
624 cgm.errorNYI(expr->getSourceRange(),
625 std::string("unimplemented AArch64 builtin call: ") +
626 getContext().BuiltinInfo.getName(builtinID));
627 return mlir::Value{};
628 }
629
630 switch (builtinID) {
631 default:
632 break;
633 case clang::AArch64::BI__builtin_arm_nop:
634 case clang::AArch64::BI__builtin_arm_yield:
635 case clang::AArch64::BI__yield:
636 case clang::AArch64::BI__builtin_arm_wfe:
637 case clang::AArch64::BI__wfe:
638 case clang::AArch64::BI__builtin_arm_wfi:
639 case clang::AArch64::BI__wfi:
640 case clang::AArch64::BI__builtin_arm_sev:
641 case clang::AArch64::BI__sev:
642 case clang::AArch64::BI__builtin_arm_sevl:
643 case clang::AArch64::BI__sevl:
644 cgm.errorNYI(expr->getSourceRange(),
645 std::string("unimplemented AArch64 builtin call: ") +
646 getContext().BuiltinInfo.getName(builtinID));
647 return mlir::Value{};
648 }
649
650 if (builtinID == clang::AArch64::BI__builtin_arm_trap) {
651 cgm.errorNYI(expr->getSourceRange(),
652 std::string("unimplemented AArch64 builtin call: ") +
653 getContext().BuiltinInfo.getName(builtinID));
654 return mlir::Value{};
655 }
656
657 if (builtinID == clang::AArch64::BI__builtin_arm_get_sme_state) {
658 cgm.errorNYI(expr->getSourceRange(),
659 std::string("unimplemented AArch64 builtin call: ") +
660 getContext().BuiltinInfo.getName(builtinID));
661 return mlir::Value{};
662 }
663
664 if (builtinID == clang::AArch64::BI__builtin_arm_rbit) {
665 cgm.errorNYI(expr->getSourceRange(),
666 std::string("unimplemented AArch64 builtin call: ") +
667 getContext().BuiltinInfo.getName(builtinID));
668 return mlir::Value{};
669 }
670 if (builtinID == clang::AArch64::BI__builtin_arm_rbit64) {
671 cgm.errorNYI(expr->getSourceRange(),
672 std::string("unimplemented AArch64 builtin call: ") +
673 getContext().BuiltinInfo.getName(builtinID));
674 return mlir::Value{};
675 }
676
677 if (builtinID == clang::AArch64::BI__builtin_arm_clz ||
678 builtinID == clang::AArch64::BI__builtin_arm_clz64) {
679 cgm.errorNYI(expr->getSourceRange(),
680 std::string("unimplemented AArch64 builtin call: ") +
681 getContext().BuiltinInfo.getName(builtinID));
682 return mlir::Value{};
683 }
684
685 if (builtinID == clang::AArch64::BI__builtin_arm_cls) {
686 cgm.errorNYI(expr->getSourceRange(),
687 std::string("unimplemented AArch64 builtin call: ") +
688 getContext().BuiltinInfo.getName(builtinID));
689 return mlir::Value{};
690 }
691 if (builtinID == clang::AArch64::BI__builtin_arm_cls64) {
692 cgm.errorNYI(expr->getSourceRange(),
693 std::string("unimplemented AArch64 builtin call: ") +
694 getContext().BuiltinInfo.getName(builtinID));
695 return mlir::Value{};
696 }
697
698 if (builtinID == clang::AArch64::BI__builtin_arm_rint32zf ||
699 builtinID == clang::AArch64::BI__builtin_arm_rint32z) {
700 cgm.errorNYI(expr->getSourceRange(),
701 std::string("unimplemented AArch64 builtin call: ") +
702 getContext().BuiltinInfo.getName(builtinID));
703 return mlir::Value{};
704 }
705
706 if (builtinID == clang::AArch64::BI__builtin_arm_rint64zf ||
707 builtinID == clang::AArch64::BI__builtin_arm_rint64z) {
708 cgm.errorNYI(expr->getSourceRange(),
709 std::string("unimplemented AArch64 builtin call: ") +
710 getContext().BuiltinInfo.getName(builtinID));
711 return mlir::Value{};
712 }
713
714 if (builtinID == clang::AArch64::BI__builtin_arm_rint32xf ||
715 builtinID == clang::AArch64::BI__builtin_arm_rint32x) {
716 cgm.errorNYI(expr->getSourceRange(),
717 std::string("unimplemented AArch64 builtin call: ") +
718 getContext().BuiltinInfo.getName(builtinID));
719 return mlir::Value{};
720 }
721
722 if (builtinID == clang::AArch64::BI__builtin_arm_rint64xf ||
723 builtinID == clang::AArch64::BI__builtin_arm_rint64x) {
724 cgm.errorNYI(expr->getSourceRange(),
725 std::string("unimplemented AArch64 builtin call: ") +
726 getContext().BuiltinInfo.getName(builtinID));
727 return mlir::Value{};
728 }
729
730 if (builtinID == clang::AArch64::BI__builtin_arm_jcvt) {
731 cgm.errorNYI(expr->getSourceRange(),
732 std::string("unimplemented AArch64 builtin call: ") +
733 getContext().BuiltinInfo.getName(builtinID));
734 return mlir::Value{};
735 }
736
737 if (builtinID == clang::AArch64::BI__builtin_arm_ld64b ||
738 builtinID == clang::AArch64::BI__builtin_arm_st64b ||
739 builtinID == clang::AArch64::BI__builtin_arm_st64bv ||
740 builtinID == clang::AArch64::BI__builtin_arm_st64bv0) {
741 cgm.errorNYI(expr->getSourceRange(),
742 std::string("unimplemented AArch64 builtin call: ") +
743 getContext().BuiltinInfo.getName(builtinID));
744 return mlir::Value{};
745 }
746
747 if (builtinID == clang::AArch64::BI__builtin_arm_rndr ||
748 builtinID == clang::AArch64::BI__builtin_arm_rndrrs) {
749 cgm.errorNYI(expr->getSourceRange(),
750 std::string("unimplemented AArch64 builtin call: ") +
751 getContext().BuiltinInfo.getName(builtinID));
752 return mlir::Value{};
753 }
754
755 if (builtinID == clang::AArch64::BI__clear_cache) {
756 cgm.errorNYI(expr->getSourceRange(),
757 std::string("unimplemented AArch64 builtin call: ") +
758 getContext().BuiltinInfo.getName(builtinID));
759 return mlir::Value{};
760 }
761
762 if ((builtinID == clang::AArch64::BI__builtin_arm_ldrex ||
763 builtinID == clang::AArch64::BI__builtin_arm_ldaex) &&
764 getContext().getTypeSize(expr->getType()) == 128) {
765 cgm.errorNYI(expr->getSourceRange(),
766 std::string("unimplemented AArch64 builtin call: ") +
767 getContext().BuiltinInfo.getName(builtinID));
768 return mlir::Value{};
769 }
770 if (builtinID == clang::AArch64::BI__builtin_arm_ldrex ||
771 builtinID == clang::AArch64::BI__builtin_arm_ldaex) {
772 cgm.errorNYI(expr->getSourceRange(),
773 std::string("unimplemented AArch64 builtin call: ") +
774 getContext().BuiltinInfo.getName(builtinID));
775 return mlir::Value{};
776 }
777
778 if ((builtinID == clang::AArch64::BI__builtin_arm_strex ||
779 builtinID == clang::AArch64::BI__builtin_arm_stlex) &&
780 getContext().getTypeSize(expr->getArg(0)->getType()) == 128) {
781 cgm.errorNYI(expr->getSourceRange(),
782 std::string("unimplemented AArch64 builtin call: ") +
783 getContext().BuiltinInfo.getName(builtinID));
784 return mlir::Value{};
785 }
786
787 if (builtinID == clang::AArch64::BI__builtin_arm_strex ||
788 builtinID == clang::AArch64::BI__builtin_arm_stlex) {
789 cgm.errorNYI(expr->getSourceRange(),
790 std::string("unimplemented AArch64 builtin call: ") +
791 getContext().BuiltinInfo.getName(builtinID));
792 return mlir::Value{};
793 }
794
795 if (builtinID == clang::AArch64::BI__getReg) {
796 cgm.errorNYI(expr->getSourceRange(),
797 std::string("unimplemented AArch64 builtin call: ") +
798 getContext().BuiltinInfo.getName(builtinID));
799 return mlir::Value{};
800 }
801
802 if (builtinID == clang::AArch64::BI__break) {
803 cgm.errorNYI(expr->getSourceRange(),
804 std::string("unimplemented AArch64 builtin call: ") +
805 getContext().BuiltinInfo.getName(builtinID));
806 return mlir::Value{};
807 }
808
809 if (builtinID == clang::AArch64::BI__builtin_arm_clrex) {
810 cgm.errorNYI(expr->getSourceRange(),
811 std::string("unimplemented AArch64 builtin call: ") +
812 getContext().BuiltinInfo.getName(builtinID));
813 return mlir::Value{};
814 }
815
816 if (builtinID == clang::AArch64::BI_ReadWriteBarrier) {
817 cgm.errorNYI(expr->getSourceRange(),
818 std::string("unimplemented AArch64 builtin call: ") +
819 getContext().BuiltinInfo.getName(builtinID));
820 return mlir::Value{};
821 }
822
823 // CRC32
824 Intrinsic::ID crcIntrinsicID = Intrinsic::not_intrinsic;
825 switch (builtinID) {
826 case clang::AArch64::BI__builtin_arm_crc32b:
827 crcIntrinsicID = Intrinsic::aarch64_crc32b;
828 break;
829 case clang::AArch64::BI__builtin_arm_crc32cb:
830 crcIntrinsicID = Intrinsic::aarch64_crc32cb;
831 break;
832 case clang::AArch64::BI__builtin_arm_crc32h:
833 crcIntrinsicID = Intrinsic::aarch64_crc32h;
834 break;
835 case clang::AArch64::BI__builtin_arm_crc32ch:
836 crcIntrinsicID = Intrinsic::aarch64_crc32ch;
837 break;
838 case clang::AArch64::BI__builtin_arm_crc32w:
839 crcIntrinsicID = Intrinsic::aarch64_crc32w;
840 break;
841 case clang::AArch64::BI__builtin_arm_crc32cw:
842 crcIntrinsicID = Intrinsic::aarch64_crc32cw;
843 break;
844 case clang::AArch64::BI__builtin_arm_crc32d:
845 crcIntrinsicID = Intrinsic::aarch64_crc32x;
846 break;
847 case clang::AArch64::BI__builtin_arm_crc32cd:
848 crcIntrinsicID = Intrinsic::aarch64_crc32cx;
849 break;
850 }
851
852 if (crcIntrinsicID != Intrinsic::not_intrinsic) {
853 cgm.errorNYI(expr->getSourceRange(),
854 std::string("unimplemented AArch64 builtin call: ") +
855 getContext().BuiltinInfo.getName(builtinID));
856 return mlir::Value{};
857 }
858
859 // Memory Operations (MOPS)
860 if (builtinID == AArch64::BI__builtin_arm_mops_memset_tag) {
861 cgm.errorNYI(expr->getSourceRange(),
862 std::string("unimplemented AArch64 builtin call: ") +
863 getContext().BuiltinInfo.getName(builtinID));
864 return mlir::Value{};
865 }
866
867 // Memory Tagging Extensions (MTE) Intrinsics
868 Intrinsic::ID mteIntrinsicID = Intrinsic::not_intrinsic;
869 switch (builtinID) {
870 case clang::AArch64::BI__builtin_arm_irg:
871 mteIntrinsicID = Intrinsic::aarch64_irg;
872 break;
873 case clang::AArch64::BI__builtin_arm_addg:
874 mteIntrinsicID = Intrinsic::aarch64_addg;
875 break;
876 case clang::AArch64::BI__builtin_arm_gmi:
877 mteIntrinsicID = Intrinsic::aarch64_gmi;
878 break;
879 case clang::AArch64::BI__builtin_arm_ldg:
880 mteIntrinsicID = Intrinsic::aarch64_ldg;
881 break;
882 case clang::AArch64::BI__builtin_arm_stg:
883 mteIntrinsicID = Intrinsic::aarch64_stg;
884 break;
885 case clang::AArch64::BI__builtin_arm_subp:
886 mteIntrinsicID = Intrinsic::aarch64_subp;
887 break;
888 }
889
890 if (mteIntrinsicID != Intrinsic::not_intrinsic) {
891 cgm.errorNYI(expr->getSourceRange(),
892 std::string("unimplemented AArch64 builtin call: ") +
893 getContext().BuiltinInfo.getName(builtinID));
894 return mlir::Value{};
895 }
896
897 if (builtinID == clang::AArch64::BI__builtin_arm_rsr ||
898 builtinID == clang::AArch64::BI__builtin_arm_rsr64 ||
899 builtinID == clang::AArch64::BI__builtin_arm_rsr128 ||
900 builtinID == clang::AArch64::BI__builtin_arm_rsrp ||
901 builtinID == clang::AArch64::BI__builtin_arm_wsr ||
902 builtinID == clang::AArch64::BI__builtin_arm_wsr64 ||
903 builtinID == clang::AArch64::BI__builtin_arm_wsr128 ||
904 builtinID == clang::AArch64::BI__builtin_arm_wsrp) {
905 cgm.errorNYI(expr->getSourceRange(),
906 std::string("unimplemented AArch64 builtin call: ") +
907 getContext().BuiltinInfo.getName(builtinID));
908 return mlir::Value{};
909 }
910
911 if (builtinID == clang::AArch64::BI_ReadStatusReg ||
912 builtinID == clang::AArch64::BI_WriteStatusReg ||
913 builtinID == clang::AArch64::BI__sys) {
914 cgm.errorNYI(expr->getSourceRange(),
915 std::string("unimplemented AArch64 builtin call: ") +
916 getContext().BuiltinInfo.getName(builtinID));
917 return mlir::Value{};
918 }
919
920 if (builtinID == clang::AArch64::BI_AddressOfReturnAddress) {
921 cgm.errorNYI(expr->getSourceRange(),
922 std::string("unimplemented AArch64 builtin call: ") +
923 getContext().BuiltinInfo.getName(builtinID));
924 return mlir::Value{};
925 }
926
927 if (builtinID == clang::AArch64::BI__builtin_sponentry) {
928 cgm.errorNYI(expr->getSourceRange(),
929 std::string("unimplemented AArch64 builtin call: ") +
930 getContext().BuiltinInfo.getName(builtinID));
931 return mlir::Value{};
932 }
933
934 if (builtinID == clang::AArch64::BI__mulh ||
935 builtinID == clang::AArch64::BI__umulh) {
936 cgm.errorNYI(expr->getSourceRange(),
937 std::string("unimplemented AArch64 builtin call: ") +
938 getContext().BuiltinInfo.getName(builtinID));
939 return mlir::Value{};
940 }
941
942 if (builtinID == AArch64::BI__writex18byte ||
943 builtinID == AArch64::BI__writex18word ||
944 builtinID == AArch64::BI__writex18dword ||
945 builtinID == AArch64::BI__writex18qword) {
946 cgm.errorNYI(expr->getSourceRange(),
947 std::string("unimplemented AArch64 builtin call: ") +
948 getContext().BuiltinInfo.getName(builtinID));
949 return mlir::Value{};
950 }
951
952 if (builtinID == AArch64::BI__readx18byte ||
953 builtinID == AArch64::BI__readx18word ||
954 builtinID == AArch64::BI__readx18dword ||
955 builtinID == AArch64::BI__readx18qword) {
956 cgm.errorNYI(expr->getSourceRange(),
957 std::string("unimplemented AArch64 builtin call: ") +
958 getContext().BuiltinInfo.getName(builtinID));
959 return mlir::Value{};
960 }
961
962 if (builtinID == AArch64::BI__addx18byte ||
963 builtinID == AArch64::BI__addx18word ||
964 builtinID == AArch64::BI__addx18dword ||
965 builtinID == AArch64::BI__addx18qword ||
966 builtinID == AArch64::BI__incx18byte ||
967 builtinID == AArch64::BI__incx18word ||
968 builtinID == AArch64::BI__incx18dword ||
969 builtinID == AArch64::BI__incx18qword) {
970 cgm.errorNYI(expr->getSourceRange(),
971 std::string("unimplemented AArch64 builtin call: ") +
972 getContext().BuiltinInfo.getName(builtinID));
973 return mlir::Value{};
974 }
975
976 if (builtinID == AArch64::BI_CopyDoubleFromInt64 ||
977 builtinID == AArch64::BI_CopyFloatFromInt32 ||
978 builtinID == AArch64::BI_CopyInt32FromFloat ||
979 builtinID == AArch64::BI_CopyInt64FromDouble) {
980 cgm.errorNYI(expr->getSourceRange(),
981 std::string("unimplemented AArch64 builtin call: ") +
982 getContext().BuiltinInfo.getName(builtinID));
983 return mlir::Value{};
984 }
985
986 if (builtinID == AArch64::BI_CountLeadingOnes ||
987 builtinID == AArch64::BI_CountLeadingOnes64 ||
988 builtinID == AArch64::BI_CountLeadingZeros ||
989 builtinID == AArch64::BI_CountLeadingZeros64) {
990 cgm.errorNYI(expr->getSourceRange(),
991 std::string("unimplemented AArch64 builtin call: ") +
992 getContext().BuiltinInfo.getName(builtinID));
993 return mlir::Value{};
994 }
995
996 if (builtinID == AArch64::BI_CountLeadingSigns ||
997 builtinID == AArch64::BI_CountLeadingSigns64) {
998 cgm.errorNYI(expr->getSourceRange(),
999 std::string("unimplemented AArch64 builtin call: ") +
1000 getContext().BuiltinInfo.getName(builtinID));
1001 return mlir::Value{};
1002 }
1003
1004 if (builtinID == AArch64::BI_CountOneBits ||
1005 builtinID == AArch64::BI_CountOneBits64) {
1006 cgm.errorNYI(expr->getSourceRange(),
1007 std::string("unimplemented AArch64 builtin call: ") +
1008 getContext().BuiltinInfo.getName(builtinID));
1009 return mlir::Value{};
1010 }
1011
1012 if (builtinID == AArch64::BI__prefetch) {
1013 cgm.errorNYI(expr->getSourceRange(),
1014 std::string("unimplemented AArch64 builtin call: ") +
1015 getContext().BuiltinInfo.getName(builtinID));
1016 return mlir::Value{};
1017 }
1018
1019 if (builtinID == AArch64::BI__hlt) {
1020 cgm.errorNYI(expr->getSourceRange(),
1021 std::string("unimplemented AArch64 builtin call: ") +
1022 getContext().BuiltinInfo.getName(builtinID));
1023 return mlir::Value{};
1024 }
1025
1026 if (builtinID == NEON::BI__builtin_neon_vcvth_bf16_f32) {
1027 cgm.errorNYI(expr->getSourceRange(),
1028 std::string("unimplemented AArch64 builtin call: ") +
1029 getContext().BuiltinInfo.getName(builtinID));
1030 return mlir::Value{};
1031 }
1032
1033 // Handle MSVC intrinsics before argument evaluation to prevent double
1034 // evaluation.
1035 assert(!cir::MissingFeatures::msvcBuiltins());
1036
1037 // Some intrinsics are equivalent - if they are use the base intrinsic ID.
1038 auto it = llvm::find_if(neonEquivalentIntrinsicMap, [builtinID](auto &p) {
1039 return p.first == builtinID;
1040 });
1041 if (it != end(neonEquivalentIntrinsicMap))
1042 builtinID = it->second;
1043
1044 // Find out if any arguments are required to be integer constant
1045 // expressions.
1046 assert(!cir::MissingFeatures::handleBuiltinICEArguments());
1047
1048 assert(!cir::MissingFeatures::neonSISDIntrinsics());
1049
1050 // Handle non-overloaded intrinsics first.
1051 switch (builtinID) {
1052 default:
1053 break;
1054 case NEON::BI__builtin_neon_vabsh_f16:
1055 case NEON::BI__builtin_neon_vaddq_p128:
1056 case NEON::BI__builtin_neon_vldrq_p128:
1057 case NEON::BI__builtin_neon_vstrq_p128:
1058 case NEON::BI__builtin_neon_vcvts_f32_u32:
1059 case NEON::BI__builtin_neon_vcvtd_f64_u64:
1060 case NEON::BI__builtin_neon_vcvts_f32_s32:
1061 case NEON::BI__builtin_neon_vcvtd_f64_s64:
1062 case NEON::BI__builtin_neon_vcvth_f16_u16:
1063 case NEON::BI__builtin_neon_vcvth_f16_u32:
1064 case NEON::BI__builtin_neon_vcvth_f16_u64:
1065 case NEON::BI__builtin_neon_vcvth_f16_s16:
1066 case NEON::BI__builtin_neon_vcvth_f16_s32:
1067 case NEON::BI__builtin_neon_vcvth_f16_s64:
1068 case NEON::BI__builtin_neon_vcvtah_u16_f16:
1069 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
1070 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
1071 case NEON::BI__builtin_neon_vcvtph_u16_f16:
1072 case NEON::BI__builtin_neon_vcvth_u16_f16:
1073 case NEON::BI__builtin_neon_vcvtah_s16_f16:
1074 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
1075 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
1076 case NEON::BI__builtin_neon_vcvtph_s16_f16:
1077 case NEON::BI__builtin_neon_vcvth_s16_f16:
1078 case NEON::BI__builtin_neon_vcaleh_f16:
1079 case NEON::BI__builtin_neon_vcalth_f16:
1080 case NEON::BI__builtin_neon_vcageh_f16:
1081 case NEON::BI__builtin_neon_vcagth_f16:
1082 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
1083 case NEON::BI__builtin_neon_vcvth_n_u16_f16:
1084 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
1085 case NEON::BI__builtin_neon_vcvth_n_f16_u16:
1086 case NEON::BI__builtin_neon_vpaddd_s64:
1087 case NEON::BI__builtin_neon_vpaddd_f64:
1088 case NEON::BI__builtin_neon_vpadds_f32:
1089 case NEON::BI__builtin_neon_vceqzd_s64:
1090 case NEON::BI__builtin_neon_vceqzd_f64:
1091 case NEON::BI__builtin_neon_vceqzs_f32:
1092 case NEON::BI__builtin_neon_vceqzh_f16:
1093 case NEON::BI__builtin_neon_vcgezd_s64:
1094 case NEON::BI__builtin_neon_vcgezd_f64:
1095 case NEON::BI__builtin_neon_vcgezs_f32:
1096 case NEON::BI__builtin_neon_vcgezh_f16:
1097 case NEON::BI__builtin_neon_vclezd_s64:
1098 case NEON::BI__builtin_neon_vclezd_f64:
1099 case NEON::BI__builtin_neon_vclezs_f32:
1100 case NEON::BI__builtin_neon_vclezh_f16:
1101 case NEON::BI__builtin_neon_vcgtzd_s64:
1102 case NEON::BI__builtin_neon_vcgtzd_f64:
1103 case NEON::BI__builtin_neon_vcgtzs_f32:
1104 case NEON::BI__builtin_neon_vcgtzh_f16:
1105 case NEON::BI__builtin_neon_vcltzd_s64:
1106 case NEON::BI__builtin_neon_vcltzd_f64:
1107 case NEON::BI__builtin_neon_vcltzs_f32:
1108 case NEON::BI__builtin_neon_vcltzh_f16:
1109 case NEON::BI__builtin_neon_vceqzd_u64:
1110 case NEON::BI__builtin_neon_vceqd_f64:
1111 case NEON::BI__builtin_neon_vcled_f64:
1112 case NEON::BI__builtin_neon_vcltd_f64:
1113 case NEON::BI__builtin_neon_vcged_f64:
1114 case NEON::BI__builtin_neon_vcgtd_f64:
1115 case NEON::BI__builtin_neon_vceqs_f32:
1116 case NEON::BI__builtin_neon_vcles_f32:
1117 case NEON::BI__builtin_neon_vclts_f32:
1118 case NEON::BI__builtin_neon_vcges_f32:
1119 case NEON::BI__builtin_neon_vcgts_f32:
1120 case NEON::BI__builtin_neon_vceqh_f16:
1121 case NEON::BI__builtin_neon_vcleh_f16:
1122 case NEON::BI__builtin_neon_vclth_f16:
1123 case NEON::BI__builtin_neon_vcgeh_f16:
1124 case NEON::BI__builtin_neon_vcgth_f16:
1125 case NEON::BI__builtin_neon_vceqd_s64:
1126 case NEON::BI__builtin_neon_vceqd_u64:
1127 case NEON::BI__builtin_neon_vcgtd_s64:
1128 case NEON::BI__builtin_neon_vcgtd_u64:
1129 case NEON::BI__builtin_neon_vcltd_s64:
1130 case NEON::BI__builtin_neon_vcltd_u64:
1131 case NEON::BI__builtin_neon_vcged_u64:
1132 case NEON::BI__builtin_neon_vcged_s64:
1133 case NEON::BI__builtin_neon_vcled_u64:
1134 case NEON::BI__builtin_neon_vcled_s64:
1135 case NEON::BI__builtin_neon_vtstd_s64:
1136 case NEON::BI__builtin_neon_vtstd_u64:
1137 case NEON::BI__builtin_neon_vset_lane_i8:
1138 case NEON::BI__builtin_neon_vset_lane_i16:
1139 case NEON::BI__builtin_neon_vset_lane_i32:
1140 case NEON::BI__builtin_neon_vset_lane_i64:
1141 case NEON::BI__builtin_neon_vset_lane_bf16:
1142 case NEON::BI__builtin_neon_vset_lane_f32:
1143 case NEON::BI__builtin_neon_vsetq_lane_i8:
1144 case NEON::BI__builtin_neon_vsetq_lane_i16:
1145 case NEON::BI__builtin_neon_vsetq_lane_i32:
1146 case NEON::BI__builtin_neon_vsetq_lane_i64:
1147 case NEON::BI__builtin_neon_vsetq_lane_bf16:
1148 case NEON::BI__builtin_neon_vsetq_lane_f32:
1149 case NEON::BI__builtin_neon_vset_lane_f64:
1150 case NEON::BI__builtin_neon_vset_lane_mf8:
1151 case NEON::BI__builtin_neon_vsetq_lane_mf8:
1152 case NEON::BI__builtin_neon_vsetq_lane_f64:
1153 case NEON::BI__builtin_neon_vget_lane_i8:
1154 case NEON::BI__builtin_neon_vdupb_lane_i8:
1155 case NEON::BI__builtin_neon_vgetq_lane_i8:
1156 case NEON::BI__builtin_neon_vdupb_laneq_i8:
1157 case NEON::BI__builtin_neon_vget_lane_mf8:
1158 case NEON::BI__builtin_neon_vdupb_lane_mf8:
1159 case NEON::BI__builtin_neon_vgetq_lane_mf8:
1160 case NEON::BI__builtin_neon_vdupb_laneq_mf8:
1161 case NEON::BI__builtin_neon_vget_lane_i16:
1162 case NEON::BI__builtin_neon_vduph_lane_i16:
1163 case NEON::BI__builtin_neon_vgetq_lane_i16:
1164 case NEON::BI__builtin_neon_vduph_laneq_i16:
1165 case NEON::BI__builtin_neon_vget_lane_i32:
1166 case NEON::BI__builtin_neon_vdups_lane_i32:
1167 case NEON::BI__builtin_neon_vdups_lane_f32:
1168 case NEON::BI__builtin_neon_vgetq_lane_i32:
1169 case NEON::BI__builtin_neon_vdups_laneq_i32:
1170 case NEON::BI__builtin_neon_vget_lane_i64:
1171 case NEON::BI__builtin_neon_vdupd_lane_i64:
1172 case NEON::BI__builtin_neon_vdupd_lane_f64:
1173 case NEON::BI__builtin_neon_vgetq_lane_i64:
1174 case NEON::BI__builtin_neon_vdupd_laneq_i64:
1175 case NEON::BI__builtin_neon_vget_lane_f32:
1176 case NEON::BI__builtin_neon_vget_lane_f64:
1177 case NEON::BI__builtin_neon_vgetq_lane_f32:
1178 case NEON::BI__builtin_neon_vdups_laneq_f32:
1179 case NEON::BI__builtin_neon_vgetq_lane_f64:
1180 case NEON::BI__builtin_neon_vdupd_laneq_f64:
1181 case NEON::BI__builtin_neon_vaddh_f16:
1182 case NEON::BI__builtin_neon_vsubh_f16:
1183 case NEON::BI__builtin_neon_vmulh_f16:
1184 case NEON::BI__builtin_neon_vdivh_f16:
1185 case NEON::BI__builtin_neon_vfmah_f16:
1186 case NEON::BI__builtin_neon_vfmsh_f16:
1187 case NEON::BI__builtin_neon_vaddd_s64:
1188 case NEON::BI__builtin_neon_vaddd_u64:
1189 case NEON::BI__builtin_neon_vsubd_s64:
1190 case NEON::BI__builtin_neon_vsubd_u64:
1191 case NEON::BI__builtin_neon_vqdmlalh_s16:
1192 case NEON::BI__builtin_neon_vqdmlslh_s16:
1193 case NEON::BI__builtin_neon_vqshlud_n_s64:
1194 case NEON::BI__builtin_neon_vqshld_n_u64:
1195 case NEON::BI__builtin_neon_vqshld_n_s64:
1196 case NEON::BI__builtin_neon_vrshrd_n_u64:
1197 case NEON::BI__builtin_neon_vrshrd_n_s64:
1198 case NEON::BI__builtin_neon_vrsrad_n_u64:
1199 case NEON::BI__builtin_neon_vrsrad_n_s64:
1200 case NEON::BI__builtin_neon_vshld_n_s64:
1201 case NEON::BI__builtin_neon_vshld_n_u64:
1202 case NEON::BI__builtin_neon_vshrd_n_s64:
1203 case NEON::BI__builtin_neon_vshrd_n_u64:
1204 case NEON::BI__builtin_neon_vsrad_n_s64:
1205 case NEON::BI__builtin_neon_vsrad_n_u64:
1206 case NEON::BI__builtin_neon_vqdmlalh_lane_s16:
1207 case NEON::BI__builtin_neon_vqdmlalh_laneq_s16:
1208 case NEON::BI__builtin_neon_vqdmlslh_lane_s16:
1209 case NEON::BI__builtin_neon_vqdmlslh_laneq_s16:
1210 case NEON::BI__builtin_neon_vqdmlals_s32:
1211 case NEON::BI__builtin_neon_vqdmlsls_s32:
1212 case NEON::BI__builtin_neon_vqdmlals_lane_s32:
1213 case NEON::BI__builtin_neon_vqdmlals_laneq_s32:
1214 case NEON::BI__builtin_neon_vqdmlsls_lane_s32:
1215 case NEON::BI__builtin_neon_vqdmlsls_laneq_s32:
1216 case NEON::BI__builtin_neon_vget_lane_bf16:
1217 case NEON::BI__builtin_neon_vduph_lane_bf16:
1218 case NEON::BI__builtin_neon_vduph_lane_f16:
1219 case NEON::BI__builtin_neon_vgetq_lane_bf16:
1220 case NEON::BI__builtin_neon_vduph_laneq_bf16:
1221 case NEON::BI__builtin_neon_vduph_laneq_f16:
1222 case NEON::BI__builtin_neon_vcvt_bf16_f32:
1223 case NEON::BI__builtin_neon_vcvtq_low_bf16_f32:
1224 case NEON::BI__builtin_neon_vcvtq_high_bf16_f32:
1225 case clang::AArch64::BI_InterlockedAdd:
1226 case clang::AArch64::BI_InterlockedAdd_acq:
1227 case clang::AArch64::BI_InterlockedAdd_rel:
1228 case clang::AArch64::BI_InterlockedAdd_nf:
1229 case clang::AArch64::BI_InterlockedAdd64:
1230 case clang::AArch64::BI_InterlockedAdd64_acq:
1231 case clang::AArch64::BI_InterlockedAdd64_rel:
1232 case clang::AArch64::BI_InterlockedAdd64_nf:
1233 cgm.errorNYI(expr->getSourceRange(),
1234 std::string("unimplemented AArch64 builtin call: ") +
1235 getContext().BuiltinInfo.getName(builtinID));
1236 return mlir::Value{};
1237 }
1238
1239 // Not all intrinsics handled by the common case work for AArch64 yet, so only
1240 // defer to common code if it's been added to our special map.
1241 assert(!cir::MissingFeatures::aarch64SIMDIntrinsics());
1242
1243 assert(!cir::MissingFeatures::aarch64TblBuiltinExpr());
1244
1245 switch (builtinID) {
1246 default:
1247 return std::nullopt;
1248 case NEON::BI__builtin_neon_vbsl_v:
1249 case NEON::BI__builtin_neon_vbslq_v:
1250 case NEON::BI__builtin_neon_vfma_lane_v:
1251 case NEON::BI__builtin_neon_vfmaq_lane_v:
1252 case NEON::BI__builtin_neon_vfma_laneq_v:
1253 case NEON::BI__builtin_neon_vfmaq_laneq_v:
1254 case NEON::BI__builtin_neon_vfmah_lane_f16:
1255 case NEON::BI__builtin_neon_vfmas_lane_f32:
1256 case NEON::BI__builtin_neon_vfmah_laneq_f16:
1257 case NEON::BI__builtin_neon_vfmas_laneq_f32:
1258 case NEON::BI__builtin_neon_vfmad_lane_f64:
1259 case NEON::BI__builtin_neon_vfmad_laneq_f64:
1260 case NEON::BI__builtin_neon_vmull_v:
1261 case NEON::BI__builtin_neon_vmax_v:
1262 case NEON::BI__builtin_neon_vmaxq_v:
1263 case NEON::BI__builtin_neon_vmaxh_f16:
1264 case NEON::BI__builtin_neon_vmin_v:
1265 case NEON::BI__builtin_neon_vminq_v:
1266 case NEON::BI__builtin_neon_vminh_f16:
1267 case NEON::BI__builtin_neon_vabd_v:
1268 case NEON::BI__builtin_neon_vabdq_v:
1269 case NEON::BI__builtin_neon_vpadal_v:
1270 case NEON::BI__builtin_neon_vpadalq_v:
1271 case NEON::BI__builtin_neon_vpmin_v:
1272 case NEON::BI__builtin_neon_vpminq_v:
1273 case NEON::BI__builtin_neon_vpmax_v:
1274 case NEON::BI__builtin_neon_vpmaxq_v:
1275 case NEON::BI__builtin_neon_vminnm_v:
1276 case NEON::BI__builtin_neon_vminnmq_v:
1277 case NEON::BI__builtin_neon_vminnmh_f16:
1278 case NEON::BI__builtin_neon_vmaxnm_v:
1279 case NEON::BI__builtin_neon_vmaxnmq_v:
1280 case NEON::BI__builtin_neon_vmaxnmh_f16:
1281 case NEON::BI__builtin_neon_vrecpss_f32:
1282 case NEON::BI__builtin_neon_vrecpsd_f64:
1283 case NEON::BI__builtin_neon_vrecpsh_f16:
1284 case NEON::BI__builtin_neon_vqshrun_n_v:
1285 case NEON::BI__builtin_neon_vqrshrun_n_v:
1286 case NEON::BI__builtin_neon_vqshrn_n_v:
1287 case NEON::BI__builtin_neon_vrshrn_n_v:
1288 case NEON::BI__builtin_neon_vqrshrn_n_v:
1289 case NEON::BI__builtin_neon_vrndah_f16:
1290 case NEON::BI__builtin_neon_vrnda_v:
1291 case NEON::BI__builtin_neon_vrndaq_v:
1292 case NEON::BI__builtin_neon_vrndih_f16:
1293 case NEON::BI__builtin_neon_vrndmh_f16:
1294 case NEON::BI__builtin_neon_vrndm_v:
1295 case NEON::BI__builtin_neon_vrndmq_v:
1296 case NEON::BI__builtin_neon_vrndnh_f16:
1297 case NEON::BI__builtin_neon_vrndn_v:
1298 case NEON::BI__builtin_neon_vrndnq_v:
1299 case NEON::BI__builtin_neon_vrndns_f32:
1300 case NEON::BI__builtin_neon_vrndph_f16:
1301 case NEON::BI__builtin_neon_vrndp_v:
1302 case NEON::BI__builtin_neon_vrndpq_v:
1303 case NEON::BI__builtin_neon_vrndxh_f16:
1304 case NEON::BI__builtin_neon_vrndx_v:
1305 case NEON::BI__builtin_neon_vrndxq_v:
1306 case NEON::BI__builtin_neon_vrndh_f16:
1307 case NEON::BI__builtin_neon_vrnd32x_f32:
1308 case NEON::BI__builtin_neon_vrnd32xq_f32:
1309 case NEON::BI__builtin_neon_vrnd32x_f64:
1310 case NEON::BI__builtin_neon_vrnd32xq_f64:
1311 case NEON::BI__builtin_neon_vrnd32z_f32:
1312 case NEON::BI__builtin_neon_vrnd32zq_f32:
1313 case NEON::BI__builtin_neon_vrnd32z_f64:
1314 case NEON::BI__builtin_neon_vrnd32zq_f64:
1315 case NEON::BI__builtin_neon_vrnd64x_f32:
1316 case NEON::BI__builtin_neon_vrnd64xq_f32:
1317 case NEON::BI__builtin_neon_vrnd64x_f64:
1318 case NEON::BI__builtin_neon_vrnd64xq_f64:
1319 case NEON::BI__builtin_neon_vrnd64z_f32:
1320 case NEON::BI__builtin_neon_vrnd64zq_f32:
1321 case NEON::BI__builtin_neon_vrnd64z_f64:
1322 case NEON::BI__builtin_neon_vrnd64zq_f64:
1323 case NEON::BI__builtin_neon_vrnd_v:
1324 case NEON::BI__builtin_neon_vrndq_v:
1325 case NEON::BI__builtin_neon_vcvt_f64_v:
1326 case NEON::BI__builtin_neon_vcvtq_f64_v:
1327 case NEON::BI__builtin_neon_vcvt_f64_f32:
1328 case NEON::BI__builtin_neon_vcvt_f32_f64:
1329 case NEON::BI__builtin_neon_vcvt_s32_v:
1330 case NEON::BI__builtin_neon_vcvt_u32_v:
1331 case NEON::BI__builtin_neon_vcvt_s64_v:
1332 case NEON::BI__builtin_neon_vcvt_u64_v:
1333 case NEON::BI__builtin_neon_vcvt_s16_f16:
1334 case NEON::BI__builtin_neon_vcvt_u16_f16:
1335 case NEON::BI__builtin_neon_vcvtq_s32_v:
1336 case NEON::BI__builtin_neon_vcvtq_u32_v:
1337 case NEON::BI__builtin_neon_vcvtq_s64_v:
1338 case NEON::BI__builtin_neon_vcvtq_u64_v:
1339 case NEON::BI__builtin_neon_vcvtq_s16_f16:
1340 case NEON::BI__builtin_neon_vcvtq_u16_f16:
1341 case NEON::BI__builtin_neon_vcvta_s16_f16:
1342 case NEON::BI__builtin_neon_vcvta_u16_f16:
1343 case NEON::BI__builtin_neon_vcvta_s32_v:
1344 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
1345 case NEON::BI__builtin_neon_vcvtaq_s32_v:
1346 case NEON::BI__builtin_neon_vcvta_u32_v:
1347 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
1348 case NEON::BI__builtin_neon_vcvtaq_u32_v:
1349 case NEON::BI__builtin_neon_vcvta_s64_v:
1350 case NEON::BI__builtin_neon_vcvtaq_s64_v:
1351 case NEON::BI__builtin_neon_vcvta_u64_v:
1352 case NEON::BI__builtin_neon_vcvtaq_u64_v:
1353 case NEON::BI__builtin_neon_vcvtm_s16_f16:
1354 case NEON::BI__builtin_neon_vcvtm_s32_v:
1355 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
1356 case NEON::BI__builtin_neon_vcvtmq_s32_v:
1357 case NEON::BI__builtin_neon_vcvtm_u16_f16:
1358 case NEON::BI__builtin_neon_vcvtm_u32_v:
1359 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
1360 case NEON::BI__builtin_neon_vcvtmq_u32_v:
1361 case NEON::BI__builtin_neon_vcvtm_s64_v:
1362 case NEON::BI__builtin_neon_vcvtmq_s64_v:
1363 case NEON::BI__builtin_neon_vcvtm_u64_v:
1364 case NEON::BI__builtin_neon_vcvtmq_u64_v:
1365 case NEON::BI__builtin_neon_vcvtn_s16_f16:
1366 case NEON::BI__builtin_neon_vcvtn_s32_v:
1367 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
1368 case NEON::BI__builtin_neon_vcvtnq_s32_v:
1369 case NEON::BI__builtin_neon_vcvtn_u16_f16:
1370 case NEON::BI__builtin_neon_vcvtn_u32_v:
1371 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
1372 case NEON::BI__builtin_neon_vcvtnq_u32_v:
1373 case NEON::BI__builtin_neon_vcvtn_s64_v:
1374 case NEON::BI__builtin_neon_vcvtnq_s64_v:
1375 case NEON::BI__builtin_neon_vcvtn_u64_v:
1376 case NEON::BI__builtin_neon_vcvtnq_u64_v:
1377 case NEON::BI__builtin_neon_vcvtp_s16_f16:
1378 case NEON::BI__builtin_neon_vcvtp_s32_v:
1379 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
1380 case NEON::BI__builtin_neon_vcvtpq_s32_v:
1381 case NEON::BI__builtin_neon_vcvtp_u16_f16:
1382 case NEON::BI__builtin_neon_vcvtp_u32_v:
1383 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
1384 case NEON::BI__builtin_neon_vcvtpq_u32_v:
1385 case NEON::BI__builtin_neon_vcvtp_s64_v:
1386 case NEON::BI__builtin_neon_vcvtpq_s64_v:
1387 case NEON::BI__builtin_neon_vcvtp_u64_v:
1388 case NEON::BI__builtin_neon_vcvtpq_u64_v:
1389 case NEON::BI__builtin_neon_vmulx_v:
1390 case NEON::BI__builtin_neon_vmulxq_v:
1391 case NEON::BI__builtin_neon_vmulxh_lane_f16:
1392 case NEON::BI__builtin_neon_vmulxh_laneq_f16:
1393 case NEON::BI__builtin_neon_vmul_lane_v:
1394 case NEON::BI__builtin_neon_vmul_laneq_v:
1395 case NEON::BI__builtin_neon_vnegd_s64:
1396 case NEON::BI__builtin_neon_vnegh_f16:
1397 case NEON::BI__builtin_neon_vpmaxnm_v:
1398 case NEON::BI__builtin_neon_vpmaxnmq_v:
1399 case NEON::BI__builtin_neon_vpminnm_v:
1400 case NEON::BI__builtin_neon_vpminnmq_v:
1401 case NEON::BI__builtin_neon_vsqrth_f16:
1402 case NEON::BI__builtin_neon_vsqrt_v:
1403 case NEON::BI__builtin_neon_vsqrtq_v:
1404 case NEON::BI__builtin_neon_vrbit_v:
1405 case NEON::BI__builtin_neon_vrbitq_v:
1406 case NEON::BI__builtin_neon_vmaxv_f16:
1407 case NEON::BI__builtin_neon_vmaxvq_f16:
1408 case NEON::BI__builtin_neon_vminv_f16:
1409 case NEON::BI__builtin_neon_vminvq_f16:
1410 case NEON::BI__builtin_neon_vmaxnmv_f16:
1411 case NEON::BI__builtin_neon_vmaxnmvq_f16:
1412 case NEON::BI__builtin_neon_vminnmv_f16:
1413 case NEON::BI__builtin_neon_vminnmvq_f16:
1414 case NEON::BI__builtin_neon_vmul_n_f64:
1415 case NEON::BI__builtin_neon_vaddlv_u8:
1416 case NEON::BI__builtin_neon_vaddlv_u16:
1417 case NEON::BI__builtin_neon_vaddlvq_u8:
1418 case NEON::BI__builtin_neon_vaddlvq_u16:
1419 case NEON::BI__builtin_neon_vaddlv_s8:
1420 case NEON::BI__builtin_neon_vaddlv_s16:
1421 case NEON::BI__builtin_neon_vaddlvq_s8:
1422 case NEON::BI__builtin_neon_vaddlvq_s16:
1423 case NEON::BI__builtin_neon_vsri_n_v:
1424 case NEON::BI__builtin_neon_vsriq_n_v:
1425 case NEON::BI__builtin_neon_vsli_n_v:
1426 case NEON::BI__builtin_neon_vsliq_n_v:
1427 case NEON::BI__builtin_neon_vsra_n_v:
1428 case NEON::BI__builtin_neon_vsraq_n_v:
1429 case NEON::BI__builtin_neon_vrsra_n_v:
1430 case NEON::BI__builtin_neon_vrsraq_n_v:
1431 case NEON::BI__builtin_neon_vld1_v:
1432 case NEON::BI__builtin_neon_vld1q_v:
1433 case NEON::BI__builtin_neon_vst1_v:
1434 case NEON::BI__builtin_neon_vst1q_v:
1435 case NEON::BI__builtin_neon_vld1_lane_v:
1436 case NEON::BI__builtin_neon_vld1q_lane_v:
1437 case NEON::BI__builtin_neon_vldap1_lane_s64:
1438 case NEON::BI__builtin_neon_vldap1q_lane_s64:
1439 case NEON::BI__builtin_neon_vld1_dup_v:
1440 case NEON::BI__builtin_neon_vld1q_dup_v:
1441 case NEON::BI__builtin_neon_vst1_lane_v:
1442 case NEON::BI__builtin_neon_vst1q_lane_v:
1443 case NEON::BI__builtin_neon_vstl1_lane_s64:
1444 case NEON::BI__builtin_neon_vstl1q_lane_s64:
1445 case NEON::BI__builtin_neon_vld2_v:
1446 case NEON::BI__builtin_neon_vld2q_v:
1447 case NEON::BI__builtin_neon_vld3_v:
1448 case NEON::BI__builtin_neon_vld3q_v:
1449 case NEON::BI__builtin_neon_vld4_v:
1450 case NEON::BI__builtin_neon_vld4q_v:
1451 case NEON::BI__builtin_neon_vld2_dup_v:
1452 case NEON::BI__builtin_neon_vld2q_dup_v:
1453 case NEON::BI__builtin_neon_vld3_dup_v:
1454 case NEON::BI__builtin_neon_vld3q_dup_v:
1455 case NEON::BI__builtin_neon_vld4_dup_v:
1456 case NEON::BI__builtin_neon_vld4q_dup_v:
1457 case NEON::BI__builtin_neon_vld2_lane_v:
1458 case NEON::BI__builtin_neon_vld2q_lane_v:
1459 case NEON::BI__builtin_neon_vld3_lane_v:
1460 case NEON::BI__builtin_neon_vld3q_lane_v:
1461 case NEON::BI__builtin_neon_vld4_lane_v:
1462 case NEON::BI__builtin_neon_vld4q_lane_v:
1463 case NEON::BI__builtin_neon_vst2_v:
1464 case NEON::BI__builtin_neon_vst2q_v:
1465 case NEON::BI__builtin_neon_vst2_lane_v:
1466 case NEON::BI__builtin_neon_vst2q_lane_v:
1467 case NEON::BI__builtin_neon_vst3_v:
1468 case NEON::BI__builtin_neon_vst3q_v:
1469 case NEON::BI__builtin_neon_vst3_lane_v:
1470 case NEON::BI__builtin_neon_vst3q_lane_v:
1471 case NEON::BI__builtin_neon_vst4_v:
1472 case NEON::BI__builtin_neon_vst4q_v:
1473 case NEON::BI__builtin_neon_vst4_lane_v:
1474 case NEON::BI__builtin_neon_vst4q_lane_v:
1475 case NEON::BI__builtin_neon_vtrn_v:
1476 case NEON::BI__builtin_neon_vtrnq_v:
1477 case NEON::BI__builtin_neon_vuzp_v:
1478 case NEON::BI__builtin_neon_vuzpq_v:
1479 case NEON::BI__builtin_neon_vzip_v:
1480 case NEON::BI__builtin_neon_vzipq_v:
1481 case NEON::BI__builtin_neon_vqtbl1q_v:
1482 case NEON::BI__builtin_neon_vqtbl2q_v:
1483 case NEON::BI__builtin_neon_vqtbl3q_v:
1484 case NEON::BI__builtin_neon_vqtbl4q_v:
1485 case NEON::BI__builtin_neon_vqtbx1q_v:
1486 case NEON::BI__builtin_neon_vqtbx2q_v:
1487 case NEON::BI__builtin_neon_vqtbx3q_v:
1488 case NEON::BI__builtin_neon_vqtbx4q_v:
1489 case NEON::BI__builtin_neon_vsqadd_v:
1490 case NEON::BI__builtin_neon_vsqaddq_v:
1491 case NEON::BI__builtin_neon_vuqadd_v:
1492 case NEON::BI__builtin_neon_vuqaddq_v:
1493 case NEON::BI__builtin_neon_vluti2_laneq_mf8:
1494 case NEON::BI__builtin_neon_vluti2_laneq_bf16:
1495 case NEON::BI__builtin_neon_vluti2_laneq_f16:
1496 case NEON::BI__builtin_neon_vluti2_laneq_p16:
1497 case NEON::BI__builtin_neon_vluti2_laneq_p8:
1498 case NEON::BI__builtin_neon_vluti2_laneq_s16:
1499 case NEON::BI__builtin_neon_vluti2_laneq_s8:
1500 case NEON::BI__builtin_neon_vluti2_laneq_u16:
1501 case NEON::BI__builtin_neon_vluti2_laneq_u8:
1502 case NEON::BI__builtin_neon_vluti2q_laneq_mf8:
1503 case NEON::BI__builtin_neon_vluti2q_laneq_bf16:
1504 case NEON::BI__builtin_neon_vluti2q_laneq_f16:
1505 case NEON::BI__builtin_neon_vluti2q_laneq_p16:
1506 case NEON::BI__builtin_neon_vluti2q_laneq_p8:
1507 case NEON::BI__builtin_neon_vluti2q_laneq_s16:
1508 case NEON::BI__builtin_neon_vluti2q_laneq_s8:
1509 case NEON::BI__builtin_neon_vluti2q_laneq_u16:
1510 case NEON::BI__builtin_neon_vluti2q_laneq_u8:
1511 case NEON::BI__builtin_neon_vluti2_lane_mf8:
1512 case NEON::BI__builtin_neon_vluti2_lane_bf16:
1513 case NEON::BI__builtin_neon_vluti2_lane_f16:
1514 case NEON::BI__builtin_neon_vluti2_lane_p16:
1515 case NEON::BI__builtin_neon_vluti2_lane_p8:
1516 case NEON::BI__builtin_neon_vluti2_lane_s16:
1517 case NEON::BI__builtin_neon_vluti2_lane_s8:
1518 case NEON::BI__builtin_neon_vluti2_lane_u16:
1519 case NEON::BI__builtin_neon_vluti2_lane_u8:
1520 case NEON::BI__builtin_neon_vluti2q_lane_mf8:
1521 case NEON::BI__builtin_neon_vluti2q_lane_bf16:
1522 case NEON::BI__builtin_neon_vluti2q_lane_f16:
1523 case NEON::BI__builtin_neon_vluti2q_lane_p16:
1524 case NEON::BI__builtin_neon_vluti2q_lane_p8:
1525 case NEON::BI__builtin_neon_vluti2q_lane_s16:
1526 case NEON::BI__builtin_neon_vluti2q_lane_s8:
1527 case NEON::BI__builtin_neon_vluti2q_lane_u16:
1528 case NEON::BI__builtin_neon_vluti2q_lane_u8:
1529 case NEON::BI__builtin_neon_vluti4q_lane_mf8:
1530 case NEON::BI__builtin_neon_vluti4q_lane_p8:
1531 case NEON::BI__builtin_neon_vluti4q_lane_s8:
1532 case NEON::BI__builtin_neon_vluti4q_lane_u8:
1533 case NEON::BI__builtin_neon_vluti4q_laneq_mf8:
1534 case NEON::BI__builtin_neon_vluti4q_laneq_p8:
1535 case NEON::BI__builtin_neon_vluti4q_laneq_s8:
1536 case NEON::BI__builtin_neon_vluti4q_laneq_u8:
1537 case NEON::BI__builtin_neon_vluti4q_lane_bf16_x2:
1538 case NEON::BI__builtin_neon_vluti4q_lane_f16_x2:
1539 case NEON::BI__builtin_neon_vluti4q_lane_p16_x2:
1540 case NEON::BI__builtin_neon_vluti4q_lane_s16_x2:
1541 case NEON::BI__builtin_neon_vluti4q_lane_u16_x2:
1542 case NEON::BI__builtin_neon_vluti4q_laneq_bf16_x2:
1543 case NEON::BI__builtin_neon_vluti4q_laneq_f16_x2:
1544 case NEON::BI__builtin_neon_vluti4q_laneq_p16_x2:
1545 case NEON::BI__builtin_neon_vluti4q_laneq_s16_x2:
1546 case NEON::BI__builtin_neon_vluti4q_laneq_u16_x2:
1547 case NEON::BI__builtin_neon_vmmlaq_f16_mf8_fpm:
1548 case NEON::BI__builtin_neon_vmmlaq_f32_mf8_fpm:
1549 case NEON::BI__builtin_neon_vcvt1_low_bf16_mf8_fpm:
1550 case NEON::BI__builtin_neon_vcvt1_bf16_mf8_fpm:
1551 case NEON::BI__builtin_neon_vcvt1_high_bf16_mf8_fpm:
1552 case NEON::BI__builtin_neon_vcvt2_low_bf16_mf8_fpm:
1553 case NEON::BI__builtin_neon_vcvt2_bf16_mf8_fpm:
1554 case NEON::BI__builtin_neon_vcvt2_high_bf16_mf8_fpm:
1555 case NEON::BI__builtin_neon_vcvt1_low_f16_mf8_fpm:
1556 case NEON::BI__builtin_neon_vcvt1_f16_mf8_fpm:
1557 case NEON::BI__builtin_neon_vcvt1_high_f16_mf8_fpm:
1558 case NEON::BI__builtin_neon_vcvt2_low_f16_mf8_fpm:
1559 case NEON::BI__builtin_neon_vcvt2_f16_mf8_fpm:
1560 case NEON::BI__builtin_neon_vcvt2_high_f16_mf8_fpm:
1561 case NEON::BI__builtin_neon_vcvt_mf8_f32_fpm:
1562 case NEON::BI__builtin_neon_vcvt_mf8_f16_fpm:
1563 case NEON::BI__builtin_neon_vcvtq_mf8_f16_fpm:
1564 case NEON::BI__builtin_neon_vcvt_high_mf8_f32_fpm:
1565 case NEON::BI__builtin_neon_vdot_f16_mf8_fpm:
1566 case NEON::BI__builtin_neon_vdotq_f16_mf8_fpm:
1567 case NEON::BI__builtin_neon_vdot_lane_f16_mf8_fpm:
1568 case NEON::BI__builtin_neon_vdotq_lane_f16_mf8_fpm:
1569 case NEON::BI__builtin_neon_vdot_laneq_f16_mf8_fpm:
1570 case NEON::BI__builtin_neon_vdotq_laneq_f16_mf8_fpm:
1571 case NEON::BI__builtin_neon_vdot_f32_mf8_fpm:
1572 case NEON::BI__builtin_neon_vdotq_f32_mf8_fpm:
1573 case NEON::BI__builtin_neon_vdot_lane_f32_mf8_fpm:
1574 case NEON::BI__builtin_neon_vdotq_lane_f32_mf8_fpm:
1575 case NEON::BI__builtin_neon_vdot_laneq_f32_mf8_fpm:
1576 case NEON::BI__builtin_neon_vdotq_laneq_f32_mf8_fpm:
1577 case NEON::BI__builtin_neon_vmlalbq_f16_mf8_fpm:
1578 case NEON::BI__builtin_neon_vmlaltq_f16_mf8_fpm:
1579 case NEON::BI__builtin_neon_vmlallbbq_f32_mf8_fpm:
1580 case NEON::BI__builtin_neon_vmlallbtq_f32_mf8_fpm:
1581 case NEON::BI__builtin_neon_vmlalltbq_f32_mf8_fpm:
1582 case NEON::BI__builtin_neon_vmlallttq_f32_mf8_fpm:
1583 case NEON::BI__builtin_neon_vmlalbq_lane_f16_mf8_fpm:
1584 case NEON::BI__builtin_neon_vmlalbq_laneq_f16_mf8_fpm:
1585 case NEON::BI__builtin_neon_vmlaltq_lane_f16_mf8_fpm:
1586 case NEON::BI__builtin_neon_vmlaltq_laneq_f16_mf8_fpm:
1587 case NEON::BI__builtin_neon_vmlallbbq_lane_f32_mf8_fpm:
1588 case NEON::BI__builtin_neon_vmlallbbq_laneq_f32_mf8_fpm:
1589 case NEON::BI__builtin_neon_vmlallbtq_lane_f32_mf8_fpm:
1590 case NEON::BI__builtin_neon_vmlallbtq_laneq_f32_mf8_fpm:
1591 case NEON::BI__builtin_neon_vmlalltbq_lane_f32_mf8_fpm:
1592 case NEON::BI__builtin_neon_vmlalltbq_laneq_f32_mf8_fpm:
1593 case NEON::BI__builtin_neon_vmlallttq_lane_f32_mf8_fpm:
1594 case NEON::BI__builtin_neon_vmlallttq_laneq_f32_mf8_fpm:
1595 case NEON::BI__builtin_neon_vamin_f16:
1596 case NEON::BI__builtin_neon_vaminq_f16:
1597 case NEON::BI__builtin_neon_vamin_f32:
1598 case NEON::BI__builtin_neon_vaminq_f32:
1599 case NEON::BI__builtin_neon_vaminq_f64:
1600 case NEON::BI__builtin_neon_vamax_f16:
1601 case NEON::BI__builtin_neon_vamaxq_f16:
1602 case NEON::BI__builtin_neon_vamax_f32:
1603 case NEON::BI__builtin_neon_vamaxq_f32:
1604 case NEON::BI__builtin_neon_vamaxq_f64:
1605 case NEON::BI__builtin_neon_vscale_f16:
1606 case NEON::BI__builtin_neon_vscaleq_f16:
1607 case NEON::BI__builtin_neon_vscale_f32:
1608 case NEON::BI__builtin_neon_vscaleq_f32:
1609 case NEON::BI__builtin_neon_vscaleq_f64:
1610 cgm.errorNYI(expr->getSourceRange(),
1611 std::string("unimplemented AArch64 builtin call: ") +
1612 getContext().BuiltinInfo.getName(builtinID));
1613 return mlir::Value{};
1614 }
1615
1616 // Unreachable: All cases in the switch above return.
1617}
Builtins.h
Defines enum values for all the target-independent builtin functions.
neonEquivalentIntrinsicMap
static const std::pair< unsigned, unsigned > neonEquivalentIntrinsicMap[]
Definition CIRGenBuiltinAArch64.cpp:217
genVscaleTimesFactor
static mlir::Value genVscaleTimesFactor(mlir::Location loc, CIRGenBuilderTy builder, mlir::Type cirTy, int32_t scalingFactor)
Definition CIRGenBuiltinAArch64.cpp:46
emitIntrinsicCallOp
static mlir::Value emitIntrinsicCallOp(CIRGenBuilderTy &builder, mlir::Location loc, const StringRef str, const mlir::Type &resTy, Operands &&...op)
Definition CIRGenBuiltinAArch64.cpp:35
TargetBuiltins.h
Enumerates target-specific builtins in their own namespaces within namespace clang.
cir::CIRBaseBuilderTy::createNUWAMul
mlir::Value createNUWAMul(mlir::Location loc, mlir::Value lhs, mlir::Value rhs)
Definition CIRBaseBuilder.h:546
clang::CIRGen::CIRGenBuilderTy::getUInt64
cir::ConstantOp getUInt64(uint64_t c, mlir::Location loc)
Definition CIRGenBuilder.h:345
clang::CIRGen::CIRGenFunction::convertType
mlir::Type convertType(clang::QualType t)
Definition CIRGenFunction.cpp:100
clang::CIRGen::CIRGenFunction::getLoc
mlir::Location getLoc(clang::SourceLocation srcLoc)
Helpers to convert Clang's SourceLocation to a MLIR Location.
Definition CIRGenFunction.cpp:104
clang::CIRGen::CIRGenFunction::returnValue
Address returnValue
The temporary alloca to hold the return value.
Definition CIRGenFunction.h:119
clang::CIRGen::CIRGenFunction::emitAArch64BuiltinExpr
std::optional< mlir::Value > emitAArch64BuiltinExpr(unsigned builtinID, const CallExpr *expr, ReturnValueSlot returnValue, llvm::Triple::ArchType arch)
Definition CIRGenBuiltinAArch64.cpp:612
clang::CIRGen::CIRGenFunction::emitAArch64SMEBuiltinExpr
std::optional< mlir::Value > emitAArch64SMEBuiltinExpr(unsigned builtinID, const CallExpr *expr)
Definition CIRGenBuiltinAArch64.cpp:206
clang::CIRGen::CIRGenFunction::getContext
clang::ASTContext & getContext() const
Definition CIRGenFunction.h:171
clang::CIRGen::CIRGenFunction::emitAArch64SVEBuiltinExpr
std::optional< mlir::Value > emitAArch64SVEBuiltinExpr(unsigned builtinID, const CallExpr *expr)
Definition CIRGenBuiltinAArch64.cpp:56
clang::CIRGen::ReturnValueSlot
Contains the address where the return value of a function can be stored, and whether the address is v...
Definition CIRGenCall.h:254
clang::CallExpr
CallExpr - Represents a function call (C99 6.5.2.2, C++ [expr.call]).
Definition Expr.h:2943
clang::ast_matchers::expr
const internal::VariadicDynCastAllOfMatcher< Stmt, Expr > expr
Matches expressions.
Definition ASTMatchersInternal.cpp:934
clang
The JSON file list parser is used to communicate input to InstallAPI.
Definition CalledOnceCheck.h:17
llvm
Diagnostic wrappers for TextAPI types for error reporting.
Definition Dominators.h:30
cir::MissingFeatures::msvcBuiltins
static bool msvcBuiltins()
Definition MissingFeatures.h:314
cir::MissingFeatures::neonSISDIntrinsics
static bool neonSISDIntrinsics()
Definition MissingFeatures.h:308
cir::MissingFeatures::handleBuiltinICEArguments
static bool handleBuiltinICEArguments()
Definition MissingFeatures.h:286
cir::MissingFeatures::aarch64SIMDIntrinsics
static bool aarch64SIMDIntrinsics()
Definition MissingFeatures.h:198
cir::MissingFeatures::aarch64SVEIntrinsics
static bool aarch64SVEIntrinsics()
Definition MissingFeatures.h:200
cir::MissingFeatures::aarch64SMEIntrinsics
static bool aarch64SMEIntrinsics()
Definition MissingFeatures.h:199
cir::MissingFeatures::aarch64TblBuiltinExpr
static bool aarch64TblBuiltinExpr()
Definition MissingFeatures.h:201
Generated on for clang by doxygen 1.14.0