clang 23.0.0git
CIRGenBuiltinAArch64.cpp
Go to the documentation of this file.
1//===---- CIRGenBuiltinAArch64.cpp - Emit CIR for AArch64 builtins --------===//
2// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
3// See https://llvm.org/LICENSE.txt for license information.
4// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5//
6//===----------------------------------------------------------------------===//
7//
8// This contains code to emit ARM64 Builtin calls as CIR or a function call
9// to be later resolved.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CIRGenBuilder.h"
14#include "CIRGenFunction.h"
15#include "clang/Basic/AArch64CodeGenUtils.h"
16#include "clang/Basic/TargetBuiltins.h"
17#include "clang/CIR/MissingFeatures.h"
18
19// TODO(cir): once all builtins are covered, decide whether we still
20// need to use LLVM intrinsics or if there's a better approach to follow. Right
21// now the intrinsics are reused to make it convenient to encode all thousands
22// of them and passing down to LLVM lowering.
23#include "llvm/IR/Intrinsics.h"
24#include "llvm/IR/IntrinsicsAArch64.h"
25
26#include "mlir/IR/Value.h"
27#include "clang/AST/GlobalDecl.h"
28#include "clang/Basic/Builtins.h"
29
30using namespace clang;
31using namespace clang::CIRGen;
32using namespace llvm;
33using namespace clang::aarch64;
34
35// Generate vscale * scalingFactor
36static mlir::Value genVscaleTimesFactor(mlir::Location loc,
37 CIRGenBuilderTy builder,
38 mlir::Type cirTy,
39 int32_t scalingFactor) {
40 mlir::Value vscale = builder.emitIntrinsicCallOp(loc, "vscale", cirTy);
41 return builder.createNUWAMul(loc, vscale,
42 builder.getUInt64(scalingFactor, loc));
43}
44
45#define SVEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
46 {#NameBase, SVE::BI__builtin_sve_##NameBase, Intrinsic::LLVMIntrinsic, 0, \
47 TypeModifier}
48
49#define SVEMAP2(NameBase, TypeModifier) \
50 {#NameBase, SVE::BI__builtin_sve_##NameBase, 0, 0, TypeModifier}
51static const ARMVectorIntrinsicInfo aarch64SVEIntrinsicMap[] = {
52#define GET_SVE_LLVM_INTRINSIC_MAP
53#include "clang/Basic/arm_sve_builtin_cg.inc"
54#undef GET_SVE_LLVM_INTRINSIC_MAP
55};
56
57static bool aarch64SIMDIntrinsicsProvenSorted = false;
58static bool aarch64SISDIntrinsicsProvenSorted = false;
59static bool aarch64SVEIntrinsicsProvenSorted = false;
60
61// Check if Builtin `builtinId` is present in `intrinsicMap`. If yes, returns
62// the corresponding info struct.
63static const ARMVectorIntrinsicInfo *
64findARMVectorIntrinsicInMap(ArrayRef<ARMVectorIntrinsicInfo> intrinsicMap,
65 unsigned builtinID, bool &mapProvenSorted) {
66
67#ifndef NDEBUG
68 if (!mapProvenSorted) {
69 assert(llvm::is_sorted(intrinsicMap));
70 mapProvenSorted = true;
71 }
72#endif
73
74 const ARMVectorIntrinsicInfo *info =
75 llvm::lower_bound(intrinsicMap, builtinID);
76
77 if (info != intrinsicMap.end() && info->BuiltinID == builtinID)
78 return info;
79
80 return nullptr;
81}
82
83//===----------------------------------------------------------------------===//
84// Emit-helpers
85//===----------------------------------------------------------------------===//
86static mlir::Value
87emitAArch64CompareBuiltinExpr(CIRGenFunction &cgf, CIRGenBuilderTy &builder,
88 mlir::Location loc, mlir::Value src,
89 mlir::Type retTy, const cir::CmpOpKind kind) {
90
91 bool scalarCmp = !isa<cir::VectorType>(src.getType());
92 if (!scalarCmp) {
93 assert(!cast<cir::VectorType>(retTy).getIsScalable() &&
94 "This is only intended for fixed-width vectors");
95 // Vector types are cast to i8 vectors. Recover original type.
96 src = builder.createBitcast(src, retTy);
97 }
98
99 mlir::Value zero = builder.getNullValue(src.getType(), loc);
100
101 if (!scalarCmp)
102 return builder.createVecCompare(loc, kind, src, zero);
103
104 // For scalars, cast !cir.bool to !cir.int<s, 1> so that the compare
105 // result is sign- rather zero-extended when casting to the output
106 // retType.
107 mlir::Value cmp = builder.createCast(
108 loc, cir::CastKind::bool_to_int,
109 builder.createCompare(loc, kind, src, zero), builder.getSIntNTy(1));
110
111 return builder.createCast(loc, cir::CastKind::integral, cmp, retTy);
112}
113
114// TODO(cir): Remove `loc` from the list of arguments once all NYIs are gone.
115static cir::VectorType getNeonType(CIRGenFunction *cgf, NeonTypeFlags typeFlags,
116 mlir::Location loc,
117 bool hasLegalHalfType = true,
118 bool v1Ty = false,
119 bool allowBFloatArgsAndRet = true) {
120 int isQuad = typeFlags.isQuad();
121 switch (typeFlags.getEltType()) {
122 case NeonTypeFlags::Int8:
123 case NeonTypeFlags::Poly8:
124 return cir::VectorType::get(typeFlags.isUnsigned() ? cgf->uInt8Ty
125 : cgf->sInt8Ty,
126 v1Ty ? 1 : (8 << isQuad));
127 case NeonTypeFlags::MFloat8:
128 cgf->getCIRGenModule().errorNYI(loc, std::string("NEON type: MFloat8"));
129 [[fallthrough]];
130 case NeonTypeFlags::Int16:
131 case NeonTypeFlags::Poly16:
132 return cir::VectorType::get(typeFlags.isUnsigned() ? cgf->uInt16Ty
133 : cgf->sInt16Ty,
134 v1Ty ? 1 : (4 << isQuad));
135 case NeonTypeFlags::BFloat16:
136 if (allowBFloatArgsAndRet)
137 return cir::VectorType::get(cgf->getCIRGenModule().bFloat16Ty,
138 v1Ty ? 1 : (4 << isQuad));
139 return cir::VectorType::get(cgf->uInt16Ty, v1Ty ? 1 : (4 << isQuad));
140 case NeonTypeFlags::Float16:
141 if (hasLegalHalfType)
142 cgf->getCIRGenModule().errorNYI(loc, std::string("NEON type: Float16"));
143 else
144 cgf->getCIRGenModule().errorNYI(loc, std::string("NEON type: Float16"));
145 [[fallthrough]];
146 case NeonTypeFlags::Int32:
147 return cir::VectorType::get(typeFlags.isUnsigned() ? cgf->uInt32Ty
148 : cgf->sInt32Ty,
149 v1Ty ? 1 : (2 << isQuad));
150 case NeonTypeFlags::Int64:
151 case NeonTypeFlags::Poly64:
152 return cir::VectorType::get(typeFlags.isUnsigned() ? cgf->uInt64Ty
153 : cgf->sInt64Ty,
154 v1Ty ? 1 : (1 << isQuad));
155 case NeonTypeFlags::Poly128:
156 // FIXME: i128 and f128 doesn't get fully support in Clang and llvm.
157 // There is a lot of i128 and f128 API missing.
158 // so we use v16i8 to represent poly128 and get pattern matched.
159 cgf->getCIRGenModule().errorNYI(loc, std::string("NEON type: Poly128"));
160 [[fallthrough]];
161 case NeonTypeFlags::Float32:
162 return cir::VectorType::get(cgf->getCIRGenModule().floatTy,
163 v1Ty ? 1 : (2 << isQuad));
164 case NeonTypeFlags::Float64:
165 return cir::VectorType::get(cgf->getCIRGenModule().doubleTy,
166 v1Ty ? 1 : (1 << isQuad));
167 }
168 llvm_unreachable("Unknown vector element type!");
169}
170
171static int64_t getIntValueFromConstOp(mlir::Value val) {
172 return val.getDefiningOp<cir::ConstantOp>().getIntValue().getSExtValue();
173}
174
175static mlir::Value emitNeonSplat(CIRGenBuilderTy &builder, mlir::Location loc,
176 mlir::Value v, mlir::Value lane,
177 unsigned int resEltCnt) {
178 assert(isa<cir::ConstantOp>(lane.getDefiningOp()) &&
179 "lane number is not a constant!");
180 int64_t laneCst = getIntValueFromConstOp(lane);
181 llvm::SmallVector<int64_t, 4> shuffleMask(resEltCnt, laneCst);
182 return builder.createVecShuffle(loc, v, shuffleMask);
183}
184
185static mlir::Value emitCommonNeonBuiltinExpr(
186 CIRGenFunction &cgf, unsigned builtinID, unsigned llvmIntrinsic,
187 unsigned altLLVMIntrinsic, const char *nameHint, unsigned modifier,
188 const CallExpr *expr, llvm::SmallVectorImpl<mlir::Value> &ops) {
189
190 mlir::Location loc = cgf.getLoc(expr->getExprLoc());
191 clang::ASTContext &ctx = cgf.getContext();
192
193 // Extract the trailing immediate argument that encodes the type discriminator
194 // for this overloaded intrinsic.
195 // TODO: Move to the parent code that takes care of argument processing.
196 const clang::Expr *arg = expr->getArg(expr->getNumArgs() - 1);
197 std::optional<llvm::APSInt> neonTypeConst = arg->getIntegerConstantExpr(ctx);
198 if (!neonTypeConst)
199 return nullptr;
200
201 // Determine the type of this overloaded NEON intrinsic.
202 NeonTypeFlags neonType(neonTypeConst->getZExtValue());
203 const bool hasLegalHalfType = cgf.getTarget().hasFastHalfType();
204
205 // The value of allowBFloatArgsAndRet is true for AArch64, but it should
206 // come from ABI info.
207 // TODO(cir): Use ABInfo to extract this information
208 const bool allowBFloatArgsAndRet = cgf.getTarget().hasFastHalfType();
209 // FIXME
210 // getTargetHooks().getABIInfo().allowBFloatArgsAndRet();
211
212 cir::VectorType vTy = getNeonType(&cgf, neonType, loc, hasLegalHalfType,
213 false, allowBFloatArgsAndRet);
214 mlir::Type ty = vTy;
215 if (!ty)
216 return nullptr;
217
218 switch (builtinID) {
219 case NEON::BI__builtin_neon_splat_lane_v:
220 case NEON::BI__builtin_neon_splat_laneq_v:
221 case NEON::BI__builtin_neon_splatq_lane_v:
222 case NEON::BI__builtin_neon_splatq_laneq_v: {
223 uint64_t numElements = vTy.getSize();
224 if (builtinID == NEON::BI__builtin_neon_splatq_lane_v)
225 numElements *= 2;
226 if (builtinID == NEON::BI__builtin_neon_splat_laneq_v)
227 numElements /= 2;
228 ops[0] = cgf.getBuilder().createBitcast(loc, ops[0], vTy);
229 return emitNeonSplat(cgf.getBuilder(), loc, ops[0], ops[1], numElements);
230 }
231 case NEON::BI__builtin_neon_vpadd_v:
232 case NEON::BI__builtin_neon_vpaddq_v:
233 case NEON::BI__builtin_neon_vabs_v:
234 case NEON::BI__builtin_neon_vabsq_v:
235 case NEON::BI__builtin_neon_vadd_v:
236 case NEON::BI__builtin_neon_vaddq_v:
237 case NEON::BI__builtin_neon_vaddhn_v:
238 case NEON::BI__builtin_neon_vcale_v:
239 case NEON::BI__builtin_neon_vcaleq_v:
240 case NEON::BI__builtin_neon_vcalt_v:
241 case NEON::BI__builtin_neon_vcaltq_v:
242 case NEON::BI__builtin_neon_vcage_v:
243 case NEON::BI__builtin_neon_vcageq_v:
244 case NEON::BI__builtin_neon_vcagt_v:
245 case NEON::BI__builtin_neon_vcagtq_v:
246 cgf.cgm.errorNYI(expr->getSourceRange(),
247 std::string("unimplemented AArch64 builtin call: ") +
248 ctx.BuiltinInfo.getName(builtinID));
249 return mlir::Value{};
250 case NEON::BI__builtin_neon_vceqz_v:
251 case NEON::BI__builtin_neon_vceqzq_v:
252 return emitAArch64CompareBuiltinExpr(cgf, cgf.getBuilder(), loc, ops[0],
253 vTy, cir::CmpOpKind::eq);
254 case NEON::BI__builtin_neon_vcgez_v:
255 case NEON::BI__builtin_neon_vcgezq_v:
256 case NEON::BI__builtin_neon_vclez_v:
257 case NEON::BI__builtin_neon_vclezq_v:
258 case NEON::BI__builtin_neon_vcgtz_v:
259 case NEON::BI__builtin_neon_vcgtzq_v:
260 case NEON::BI__builtin_neon_vcltz_v:
261 case NEON::BI__builtin_neon_vcltzq_v:
262 case NEON::BI__builtin_neon_vclz_v:
263 case NEON::BI__builtin_neon_vclzq_v:
264 case NEON::BI__builtin_neon_vcvt_f32_v:
265 case NEON::BI__builtin_neon_vcvtq_f32_v:
266 case NEON::BI__builtin_neon_vcvt_f16_s16:
267 case NEON::BI__builtin_neon_vcvt_f16_u16:
268 case NEON::BI__builtin_neon_vcvtq_f16_s16:
269 case NEON::BI__builtin_neon_vcvtq_f16_u16:
270 case NEON::BI__builtin_neon_vcvt_n_f16_s16:
271 case NEON::BI__builtin_neon_vcvt_n_f16_u16:
272 case NEON::BI__builtin_neon_vcvtq_n_f16_s16:
273 case NEON::BI__builtin_neon_vcvtq_n_f16_u16:
274 case NEON::BI__builtin_neon_vcvt_n_f32_v:
275 case NEON::BI__builtin_neon_vcvt_n_f64_v:
276 case NEON::BI__builtin_neon_vcvtq_n_f32_v:
277 case NEON::BI__builtin_neon_vcvtq_n_f64_v:
278 case NEON::BI__builtin_neon_vcvt_n_s16_f16:
279 case NEON::BI__builtin_neon_vcvt_n_s32_v:
280 case NEON::BI__builtin_neon_vcvt_n_u16_f16:
281 case NEON::BI__builtin_neon_vcvt_n_u32_v:
282 case NEON::BI__builtin_neon_vcvt_n_s64_v:
283 case NEON::BI__builtin_neon_vcvt_n_u64_v:
284 case NEON::BI__builtin_neon_vcvtq_n_s16_f16:
285 case NEON::BI__builtin_neon_vcvtq_n_s32_v:
286 case NEON::BI__builtin_neon_vcvtq_n_u16_f16:
287 case NEON::BI__builtin_neon_vcvtq_n_u32_v:
288 case NEON::BI__builtin_neon_vcvtq_n_s64_v:
289 case NEON::BI__builtin_neon_vcvtq_n_u64_v:
290 case NEON::BI__builtin_neon_vcvt_s32_v:
291 case NEON::BI__builtin_neon_vcvt_u32_v:
292 case NEON::BI__builtin_neon_vcvt_s64_v:
293 case NEON::BI__builtin_neon_vcvt_u64_v:
294 case NEON::BI__builtin_neon_vcvt_s16_f16:
295 case NEON::BI__builtin_neon_vcvt_u16_f16:
296 case NEON::BI__builtin_neon_vcvtq_s32_v:
297 case NEON::BI__builtin_neon_vcvtq_u32_v:
298 case NEON::BI__builtin_neon_vcvtq_s64_v:
299 case NEON::BI__builtin_neon_vcvtq_u64_v:
300 case NEON::BI__builtin_neon_vcvtq_s16_f16:
301 case NEON::BI__builtin_neon_vcvtq_u16_f16:
302 case NEON::BI__builtin_neon_vcvta_s16_f16:
303 case NEON::BI__builtin_neon_vcvta_s32_v:
304 case NEON::BI__builtin_neon_vcvta_s64_v:
305 case NEON::BI__builtin_neon_vcvta_u16_f16:
306 case NEON::BI__builtin_neon_vcvta_u32_v:
307 case NEON::BI__builtin_neon_vcvta_u64_v:
308 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
309 case NEON::BI__builtin_neon_vcvtaq_s32_v:
310 case NEON::BI__builtin_neon_vcvtaq_s64_v:
311 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
312 case NEON::BI__builtin_neon_vcvtaq_u32_v:
313 case NEON::BI__builtin_neon_vcvtaq_u64_v:
314 case NEON::BI__builtin_neon_vcvtn_s16_f16:
315 case NEON::BI__builtin_neon_vcvtn_s32_v:
316 case NEON::BI__builtin_neon_vcvtn_s64_v:
317 case NEON::BI__builtin_neon_vcvtn_u16_f16:
318 case NEON::BI__builtin_neon_vcvtn_u32_v:
319 case NEON::BI__builtin_neon_vcvtn_u64_v:
320 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
321 case NEON::BI__builtin_neon_vcvtnq_s32_v:
322 case NEON::BI__builtin_neon_vcvtnq_s64_v:
323 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
324 case NEON::BI__builtin_neon_vcvtnq_u32_v:
325 case NEON::BI__builtin_neon_vcvtnq_u64_v:
326 case NEON::BI__builtin_neon_vcvtp_s16_f16:
327 case NEON::BI__builtin_neon_vcvtp_s32_v:
328 case NEON::BI__builtin_neon_vcvtp_s64_v:
329 case NEON::BI__builtin_neon_vcvtp_u16_f16:
330 case NEON::BI__builtin_neon_vcvtp_u32_v:
331 case NEON::BI__builtin_neon_vcvtp_u64_v:
332 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
333 case NEON::BI__builtin_neon_vcvtpq_s32_v:
334 case NEON::BI__builtin_neon_vcvtpq_s64_v:
335 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
336 case NEON::BI__builtin_neon_vcvtpq_u32_v:
337 case NEON::BI__builtin_neon_vcvtpq_u64_v:
338 case NEON::BI__builtin_neon_vcvtm_s16_f16:
339 case NEON::BI__builtin_neon_vcvtm_s32_v:
340 case NEON::BI__builtin_neon_vcvtm_s64_v:
341 case NEON::BI__builtin_neon_vcvtm_u16_f16:
342 case NEON::BI__builtin_neon_vcvtm_u32_v:
343 case NEON::BI__builtin_neon_vcvtm_u64_v:
344 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
345 case NEON::BI__builtin_neon_vcvtmq_s32_v:
346 case NEON::BI__builtin_neon_vcvtmq_s64_v:
347 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
348 case NEON::BI__builtin_neon_vcvtmq_u32_v:
349 case NEON::BI__builtin_neon_vcvtmq_u64_v:
350 case NEON::BI__builtin_neon_vcvtx_f32_v:
351 case NEON::BI__builtin_neon_vext_v:
352 case NEON::BI__builtin_neon_vextq_v:
353 case NEON::BI__builtin_neon_vfma_v:
354 case NEON::BI__builtin_neon_vfmaq_v:
355 case NEON::BI__builtin_neon_vld1_v:
356 case NEON::BI__builtin_neon_vld1q_v:
357 case NEON::BI__builtin_neon_vld1_x2_v:
358 case NEON::BI__builtin_neon_vld1q_x2_v:
359 case NEON::BI__builtin_neon_vld1_x3_v:
360 case NEON::BI__builtin_neon_vld1q_x3_v:
361 case NEON::BI__builtin_neon_vld1_x4_v:
362 case NEON::BI__builtin_neon_vld1q_x4_v:
363 case NEON::BI__builtin_neon_vld2_v:
364 case NEON::BI__builtin_neon_vld2q_v:
365 case NEON::BI__builtin_neon_vld3_v:
366 case NEON::BI__builtin_neon_vld3q_v:
367 case NEON::BI__builtin_neon_vld4_v:
368 case NEON::BI__builtin_neon_vld4q_v:
369 case NEON::BI__builtin_neon_vld2_dup_v:
370 case NEON::BI__builtin_neon_vld2q_dup_v:
371 case NEON::BI__builtin_neon_vld3_dup_v:
372 case NEON::BI__builtin_neon_vld3q_dup_v:
373 case NEON::BI__builtin_neon_vld4_dup_v:
374 case NEON::BI__builtin_neon_vld4q_dup_v:
375 case NEON::BI__builtin_neon_vld1_dup_v:
376 case NEON::BI__builtin_neon_vld1q_dup_v:
377 case NEON::BI__builtin_neon_vld2_lane_v:
378 case NEON::BI__builtin_neon_vld2q_lane_v:
379 case NEON::BI__builtin_neon_vld3_lane_v:
380 case NEON::BI__builtin_neon_vld3q_lane_v:
381 case NEON::BI__builtin_neon_vld4_lane_v:
382 case NEON::BI__builtin_neon_vld4q_lane_v:
383 case NEON::BI__builtin_neon_vmovl_v:
384 case NEON::BI__builtin_neon_vmovn_v:
385 case NEON::BI__builtin_neon_vmull_v:
386 case NEON::BI__builtin_neon_vpadal_v:
387 case NEON::BI__builtin_neon_vpadalq_v:
388 case NEON::BI__builtin_neon_vpaddl_v:
389 case NEON::BI__builtin_neon_vpaddlq_v:
390 case NEON::BI__builtin_neon_vqdmlal_v:
391 case NEON::BI__builtin_neon_vqdmlsl_v:
392 case NEON::BI__builtin_neon_vqdmulhq_lane_v:
393 case NEON::BI__builtin_neon_vqdmulh_lane_v:
394 case NEON::BI__builtin_neon_vqrdmulhq_lane_v:
395 case NEON::BI__builtin_neon_vqrdmulh_lane_v:
396 case NEON::BI__builtin_neon_vqdmulhq_laneq_v:
397 case NEON::BI__builtin_neon_vqdmulh_laneq_v:
398 case NEON::BI__builtin_neon_vqrdmulhq_laneq_v:
399 case NEON::BI__builtin_neon_vqrdmulh_laneq_v:
400 case NEON::BI__builtin_neon_vqshl_n_v:
401 case NEON::BI__builtin_neon_vqshlq_n_v:
402 case NEON::BI__builtin_neon_vqshlu_n_v:
403 case NEON::BI__builtin_neon_vqshluq_n_v:
404 case NEON::BI__builtin_neon_vrecpe_v:
405 case NEON::BI__builtin_neon_vrecpeq_v:
406 case NEON::BI__builtin_neon_vrsqrte_v:
407 case NEON::BI__builtin_neon_vrsqrteq_v:
408 case NEON::BI__builtin_neon_vrndi_v:
409 case NEON::BI__builtin_neon_vrndiq_v:
410 case NEON::BI__builtin_neon_vrshr_n_v:
411 case NEON::BI__builtin_neon_vrshrq_n_v:
412 case NEON::BI__builtin_neon_vsha512hq_u64:
413 case NEON::BI__builtin_neon_vsha512h2q_u64:
414 case NEON::BI__builtin_neon_vsha512su0q_u64:
415 case NEON::BI__builtin_neon_vsha512su1q_u64:
416 case NEON::BI__builtin_neon_vshl_n_v:
417 case NEON::BI__builtin_neon_vshlq_n_v:
418 case NEON::BI__builtin_neon_vshll_n_v:
419 case NEON::BI__builtin_neon_vshrn_n_v:
420 case NEON::BI__builtin_neon_vshr_n_v:
421 case NEON::BI__builtin_neon_vshrq_n_v:
422 case NEON::BI__builtin_neon_vst1_v:
423 case NEON::BI__builtin_neon_vst1q_v:
424 case NEON::BI__builtin_neon_vst2_v:
425 case NEON::BI__builtin_neon_vst2q_v:
426 case NEON::BI__builtin_neon_vst3_v:
427 case NEON::BI__builtin_neon_vst3q_v:
428 case NEON::BI__builtin_neon_vst4_v:
429 case NEON::BI__builtin_neon_vst4q_v:
430 case NEON::BI__builtin_neon_vst2_lane_v:
431 case NEON::BI__builtin_neon_vst2q_lane_v:
432 case NEON::BI__builtin_neon_vst3_lane_v:
433 case NEON::BI__builtin_neon_vst3q_lane_v:
434 case NEON::BI__builtin_neon_vst4_lane_v:
435 case NEON::BI__builtin_neon_vst4q_lane_v:
436 case NEON::BI__builtin_neon_vsm3partw1q_u32:
437 case NEON::BI__builtin_neon_vsm3partw2q_u32:
438 case NEON::BI__builtin_neon_vsm3ss1q_u32:
439 case NEON::BI__builtin_neon_vsm4ekeyq_u32:
440 case NEON::BI__builtin_neon_vsm4eq_u32:
441 case NEON::BI__builtin_neon_vsm3tt1aq_u32:
442 case NEON::BI__builtin_neon_vsm3tt1bq_u32:
443 case NEON::BI__builtin_neon_vsm3tt2aq_u32:
444 case NEON::BI__builtin_neon_vsm3tt2bq_u32:
445 case NEON::BI__builtin_neon_vst1_x2_v:
446 case NEON::BI__builtin_neon_vst1q_x2_v:
447 case NEON::BI__builtin_neon_vst1_x3_v:
448 case NEON::BI__builtin_neon_vst1q_x3_v:
449 case NEON::BI__builtin_neon_vst1_x4_v:
450 case NEON::BI__builtin_neon_vst1q_x4_v:
451 case NEON::BI__builtin_neon_vsubhn_v:
452 case NEON::BI__builtin_neon_vtrn_v:
453 case NEON::BI__builtin_neon_vtrnq_v:
454 case NEON::BI__builtin_neon_vtst_v:
455 case NEON::BI__builtin_neon_vtstq_v:
456 case NEON::BI__builtin_neon_vuzp_v:
457 case NEON::BI__builtin_neon_vuzpq_v:
458 case NEON::BI__builtin_neon_vxarq_u64:
459 case NEON::BI__builtin_neon_vzip_v:
460 case NEON::BI__builtin_neon_vzipq_v:
461 case NEON::BI__builtin_neon_vdot_s32:
462 case NEON::BI__builtin_neon_vdot_u32:
463 case NEON::BI__builtin_neon_vdotq_s32:
464 case NEON::BI__builtin_neon_vdotq_u32:
465 case NEON::BI__builtin_neon_vfmlal_low_f16:
466 case NEON::BI__builtin_neon_vfmlalq_low_f16:
467 case NEON::BI__builtin_neon_vfmlsl_low_f16:
468 case NEON::BI__builtin_neon_vfmlslq_low_f16:
469 case NEON::BI__builtin_neon_vfmlal_high_f16:
470 case NEON::BI__builtin_neon_vfmlalq_high_f16:
471 case NEON::BI__builtin_neon_vfmlsl_high_f16:
472 case NEON::BI__builtin_neon_vfmlslq_high_f16:
473 case NEON::BI__builtin_neon_vmmlaq_s32:
474 case NEON::BI__builtin_neon_vmmlaq_u32:
475 cgf.cgm.errorNYI(expr->getSourceRange(),
476 std::string("unimplemented AArch64 builtin call: ") +
477 ctx.BuiltinInfo.getName(builtinID));
478 return mlir::Value{};
479 case NEON::BI__builtin_neon_vmul_v:
480 case NEON::BI__builtin_neon_vmulq_v:
481 return cgf.getBuilder().emitIntrinsicCallOp(loc, "aarch64.neon.pmul", vTy,
482 ops);
483 case NEON::BI__builtin_neon_vusmmlaq_s32:
484 case NEON::BI__builtin_neon_vusdot_s32:
485 case NEON::BI__builtin_neon_vusdotq_s32:
486 case NEON::BI__builtin_neon_vbfdot_f32:
487 case NEON::BI__builtin_neon_vbfdotq_f32:
488 case NEON::BI__builtin_neon___a32_vcvt_bf16_f32:
489 default:
490 cgf.cgm.errorNYI(expr->getSourceRange(),
491 std::string("unimplemented AArch64 builtin call: ") +
492 ctx.BuiltinInfo.getName(builtinID));
493 return mlir::Value{};
494
495 cgf.cgm.errorNYI(expr->getSourceRange(),
496 std::string("unimplemented AArch64 builtin call: ") +
497 ctx.BuiltinInfo.getName(builtinID));
498 return mlir::Value{};
499 }
500}
501
502// Emit an intrinsic where all operands are of the same type as the result.
503// Depending on mode, this may be a constrained floating-point intrinsic.
504static mlir::Value
505emitCallMaybeConstrainedBuiltin(CIRGenBuilderTy &builder, mlir::Location loc,
506 StringRef intrName, mlir::Type retTy,
507 llvm::SmallVector<mlir::Value> &ops) {
508 assert(!cir::MissingFeatures::emitConstrainedFPCall());
509
510 return builder.emitIntrinsicCallOp(loc, intrName, retTy, ops);
511}
512
513bool CIRGenFunction::getAArch64SVEProcessedOperands(
514 unsigned builtinID, const CallExpr *expr, SmallVectorImpl<mlir::Value> &ops,
515 SVETypeFlags typeFlags) {
516 // Find out if any arguments are required to be integer constant expressions.
517 unsigned iceArguments = 0;
518 ASTContext::GetBuiltinTypeError error;
519 getContext().GetBuiltinType(builtinID, error, &iceArguments);
520 assert(error == ASTContext::GE_None && "Should not codegen an error");
521
522 for (unsigned i = 0, e = expr->getNumArgs(); i != e; i++) {
523 bool isIce = iceArguments & (1 << i);
524 mlir::Value arg = emitScalarExpr(expr->getArg(i));
525
526 if (isIce) {
527 cgm.errorNYI(expr->getSourceRange(),
528 std::string("unimplemented AArch64 builtin call: ") +
529 getContext().BuiltinInfo.getName(builtinID));
530 }
531
532 // FIXME: Handle types like svint16x2_t, which are currently incorrectly
533 // converted to i32. These should be treated as structs and unpacked.
534
535 ops.push_back(arg);
536 }
537 return true;
538}
539
540static llvm::StringRef getLLVMIntrNameNoPrefix(llvm::Intrinsic::ID intrID) {
541 llvm::StringRef llvmIntrName = llvm::Intrinsic::getBaseName(intrID);
542 assert(llvmIntrName.starts_with("llvm.") && "Not an LLVM intrinsic!");
543 return llvmIntrName.drop_front(/*strlen("llvm.")=*/5);
544}
545
546// Reinterpret the input predicate so that it can be used to correctly isolate
547// the elements of the specified datatype.
548mlir::Value CIRGenFunction::emitSVEPredicateCast(mlir::Value pred,
549 unsigned minNumElts,
550 mlir::Location loc) {
551
552 // TODO: Handle "aarch64.svcount" once we get round to supporting SME.
553
554 auto retTy = cir::VectorType::get(builder.getUIntNTy(1), minNumElts,
555 /*is_scalable=*/true);
556 if (pred.getType() == retTy)
557 return pred;
558
559 llvm::Intrinsic::ID intID;
560 switch (minNumElts) {
561 default:
562 llvm_unreachable("unsupported element count!");
563 case 1:
564 case 2:
565 case 4:
566 case 8:
567 intID = Intrinsic::aarch64_sve_convert_from_svbool;
568 break;
569 case 16:
570 intID = Intrinsic::aarch64_sve_convert_to_svbool;
571 break;
572 }
573
574 llvm::StringRef llvmIntrName = getLLVMIntrNameNoPrefix(intID);
575 auto call = builder.emitIntrinsicCallOp(loc, llvmIntrName, retTy,
576 mlir::ValueRange{pred});
577 assert(call.getType() == retTy && "Unexpected return type!");
578 return call;
579}
580
581//===----------------------------------------------------------------------===//
582// SVE helpers
583//===----------------------------------------------------------------------===//
584// Get the minimum number of elements in an SVE vector for the given element
585// type. The actual number of elements in the vector would be an integer (power
586// of two) multiple of this value.
587static unsigned getSVEMinEltCount(clang::SVETypeFlags::EltType sveType) {
588 switch (sveType) {
589 default:
590 llvm_unreachable("Invalid SVETypeFlag!");
591
592 case SVETypeFlags::EltTyInt8:
593 return 16;
594 case SVETypeFlags::EltTyInt16:
595 return 8;
596 case SVETypeFlags::EltTyInt32:
597 return 4;
598 case SVETypeFlags::EltTyInt64:
599 return 2;
600
601 case SVETypeFlags::EltTyMFloat8:
602 return 16;
603 case SVETypeFlags::EltTyFloat16:
604 case SVETypeFlags::EltTyBFloat16:
605 return 8;
606 case SVETypeFlags::EltTyFloat32:
607 return 4;
608 case SVETypeFlags::EltTyFloat64:
609 return 2;
610
611 case SVETypeFlags::EltTyBool8:
612 return 16;
613 case SVETypeFlags::EltTyBool16:
614 return 8;
615 case SVETypeFlags::EltTyBool32:
616 return 4;
617 case SVETypeFlags::EltTyBool64:
618 return 2;
619 }
620}
621
622// TODO(cir): Share with OGCG
623constexpr unsigned sveBitsPerBlock = 128;
624
625static cir::VectorType getSVEVectorForElementType(CIRGenModule &cgm,
626 mlir::Type eltTy) {
627 unsigned numElts =
628 sveBitsPerBlock / cgm.getDataLayout().getTypeSizeInBits(eltTy);
629 return cir::VectorType::get(eltTy, numElts, /*is_scalable=*/true);
630}
631
632//===----------------------------------------------------------------------===//
633// NEON helpers
634//===----------------------------------------------------------------------===//
635/// Return true if BuiltinID is an overloaded Neon intrinsic with an extra
636/// argument that specifies the vector type. The additional argument is meant
637/// for Sema checking (see `CheckNeonBuiltinFunctionCall`) and this function
638/// should be kept consistent with the logic in Sema.
639/// TODO: Make this return false for SISD builtins.
640/// TODO(cir): Share this with ARM.cpp
641static bool hasExtraNeonArgument(unsigned builtinID) {
642 // Required by the headers included below, but not in this particular
643 // function.
644 [[maybe_unused]] int PtrArgNum = -1;
645 [[maybe_unused]] bool HasConstPtr = false;
646
647 // The mask encodes the type. We don't care about the actual value. Instead,
648 // we just check whether its been set.
649 uint64_t mask = 0;
650 switch (builtinID) {
651#define GET_NEON_OVERLOAD_CHECK
652#include "clang/Basic/arm_fp16.inc"
653#include "clang/Basic/arm_neon.inc"
654#undef GET_NEON_OVERLOAD_CHECK
655 // Non-neon builtins for controling VFP that take extra argument for
656 // discriminating the type.
657 case ARM::BI__builtin_arm_vcvtr_f:
658 case ARM::BI__builtin_arm_vcvtr_d:
659 mask = 1;
660 }
661 switch (builtinID) {
662 default:
663 break;
664 }
665
666 return mask != 0;
667}
668
669// TODO(cir): Remove `cgm` from the list of arguments once all NYI(s) are gone.
670template <typename Operation>
671static mlir::Value
672emitNeonCallToOp(CIRGenModule &cgm, CIRGenBuilderTy &builder,
673 llvm::SmallVector<mlir::Type> argTypes,
674 llvm::SmallVectorImpl<mlir::Value> &args,
675 std::optional<llvm::StringRef> intrinsicName,
676 mlir::Type funcResTy, mlir::Location loc,
677 bool isConstrainedFPIntrinsic = false, unsigned shift = 0,
678 bool rightshift = false) {
679 // TODO(cir): Consider removing the following unreachable when we have
680 // emitConstrainedFPCall feature implemented
681 assert(!cir::MissingFeatures::emitConstrainedFPCall());
682 if (isConstrainedFPIntrinsic)
683 cgm.errorNYI(loc, std::string("constrained FP intrinsic"));
684
685 for (unsigned j = 0; j < argTypes.size(); ++j) {
686 if (isConstrainedFPIntrinsic) {
687 assert(!cir::MissingFeatures::emitConstrainedFPCall());
688 }
689 if (shift > 0 && shift == j) {
690 cgm.errorNYI(loc, std::string("intrinsic requiring a shift Op"));
691 } else {
692 args[j] = builder.createBitcast(args[j], argTypes[j]);
693 }
694 }
695 if (isConstrainedFPIntrinsic) {
696 assert(!cir::MissingFeatures::emitConstrainedFPCall());
697 return nullptr;
698 }
699 if constexpr (std::is_same_v<Operation, cir::LLVMIntrinsicCallOp>) {
700 return Operation::create(builder, loc,
701 builder.getStringAttr(intrinsicName.value()),
702 funcResTy, args)
703 .getResult();
704 } else {
705 return Operation::create(builder, loc, funcResTy, args).getResult();
706 }
707}
708
709// TODO(cir): Remove `cgm` from the list of arguments once all NYI(s) are gone.
710static mlir::Value emitNeonCall(CIRGenModule &cgm, CIRGenBuilderTy &builder,
711 llvm::SmallVector<mlir::Type> argTypes,
712 llvm::SmallVectorImpl<mlir::Value> &args,
713 llvm::StringRef intrinsicName,
714 mlir::Type funcResTy, mlir::Location loc,
715 bool isConstrainedFPIntrinsic = false,
716 unsigned shift = 0, bool rightshift = false) {
717 return emitNeonCallToOp<cir::LLVMIntrinsicCallOp>(
718 cgm, builder, std::move(argTypes), args, intrinsicName, funcResTy, loc,
719 isConstrainedFPIntrinsic, shift, rightshift);
720}
721
722static mlir::Value emitCommonNeonSISDBuiltinExpr(
723 CIRGenFunction &cgf, const ARMVectorIntrinsicInfo &info,
724 llvm::SmallVectorImpl<mlir::Value> &ops, const CallExpr *expr) {
725 assert(info.LLVMIntrinsic && "Generic code assumes a valid intrinsic");
726
727 switch (info.BuiltinID) {
728 case NEON::BI__builtin_neon_vcled_s64:
729 case NEON::BI__builtin_neon_vcled_u64:
730 case NEON::BI__builtin_neon_vcles_f32:
731 case NEON::BI__builtin_neon_vcled_f64:
732 case NEON::BI__builtin_neon_vcltd_s64:
733 case NEON::BI__builtin_neon_vcltd_u64:
734 case NEON::BI__builtin_neon_vclts_f32:
735 case NEON::BI__builtin_neon_vcltd_f64:
736 case NEON::BI__builtin_neon_vcales_f32:
737 case NEON::BI__builtin_neon_vcaled_f64:
738 case NEON::BI__builtin_neon_vcalts_f32:
739 case NEON::BI__builtin_neon_vcaltd_f64:
740 cgf.cgm.errorNYI(expr->getSourceRange(),
741 std::string("unimplemented AArch64 builtin call: ") +
742 cgf.getContext().BuiltinInfo.getName(info.BuiltinID));
743 break;
744 }
745
746 llvm::StringRef llvmIntrName = getLLVMIntrNameNoPrefix(
747 static_cast<llvm::Intrinsic::ID>(info.LLVMIntrinsic));
748 mlir::Location loc = cgf.getLoc(expr->getExprLoc());
749
750 // The switch stmt is intended to help catch NYI cases and will be removed
751 // once the CIR implementation is complete. Avoid adding specialized
752 // code in cases - that should only be required for a handful of examples.
753 switch (info.BuiltinID) {
754 default:
755 cgf.cgm.errorNYI(expr->getSourceRange(),
756 std::string("unimplemented AArch64 builtin call: ") +
757 cgf.getContext().BuiltinInfo.getName(info.BuiltinID));
758 break;
759 case NEON::BI__builtin_neon_vabdd_f64:
760 case NEON::BI__builtin_neon_vabds_f32:
761 case NEON::BI__builtin_neon_vshld_s64:
762 case NEON::BI__builtin_neon_vshld_u64:
763 return emitNeonCall(cgf.cgm, cgf.getBuilder(),
764 {cgf.convertType(expr->getArg(0)->getType())}, ops,
765 llvmIntrName, cgf.convertType(expr->getType()), loc);
766 }
767
768 return nullptr;
769}
770
771//===----------------------------------------------------------------------===//
772// SVE helpers
773//===----------------------------------------------------------------------===//
774std::optional<mlir::Value>
775CIRGenFunction::emitAArch64SVEBuiltinExpr(unsigned builtinID,
776 const CallExpr *expr) {
777 mlir::Type ty = convertType(expr->getType());
778
779 if (builtinID >= SVE::BI__builtin_sve_reinterpret_s8_s8 &&
780 builtinID <= SVE::BI__builtin_sve_reinterpret_f64_f64_x4) {
781 cgm.errorNYI(expr->getSourceRange(),
782 std::string("unimplemented AArch64 builtin call: ") +
783 getContext().BuiltinInfo.getName(builtinID));
784 return mlir::Value{};
785 }
786
787 assert(!cir::MissingFeatures::aarch64SVEIntrinsics());
788
789 auto *builtinIntrInfo = findARMVectorIntrinsicInMap(
790 aarch64SVEIntrinsicMap, builtinID, aarch64SVEIntrinsicsProvenSorted);
791
792 // The operands of the builtin call
793 llvm::SmallVector<mlir::Value> ops;
794
795 SVETypeFlags typeFlags(builtinIntrInfo->TypeModifier);
796 if (!CIRGenFunction::getAArch64SVEProcessedOperands(builtinID, expr, ops,
797 typeFlags))
798 return mlir::Value{};
799
800 if (typeFlags.isLoad() || typeFlags.isStore() || typeFlags.isGatherLoad() ||
801 typeFlags.isScatterStore() || typeFlags.isPrefetch() ||
802 typeFlags.isGatherPrefetch() || typeFlags.isStructLoad() ||
803 typeFlags.isStructStore() || typeFlags.isTupleSet() ||
804 typeFlags.isTupleGet() || typeFlags.isTupleCreate() ||
805 typeFlags.isUndef())
806 cgm.errorNYI(expr->getSourceRange(),
807 std::string("unimplemented AArch64 builtin call: ") +
808 getContext().BuiltinInfo.getName(builtinID));
809
810 mlir::Location loc = getLoc(expr->getExprLoc());
811
812 // Handle built-ins for which there is a corresponding LLVM Intrinsic.
813 // -------------------------------------------------------------------
814 if (builtinIntrInfo->LLVMIntrinsic != 0) {
815 // Emit set FPMR for intrinsics that require it.
816 if (typeFlags.setsFPMR())
817 cgm.errorNYI(expr->getSourceRange(),
818 std::string("unimplemented AArch64 builtin call: ") +
819 getContext().BuiltinInfo.getName(builtinID));
820
821 // Zero-ing predication
822 if (typeFlags.getMergeType() == SVETypeFlags::MergeZeroExp) {
823 auto null = builder.getNullValue(convertType(expr->getType()),
824 getLoc(expr->getExprLoc()));
825 ops.insert(ops.begin(), null);
826 }
827
828 if (typeFlags.getMergeType() == SVETypeFlags::MergeAnyExp)
829 ops.insert(ops.begin(),
830 builder.getConstant(loc, cir::UndefAttr::get(ty)));
831
832 // Some ACLE builtins leave out the argument to specify the predicate
833 // pattern, which is expected to be expanded to an SV_ALL pattern.
834 if (typeFlags.isAppendSVALL())
835 cgm.errorNYI(expr->getSourceRange(),
836 std::string("unimplemented AArch64 builtin call: ") +
837 getContext().BuiltinInfo.getName(builtinID));
838 if (typeFlags.isInsertOp1SVALL())
839 cgm.errorNYI(expr->getSourceRange(),
840 std::string("unimplemented AArch64 builtin call: ") +
841 getContext().BuiltinInfo.getName(builtinID));
842
843 // Predicates must match the main datatype.
844 for (mlir::Value &op : ops)
845 if (auto predTy = dyn_cast<cir::VectorType>(op.getType()))
846 if (auto cirInt = dyn_cast<cir::IntType>(predTy.getElementType()))
847 if (cirInt.getWidth() == 1)
848 op = emitSVEPredicateCast(
849 op, getSVEMinEltCount(typeFlags.getEltType()), loc);
850
851 // Splat scalar operand to vector (intrinsics with _n infix)
852 if (typeFlags.hasSplatOperand()) {
853 unsigned opNo = typeFlags.getSplatOperand();
854 ops[opNo] = cir::VecSplatOp::create(
855 builder, loc, getSVEVectorForElementType(cgm, ops[opNo].getType()),
856 ops[opNo]);
857 }
858
859 if (typeFlags.isReverseCompare())
860 cgm.errorNYI(expr->getSourceRange(),
861 std::string("unimplemented AArch64 builtin call: ") +
862 getContext().BuiltinInfo.getName(builtinID));
863 if (typeFlags.isReverseUSDOT())
864 cgm.errorNYI(expr->getSourceRange(),
865 std::string("unimplemented AArch64 builtin call: ") +
866 getContext().BuiltinInfo.getName(builtinID));
867 if (typeFlags.isReverseMergeAnyBinOp() &&
868 typeFlags.getMergeType() == SVETypeFlags::MergeAny)
869 cgm.errorNYI(expr->getSourceRange(),
870 std::string("unimplemented AArch64 builtin call: ") +
871 getContext().BuiltinInfo.getName(builtinID));
872 if (typeFlags.isReverseMergeAnyAccOp() &&
873 typeFlags.getMergeType() == SVETypeFlags::MergeAny)
874 cgm.errorNYI(expr->getSourceRange(),
875 std::string("unimplemented AArch64 builtin call: ") +
876 getContext().BuiltinInfo.getName(builtinID));
877
878 // Predicated intrinsics with _z suffix.
879 if (typeFlags.getMergeType() == SVETypeFlags::MergeZero) {
880 cgm.errorNYI(expr->getSourceRange(),
881 std::string("unimplemented AArch64 builtin call: ") +
882 getContext().BuiltinInfo.getName(builtinID));
883 }
884
885 llvm::StringRef llvmIntrName = getLLVMIntrNameNoPrefix(
886 static_cast<llvm::Intrinsic::ID>(builtinIntrInfo->LLVMIntrinsic));
887 auto retTy = convertType(expr->getType());
888
889 auto call = builder.emitIntrinsicCallOp(loc, llvmIntrName, retTy,
890 mlir::ValueRange{ops});
891 if (call.getType() == retTy)
892 return call;
893
894 // Predicate results must be converted to svbool_t.
895 if (isa<mlir::VectorType>(retTy) &&
896 cast<mlir::VectorType>(retTy).isScalable())
897 cgm.errorNYI(expr->getSourceRange(),
898 std::string("unimplemented AArch64 builtin call: ") +
899 getContext().BuiltinInfo.getName(builtinID));
900 // TODO Handle struct types, e.g. svint8x2_t (update the converter first).
901
902 llvm_unreachable("unsupported element count!");
903 }
904
905 // Handle the remaining built-ins.
906 // -------------------------------
907 switch (builtinID) {
908 default:
909 return std::nullopt;
910
911 case SVE::BI__builtin_sve_svreinterpret_b:
912 case SVE::BI__builtin_sve_svreinterpret_c:
913 case SVE::BI__builtin_sve_svpsel_lane_b8:
914 case SVE::BI__builtin_sve_svpsel_lane_b16:
915 case SVE::BI__builtin_sve_svpsel_lane_b32:
916 case SVE::BI__builtin_sve_svpsel_lane_b64:
917 case SVE::BI__builtin_sve_svpsel_lane_c8:
918 case SVE::BI__builtin_sve_svpsel_lane_c16:
919 case SVE::BI__builtin_sve_svpsel_lane_c32:
920 case SVE::BI__builtin_sve_svpsel_lane_c64:
921 case SVE::BI__builtin_sve_svmov_b_z:
922 case SVE::BI__builtin_sve_svnot_b_z:
923 case SVE::BI__builtin_sve_svmovlb_u16:
924 case SVE::BI__builtin_sve_svmovlb_u32:
925 case SVE::BI__builtin_sve_svmovlb_u64:
926 case SVE::BI__builtin_sve_svmovlb_s16:
927 case SVE::BI__builtin_sve_svmovlb_s32:
928 case SVE::BI__builtin_sve_svmovlb_s64:
929 case SVE::BI__builtin_sve_svmovlt_u16:
930 case SVE::BI__builtin_sve_svmovlt_u32:
931 case SVE::BI__builtin_sve_svmovlt_u64:
932 case SVE::BI__builtin_sve_svmovlt_s16:
933 case SVE::BI__builtin_sve_svmovlt_s32:
934 case SVE::BI__builtin_sve_svmovlt_s64:
935 case SVE::BI__builtin_sve_svpmullt_u16:
936 case SVE::BI__builtin_sve_svpmullt_u64:
937 case SVE::BI__builtin_sve_svpmullt_n_u16:
938 case SVE::BI__builtin_sve_svpmullt_n_u64:
939 case SVE::BI__builtin_sve_svpmullb_u16:
940 case SVE::BI__builtin_sve_svpmullb_u64:
941 case SVE::BI__builtin_sve_svpmullb_n_u16:
942 case SVE::BI__builtin_sve_svpmullb_n_u64:
943
944 case SVE::BI__builtin_sve_svdup_n_b8:
945 case SVE::BI__builtin_sve_svdup_n_b16:
946 case SVE::BI__builtin_sve_svdup_n_b32:
947 case SVE::BI__builtin_sve_svdup_n_b64:
948
949 case SVE::BI__builtin_sve_svdupq_n_b8:
950 case SVE::BI__builtin_sve_svdupq_n_b16:
951 case SVE::BI__builtin_sve_svdupq_n_b32:
952 case SVE::BI__builtin_sve_svdupq_n_b64:
953 case SVE::BI__builtin_sve_svdupq_n_u8:
954 case SVE::BI__builtin_sve_svdupq_n_s8:
955 case SVE::BI__builtin_sve_svdupq_n_u64:
956 case SVE::BI__builtin_sve_svdupq_n_f64:
957 case SVE::BI__builtin_sve_svdupq_n_s64:
958 case SVE::BI__builtin_sve_svdupq_n_u16:
959 case SVE::BI__builtin_sve_svdupq_n_f16:
960 case SVE::BI__builtin_sve_svdupq_n_bf16:
961 case SVE::BI__builtin_sve_svdupq_n_s16:
962 case SVE::BI__builtin_sve_svdupq_n_u32:
963 case SVE::BI__builtin_sve_svdupq_n_f32:
964 case SVE::BI__builtin_sve_svdupq_n_s32:
965 case SVE::BI__builtin_sve_svpfalse_b:
966 case SVE::BI__builtin_sve_svpfalse_c:
967 cgm.errorNYI(expr->getSourceRange(),
968 std::string("unimplemented AArch64 builtin call: ") +
969 getContext().BuiltinInfo.getName(builtinID));
970 return mlir::Value{};
971
972 case SVE::BI__builtin_sve_svlen_u8:
973 case SVE::BI__builtin_sve_svlen_s8:
974 return genVscaleTimesFactor(loc, builder, convertType(expr->getType()), 16);
975
976 case SVE::BI__builtin_sve_svlen_u16:
977 case SVE::BI__builtin_sve_svlen_s16:
978 case SVE::BI__builtin_sve_svlen_f16:
979 case SVE::BI__builtin_sve_svlen_bf16:
980 return genVscaleTimesFactor(loc, builder, convertType(expr->getType()), 8);
981
982 case SVE::BI__builtin_sve_svlen_u32:
983 case SVE::BI__builtin_sve_svlen_s32:
984 case SVE::BI__builtin_sve_svlen_f32:
985 return genVscaleTimesFactor(loc, builder, convertType(expr->getType()), 4);
986
987 case SVE::BI__builtin_sve_svlen_u64:
988 case SVE::BI__builtin_sve_svlen_s64:
989 case SVE::BI__builtin_sve_svlen_f64:
990 return genVscaleTimesFactor(loc, builder, convertType(expr->getType()), 2);
991
992 case SVE::BI__builtin_sve_svtbl2_u8:
993 case SVE::BI__builtin_sve_svtbl2_s8:
994 case SVE::BI__builtin_sve_svtbl2_u16:
995 case SVE::BI__builtin_sve_svtbl2_s16:
996 case SVE::BI__builtin_sve_svtbl2_u32:
997 case SVE::BI__builtin_sve_svtbl2_s32:
998 case SVE::BI__builtin_sve_svtbl2_u64:
999 case SVE::BI__builtin_sve_svtbl2_s64:
1000 case SVE::BI__builtin_sve_svtbl2_f16:
1001 case SVE::BI__builtin_sve_svtbl2_bf16:
1002 case SVE::BI__builtin_sve_svtbl2_f32:
1003 case SVE::BI__builtin_sve_svtbl2_f64:
1004 case SVE::BI__builtin_sve_svset_neonq_s8:
1005 case SVE::BI__builtin_sve_svset_neonq_s16:
1006 case SVE::BI__builtin_sve_svset_neonq_s32:
1007 case SVE::BI__builtin_sve_svset_neonq_s64:
1008 case SVE::BI__builtin_sve_svset_neonq_u8:
1009 case SVE::BI__builtin_sve_svset_neonq_u16:
1010 case SVE::BI__builtin_sve_svset_neonq_u32:
1011 case SVE::BI__builtin_sve_svset_neonq_u64:
1012 case SVE::BI__builtin_sve_svset_neonq_f16:
1013 case SVE::BI__builtin_sve_svset_neonq_f32:
1014 case SVE::BI__builtin_sve_svset_neonq_f64:
1015 case SVE::BI__builtin_sve_svset_neonq_bf16:
1016 case SVE::BI__builtin_sve_svget_neonq_s8:
1017 case SVE::BI__builtin_sve_svget_neonq_s16:
1018 case SVE::BI__builtin_sve_svget_neonq_s32:
1019 case SVE::BI__builtin_sve_svget_neonq_s64:
1020 case SVE::BI__builtin_sve_svget_neonq_u8:
1021 case SVE::BI__builtin_sve_svget_neonq_u16:
1022 case SVE::BI__builtin_sve_svget_neonq_u32:
1023 case SVE::BI__builtin_sve_svget_neonq_u64:
1024 case SVE::BI__builtin_sve_svget_neonq_f16:
1025 case SVE::BI__builtin_sve_svget_neonq_f32:
1026 case SVE::BI__builtin_sve_svget_neonq_f64:
1027 case SVE::BI__builtin_sve_svget_neonq_bf16:
1028 case SVE::BI__builtin_sve_svdup_neonq_s8:
1029 case SVE::BI__builtin_sve_svdup_neonq_s16:
1030 case SVE::BI__builtin_sve_svdup_neonq_s32:
1031 case SVE::BI__builtin_sve_svdup_neonq_s64:
1032 case SVE::BI__builtin_sve_svdup_neonq_u8:
1033 case SVE::BI__builtin_sve_svdup_neonq_u16:
1034 case SVE::BI__builtin_sve_svdup_neonq_u32:
1035 case SVE::BI__builtin_sve_svdup_neonq_u64:
1036 case SVE::BI__builtin_sve_svdup_neonq_f16:
1037 case SVE::BI__builtin_sve_svdup_neonq_f32:
1038 case SVE::BI__builtin_sve_svdup_neonq_f64:
1039 case SVE::BI__builtin_sve_svdup_neonq_bf16:
1040 cgm.errorNYI(expr->getSourceRange(),
1041 std::string("unimplemented AArch64 builtin call: ") +
1042 getContext().BuiltinInfo.getName(builtinID));
1043 return mlir::Value{};
1044 }
1045
1046 // Unreachable: All cases in the switch above return.
1047}
1048
1049std::optional<mlir::Value>
1050CIRGenFunction::emitAArch64SMEBuiltinExpr(unsigned builtinID,
1051 const CallExpr *expr) {
1052 assert(!cir::MissingFeatures::aarch64SMEIntrinsics());
1053
1054 cgm.errorNYI(expr->getSourceRange(),
1055 std::string("unimplemented AArch64 builtin call: ") +
1056 getContext().BuiltinInfo.getName(builtinID));
1057 return mlir::Value{};
1058}
1059
1060// Some intrinsics are equivalent for codegen.
1061static const std::pair<unsigned, unsigned> neonEquivalentIntrinsicMap[] = {
1062 {
1063 NEON::BI__builtin_neon_splat_lane_bf16,
1064 NEON::BI__builtin_neon_splat_lane_v,
1065 },
1066 {
1067 NEON::BI__builtin_neon_splat_laneq_bf16,
1068 NEON::BI__builtin_neon_splat_laneq_v,
1069 },
1070 {
1071 NEON::BI__builtin_neon_splatq_lane_bf16,
1072 NEON::BI__builtin_neon_splatq_lane_v,
1073 },
1074 {
1075 NEON::BI__builtin_neon_splatq_laneq_bf16,
1076 NEON::BI__builtin_neon_splatq_laneq_v,
1077 },
1078 {
1079 NEON::BI__builtin_neon_vabd_f16,
1080 NEON::BI__builtin_neon_vabd_v,
1081 },
1082 {
1083 NEON::BI__builtin_neon_vabdq_f16,
1084 NEON::BI__builtin_neon_vabdq_v,
1085 },
1086 {
1087 NEON::BI__builtin_neon_vabs_f16,
1088 NEON::BI__builtin_neon_vabs_v,
1089 },
1090 {
1091 NEON::BI__builtin_neon_vabsq_f16,
1092 NEON::BI__builtin_neon_vabsq_v,
1093 },
1094 {
1095 NEON::BI__builtin_neon_vcage_f16,
1096 NEON::BI__builtin_neon_vcage_v,
1097 },
1098 {
1099 NEON::BI__builtin_neon_vcageq_f16,
1100 NEON::BI__builtin_neon_vcageq_v,
1101 },
1102 {
1103 NEON::BI__builtin_neon_vcagt_f16,
1104 NEON::BI__builtin_neon_vcagt_v,
1105 },
1106 {
1107 NEON::BI__builtin_neon_vcagtq_f16,
1108 NEON::BI__builtin_neon_vcagtq_v,
1109 },
1110 {
1111 NEON::BI__builtin_neon_vcale_f16,
1112 NEON::BI__builtin_neon_vcale_v,
1113 },
1114 {
1115 NEON::BI__builtin_neon_vcaleq_f16,
1116 NEON::BI__builtin_neon_vcaleq_v,
1117 },
1118 {
1119 NEON::BI__builtin_neon_vcalt_f16,
1120 NEON::BI__builtin_neon_vcalt_v,
1121 },
1122 {
1123 NEON::BI__builtin_neon_vcaltq_f16,
1124 NEON::BI__builtin_neon_vcaltq_v,
1125 },
1126 {
1127 NEON::BI__builtin_neon_vceqz_f16,
1128 NEON::BI__builtin_neon_vceqz_v,
1129 },
1130 {
1131 NEON::BI__builtin_neon_vceqzq_f16,
1132 NEON::BI__builtin_neon_vceqzq_v,
1133 },
1134 {
1135 NEON::BI__builtin_neon_vcgez_f16,
1136 NEON::BI__builtin_neon_vcgez_v,
1137 },
1138 {
1139 NEON::BI__builtin_neon_vcgezq_f16,
1140 NEON::BI__builtin_neon_vcgezq_v,
1141 },
1142 {
1143 NEON::BI__builtin_neon_vcgtz_f16,
1144 NEON::BI__builtin_neon_vcgtz_v,
1145 },
1146 {
1147 NEON::BI__builtin_neon_vcgtzq_f16,
1148 NEON::BI__builtin_neon_vcgtzq_v,
1149 },
1150 {
1151 NEON::BI__builtin_neon_vclez_f16,
1152 NEON::BI__builtin_neon_vclez_v,
1153 },
1154 {
1155 NEON::BI__builtin_neon_vclezq_f16,
1156 NEON::BI__builtin_neon_vclezq_v,
1157 },
1158 {
1159 NEON::BI__builtin_neon_vcltz_f16,
1160 NEON::BI__builtin_neon_vcltz_v,
1161 },
1162 {
1163 NEON::BI__builtin_neon_vcltzq_f16,
1164 NEON::BI__builtin_neon_vcltzq_v,
1165 },
1166 {
1167 NEON::BI__builtin_neon_vfma_f16,
1168 NEON::BI__builtin_neon_vfma_v,
1169 },
1170 {
1171 NEON::BI__builtin_neon_vfma_lane_f16,
1172 NEON::BI__builtin_neon_vfma_lane_v,
1173 },
1174 {
1175 NEON::BI__builtin_neon_vfma_laneq_f16,
1176 NEON::BI__builtin_neon_vfma_laneq_v,
1177 },
1178 {
1179 NEON::BI__builtin_neon_vfmaq_f16,
1180 NEON::BI__builtin_neon_vfmaq_v,
1181 },
1182 {
1183 NEON::BI__builtin_neon_vfmaq_lane_f16,
1184 NEON::BI__builtin_neon_vfmaq_lane_v,
1185 },
1186 {
1187 NEON::BI__builtin_neon_vfmaq_laneq_f16,
1188 NEON::BI__builtin_neon_vfmaq_laneq_v,
1189 },
1190 {NEON::BI__builtin_neon_vld1_bf16_x2, NEON::BI__builtin_neon_vld1_x2_v},
1191 {NEON::BI__builtin_neon_vld1_bf16_x3, NEON::BI__builtin_neon_vld1_x3_v},
1192 {NEON::BI__builtin_neon_vld1_bf16_x4, NEON::BI__builtin_neon_vld1_x4_v},
1193 {NEON::BI__builtin_neon_vld1_bf16, NEON::BI__builtin_neon_vld1_v},
1194 {NEON::BI__builtin_neon_vld1_dup_bf16, NEON::BI__builtin_neon_vld1_dup_v},
1195 {NEON::BI__builtin_neon_vld1_lane_bf16, NEON::BI__builtin_neon_vld1_lane_v},
1196 {NEON::BI__builtin_neon_vld1q_bf16_x2, NEON::BI__builtin_neon_vld1q_x2_v},
1197 {NEON::BI__builtin_neon_vld1q_bf16_x3, NEON::BI__builtin_neon_vld1q_x3_v},
1198 {NEON::BI__builtin_neon_vld1q_bf16_x4, NEON::BI__builtin_neon_vld1q_x4_v},
1199 {NEON::BI__builtin_neon_vld1q_bf16, NEON::BI__builtin_neon_vld1q_v},
1200 {NEON::BI__builtin_neon_vld1q_dup_bf16, NEON::BI__builtin_neon_vld1q_dup_v},
1201 {NEON::BI__builtin_neon_vld1q_lane_bf16,
1202 NEON::BI__builtin_neon_vld1q_lane_v},
1203 {NEON::BI__builtin_neon_vld2_bf16, NEON::BI__builtin_neon_vld2_v},
1204 {NEON::BI__builtin_neon_vld2_dup_bf16, NEON::BI__builtin_neon_vld2_dup_v},
1205 {NEON::BI__builtin_neon_vld2_lane_bf16, NEON::BI__builtin_neon_vld2_lane_v},
1206 {NEON::BI__builtin_neon_vld2q_bf16, NEON::BI__builtin_neon_vld2q_v},
1207 {NEON::BI__builtin_neon_vld2q_dup_bf16, NEON::BI__builtin_neon_vld2q_dup_v},
1208 {NEON::BI__builtin_neon_vld2q_lane_bf16,
1209 NEON::BI__builtin_neon_vld2q_lane_v},
1210 {NEON::BI__builtin_neon_vld3_bf16, NEON::BI__builtin_neon_vld3_v},
1211 {NEON::BI__builtin_neon_vld3_dup_bf16, NEON::BI__builtin_neon_vld3_dup_v},
1212 {NEON::BI__builtin_neon_vld3_lane_bf16, NEON::BI__builtin_neon_vld3_lane_v},
1213 {NEON::BI__builtin_neon_vld3q_bf16, NEON::BI__builtin_neon_vld3q_v},
1214 {NEON::BI__builtin_neon_vld3q_dup_bf16, NEON::BI__builtin_neon_vld3q_dup_v},
1215 {NEON::BI__builtin_neon_vld3q_lane_bf16,
1216 NEON::BI__builtin_neon_vld3q_lane_v},
1217 {NEON::BI__builtin_neon_vld4_bf16, NEON::BI__builtin_neon_vld4_v},
1218 {NEON::BI__builtin_neon_vld4_dup_bf16, NEON::BI__builtin_neon_vld4_dup_v},
1219 {NEON::BI__builtin_neon_vld4_lane_bf16, NEON::BI__builtin_neon_vld4_lane_v},
1220 {NEON::BI__builtin_neon_vld4q_bf16, NEON::BI__builtin_neon_vld4q_v},
1221 {NEON::BI__builtin_neon_vld4q_dup_bf16, NEON::BI__builtin_neon_vld4q_dup_v},
1222 {NEON::BI__builtin_neon_vld4q_lane_bf16,
1223 NEON::BI__builtin_neon_vld4q_lane_v},
1224 {
1225 NEON::BI__builtin_neon_vmax_f16,
1226 NEON::BI__builtin_neon_vmax_v,
1227 },
1228 {
1229 NEON::BI__builtin_neon_vmaxnm_f16,
1230 NEON::BI__builtin_neon_vmaxnm_v,
1231 },
1232 {
1233 NEON::BI__builtin_neon_vmaxnmq_f16,
1234 NEON::BI__builtin_neon_vmaxnmq_v,
1235 },
1236 {
1237 NEON::BI__builtin_neon_vmaxq_f16,
1238 NEON::BI__builtin_neon_vmaxq_v,
1239 },
1240 {
1241 NEON::BI__builtin_neon_vmin_f16,
1242 NEON::BI__builtin_neon_vmin_v,
1243 },
1244 {
1245 NEON::BI__builtin_neon_vminnm_f16,
1246 NEON::BI__builtin_neon_vminnm_v,
1247 },
1248 {
1249 NEON::BI__builtin_neon_vminnmq_f16,
1250 NEON::BI__builtin_neon_vminnmq_v,
1251 },
1252 {
1253 NEON::BI__builtin_neon_vminq_f16,
1254 NEON::BI__builtin_neon_vminq_v,
1255 },
1256 {
1257 NEON::BI__builtin_neon_vmulx_f16,
1258 NEON::BI__builtin_neon_vmulx_v,
1259 },
1260 {
1261 NEON::BI__builtin_neon_vmulxq_f16,
1262 NEON::BI__builtin_neon_vmulxq_v,
1263 },
1264 {
1265 NEON::BI__builtin_neon_vpadd_f16,
1266 NEON::BI__builtin_neon_vpadd_v,
1267 },
1268 {
1269 NEON::BI__builtin_neon_vpaddq_f16,
1270 NEON::BI__builtin_neon_vpaddq_v,
1271 },
1272 {
1273 NEON::BI__builtin_neon_vpmax_f16,
1274 NEON::BI__builtin_neon_vpmax_v,
1275 },
1276 {
1277 NEON::BI__builtin_neon_vpmaxnm_f16,
1278 NEON::BI__builtin_neon_vpmaxnm_v,
1279 },
1280 {
1281 NEON::BI__builtin_neon_vpmaxnmq_f16,
1282 NEON::BI__builtin_neon_vpmaxnmq_v,
1283 },
1284 {
1285 NEON::BI__builtin_neon_vpmaxq_f16,
1286 NEON::BI__builtin_neon_vpmaxq_v,
1287 },
1288 {
1289 NEON::BI__builtin_neon_vpmin_f16,
1290 NEON::BI__builtin_neon_vpmin_v,
1291 },
1292 {
1293 NEON::BI__builtin_neon_vpminnm_f16,
1294 NEON::BI__builtin_neon_vpminnm_v,
1295 },
1296 {
1297 NEON::BI__builtin_neon_vpminnmq_f16,
1298 NEON::BI__builtin_neon_vpminnmq_v,
1299 },
1300 {
1301 NEON::BI__builtin_neon_vpminq_f16,
1302 NEON::BI__builtin_neon_vpminq_v,
1303 },
1304 {
1305 NEON::BI__builtin_neon_vrecpe_f16,
1306 NEON::BI__builtin_neon_vrecpe_v,
1307 },
1308 {
1309 NEON::BI__builtin_neon_vrecpeq_f16,
1310 NEON::BI__builtin_neon_vrecpeq_v,
1311 },
1312 {
1313 NEON::BI__builtin_neon_vrecps_f16,
1314 NEON::BI__builtin_neon_vrecps_v,
1315 },
1316 {
1317 NEON::BI__builtin_neon_vrecpsq_f16,
1318 NEON::BI__builtin_neon_vrecpsq_v,
1319 },
1320 {
1321 NEON::BI__builtin_neon_vrnd_f16,
1322 NEON::BI__builtin_neon_vrnd_v,
1323 },
1324 {
1325 NEON::BI__builtin_neon_vrnda_f16,
1326 NEON::BI__builtin_neon_vrnda_v,
1327 },
1328 {
1329 NEON::BI__builtin_neon_vrndaq_f16,
1330 NEON::BI__builtin_neon_vrndaq_v,
1331 },
1332 {
1333 NEON::BI__builtin_neon_vrndi_f16,
1334 NEON::BI__builtin_neon_vrndi_v,
1335 },
1336 {
1337 NEON::BI__builtin_neon_vrndiq_f16,
1338 NEON::BI__builtin_neon_vrndiq_v,
1339 },
1340 {
1341 NEON::BI__builtin_neon_vrndm_f16,
1342 NEON::BI__builtin_neon_vrndm_v,
1343 },
1344 {
1345 NEON::BI__builtin_neon_vrndmq_f16,
1346 NEON::BI__builtin_neon_vrndmq_v,
1347 },
1348 {
1349 NEON::BI__builtin_neon_vrndn_f16,
1350 NEON::BI__builtin_neon_vrndn_v,
1351 },
1352 {
1353 NEON::BI__builtin_neon_vrndnq_f16,
1354 NEON::BI__builtin_neon_vrndnq_v,
1355 },
1356 {
1357 NEON::BI__builtin_neon_vrndp_f16,
1358 NEON::BI__builtin_neon_vrndp_v,
1359 },
1360 {
1361 NEON::BI__builtin_neon_vrndpq_f16,
1362 NEON::BI__builtin_neon_vrndpq_v,
1363 },
1364 {
1365 NEON::BI__builtin_neon_vrndq_f16,
1366 NEON::BI__builtin_neon_vrndq_v,
1367 },
1368 {
1369 NEON::BI__builtin_neon_vrndx_f16,
1370 NEON::BI__builtin_neon_vrndx_v,
1371 },
1372 {
1373 NEON::BI__builtin_neon_vrndxq_f16,
1374 NEON::BI__builtin_neon_vrndxq_v,
1375 },
1376 {
1377 NEON::BI__builtin_neon_vrsqrte_f16,
1378 NEON::BI__builtin_neon_vrsqrte_v,
1379 },
1380 {
1381 NEON::BI__builtin_neon_vrsqrteq_f16,
1382 NEON::BI__builtin_neon_vrsqrteq_v,
1383 },
1384 {
1385 NEON::BI__builtin_neon_vrsqrts_f16,
1386 NEON::BI__builtin_neon_vrsqrts_v,
1387 },
1388 {
1389 NEON::BI__builtin_neon_vrsqrtsq_f16,
1390 NEON::BI__builtin_neon_vrsqrtsq_v,
1391 },
1392 {
1393 NEON::BI__builtin_neon_vsqrt_f16,
1394 NEON::BI__builtin_neon_vsqrt_v,
1395 },
1396 {
1397 NEON::BI__builtin_neon_vsqrtq_f16,
1398 NEON::BI__builtin_neon_vsqrtq_v,
1399 },
1400 {NEON::BI__builtin_neon_vst1_bf16_x2, NEON::BI__builtin_neon_vst1_x2_v},
1401 {NEON::BI__builtin_neon_vst1_bf16_x3, NEON::BI__builtin_neon_vst1_x3_v},
1402 {NEON::BI__builtin_neon_vst1_bf16_x4, NEON::BI__builtin_neon_vst1_x4_v},
1403 {NEON::BI__builtin_neon_vst1_bf16, NEON::BI__builtin_neon_vst1_v},
1404 {NEON::BI__builtin_neon_vst1_lane_bf16, NEON::BI__builtin_neon_vst1_lane_v},
1405 {NEON::BI__builtin_neon_vst1q_bf16_x2, NEON::BI__builtin_neon_vst1q_x2_v},
1406 {NEON::BI__builtin_neon_vst1q_bf16_x3, NEON::BI__builtin_neon_vst1q_x3_v},
1407 {NEON::BI__builtin_neon_vst1q_bf16_x4, NEON::BI__builtin_neon_vst1q_x4_v},
1408 {NEON::BI__builtin_neon_vst1q_bf16, NEON::BI__builtin_neon_vst1q_v},
1409 {NEON::BI__builtin_neon_vst1q_lane_bf16,
1410 NEON::BI__builtin_neon_vst1q_lane_v},
1411 {NEON::BI__builtin_neon_vst2_bf16, NEON::BI__builtin_neon_vst2_v},
1412 {NEON::BI__builtin_neon_vst2_lane_bf16, NEON::BI__builtin_neon_vst2_lane_v},
1413 {NEON::BI__builtin_neon_vst2q_bf16, NEON::BI__builtin_neon_vst2q_v},
1414 {NEON::BI__builtin_neon_vst2q_lane_bf16,
1415 NEON::BI__builtin_neon_vst2q_lane_v},
1416 {NEON::BI__builtin_neon_vst3_bf16, NEON::BI__builtin_neon_vst3_v},
1417 {NEON::BI__builtin_neon_vst3_lane_bf16, NEON::BI__builtin_neon_vst3_lane_v},
1418 {NEON::BI__builtin_neon_vst3q_bf16, NEON::BI__builtin_neon_vst3q_v},
1419 {NEON::BI__builtin_neon_vst3q_lane_bf16,
1420 NEON::BI__builtin_neon_vst3q_lane_v},
1421 {NEON::BI__builtin_neon_vst4_bf16, NEON::BI__builtin_neon_vst4_v},
1422 {NEON::BI__builtin_neon_vst4_lane_bf16, NEON::BI__builtin_neon_vst4_lane_v},
1423 {NEON::BI__builtin_neon_vst4q_bf16, NEON::BI__builtin_neon_vst4q_v},
1424 {NEON::BI__builtin_neon_vst4q_lane_bf16,
1425 NEON::BI__builtin_neon_vst4q_lane_v},
1426 // The mangling rules cause us to have one ID for each type for
1427 // vldap1(q)_lane and vstl1(q)_lane, but codegen is equivalent for all of
1428 // them. Choose an arbitrary one to be handled as tha canonical variation.
1429 {NEON::BI__builtin_neon_vldap1_lane_u64,
1430 NEON::BI__builtin_neon_vldap1_lane_s64},
1431 {NEON::BI__builtin_neon_vldap1_lane_f64,
1432 NEON::BI__builtin_neon_vldap1_lane_s64},
1433 {NEON::BI__builtin_neon_vldap1_lane_p64,
1434 NEON::BI__builtin_neon_vldap1_lane_s64},
1435 {NEON::BI__builtin_neon_vldap1q_lane_u64,
1436 NEON::BI__builtin_neon_vldap1q_lane_s64},
1437 {NEON::BI__builtin_neon_vldap1q_lane_f64,
1438 NEON::BI__builtin_neon_vldap1q_lane_s64},
1439 {NEON::BI__builtin_neon_vldap1q_lane_p64,
1440 NEON::BI__builtin_neon_vldap1q_lane_s64},
1441 {NEON::BI__builtin_neon_vstl1_lane_u64,
1442 NEON::BI__builtin_neon_vstl1_lane_s64},
1443 {NEON::BI__builtin_neon_vstl1_lane_f64,
1444 NEON::BI__builtin_neon_vstl1_lane_s64},
1445 {NEON::BI__builtin_neon_vstl1_lane_p64,
1446 NEON::BI__builtin_neon_vstl1_lane_s64},
1447 {NEON::BI__builtin_neon_vstl1q_lane_u64,
1448 NEON::BI__builtin_neon_vstl1q_lane_s64},
1449 {NEON::BI__builtin_neon_vstl1q_lane_f64,
1450 NEON::BI__builtin_neon_vstl1q_lane_s64},
1451 {NEON::BI__builtin_neon_vstl1q_lane_p64,
1452 NEON::BI__builtin_neon_vstl1q_lane_s64},
1453};
1454
1455std::optional<mlir::Value>
1456CIRGenFunction::emitAArch64BuiltinExpr(unsigned builtinID, const CallExpr *expr,
1457 ReturnValueSlot returnValue,
1458 llvm::Triple::ArchType arch) {
1459 if (builtinID >= clang::AArch64::FirstSVEBuiltin &&
1460 builtinID <= clang::AArch64::LastSVEBuiltin)
1461 return emitAArch64SVEBuiltinExpr(builtinID, expr);
1462
1463 if (builtinID >= clang::AArch64::FirstSMEBuiltin &&
1464 builtinID <= clang::AArch64::LastSMEBuiltin)
1465 return emitAArch64SMEBuiltinExpr(builtinID, expr);
1466
1467 if (builtinID == Builtin::BI__builtin_cpu_supports) {
1468 cgm.errorNYI(expr->getSourceRange(),
1469 std::string("unimplemented AArch64 builtin call: ") +
1470 getContext().BuiltinInfo.getName(builtinID));
1471 return mlir::Value{};
1472 }
1473
1474 switch (builtinID) {
1475 default:
1476 break;
1477 case clang::AArch64::BI__builtin_arm_nop:
1478 case clang::AArch64::BI__builtin_arm_yield:
1479 case clang::AArch64::BI__yield:
1480 case clang::AArch64::BI__builtin_arm_wfe:
1481 case clang::AArch64::BI__wfe:
1482 case clang::AArch64::BI__builtin_arm_wfi:
1483 case clang::AArch64::BI__wfi:
1484 case clang::AArch64::BI__builtin_arm_sev:
1485 case clang::AArch64::BI__sev:
1486 case clang::AArch64::BI__builtin_arm_sevl:
1487 case clang::AArch64::BI__sevl:
1488 cgm.errorNYI(expr->getSourceRange(),
1489 std::string("unimplemented AArch64 builtin call: ") +
1490 getContext().BuiltinInfo.getName(builtinID));
1491 return mlir::Value{};
1492 }
1493
1494 if (builtinID == clang::AArch64::BI__builtin_arm_trap) {
1495 cgm.errorNYI(expr->getSourceRange(),
1496 std::string("unimplemented AArch64 builtin call: ") +
1497 getContext().BuiltinInfo.getName(builtinID));
1498 return mlir::Value{};
1499 }
1500
1501 if (builtinID == clang::AArch64::BI__builtin_arm_get_sme_state) {
1502 cgm.errorNYI(expr->getSourceRange(),
1503 std::string("unimplemented AArch64 builtin call: ") +
1504 getContext().BuiltinInfo.getName(builtinID));
1505 return mlir::Value{};
1506 }
1507
1508 if (builtinID == clang::AArch64::BI__builtin_arm_rbit) {
1509 cgm.errorNYI(expr->getSourceRange(),
1510 std::string("unimplemented AArch64 builtin call: ") +
1511 getContext().BuiltinInfo.getName(builtinID));
1512 return mlir::Value{};
1513 }
1514 if (builtinID == clang::AArch64::BI__builtin_arm_rbit64) {
1515 cgm.errorNYI(expr->getSourceRange(),
1516 std::string("unimplemented AArch64 builtin call: ") +
1517 getContext().BuiltinInfo.getName(builtinID));
1518 return mlir::Value{};
1519 }
1520
1521 if (builtinID == clang::AArch64::BI__builtin_arm_clz ||
1522 builtinID == clang::AArch64::BI__builtin_arm_clz64) {
1523 cgm.errorNYI(expr->getSourceRange(),
1524 std::string("unimplemented AArch64 builtin call: ") +
1525 getContext().BuiltinInfo.getName(builtinID));
1526 return mlir::Value{};
1527 }
1528
1529 if (builtinID == clang::AArch64::BI__builtin_arm_cls) {
1530 cgm.errorNYI(expr->getSourceRange(),
1531 std::string("unimplemented AArch64 builtin call: ") +
1532 getContext().BuiltinInfo.getName(builtinID));
1533 return mlir::Value{};
1534 }
1535 if (builtinID == clang::AArch64::BI__builtin_arm_cls64) {
1536 cgm.errorNYI(expr->getSourceRange(),
1537 std::string("unimplemented AArch64 builtin call: ") +
1538 getContext().BuiltinInfo.getName(builtinID));
1539 return mlir::Value{};
1540 }
1541
1542 if (builtinID == clang::AArch64::BI__builtin_arm_rint32zf ||
1543 builtinID == clang::AArch64::BI__builtin_arm_rint32z) {
1544 cgm.errorNYI(expr->getSourceRange(),
1545 std::string("unimplemented AArch64 builtin call: ") +
1546 getContext().BuiltinInfo.getName(builtinID));
1547 return mlir::Value{};
1548 }
1549
1550 if (builtinID == clang::AArch64::BI__builtin_arm_rint64zf ||
1551 builtinID == clang::AArch64::BI__builtin_arm_rint64z) {
1552 cgm.errorNYI(expr->getSourceRange(),
1553 std::string("unimplemented AArch64 builtin call: ") +
1554 getContext().BuiltinInfo.getName(builtinID));
1555 return mlir::Value{};
1556 }
1557
1558 if (builtinID == clang::AArch64::BI__builtin_arm_rint32xf ||
1559 builtinID == clang::AArch64::BI__builtin_arm_rint32x) {
1560 cgm.errorNYI(expr->getSourceRange(),
1561 std::string("unimplemented AArch64 builtin call: ") +
1562 getContext().BuiltinInfo.getName(builtinID));
1563 return mlir::Value{};
1564 }
1565
1566 if (builtinID == clang::AArch64::BI__builtin_arm_rint64xf ||
1567 builtinID == clang::AArch64::BI__builtin_arm_rint64x) {
1568 cgm.errorNYI(expr->getSourceRange(),
1569 std::string("unimplemented AArch64 builtin call: ") +
1570 getContext().BuiltinInfo.getName(builtinID));
1571 return mlir::Value{};
1572 }
1573
1574 if (builtinID == clang::AArch64::BI__builtin_arm_jcvt) {
1575 cgm.errorNYI(expr->getSourceRange(),
1576 std::string("unimplemented AArch64 builtin call: ") +
1577 getContext().BuiltinInfo.getName(builtinID));
1578 return mlir::Value{};
1579 }
1580
1581 if (builtinID == clang::AArch64::BI__builtin_arm_ld64b ||
1582 builtinID == clang::AArch64::BI__builtin_arm_st64b ||
1583 builtinID == clang::AArch64::BI__builtin_arm_st64bv ||
1584 builtinID == clang::AArch64::BI__builtin_arm_st64bv0) {
1585 cgm.errorNYI(expr->getSourceRange(),
1586 std::string("unimplemented AArch64 builtin call: ") +
1587 getContext().BuiltinInfo.getName(builtinID));
1588 return mlir::Value{};
1589 }
1590
1591 if (builtinID == clang::AArch64::BI__builtin_arm_atomic_store_with_stshh) {
1592 cgm.errorNYI(expr->getSourceRange(),
1593 std::string("unimplemented AArch64 builtin call: ") +
1594 getContext().BuiltinInfo.getName(builtinID));
1595 return mlir::Value{};
1596 }
1597
1598 if (builtinID == clang::AArch64::BI__builtin_arm_rndr ||
1599 builtinID == clang::AArch64::BI__builtin_arm_rndrrs) {
1600 cgm.errorNYI(expr->getSourceRange(),
1601 std::string("unimplemented AArch64 builtin call: ") +
1602 getContext().BuiltinInfo.getName(builtinID));
1603 return mlir::Value{};
1604 }
1605
1606 if (builtinID == clang::AArch64::BI__clear_cache) {
1607 cgm.errorNYI(expr->getSourceRange(),
1608 std::string("unimplemented AArch64 builtin call: ") +
1609 getContext().BuiltinInfo.getName(builtinID));
1610 return mlir::Value{};
1611 }
1612
1613 if ((builtinID == clang::AArch64::BI__builtin_arm_ldrex ||
1614 builtinID == clang::AArch64::BI__builtin_arm_ldaex) &&
1615 getContext().getTypeSize(expr->getType()) == 128) {
1616 cgm.errorNYI(expr->getSourceRange(),
1617 std::string("unimplemented AArch64 builtin call: ") +
1618 getContext().BuiltinInfo.getName(builtinID));
1619 return mlir::Value{};
1620 }
1621 if (builtinID == clang::AArch64::BI__builtin_arm_ldrex ||
1622 builtinID == clang::AArch64::BI__builtin_arm_ldaex) {
1623 cgm.errorNYI(expr->getSourceRange(),
1624 std::string("unimplemented AArch64 builtin call: ") +
1625 getContext().BuiltinInfo.getName(builtinID));
1626 return mlir::Value{};
1627 }
1628
1629 if ((builtinID == clang::AArch64::BI__builtin_arm_strex ||
1630 builtinID == clang::AArch64::BI__builtin_arm_stlex) &&
1631 getContext().getTypeSize(expr->getArg(0)->getType()) == 128) {
1632 cgm.errorNYI(expr->getSourceRange(),
1633 std::string("unimplemented AArch64 builtin call: ") +
1634 getContext().BuiltinInfo.getName(builtinID));
1635 return mlir::Value{};
1636 }
1637
1638 if (builtinID == clang::AArch64::BI__builtin_arm_strex ||
1639 builtinID == clang::AArch64::BI__builtin_arm_stlex) {
1640 cgm.errorNYI(expr->getSourceRange(),
1641 std::string("unimplemented AArch64 builtin call: ") +
1642 getContext().BuiltinInfo.getName(builtinID));
1643 return mlir::Value{};
1644 }
1645
1646 if (builtinID == clang::AArch64::BI__getReg) {
1647 cgm.errorNYI(expr->getSourceRange(),
1648 std::string("unimplemented AArch64 builtin call: ") +
1649 getContext().BuiltinInfo.getName(builtinID));
1650 return mlir::Value{};
1651 }
1652
1653 if (builtinID == clang::AArch64::BI__break) {
1654 cgm.errorNYI(expr->getSourceRange(),
1655 std::string("unimplemented AArch64 builtin call: ") +
1656 getContext().BuiltinInfo.getName(builtinID));
1657 return mlir::Value{};
1658 }
1659
1660 if (builtinID == clang::AArch64::BI__builtin_arm_clrex) {
1661 cgm.errorNYI(expr->getSourceRange(),
1662 std::string("unimplemented AArch64 builtin call: ") +
1663 getContext().BuiltinInfo.getName(builtinID));
1664 return mlir::Value{};
1665 }
1666
1667 if (builtinID == clang::AArch64::BI_ReadWriteBarrier) {
1668 cgm.errorNYI(expr->getSourceRange(),
1669 std::string("unimplemented AArch64 builtin call: ") +
1670 getContext().BuiltinInfo.getName(builtinID));
1671 return mlir::Value{};
1672 }
1673
1674 // CRC32
1675 Intrinsic::ID crcIntrinsicID = Intrinsic::not_intrinsic;
1676 switch (builtinID) {
1677 case clang::AArch64::BI__builtin_arm_crc32b:
1678 crcIntrinsicID = Intrinsic::aarch64_crc32b;
1679 break;
1680 case clang::AArch64::BI__builtin_arm_crc32cb:
1681 crcIntrinsicID = Intrinsic::aarch64_crc32cb;
1682 break;
1683 case clang::AArch64::BI__builtin_arm_crc32h:
1684 crcIntrinsicID = Intrinsic::aarch64_crc32h;
1685 break;
1686 case clang::AArch64::BI__builtin_arm_crc32ch:
1687 crcIntrinsicID = Intrinsic::aarch64_crc32ch;
1688 break;
1689 case clang::AArch64::BI__builtin_arm_crc32w:
1690 crcIntrinsicID = Intrinsic::aarch64_crc32w;
1691 break;
1692 case clang::AArch64::BI__builtin_arm_crc32cw:
1693 crcIntrinsicID = Intrinsic::aarch64_crc32cw;
1694 break;
1695 case clang::AArch64::BI__builtin_arm_crc32d:
1696 crcIntrinsicID = Intrinsic::aarch64_crc32x;
1697 break;
1698 case clang::AArch64::BI__builtin_arm_crc32cd:
1699 crcIntrinsicID = Intrinsic::aarch64_crc32cx;
1700 break;
1701 }
1702
1703 if (crcIntrinsicID != Intrinsic::not_intrinsic) {
1704 cgm.errorNYI(expr->getSourceRange(),
1705 std::string("unimplemented AArch64 builtin call: ") +
1706 getContext().BuiltinInfo.getName(builtinID));
1707 return mlir::Value{};
1708 }
1709
1710 // Memory Operations (MOPS)
1711 if (builtinID == AArch64::BI__builtin_arm_mops_memset_tag) {
1712 cgm.errorNYI(expr->getSourceRange(),
1713 std::string("unimplemented AArch64 builtin call: ") +
1714 getContext().BuiltinInfo.getName(builtinID));
1715 return mlir::Value{};
1716 }
1717
1718 // Memory Tagging Extensions (MTE) Intrinsics
1719 Intrinsic::ID mteIntrinsicID = Intrinsic::not_intrinsic;
1720 switch (builtinID) {
1721 case clang::AArch64::BI__builtin_arm_irg:
1722 mteIntrinsicID = Intrinsic::aarch64_irg;
1723 break;
1724 case clang::AArch64::BI__builtin_arm_addg:
1725 mteIntrinsicID = Intrinsic::aarch64_addg;
1726 break;
1727 case clang::AArch64::BI__builtin_arm_gmi:
1728 mteIntrinsicID = Intrinsic::aarch64_gmi;
1729 break;
1730 case clang::AArch64::BI__builtin_arm_ldg:
1731 mteIntrinsicID = Intrinsic::aarch64_ldg;
1732 break;
1733 case clang::AArch64::BI__builtin_arm_stg:
1734 mteIntrinsicID = Intrinsic::aarch64_stg;
1735 break;
1736 case clang::AArch64::BI__builtin_arm_subp:
1737 mteIntrinsicID = Intrinsic::aarch64_subp;
1738 break;
1739 }
1740
1741 if (mteIntrinsicID != Intrinsic::not_intrinsic) {
1742 cgm.errorNYI(expr->getSourceRange(),
1743 std::string("unimplemented AArch64 builtin call: ") +
1744 getContext().BuiltinInfo.getName(builtinID));
1745 return mlir::Value{};
1746 }
1747
1748 if (builtinID == clang::AArch64::BI__builtin_arm_rsr ||
1749 builtinID == clang::AArch64::BI__builtin_arm_rsr64 ||
1750 builtinID == clang::AArch64::BI__builtin_arm_rsr128 ||
1751 builtinID == clang::AArch64::BI__builtin_arm_rsrp ||
1752 builtinID == clang::AArch64::BI__builtin_arm_wsr ||
1753 builtinID == clang::AArch64::BI__builtin_arm_wsr64 ||
1754 builtinID == clang::AArch64::BI__builtin_arm_wsr128 ||
1755 builtinID == clang::AArch64::BI__builtin_arm_wsrp) {
1756 cgm.errorNYI(expr->getSourceRange(),
1757 std::string("unimplemented AArch64 builtin call: ") +
1758 getContext().BuiltinInfo.getName(builtinID));
1759 return mlir::Value{};
1760 }
1761
1762 if (builtinID == clang::AArch64::BI_ReadStatusReg ||
1763 builtinID == clang::AArch64::BI_WriteStatusReg ||
1764 builtinID == clang::AArch64::BI__sys) {
1765 cgm.errorNYI(expr->getSourceRange(),
1766 std::string("unimplemented AArch64 builtin call: ") +
1767 getContext().BuiltinInfo.getName(builtinID));
1768 return mlir::Value{};
1769 }
1770
1771 if (builtinID == clang::AArch64::BI_AddressOfReturnAddress) {
1772 cgm.errorNYI(expr->getSourceRange(),
1773 std::string("unimplemented AArch64 builtin call: ") +
1774 getContext().BuiltinInfo.getName(builtinID));
1775 return mlir::Value{};
1776 }
1777
1778 if (builtinID == clang::AArch64::BI__builtin_sponentry) {
1779 cgm.errorNYI(expr->getSourceRange(),
1780 std::string("unimplemented AArch64 builtin call: ") +
1781 getContext().BuiltinInfo.getName(builtinID));
1782 return mlir::Value{};
1783 }
1784
1785 if (builtinID == clang::AArch64::BI__mulh ||
1786 builtinID == clang::AArch64::BI__umulh) {
1787 cgm.errorNYI(expr->getSourceRange(),
1788 std::string("unimplemented AArch64 builtin call: ") +
1789 getContext().BuiltinInfo.getName(builtinID));
1790 return mlir::Value{};
1791 }
1792
1793 if (builtinID == AArch64::BI__writex18byte ||
1794 builtinID == AArch64::BI__writex18word ||
1795 builtinID == AArch64::BI__writex18dword ||
1796 builtinID == AArch64::BI__writex18qword) {
1797 cgm.errorNYI(expr->getSourceRange(),
1798 std::string("unimplemented AArch64 builtin call: ") +
1799 getContext().BuiltinInfo.getName(builtinID));
1800 return mlir::Value{};
1801 }
1802
1803 if (builtinID == AArch64::BI__readx18byte ||
1804 builtinID == AArch64::BI__readx18word ||
1805 builtinID == AArch64::BI__readx18dword ||
1806 builtinID == AArch64::BI__readx18qword) {
1807 cgm.errorNYI(expr->getSourceRange(),
1808 std::string("unimplemented AArch64 builtin call: ") +
1809 getContext().BuiltinInfo.getName(builtinID));
1810 return mlir::Value{};
1811 }
1812
1813 if (builtinID == AArch64::BI__addx18byte ||
1814 builtinID == AArch64::BI__addx18word ||
1815 builtinID == AArch64::BI__addx18dword ||
1816 builtinID == AArch64::BI__addx18qword ||
1817 builtinID == AArch64::BI__incx18byte ||
1818 builtinID == AArch64::BI__incx18word ||
1819 builtinID == AArch64::BI__incx18dword ||
1820 builtinID == AArch64::BI__incx18qword) {
1821 cgm.errorNYI(expr->getSourceRange(),
1822 std::string("unimplemented AArch64 builtin call: ") +
1823 getContext().BuiltinInfo.getName(builtinID));
1824 return mlir::Value{};
1825 }
1826
1827 if (builtinID == AArch64::BI_CopyDoubleFromInt64 ||
1828 builtinID == AArch64::BI_CopyFloatFromInt32 ||
1829 builtinID == AArch64::BI_CopyInt32FromFloat ||
1830 builtinID == AArch64::BI_CopyInt64FromDouble) {
1831 cgm.errorNYI(expr->getSourceRange(),
1832 std::string("unimplemented AArch64 builtin call: ") +
1833 getContext().BuiltinInfo.getName(builtinID));
1834 return mlir::Value{};
1835 }
1836
1837 if (builtinID == AArch64::BI_CountLeadingOnes ||
1838 builtinID == AArch64::BI_CountLeadingOnes64 ||
1839 builtinID == AArch64::BI_CountLeadingZeros ||
1840 builtinID == AArch64::BI_CountLeadingZeros64) {
1841 cgm.errorNYI(expr->getSourceRange(),
1842 std::string("unimplemented AArch64 builtin call: ") +
1843 getContext().BuiltinInfo.getName(builtinID));
1844 return mlir::Value{};
1845 }
1846
1847 if (builtinID == AArch64::BI_CountLeadingSigns ||
1848 builtinID == AArch64::BI_CountLeadingSigns64) {
1849 cgm.errorNYI(expr->getSourceRange(),
1850 std::string("unimplemented AArch64 builtin call: ") +
1851 getContext().BuiltinInfo.getName(builtinID));
1852 return mlir::Value{};
1853 }
1854
1855 if (builtinID == AArch64::BI_CountOneBits ||
1856 builtinID == AArch64::BI_CountOneBits64) {
1857 cgm.errorNYI(expr->getSourceRange(),
1858 std::string("unimplemented AArch64 builtin call: ") +
1859 getContext().BuiltinInfo.getName(builtinID));
1860 return mlir::Value{};
1861 }
1862
1863 if (builtinID == AArch64::BI__prefetch) {
1864 cgm.errorNYI(expr->getSourceRange(),
1865 std::string("unimplemented AArch64 builtin call: ") +
1866 getContext().BuiltinInfo.getName(builtinID));
1867 return mlir::Value{};
1868 }
1869
1870 if (builtinID == AArch64::BI__hlt) {
1871 cgm.errorNYI(expr->getSourceRange(),
1872 std::string("unimplemented AArch64 builtin call: ") +
1873 getContext().BuiltinInfo.getName(builtinID));
1874 return mlir::Value{};
1875 }
1876
1877 if (builtinID == NEON::BI__builtin_neon_vcvth_bf16_f32) {
1878 cgm.errorNYI(expr->getSourceRange(),
1879 std::string("unimplemented AArch64 builtin call: ") +
1880 getContext().BuiltinInfo.getName(builtinID));
1881 return mlir::Value{};
1882 }
1883
1884 // Handle MSVC intrinsics before argument evaluation to prevent double
1885 // evaluation.
1886 assert(!cir::MissingFeatures::msvcBuiltins());
1887
1888 // Some intrinsics are equivalent - if they are use the base intrinsic ID.
1889 auto it = llvm::find_if(neonEquivalentIntrinsicMap, [builtinID](auto &p) {
1890 return p.first == builtinID;
1891 });
1892 if (it != end(neonEquivalentIntrinsicMap))
1893 builtinID = it->second;
1894
1895 // Find out if any arguments are required to be integer constant
1896 // expressions.
1897 assert(!cir::MissingFeatures::handleBuiltinICEArguments());
1898 unsigned iceArguments = 0;
1899 ASTContext::GetBuiltinTypeError error;
1900 getContext().GetBuiltinType(builtinID, error, &iceArguments);
1901 assert(error == ASTContext::GE_None && "Should not codegen an error");
1902 llvm::SmallVector<mlir::Value> ops;
1903
1904 // Skip extra arguments used to discriminate vector types and that are
1905 // intended for Sema checking.
1906 bool hasExtraArg = hasExtraNeonArgument(builtinID);
1907 unsigned numArgs = expr->getNumArgs() - (hasExtraArg ? 1 : 0);
1908 for (unsigned i = 0, e = numArgs; i != e; i++) {
1909 if (i == 0) {
1910 switch (builtinID) {
1911 case NEON::BI__builtin_neon_vld1_v:
1912 case NEON::BI__builtin_neon_vld1q_v:
1913 case NEON::BI__builtin_neon_vld1_dup_v:
1914 case NEON::BI__builtin_neon_vld1q_dup_v:
1915 case NEON::BI__builtin_neon_vld1_lane_v:
1916 case NEON::BI__builtin_neon_vld1q_lane_v:
1917 case NEON::BI__builtin_neon_vst1_v:
1918 case NEON::BI__builtin_neon_vst1q_v:
1919 case NEON::BI__builtin_neon_vst1_lane_v:
1920 case NEON::BI__builtin_neon_vst1q_lane_v:
1921 case NEON::BI__builtin_neon_vldap1_lane_s64:
1922 case NEON::BI__builtin_neon_vldap1q_lane_s64:
1923 case NEON::BI__builtin_neon_vstl1_lane_s64:
1924 case NEON::BI__builtin_neon_vstl1q_lane_s64:
1925 // Get the alignment for the argument in addition to the value;
1926 // we'll use it later.
1927 cgm.errorNYI(
1928 expr->getSourceRange(),
1929 std::string("unimplemented AArch64 builtin argument handling ") +
1930 getContext().BuiltinInfo.getName(builtinID));
1931 }
1932 }
1933 ops.push_back(
1934 emitScalarOrConstFoldImmArg(iceArguments, i, expr->getArg(i)));
1935 }
1936
1937 const ARMVectorIntrinsicInfo *builtin = findARMVectorIntrinsicInMap(
1938 AArch64SISDIntrinsicMap, builtinID, aarch64SISDIntrinsicsProvenSorted);
1939 if (builtin)
1940 return emitCommonNeonSISDBuiltinExpr(*this, *builtin, ops, expr);
1941
1942 // Not all intrinsics handled by the common case work for AArch64 yet, so only
1943 // defer to common code if it's been added to our special map.
1944 assert(!cir::MissingFeatures::aarch64SIMDIntrinsics());
1945
1946 assert(!cir::MissingFeatures::aarch64TblBuiltinExpr());
1947
1948 const Expr *arg = expr->getArg(expr->getNumArgs() - 1);
1949 NeonTypeFlags type(0);
1950 // A trailing constant integer is used for discriminating overloaded builtin
1951 // calls. Use it to determine the type of this overloaded NEON intrinsic.
1952 if (std::optional<llvm::APSInt> result =
1953 arg->getIntegerConstantExpr(getContext()))
1954 type = NeonTypeFlags(result->getZExtValue());
1955
1956 bool usgn = type.isUnsigned();
1957
1958 mlir::Location loc = getLoc(expr->getExprLoc());
1959
1960 // Not all intrinsics handled by the common case work for AArch64 yet, so only
1961 // defer to common code if it's been added to our special map.
1962 builtin = findARMVectorIntrinsicInMap(AArch64SIMDIntrinsicMap, builtinID,
1963 aarch64SIMDIntrinsicsProvenSorted);
1964
1965 if (builtin)
1966 return emitCommonNeonBuiltinExpr(
1967 *this, builtin->BuiltinID, builtin->LLVMIntrinsic,
1968 builtin->AltLLVMIntrinsic, builtin->NameHint, builtin->TypeModifier,
1969 expr, ops);
1970
1971 // Handle non-overloaded intrinsics first.
1972 switch (builtinID) {
1973 default:
1974 break;
1975 case NEON::BI__builtin_neon_vabsh_f16: {
1976 return cir::FAbsOp::create(builder, loc, ops);
1977 }
1978 case NEON::BI__builtin_neon_vaddq_p128:
1979 case NEON::BI__builtin_neon_vldrq_p128:
1980 case NEON::BI__builtin_neon_vstrq_p128:
1981 case NEON::BI__builtin_neon_vcvts_f32_u32:
1982 case NEON::BI__builtin_neon_vcvtd_f64_u64:
1983 case NEON::BI__builtin_neon_vcvts_f32_s32:
1984 case NEON::BI__builtin_neon_vcvtd_f64_s64:
1985 case NEON::BI__builtin_neon_vcvth_f16_u16:
1986 case NEON::BI__builtin_neon_vcvth_f16_u32:
1987 case NEON::BI__builtin_neon_vcvth_f16_u64:
1988 case NEON::BI__builtin_neon_vcvth_f16_s16:
1989 case NEON::BI__builtin_neon_vcvth_f16_s32:
1990 case NEON::BI__builtin_neon_vcvth_f16_s64:
1991 case NEON::BI__builtin_neon_vcvtah_u16_f16:
1992 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
1993 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
1994 case NEON::BI__builtin_neon_vcvtph_u16_f16:
1995 case NEON::BI__builtin_neon_vcvth_u16_f16:
1996 case NEON::BI__builtin_neon_vcvtah_s16_f16:
1997 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
1998 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
1999 case NEON::BI__builtin_neon_vcvtph_s16_f16:
2000 case NEON::BI__builtin_neon_vcvth_s16_f16:
2001 case NEON::BI__builtin_neon_vcaleh_f16:
2002 case NEON::BI__builtin_neon_vcalth_f16:
2003 case NEON::BI__builtin_neon_vcageh_f16:
2004 case NEON::BI__builtin_neon_vcagth_f16:
2005 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
2006 case NEON::BI__builtin_neon_vcvth_n_u16_f16:
2007 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
2008 case NEON::BI__builtin_neon_vcvth_n_f16_u16:
2009 case NEON::BI__builtin_neon_vpaddd_s64:
2010 case NEON::BI__builtin_neon_vpaddd_f64:
2011 case NEON::BI__builtin_neon_vpadds_f32:
2012 cgm.errorNYI(expr->getSourceRange(),
2013 std::string("unimplemented AArch64 builtin call: ") +
2014 getContext().BuiltinInfo.getName(builtinID));
2015 return mlir::Value{};
2016 case NEON::BI__builtin_neon_vceqzd_s64:
2017 case NEON::BI__builtin_neon_vceqzd_f64:
2018 case NEON::BI__builtin_neon_vceqzs_f32:
2019 case NEON::BI__builtin_neon_vceqzh_f16:
2020 return emitAArch64CompareBuiltinExpr(
2021 *this, builder, loc, ops[0],
2022 convertType(expr->getCallReturnType(getContext())), cir::CmpOpKind::eq);
2023 case NEON::BI__builtin_neon_vcgezd_s64:
2024 case NEON::BI__builtin_neon_vcgezd_f64:
2025 case NEON::BI__builtin_neon_vcgezs_f32:
2026 case NEON::BI__builtin_neon_vcgezh_f16:
2027 case NEON::BI__builtin_neon_vclezd_s64:
2028 case NEON::BI__builtin_neon_vclezd_f64:
2029 case NEON::BI__builtin_neon_vclezs_f32:
2030 case NEON::BI__builtin_neon_vclezh_f16:
2031 case NEON::BI__builtin_neon_vcgtzd_s64:
2032 case NEON::BI__builtin_neon_vcgtzd_f64:
2033 case NEON::BI__builtin_neon_vcgtzs_f32:
2034 case NEON::BI__builtin_neon_vcgtzh_f16:
2035 case NEON::BI__builtin_neon_vcltzd_s64:
2036 case NEON::BI__builtin_neon_vcltzd_f64:
2037 case NEON::BI__builtin_neon_vcltzs_f32:
2038 case NEON::BI__builtin_neon_vcltzh_f16:
2039 case NEON::BI__builtin_neon_vceqzd_u64: {
2040 return emitAArch64CompareBuiltinExpr(
2041 *this, builder, loc, ops[0],
2042 convertType(expr->getCallReturnType(getContext())), cir::CmpOpKind::eq);
2043 }
2044 case NEON::BI__builtin_neon_vceqd_f64:
2045 case NEON::BI__builtin_neon_vcled_f64:
2046 case NEON::BI__builtin_neon_vcltd_f64:
2047 case NEON::BI__builtin_neon_vcged_f64:
2048 case NEON::BI__builtin_neon_vcgtd_f64:
2049 case NEON::BI__builtin_neon_vceqs_f32:
2050 case NEON::BI__builtin_neon_vcles_f32:
2051 case NEON::BI__builtin_neon_vclts_f32:
2052 case NEON::BI__builtin_neon_vcges_f32:
2053 case NEON::BI__builtin_neon_vcgts_f32:
2054 case NEON::BI__builtin_neon_vceqh_f16:
2055 case NEON::BI__builtin_neon_vcleh_f16:
2056 case NEON::BI__builtin_neon_vclth_f16:
2057 case NEON::BI__builtin_neon_vcgeh_f16:
2058 case NEON::BI__builtin_neon_vcgth_f16:
2059 case NEON::BI__builtin_neon_vceqd_s64:
2060 case NEON::BI__builtin_neon_vceqd_u64:
2061 case NEON::BI__builtin_neon_vcgtd_s64:
2062 case NEON::BI__builtin_neon_vcgtd_u64:
2063 case NEON::BI__builtin_neon_vcltd_s64:
2064 case NEON::BI__builtin_neon_vcltd_u64:
2065 case NEON::BI__builtin_neon_vcged_u64:
2066 case NEON::BI__builtin_neon_vcged_s64:
2067 case NEON::BI__builtin_neon_vcled_u64:
2068 case NEON::BI__builtin_neon_vcled_s64:
2069 cgm.errorNYI(expr->getSourceRange(),
2070 std::string("unimplemented AArch64 builtin call: ") +
2071 getContext().BuiltinInfo.getName(builtinID));
2072 return mlir::Value{};
2073 case NEON::BI__builtin_neon_vnegd_s64: {
2074 return builder.createNeg(ops[0]);
2075 }
2076 case NEON::BI__builtin_neon_vnegh_f16: {
2077 return builder.createFNeg(ops[0]);
2078 }
2079 case NEON::BI__builtin_neon_vtstd_s64:
2080 case NEON::BI__builtin_neon_vtstd_u64:
2081 case NEON::BI__builtin_neon_vset_lane_i8:
2082 case NEON::BI__builtin_neon_vset_lane_i16:
2083 case NEON::BI__builtin_neon_vset_lane_i32:
2084 case NEON::BI__builtin_neon_vset_lane_i64:
2085 case NEON::BI__builtin_neon_vset_lane_bf16:
2086 case NEON::BI__builtin_neon_vset_lane_f32:
2087 case NEON::BI__builtin_neon_vsetq_lane_i8:
2088 case NEON::BI__builtin_neon_vsetq_lane_i16:
2089 case NEON::BI__builtin_neon_vsetq_lane_i32:
2090 case NEON::BI__builtin_neon_vsetq_lane_i64:
2091 case NEON::BI__builtin_neon_vsetq_lane_bf16:
2092 case NEON::BI__builtin_neon_vsetq_lane_f32:
2093 case NEON::BI__builtin_neon_vset_lane_f64:
2094 case NEON::BI__builtin_neon_vset_lane_mf8:
2095 case NEON::BI__builtin_neon_vsetq_lane_mf8:
2096 case NEON::BI__builtin_neon_vsetq_lane_f64:
2097 case NEON::BI__builtin_neon_vget_lane_i8:
2098 case NEON::BI__builtin_neon_vdupb_lane_i8:
2099 case NEON::BI__builtin_neon_vgetq_lane_i8:
2100 case NEON::BI__builtin_neon_vdupb_laneq_i8:
2101 case NEON::BI__builtin_neon_vget_lane_mf8:
2102 case NEON::BI__builtin_neon_vdupb_lane_mf8:
2103 case NEON::BI__builtin_neon_vgetq_lane_mf8:
2104 case NEON::BI__builtin_neon_vdupb_laneq_mf8:
2105 case NEON::BI__builtin_neon_vget_lane_i16:
2106 case NEON::BI__builtin_neon_vduph_lane_i16:
2107 case NEON::BI__builtin_neon_vgetq_lane_i16:
2108 case NEON::BI__builtin_neon_vduph_laneq_i16:
2109 case NEON::BI__builtin_neon_vget_lane_i32:
2110 case NEON::BI__builtin_neon_vdups_lane_i32:
2111 case NEON::BI__builtin_neon_vdups_lane_f32:
2112 case NEON::BI__builtin_neon_vgetq_lane_i32:
2113 case NEON::BI__builtin_neon_vdups_laneq_i32:
2114 case NEON::BI__builtin_neon_vget_lane_i64:
2115 case NEON::BI__builtin_neon_vdupd_lane_i64:
2116 case NEON::BI__builtin_neon_vdupd_lane_f64:
2117 case NEON::BI__builtin_neon_vgetq_lane_i64:
2118 case NEON::BI__builtin_neon_vdupd_laneq_i64:
2119 case NEON::BI__builtin_neon_vget_lane_f32:
2120 case NEON::BI__builtin_neon_vget_lane_f64:
2121 case NEON::BI__builtin_neon_vgetq_lane_f32:
2122 case NEON::BI__builtin_neon_vdups_laneq_f32:
2123 case NEON::BI__builtin_neon_vgetq_lane_f64:
2124 case NEON::BI__builtin_neon_vdupd_laneq_f64:
2125 case NEON::BI__builtin_neon_vaddh_f16:
2126 case NEON::BI__builtin_neon_vsubh_f16:
2127 case NEON::BI__builtin_neon_vmulh_f16:
2128 case NEON::BI__builtin_neon_vdivh_f16:
2129 cgm.errorNYI(expr->getSourceRange(),
2130 std::string("unimplemented AArch64 builtin call: ") +
2131 getContext().BuiltinInfo.getName(builtinID));
2132 return mlir::Value{};
2133 case NEON::BI__builtin_neon_vfmah_f16:
2134 // NEON intrinsic puts accumulator first, unlike the LLVM fma.
2135 std::rotate(ops.begin(), ops.begin() + 1, ops.end());
2136 return emitCallMaybeConstrainedBuiltin(builder, loc, "fma",
2137 convertType(expr->getType()), ops);
2138 break;
2139 case NEON::BI__builtin_neon_vfmsh_f16:
2140 // NEON intrinsic puts accumulator first, unlike the LLVM fma.
2141 std::rotate(ops.begin(), ops.begin() + 1, ops.end());
2142 ops[0] = builder.createFNeg(ops[0]);
2143 return emitCallMaybeConstrainedBuiltin(builder, loc, "fma",
2144 convertType(expr->getType()), ops);
2145 case NEON::BI__builtin_neon_vaddd_s64:
2146 case NEON::BI__builtin_neon_vaddd_u64:
2147 case NEON::BI__builtin_neon_vsubd_s64:
2148 case NEON::BI__builtin_neon_vsubd_u64:
2149 case NEON::BI__builtin_neon_vqdmlalh_s16:
2150 case NEON::BI__builtin_neon_vqdmlslh_s16:
2151 case NEON::BI__builtin_neon_vqshlud_n_s64:
2152 case NEON::BI__builtin_neon_vqshld_n_u64:
2153 case NEON::BI__builtin_neon_vqshld_n_s64:
2154 case NEON::BI__builtin_neon_vrshrd_n_u64:
2155 case NEON::BI__builtin_neon_vrshrd_n_s64:
2156 case NEON::BI__builtin_neon_vrsrad_n_u64:
2157 case NEON::BI__builtin_neon_vrsrad_n_s64:
2158 cgm.errorNYI(expr->getSourceRange(),
2159 std::string("unimplemented AArch64 builtin call: ") +
2160 getContext().BuiltinInfo.getName(builtinID));
2161 return mlir::Value{};
2162 case NEON::BI__builtin_neon_vshld_n_s64:
2163 case NEON::BI__builtin_neon_vshld_n_u64: {
2164 auto loc = getLoc(expr->getExprLoc());
2165 std::optional<llvm::APSInt> amt =
2166 expr->getArg(1)->getIntegerConstantExpr(getContext());
2167 assert(amt && "Expected argument to be a constant");
2168 return builder.createShiftLeft(loc, ops[0], amt->getZExtValue());
2169 }
2170 case NEON::BI__builtin_neon_vshrd_n_s64:
2171 case NEON::BI__builtin_neon_vshrd_n_u64:
2172 case NEON::BI__builtin_neon_vsrad_n_s64:
2173 case NEON::BI__builtin_neon_vsrad_n_u64:
2174 case NEON::BI__builtin_neon_vqdmlalh_lane_s16:
2175 case NEON::BI__builtin_neon_vqdmlalh_laneq_s16:
2176 case NEON::BI__builtin_neon_vqdmlslh_lane_s16:
2177 case NEON::BI__builtin_neon_vqdmlslh_laneq_s16:
2178 case NEON::BI__builtin_neon_vqdmlals_s32:
2179 case NEON::BI__builtin_neon_vqdmlsls_s32:
2180 case NEON::BI__builtin_neon_vqdmlals_lane_s32:
2181 case NEON::BI__builtin_neon_vqdmlals_laneq_s32:
2182 case NEON::BI__builtin_neon_vqdmlsls_lane_s32:
2183 case NEON::BI__builtin_neon_vqdmlsls_laneq_s32:
2184 case NEON::BI__builtin_neon_vget_lane_bf16:
2185 cgm.errorNYI(expr->getSourceRange(),
2186 std::string("unimplemented AArch64 builtin call: ") +
2187 getContext().BuiltinInfo.getName(builtinID));
2188 return mlir::Value{};
2189 case NEON::BI__builtin_neon_vduph_lane_bf16: {
2190 return cir::VecExtractOp::create(builder, loc, ops[0], ops[1]);
2191 }
2192 case NEON::BI__builtin_neon_vduph_lane_f16:
2193 case NEON::BI__builtin_neon_vgetq_lane_bf16:
2194 cgm.errorNYI(expr->getSourceRange(),
2195 std::string("unimplemented AArch64 builtin call: ") +
2196 getContext().BuiltinInfo.getName(builtinID));
2197 return mlir::Value{};
2198 case NEON::BI__builtin_neon_vduph_laneq_bf16: {
2199 return cir::VecExtractOp::create(builder, loc, ops[0], ops[1]);
2200 }
2201 case NEON::BI__builtin_neon_vduph_laneq_f16:
2202 case NEON::BI__builtin_neon_vcvt_bf16_f32:
2203 case NEON::BI__builtin_neon_vcvtq_low_bf16_f32:
2204 case NEON::BI__builtin_neon_vcvtq_high_bf16_f32:
2205 case clang::AArch64::BI_InterlockedAdd:
2206 case clang::AArch64::BI_InterlockedAdd_acq:
2207 case clang::AArch64::BI_InterlockedAdd_rel:
2208 case clang::AArch64::BI_InterlockedAdd_nf:
2209 case clang::AArch64::BI_InterlockedAdd64:
2210 case clang::AArch64::BI_InterlockedAdd64_acq:
2211 case clang::AArch64::BI_InterlockedAdd64_rel:
2212 case clang::AArch64::BI_InterlockedAdd64_nf:
2213 cgm.errorNYI(expr->getSourceRange(),
2214 std::string("unimplemented AArch64 builtin call: ") +
2215 getContext().BuiltinInfo.getName(builtinID));
2216 return mlir::Value{};
2217 }
2218
2219 cir::VectorType ty = getNeonType(this, type, loc);
2220 if (!ty)
2221 return nullptr;
2222
2223 llvm::StringRef intrName;
2224
2225 switch (builtinID) {
2226 default:
2227 return std::nullopt;
2228 case NEON::BI__builtin_neon_vbsl_v:
2229 case NEON::BI__builtin_neon_vbslq_v:
2230 case NEON::BI__builtin_neon_vfma_lane_v:
2231 case NEON::BI__builtin_neon_vfmaq_lane_v:
2232 case NEON::BI__builtin_neon_vfma_laneq_v:
2233 case NEON::BI__builtin_neon_vfmaq_laneq_v:
2234 case NEON::BI__builtin_neon_vfmah_lane_f16:
2235 case NEON::BI__builtin_neon_vfmas_lane_f32:
2236 case NEON::BI__builtin_neon_vfmah_laneq_f16:
2237 case NEON::BI__builtin_neon_vfmas_laneq_f32:
2238 case NEON::BI__builtin_neon_vfmad_lane_f64:
2239 case NEON::BI__builtin_neon_vfmad_laneq_f64:
2240 cgm.errorNYI(expr->getSourceRange(),
2241 std::string("unimplemented AArch64 builtin call: ") +
2242 getContext().BuiltinInfo.getName(builtinID));
2243 return mlir::Value{};
2244 case NEON::BI__builtin_neon_vmull_v: {
2245 intrName = usgn ? "aarch64.neon.umull" : "aarch64.neon.smull";
2246 if (type.isPoly())
2247 intrName = "aarch64.neon.pmull";
2248 cir::VectorType argTy = builder.getExtendedOrTruncatedElementVectorType(
2249 ty, /*isExtended*/ false, !usgn);
2250 return emitNeonCall(cgm, builder, {argTy, argTy}, ops, intrName, ty, loc);
2251 }
2252 case NEON::BI__builtin_neon_vmax_v:
2253 case NEON::BI__builtin_neon_vmaxq_v:
2254 intrName = usgn ? "aarch64.neon.umax" : "aarch64.neon.smax";
2255 if (cir::isFPOrVectorOfFPType(ty))
2256 intrName = "aarch64.neon.fmax";
2257 return emitNeonCall(cgm, builder, {ty, ty}, ops, intrName, ty, loc);
2258 case NEON::BI__builtin_neon_vmaxh_f16:
2259 case NEON::BI__builtin_neon_vmin_v:
2260 case NEON::BI__builtin_neon_vminq_v:
2261 case NEON::BI__builtin_neon_vminh_f16:
2262 cgm.errorNYI(expr->getSourceRange(),
2263 std::string("unimplemented AArch64 builtin call: ") +
2264 getContext().BuiltinInfo.getName(builtinID));
2265 return mlir::Value{};
2266 case NEON::BI__builtin_neon_vabd_v:
2267 case NEON::BI__builtin_neon_vabdq_v:
2268 intrName = usgn ? "aarch64.neon.uabd" : "aarch64.neon.sabd";
2269 if (cir::isFPOrVectorOfFPType(ty))
2270 intrName = "aarch64.neon.fabd";
2271 return emitNeonCall(cgm, builder, {ty, ty}, ops, intrName, ty, loc);
2272 case NEON::BI__builtin_neon_vpadal_v:
2273 case NEON::BI__builtin_neon_vpadalq_v:
2274 case NEON::BI__builtin_neon_vpmin_v:
2275 case NEON::BI__builtin_neon_vpminq_v:
2276 case NEON::BI__builtin_neon_vpmax_v:
2277 case NEON::BI__builtin_neon_vpmaxq_v:
2278 case NEON::BI__builtin_neon_vminnm_v:
2279 case NEON::BI__builtin_neon_vminnmq_v:
2280 case NEON::BI__builtin_neon_vminnmh_f16:
2281 cgm.errorNYI(expr->getSourceRange(),
2282 std::string("unimplemented AArch64 builtin call: ") +
2283 getContext().BuiltinInfo.getName(builtinID));
2284 return mlir::Value{};
2285 case NEON::BI__builtin_neon_vmaxnm_v:
2286 case NEON::BI__builtin_neon_vmaxnmq_v:
2287 intrName = "aarch64.neon.fmaxnm";
2288 return emitNeonCall(cgm, builder, {ty, ty}, ops, intrName, ty, loc);
2289 case NEON::BI__builtin_neon_vmaxnmh_f16:
2290 case NEON::BI__builtin_neon_vrecpss_f32:
2291 case NEON::BI__builtin_neon_vrecpsd_f64:
2292 case NEON::BI__builtin_neon_vrecpsh_f16:
2293 case NEON::BI__builtin_neon_vqshrun_n_v:
2294 case NEON::BI__builtin_neon_vqrshrun_n_v:
2295 case NEON::BI__builtin_neon_vqshrn_n_v:
2296 case NEON::BI__builtin_neon_vrshrn_n_v:
2297 case NEON::BI__builtin_neon_vqrshrn_n_v:
2298 case NEON::BI__builtin_neon_vrndah_f16:
2299 case NEON::BI__builtin_neon_vrnda_v:
2300 case NEON::BI__builtin_neon_vrndaq_v:
2301 case NEON::BI__builtin_neon_vrndih_f16:
2302 case NEON::BI__builtin_neon_vrndmh_f16:
2303 case NEON::BI__builtin_neon_vrndm_v:
2304 case NEON::BI__builtin_neon_vrndmq_v:
2305 case NEON::BI__builtin_neon_vrndnh_f16:
2306 case NEON::BI__builtin_neon_vrndn_v:
2307 case NEON::BI__builtin_neon_vrndnq_v:
2308 case NEON::BI__builtin_neon_vrndns_f32:
2309 case NEON::BI__builtin_neon_vrndph_f16:
2310 case NEON::BI__builtin_neon_vrndp_v:
2311 case NEON::BI__builtin_neon_vrndpq_v:
2312 case NEON::BI__builtin_neon_vrndxh_f16:
2313 case NEON::BI__builtin_neon_vrndx_v:
2314 case NEON::BI__builtin_neon_vrndxq_v:
2315 case NEON::BI__builtin_neon_vrndh_f16:
2316 case NEON::BI__builtin_neon_vrnd32x_f32:
2317 case NEON::BI__builtin_neon_vrnd32xq_f32:
2318 case NEON::BI__builtin_neon_vrnd32x_f64:
2319 case NEON::BI__builtin_neon_vrnd32xq_f64:
2320 case NEON::BI__builtin_neon_vrnd32z_f32:
2321 case NEON::BI__builtin_neon_vrnd32zq_f32:
2322 case NEON::BI__builtin_neon_vrnd32z_f64:
2323 case NEON::BI__builtin_neon_vrnd32zq_f64:
2324 case NEON::BI__builtin_neon_vrnd64x_f32:
2325 case NEON::BI__builtin_neon_vrnd64xq_f32:
2326 case NEON::BI__builtin_neon_vrnd64x_f64:
2327 case NEON::BI__builtin_neon_vrnd64xq_f64:
2328 case NEON::BI__builtin_neon_vrnd64z_f32:
2329 case NEON::BI__builtin_neon_vrnd64zq_f32:
2330 case NEON::BI__builtin_neon_vrnd64z_f64:
2331 case NEON::BI__builtin_neon_vrnd64zq_f64:
2332 case NEON::BI__builtin_neon_vrnd_v:
2333 case NEON::BI__builtin_neon_vrndq_v:
2334 case NEON::BI__builtin_neon_vcvt_f64_v:
2335 case NEON::BI__builtin_neon_vcvtq_f64_v:
2336 case NEON::BI__builtin_neon_vcvt_f64_f32:
2337 case NEON::BI__builtin_neon_vcvt_f32_f64:
2338 case NEON::BI__builtin_neon_vcvt_s32_v:
2339 case NEON::BI__builtin_neon_vcvt_u32_v:
2340 case NEON::BI__builtin_neon_vcvt_s64_v:
2341 case NEON::BI__builtin_neon_vcvt_u64_v:
2342 case NEON::BI__builtin_neon_vcvt_s16_f16:
2343 case NEON::BI__builtin_neon_vcvt_u16_f16:
2344 case NEON::BI__builtin_neon_vcvtq_s32_v:
2345 case NEON::BI__builtin_neon_vcvtq_u32_v:
2346 case NEON::BI__builtin_neon_vcvtq_s64_v:
2347 case NEON::BI__builtin_neon_vcvtq_u64_v:
2348 case NEON::BI__builtin_neon_vcvtq_s16_f16:
2349 case NEON::BI__builtin_neon_vcvtq_u16_f16:
2350 case NEON::BI__builtin_neon_vcvta_s16_f16:
2351 case NEON::BI__builtin_neon_vcvta_u16_f16:
2352 case NEON::BI__builtin_neon_vcvta_s32_v:
2353 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
2354 case NEON::BI__builtin_neon_vcvtaq_s32_v:
2355 case NEON::BI__builtin_neon_vcvta_u32_v:
2356 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
2357 case NEON::BI__builtin_neon_vcvtaq_u32_v:
2358 case NEON::BI__builtin_neon_vcvta_s64_v:
2359 case NEON::BI__builtin_neon_vcvtaq_s64_v:
2360 case NEON::BI__builtin_neon_vcvta_u64_v:
2361 case NEON::BI__builtin_neon_vcvtaq_u64_v:
2362 case NEON::BI__builtin_neon_vcvtm_s16_f16:
2363 case NEON::BI__builtin_neon_vcvtm_s32_v:
2364 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
2365 case NEON::BI__builtin_neon_vcvtmq_s32_v:
2366 case NEON::BI__builtin_neon_vcvtm_u16_f16:
2367 case NEON::BI__builtin_neon_vcvtm_u32_v:
2368 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
2369 case NEON::BI__builtin_neon_vcvtmq_u32_v:
2370 case NEON::BI__builtin_neon_vcvtm_s64_v:
2371 case NEON::BI__builtin_neon_vcvtmq_s64_v:
2372 case NEON::BI__builtin_neon_vcvtm_u64_v:
2373 case NEON::BI__builtin_neon_vcvtmq_u64_v:
2374 case NEON::BI__builtin_neon_vcvtn_s16_f16:
2375 case NEON::BI__builtin_neon_vcvtn_s32_v:
2376 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
2377 case NEON::BI__builtin_neon_vcvtnq_s32_v:
2378 case NEON::BI__builtin_neon_vcvtn_u16_f16:
2379 case NEON::BI__builtin_neon_vcvtn_u32_v:
2380 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
2381 case NEON::BI__builtin_neon_vcvtnq_u32_v:
2382 case NEON::BI__builtin_neon_vcvtn_s64_v:
2383 case NEON::BI__builtin_neon_vcvtnq_s64_v:
2384 case NEON::BI__builtin_neon_vcvtn_u64_v:
2385 case NEON::BI__builtin_neon_vcvtnq_u64_v:
2386 case NEON::BI__builtin_neon_vcvtp_s16_f16:
2387 case NEON::BI__builtin_neon_vcvtp_s32_v:
2388 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
2389 case NEON::BI__builtin_neon_vcvtpq_s32_v:
2390 case NEON::BI__builtin_neon_vcvtp_u16_f16:
2391 case NEON::BI__builtin_neon_vcvtp_u32_v:
2392 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
2393 case NEON::BI__builtin_neon_vcvtpq_u32_v:
2394 case NEON::BI__builtin_neon_vcvtp_s64_v:
2395 case NEON::BI__builtin_neon_vcvtpq_s64_v:
2396 case NEON::BI__builtin_neon_vcvtp_u64_v:
2397 case NEON::BI__builtin_neon_vcvtpq_u64_v:
2398 case NEON::BI__builtin_neon_vmulx_v:
2399 case NEON::BI__builtin_neon_vmulxq_v:
2400 case NEON::BI__builtin_neon_vmulxh_lane_f16:
2401 case NEON::BI__builtin_neon_vmulxh_laneq_f16:
2402 case NEON::BI__builtin_neon_vmul_lane_v:
2403 case NEON::BI__builtin_neon_vmul_laneq_v:
2404 case NEON::BI__builtin_neon_vpmaxnm_v:
2405 case NEON::BI__builtin_neon_vpmaxnmq_v:
2406 case NEON::BI__builtin_neon_vpminnm_v:
2407 case NEON::BI__builtin_neon_vpminnmq_v:
2408 case NEON::BI__builtin_neon_vsqrth_f16:
2409 case NEON::BI__builtin_neon_vsqrt_v:
2410 case NEON::BI__builtin_neon_vsqrtq_v:
2411 case NEON::BI__builtin_neon_vrbit_v:
2412 case NEON::BI__builtin_neon_vrbitq_v:
2413 case NEON::BI__builtin_neon_vmaxv_f16:
2414 case NEON::BI__builtin_neon_vmaxvq_f16:
2415 case NEON::BI__builtin_neon_vminv_f16:
2416 case NEON::BI__builtin_neon_vminvq_f16:
2417 case NEON::BI__builtin_neon_vmaxnmv_f16:
2418 case NEON::BI__builtin_neon_vmaxnmvq_f16:
2419 case NEON::BI__builtin_neon_vminnmv_f16:
2420 case NEON::BI__builtin_neon_vminnmvq_f16:
2421 case NEON::BI__builtin_neon_vmul_n_f64:
2422 case NEON::BI__builtin_neon_vaddlv_u8:
2423 case NEON::BI__builtin_neon_vaddlv_u16:
2424 case NEON::BI__builtin_neon_vaddlvq_u8:
2425 case NEON::BI__builtin_neon_vaddlvq_u16:
2426 case NEON::BI__builtin_neon_vaddlv_s8:
2427 case NEON::BI__builtin_neon_vaddlv_s16:
2428 case NEON::BI__builtin_neon_vaddlvq_s8:
2429 case NEON::BI__builtin_neon_vaddlvq_s16:
2430 case NEON::BI__builtin_neon_vsri_n_v:
2431 case NEON::BI__builtin_neon_vsriq_n_v:
2432 case NEON::BI__builtin_neon_vsli_n_v:
2433 case NEON::BI__builtin_neon_vsliq_n_v:
2434 case NEON::BI__builtin_neon_vsra_n_v:
2435 case NEON::BI__builtin_neon_vsraq_n_v:
2436 case NEON::BI__builtin_neon_vrsra_n_v:
2437 case NEON::BI__builtin_neon_vrsraq_n_v:
2438 case NEON::BI__builtin_neon_vld1_v:
2439 case NEON::BI__builtin_neon_vld1q_v:
2440 case NEON::BI__builtin_neon_vst1_v:
2441 case NEON::BI__builtin_neon_vst1q_v:
2442 case NEON::BI__builtin_neon_vld1_lane_v:
2443 case NEON::BI__builtin_neon_vld1q_lane_v:
2444 case NEON::BI__builtin_neon_vldap1_lane_s64:
2445 case NEON::BI__builtin_neon_vldap1q_lane_s64:
2446 case NEON::BI__builtin_neon_vld1_dup_v:
2447 case NEON::BI__builtin_neon_vld1q_dup_v:
2448 case NEON::BI__builtin_neon_vst1_lane_v:
2449 case NEON::BI__builtin_neon_vst1q_lane_v:
2450 case NEON::BI__builtin_neon_vstl1_lane_s64:
2451 case NEON::BI__builtin_neon_vstl1q_lane_s64:
2452 case NEON::BI__builtin_neon_vld2_v:
2453 case NEON::BI__builtin_neon_vld2q_v:
2454 case NEON::BI__builtin_neon_vld3_v:
2455 case NEON::BI__builtin_neon_vld3q_v:
2456 case NEON::BI__builtin_neon_vld4_v:
2457 case NEON::BI__builtin_neon_vld4q_v:
2458 case NEON::BI__builtin_neon_vld2_dup_v:
2459 case NEON::BI__builtin_neon_vld2q_dup_v:
2460 case NEON::BI__builtin_neon_vld3_dup_v:
2461 case NEON::BI__builtin_neon_vld3q_dup_v:
2462 case NEON::BI__builtin_neon_vld4_dup_v:
2463 case NEON::BI__builtin_neon_vld4q_dup_v:
2464 case NEON::BI__builtin_neon_vld2_lane_v:
2465 case NEON::BI__builtin_neon_vld2q_lane_v:
2466 case NEON::BI__builtin_neon_vld3_lane_v:
2467 case NEON::BI__builtin_neon_vld3q_lane_v:
2468 case NEON::BI__builtin_neon_vld4_lane_v:
2469 case NEON::BI__builtin_neon_vld4q_lane_v:
2470 case NEON::BI__builtin_neon_vst2_v:
2471 case NEON::BI__builtin_neon_vst2q_v:
2472 case NEON::BI__builtin_neon_vst2_lane_v:
2473 case NEON::BI__builtin_neon_vst2q_lane_v:
2474 case NEON::BI__builtin_neon_vst3_v:
2475 case NEON::BI__builtin_neon_vst3q_v:
2476 case NEON::BI__builtin_neon_vst3_lane_v:
2477 case NEON::BI__builtin_neon_vst3q_lane_v:
2478 case NEON::BI__builtin_neon_vst4_v:
2479 case NEON::BI__builtin_neon_vst4q_v:
2480 case NEON::BI__builtin_neon_vst4_lane_v:
2481 case NEON::BI__builtin_neon_vst4q_lane_v:
2482 case NEON::BI__builtin_neon_vtrn_v:
2483 case NEON::BI__builtin_neon_vtrnq_v:
2484 case NEON::BI__builtin_neon_vuzp_v:
2485 case NEON::BI__builtin_neon_vuzpq_v:
2486 case NEON::BI__builtin_neon_vzip_v:
2487 case NEON::BI__builtin_neon_vzipq_v:
2488 case NEON::BI__builtin_neon_vqtbl1q_v:
2489 case NEON::BI__builtin_neon_vqtbl2q_v:
2490 case NEON::BI__builtin_neon_vqtbl3q_v:
2491 case NEON::BI__builtin_neon_vqtbl4q_v:
2492 case NEON::BI__builtin_neon_vqtbx1q_v:
2493 case NEON::BI__builtin_neon_vqtbx2q_v:
2494 case NEON::BI__builtin_neon_vqtbx3q_v:
2495 case NEON::BI__builtin_neon_vqtbx4q_v:
2496 case NEON::BI__builtin_neon_vsqadd_v:
2497 case NEON::BI__builtin_neon_vsqaddq_v:
2498 case NEON::BI__builtin_neon_vuqadd_v:
2499 case NEON::BI__builtin_neon_vuqaddq_v:
2500 case NEON::BI__builtin_neon_vluti2_laneq_mf8:
2501 case NEON::BI__builtin_neon_vluti2_laneq_bf16:
2502 case NEON::BI__builtin_neon_vluti2_laneq_f16:
2503 case NEON::BI__builtin_neon_vluti2_laneq_p16:
2504 case NEON::BI__builtin_neon_vluti2_laneq_p8:
2505 case NEON::BI__builtin_neon_vluti2_laneq_s16:
2506 case NEON::BI__builtin_neon_vluti2_laneq_s8:
2507 case NEON::BI__builtin_neon_vluti2_laneq_u16:
2508 case NEON::BI__builtin_neon_vluti2_laneq_u8:
2509 case NEON::BI__builtin_neon_vluti2q_laneq_mf8:
2510 case NEON::BI__builtin_neon_vluti2q_laneq_bf16:
2511 case NEON::BI__builtin_neon_vluti2q_laneq_f16:
2512 case NEON::BI__builtin_neon_vluti2q_laneq_p16:
2513 case NEON::BI__builtin_neon_vluti2q_laneq_p8:
2514 case NEON::BI__builtin_neon_vluti2q_laneq_s16:
2515 case NEON::BI__builtin_neon_vluti2q_laneq_s8:
2516 case NEON::BI__builtin_neon_vluti2q_laneq_u16:
2517 case NEON::BI__builtin_neon_vluti2q_laneq_u8:
2518 case NEON::BI__builtin_neon_vluti2_lane_mf8:
2519 case NEON::BI__builtin_neon_vluti2_lane_bf16:
2520 case NEON::BI__builtin_neon_vluti2_lane_f16:
2521 case NEON::BI__builtin_neon_vluti2_lane_p16:
2522 case NEON::BI__builtin_neon_vluti2_lane_p8:
2523 case NEON::BI__builtin_neon_vluti2_lane_s16:
2524 case NEON::BI__builtin_neon_vluti2_lane_s8:
2525 case NEON::BI__builtin_neon_vluti2_lane_u16:
2526 case NEON::BI__builtin_neon_vluti2_lane_u8:
2527 case NEON::BI__builtin_neon_vluti2q_lane_mf8:
2528 case NEON::BI__builtin_neon_vluti2q_lane_bf16:
2529 case NEON::BI__builtin_neon_vluti2q_lane_f16:
2530 case NEON::BI__builtin_neon_vluti2q_lane_p16:
2531 case NEON::BI__builtin_neon_vluti2q_lane_p8:
2532 case NEON::BI__builtin_neon_vluti2q_lane_s16:
2533 case NEON::BI__builtin_neon_vluti2q_lane_s8:
2534 case NEON::BI__builtin_neon_vluti2q_lane_u16:
2535 case NEON::BI__builtin_neon_vluti2q_lane_u8:
2536 case NEON::BI__builtin_neon_vluti4q_lane_mf8:
2537 case NEON::BI__builtin_neon_vluti4q_lane_p8:
2538 case NEON::BI__builtin_neon_vluti4q_lane_s8:
2539 case NEON::BI__builtin_neon_vluti4q_lane_u8:
2540 case NEON::BI__builtin_neon_vluti4q_laneq_mf8:
2541 case NEON::BI__builtin_neon_vluti4q_laneq_p8:
2542 case NEON::BI__builtin_neon_vluti4q_laneq_s8:
2543 case NEON::BI__builtin_neon_vluti4q_laneq_u8:
2544 case NEON::BI__builtin_neon_vluti4q_lane_bf16_x2:
2545 case NEON::BI__builtin_neon_vluti4q_lane_f16_x2:
2546 case NEON::BI__builtin_neon_vluti4q_lane_p16_x2:
2547 case NEON::BI__builtin_neon_vluti4q_lane_s16_x2:
2548 case NEON::BI__builtin_neon_vluti4q_lane_u16_x2:
2549 case NEON::BI__builtin_neon_vluti4q_laneq_bf16_x2:
2550 case NEON::BI__builtin_neon_vluti4q_laneq_f16_x2:
2551 case NEON::BI__builtin_neon_vluti4q_laneq_p16_x2:
2552 case NEON::BI__builtin_neon_vluti4q_laneq_s16_x2:
2553 case NEON::BI__builtin_neon_vluti4q_laneq_u16_x2:
2554 case NEON::BI__builtin_neon_vmmlaq_f16_mf8_fpm:
2555 case NEON::BI__builtin_neon_vmmlaq_f32_mf8_fpm:
2556 case NEON::BI__builtin_neon_vcvt1_low_bf16_mf8_fpm:
2557 case NEON::BI__builtin_neon_vcvt1_bf16_mf8_fpm:
2558 case NEON::BI__builtin_neon_vcvt1_high_bf16_mf8_fpm:
2559 case NEON::BI__builtin_neon_vcvt2_low_bf16_mf8_fpm:
2560 case NEON::BI__builtin_neon_vcvt2_bf16_mf8_fpm:
2561 case NEON::BI__builtin_neon_vcvt2_high_bf16_mf8_fpm:
2562 case NEON::BI__builtin_neon_vcvt1_low_f16_mf8_fpm:
2563 case NEON::BI__builtin_neon_vcvt1_f16_mf8_fpm:
2564 case NEON::BI__builtin_neon_vcvt1_high_f16_mf8_fpm:
2565 case NEON::BI__builtin_neon_vcvt2_low_f16_mf8_fpm:
2566 case NEON::BI__builtin_neon_vcvt2_f16_mf8_fpm:
2567 case NEON::BI__builtin_neon_vcvt2_high_f16_mf8_fpm:
2568 case NEON::BI__builtin_neon_vcvt_mf8_f32_fpm:
2569 case NEON::BI__builtin_neon_vcvt_mf8_f16_fpm:
2570 case NEON::BI__builtin_neon_vcvtq_mf8_f16_fpm:
2571 case NEON::BI__builtin_neon_vcvt_high_mf8_f32_fpm:
2572 case NEON::BI__builtin_neon_vdot_f16_mf8_fpm:
2573 case NEON::BI__builtin_neon_vdotq_f16_mf8_fpm:
2574 case NEON::BI__builtin_neon_vdot_lane_f16_mf8_fpm:
2575 case NEON::BI__builtin_neon_vdotq_lane_f16_mf8_fpm:
2576 case NEON::BI__builtin_neon_vdot_laneq_f16_mf8_fpm:
2577 case NEON::BI__builtin_neon_vdotq_laneq_f16_mf8_fpm:
2578 case NEON::BI__builtin_neon_vdot_f32_mf8_fpm:
2579 case NEON::BI__builtin_neon_vdotq_f32_mf8_fpm:
2580 case NEON::BI__builtin_neon_vdot_lane_f32_mf8_fpm:
2581 case NEON::BI__builtin_neon_vdotq_lane_f32_mf8_fpm:
2582 case NEON::BI__builtin_neon_vdot_laneq_f32_mf8_fpm:
2583 case NEON::BI__builtin_neon_vdotq_laneq_f32_mf8_fpm:
2584 case NEON::BI__builtin_neon_vmlalbq_f16_mf8_fpm:
2585 case NEON::BI__builtin_neon_vmlaltq_f16_mf8_fpm:
2586 case NEON::BI__builtin_neon_vmlallbbq_f32_mf8_fpm:
2587 case NEON::BI__builtin_neon_vmlallbtq_f32_mf8_fpm:
2588 case NEON::BI__builtin_neon_vmlalltbq_f32_mf8_fpm:
2589 case NEON::BI__builtin_neon_vmlallttq_f32_mf8_fpm:
2590 case NEON::BI__builtin_neon_vmlalbq_lane_f16_mf8_fpm:
2591 case NEON::BI__builtin_neon_vmlalbq_laneq_f16_mf8_fpm:
2592 case NEON::BI__builtin_neon_vmlaltq_lane_f16_mf8_fpm:
2593 case NEON::BI__builtin_neon_vmlaltq_laneq_f16_mf8_fpm:
2594 case NEON::BI__builtin_neon_vmlallbbq_lane_f32_mf8_fpm:
2595 case NEON::BI__builtin_neon_vmlallbbq_laneq_f32_mf8_fpm:
2596 case NEON::BI__builtin_neon_vmlallbtq_lane_f32_mf8_fpm:
2597 case NEON::BI__builtin_neon_vmlallbtq_laneq_f32_mf8_fpm:
2598 case NEON::BI__builtin_neon_vmlalltbq_lane_f32_mf8_fpm:
2599 case NEON::BI__builtin_neon_vmlalltbq_laneq_f32_mf8_fpm:
2600 case NEON::BI__builtin_neon_vmlallttq_lane_f32_mf8_fpm:
2601 case NEON::BI__builtin_neon_vmlallttq_laneq_f32_mf8_fpm:
2602 case NEON::BI__builtin_neon_vamin_f16:
2603 case NEON::BI__builtin_neon_vaminq_f16:
2604 case NEON::BI__builtin_neon_vamin_f32:
2605 case NEON::BI__builtin_neon_vaminq_f32:
2606 case NEON::BI__builtin_neon_vaminq_f64:
2607 case NEON::BI__builtin_neon_vamax_f16:
2608 case NEON::BI__builtin_neon_vamaxq_f16:
2609 case NEON::BI__builtin_neon_vamax_f32:
2610 case NEON::BI__builtin_neon_vamaxq_f32:
2611 case NEON::BI__builtin_neon_vamaxq_f64:
2612 case NEON::BI__builtin_neon_vscale_f16:
2613 case NEON::BI__builtin_neon_vscaleq_f16:
2614 case NEON::BI__builtin_neon_vscale_f32:
2615 case NEON::BI__builtin_neon_vscaleq_f32:
2616 case NEON::BI__builtin_neon_vscaleq_f64:
2617 cgm.errorNYI(expr->getSourceRange(),
2618 std::string("unimplemented AArch64 builtin call: ") +
2619 getContext().BuiltinInfo.getName(builtinID));
2620 return mlir::Value{};
2621 }
2622
2623 // Unreachable: All cases in the switch above return.
2624}
AArch64CodeGenUtils.h
Utilities used for generating code for AArch64 that are shared between the classic and ClangIR code-g...
Builtins.h
Defines enum values for all the target-independent builtin functions.
emitCommonNeonSISDBuiltinExpr
static mlir::Value emitCommonNeonSISDBuiltinExpr(CIRGenFunction &cgf, const ARMVectorIntrinsicInfo &info, llvm::SmallVectorImpl< mlir::Value > &ops, const CallExpr *expr)
Definition CIRGenBuiltinAArch64.cpp:722
hasExtraNeonArgument
static bool hasExtraNeonArgument(unsigned builtinID)
Return true if BuiltinID is an overloaded Neon intrinsic with an extra argument that specifies the ve...
Definition CIRGenBuiltinAArch64.cpp:641
aarch64SVEIntrinsicsProvenSorted
static bool aarch64SVEIntrinsicsProvenSorted
Definition CIRGenBuiltinAArch64.cpp:59
neonEquivalentIntrinsicMap
static const std::pair< unsigned, unsigned > neonEquivalentIntrinsicMap[]
Definition CIRGenBuiltinAArch64.cpp:1061
emitNeonSplat
static mlir::Value emitNeonSplat(CIRGenBuilderTy &builder, mlir::Location loc, mlir::Value v, mlir::Value lane, unsigned int resEltCnt)
Definition CIRGenBuiltinAArch64.cpp:175
emitNeonCallToOp
static mlir::Value emitNeonCallToOp(CIRGenModule &cgm, CIRGenBuilderTy &builder, llvm::SmallVector< mlir::Type > argTypes, llvm::SmallVectorImpl< mlir::Value > &args, std::optional< llvm::StringRef > intrinsicName, mlir::Type funcResTy, mlir::Location loc, bool isConstrainedFPIntrinsic=false, unsigned shift=0, bool rightshift=false)
Definition CIRGenBuiltinAArch64.cpp:672
aarch64SVEIntrinsicMap
static const ARMVectorIntrinsicInfo aarch64SVEIntrinsicMap[]
Definition CIRGenBuiltinAArch64.cpp:51
getSVEVectorForElementType
static cir::VectorType getSVEVectorForElementType(CIRGenModule &cgm, mlir::Type eltTy)
Definition CIRGenBuiltinAArch64.cpp:625
getSVEMinEltCount
static unsigned getSVEMinEltCount(clang::SVETypeFlags::EltType sveType)
Definition CIRGenBuiltinAArch64.cpp:587
genVscaleTimesFactor
static mlir::Value genVscaleTimesFactor(mlir::Location loc, CIRGenBuilderTy builder, mlir::Type cirTy, int32_t scalingFactor)
Definition CIRGenBuiltinAArch64.cpp:36
getLLVMIntrNameNoPrefix
static llvm::StringRef getLLVMIntrNameNoPrefix(llvm::Intrinsic::ID intrID)
Definition CIRGenBuiltinAArch64.cpp:540
getIntValueFromConstOp
static int64_t getIntValueFromConstOp(mlir::Value val)
Definition CIRGenBuiltinAArch64.cpp:171
emitCallMaybeConstrainedBuiltin
static mlir::Value emitCallMaybeConstrainedBuiltin(CIRGenBuilderTy &builder, mlir::Location loc, StringRef intrName, mlir::Type retTy, llvm::SmallVector< mlir::Value > &ops)
Definition CIRGenBuiltinAArch64.cpp:505
emitCommonNeonBuiltinExpr
static mlir::Value emitCommonNeonBuiltinExpr(CIRGenFunction &cgf, unsigned builtinID, unsigned llvmIntrinsic, unsigned altLLVMIntrinsic, const char *nameHint, unsigned modifier, const CallExpr *expr, llvm::SmallVectorImpl< mlir::Value > &ops)
Definition CIRGenBuiltinAArch64.cpp:185
emitNeonCall
static mlir::Value emitNeonCall(CIRGenModule &cgm, CIRGenBuilderTy &builder, llvm::SmallVector< mlir::Type > argTypes, llvm::SmallVectorImpl< mlir::Value > &args, llvm::StringRef intrinsicName, mlir::Type funcResTy, mlir::Location loc, bool isConstrainedFPIntrinsic=false, unsigned shift=0, bool rightshift=false)
Definition CIRGenBuiltinAArch64.cpp:710
findARMVectorIntrinsicInMap
static const ARMVectorIntrinsicInfo * findARMVectorIntrinsicInMap(ArrayRef< ARMVectorIntrinsicInfo > intrinsicMap, unsigned builtinID, bool &mapProvenSorted)
Definition CIRGenBuiltinAArch64.cpp:64
aarch64SIMDIntrinsicsProvenSorted
static bool aarch64SIMDIntrinsicsProvenSorted
Definition CIRGenBuiltinAArch64.cpp:57
sveBitsPerBlock
constexpr unsigned sveBitsPerBlock
Definition CIRGenBuiltinAArch64.cpp:623
emitAArch64CompareBuiltinExpr
static mlir::Value emitAArch64CompareBuiltinExpr(CIRGenFunction &cgf, CIRGenBuilderTy &builder, mlir::Location loc, mlir::Value src, mlir::Type retTy, const cir::CmpOpKind kind)
Definition CIRGenBuiltinAArch64.cpp:87
getNeonType
static cir::VectorType getNeonType(CIRGenFunction *cgf, NeonTypeFlags typeFlags, mlir::Location loc, bool hasLegalHalfType=true, bool v1Ty=false, bool allowBFloatArgsAndRet=true)
Definition CIRGenBuiltinAArch64.cpp:115
aarch64SISDIntrinsicsProvenSorted
static bool aarch64SISDIntrinsicsProvenSorted
Definition CIRGenBuiltinAArch64.cpp:58
getType
TokenType getType() const
Returns the token's type, e.g.
Definition FormatToken.h:984
kind
*collection of selector each with an associated kind and an ordered *collection of selectors A selector has a kind
Definition OpenMPClause.h:9714
TargetBuiltins.h
Enumerates target-specific builtins in their own namespaces within namespace clang.
cir::CIRBaseBuilderTy::getNullValue
cir::ConstantOp getNullValue(mlir::Type ty, mlir::Location loc)
Definition CIRBaseBuilder.h:98
cir::CIRBaseBuilderTy::createCast
mlir::Value createCast(mlir::Location loc, cir::CastKind kind, mlir::Value src, mlir::Type newTy)
Definition CIRBaseBuilder.h:497
cir::CIRBaseBuilderTy::createNUWAMul
mlir::Value createNUWAMul(mlir::Location loc, mlir::Value lhs, mlir::Value rhs)
Definition CIRBaseBuilder.h:635
cir::CIRBaseBuilderTy::createVecCompare
cir::VecCmpOp createVecCompare(mlir::Location loc, cir::CmpOpKind kind, mlir::Value lhs, mlir::Value rhs)
Definition CIRBaseBuilder.h:699
cir::CIRBaseBuilderTy::createBitcast
mlir::Value createBitcast(mlir::Value src, mlir::Type newTy)
Definition CIRBaseBuilder.h:531
cir::CIRBaseBuilderTy::createCompare
cir::CmpOp createCompare(mlir::Location loc, cir::CmpOpKind kind, mlir::Value lhs, mlir::Value rhs)
Definition CIRBaseBuilder.h:694
cir::CIRDataLayout::getTypeSizeInBits
llvm::TypeSize getTypeSizeInBits(mlir::Type ty) const
Definition CIRDataLayout.cpp:46
clang::ASTContext
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition ASTContext.h:226
clang::ASTContext::BuiltinInfo
Builtin::Context & BuiltinInfo
Definition ASTContext.h:800
clang::ASTContext::GetBuiltinType
QualType GetBuiltinType(unsigned ID, GetBuiltinTypeError &Error, unsigned *IntegerConstantArgs=nullptr) const
Return the type for the specified builtin.
Definition ASTContext.cpp:12883
clang::ASTContext::GE_None
@ GE_None
No error.
Definition ASTContext.h:2605
clang::Builtin::Context::getName
std::string getName(unsigned ID) const
Return the identifier name for the specified builtin, e.g.
Definition Builtins.cpp:80
clang::CIRGen::CIRGenBuilderTy::getUInt64
cir::ConstantOp getUInt64(uint64_t c, mlir::Location loc)
Definition CIRGenBuilder.h:460
clang::CIRGen::CIRGenBuilderTy::emitIntrinsicCallOp
mlir::Value emitIntrinsicCallOp(mlir::Location loc, const llvm::StringRef str, const mlir::Type &resTy, Operands &&...op)
Definition CIRGenBuilder.h:869
clang::CIRGen::CIRGenBuilderTy::getSIntNTy
cir::IntType getSIntNTy(int n)
Definition CIRGenBuilder.h:344
clang::CIRGen::CIRGenBuilderTy::createVecShuffle
cir::VecShuffleOp createVecShuffle(mlir::Location loc, mlir::Value vec1, mlir::Value vec2, llvm::ArrayRef< mlir::Attribute > maskAttrs)
Definition CIRGenBuilder.h:840
clang::CIRGen::CIRGenFunction::convertType
mlir::Type convertType(clang::QualType t)
Definition CIRGenFunction.cpp:102
clang::CIRGen::CIRGenFunction::getTarget
const TargetInfo & getTarget() const
Definition CIRGenFunction.h:233
clang::CIRGen::CIRGenFunction::getLoc
mlir::Location getLoc(clang::SourceLocation srcLoc)
Helpers to convert Clang's SourceLocation to a MLIR Location.
Definition CIRGenFunction.cpp:106
clang::CIRGen::CIRGenFunction::emitSVEPredicateCast
mlir::Value emitSVEPredicateCast(mlir::Value pred, unsigned minNumElts, mlir::Location loc)
Definition CIRGenBuiltinAArch64.cpp:548
clang::CIRGen::CIRGenFunction::getAArch64SVEProcessedOperands
bool getAArch64SVEProcessedOperands(unsigned builtinID, const CallExpr *expr, SmallVectorImpl< mlir::Value > &ops, clang::SVETypeFlags typeFlags)
Definition CIRGenBuiltinAArch64.cpp:513
clang::CIRGen::CIRGenFunction::returnValue
Address returnValue
The temporary alloca to hold the return value.
Definition CIRGenFunction.h:90
clang::CIRGen::CIRGenFunction::emitAArch64BuiltinExpr
std::optional< mlir::Value > emitAArch64BuiltinExpr(unsigned builtinID, const CallExpr *expr, ReturnValueSlot returnValue, llvm::Triple::ArchType arch)
Definition CIRGenBuiltinAArch64.cpp:1456
clang::CIRGen::CIRGenFunction::emitAArch64SMEBuiltinExpr
std::optional< mlir::Value > emitAArch64SMEBuiltinExpr(unsigned builtinID, const CallExpr *expr)
Definition CIRGenBuiltinAArch64.cpp:1050
clang::CIRGen::CIRGenFunction::emitScalarExpr
mlir::Value emitScalarExpr(const clang::Expr *e, bool ignoreResultAssign=false)
Emit the computation of the specified expression of scalar type.
Definition CIRGenExprScalar.cpp:1658
clang::CIRGen::CIRGenFunction::getBuilder
CIRGenBuilderTy & getBuilder()
Definition CIRGenFunction.h:145
clang::CIRGen::CIRGenFunction::getContext
clang::ASTContext & getContext() const
Definition CIRGenFunction.h:143
clang::CIRGen::CIRGenFunction::emitAArch64SVEBuiltinExpr
std::optional< mlir::Value > emitAArch64SVEBuiltinExpr(unsigned builtinID, const CallExpr *expr)
Definition CIRGenBuiltinAArch64.cpp:775
clang::CIRGen::CIRGenFunction::emitScalarOrConstFoldImmArg
mlir::Value emitScalarOrConstFoldImmArg(unsigned iceArguments, unsigned idx, const Expr *argExpr)
Definition CIRGenBuiltin.cpp:2412
clang::CIRGen::CIRGenModule
This class organizes the cross-function state that is used while generating CIR code.
Definition CIRGenModule.h:58
clang::CIRGen::CIRGenModule::errorNYI
DiagnosticBuilder errorNYI(SourceLocation, llvm::StringRef)
Helpers to emit "not yet implemented" error diagnostics.
Definition CIRGenModule.cpp:3172
clang::CIRGen::CIRGenModule::getDataLayout
const cir::CIRDataLayout getDataLayout() const
Definition CIRGenModule.h:145
clang::CIRGen::ReturnValueSlot
Contains the address where the return value of a function can be stored, and whether the address is v...
Definition CIRGenCall.h:260
clang::CallExpr
CallExpr - Represents a function call (C99 6.5.2.2, C++ [expr.call]).
Definition Expr.h:2946
clang::Expr
This represents one expression.
Definition Expr.h:112
clang::NeonTypeFlags
Flags to identify the types for overloaded Neon builtins.
Definition TargetBuiltins.h:253
clang::NeonTypeFlags::getEltType
EltType getEltType() const
Definition TargetBuiltins.h:286
clang::SVETypeFlags
Flags to identify the types for overloaded SVE builtins.
Definition TargetBuiltins.h:330
clang::SVETypeFlags::isReverseUSDOT
bool isReverseUSDOT() const
Definition TargetBuiltins.h:409
clang::SVETypeFlags::isGatherLoad
bool isGatherLoad() const
Definition TargetBuiltins.h:389
clang::SVETypeFlags::getEltType
EltType getEltType() const
Definition TargetBuiltins.h:367
clang::SVETypeFlags::isPrefetch
bool isPrefetch() const
Definition TargetBuiltins.h:404
clang::SVETypeFlags::isTupleSet
bool isTupleSet() const
Definition TargetBuiltins.h:415
clang::SVETypeFlags::isReverseMergeAnyAccOp
bool isReverseMergeAnyAccOp() const
Definition TargetBuiltins.h:411
clang::SVETypeFlags::isTupleGet
bool isTupleGet() const
Definition TargetBuiltins.h:414
clang::SVETypeFlags::isInsertOp1SVALL
bool isInsertOp1SVALL() const
Definition TargetBuiltins.h:407
clang::SVETypeFlags::isAppendSVALL
bool isAppendSVALL() const
Definition TargetBuiltins.h:406
clang::SVETypeFlags::isReverseMergeAnyBinOp
bool isReverseMergeAnyBinOp() const
Definition TargetBuiltins.h:410
clang::SVETypeFlags::isStructStore
bool isStructStore() const
Definition TargetBuiltins.h:392
clang::SVETypeFlags::isTupleCreate
bool isTupleCreate() const
Definition TargetBuiltins.h:413
clang::SVETypeFlags::isGatherPrefetch
bool isGatherPrefetch() const
Definition TargetBuiltins.h:408
clang::SVETypeFlags::hasSplatOperand
bool hasSplatOperand() const
Definition TargetBuiltins.h:383
clang::SVETypeFlags::getMergeType
MergeType getMergeType() const
Definition TargetBuiltins.h:375
clang::SVETypeFlags::isStructLoad
bool isStructLoad() const
Definition TargetBuiltins.h:391
clang::SVETypeFlags::getSplatOperand
unsigned getSplatOperand() const
Definition TargetBuiltins.h:379
clang::SVETypeFlags::isScatterStore
bool isScatterStore() const
Definition TargetBuiltins.h:390
clang::SVETypeFlags::isReverseCompare
bool isReverseCompare() const
Definition TargetBuiltins.h:405
clang::TargetInfo::hasFastHalfType
virtual bool hasFastHalfType() const
Determine whether the target has fast native support for operations on half types.
Definition TargetInfo.h:712
clang::aarch64::AArch64SISDIntrinsicMap
const ARMVectorIntrinsicInfo AArch64SISDIntrinsicMap[]
Definition AArch64CodeGenUtils.h:364
clang::aarch64::AArch64SIMDIntrinsicMap
const ARMVectorIntrinsicInfo AArch64SIMDIntrinsicMap[]
Definition AArch64CodeGenUtils.h:80
clang::ast_matchers::type
const internal::VariadicAllOfMatcher< Type > type
Matches Types in the clang AST.
Definition ASTMatchersInternal.cpp:809
clang::ast_matchers::expr
const internal::VariadicDynCastAllOfMatcher< Stmt, Expr > expr
Matches expressions.
Definition ASTMatchersInternal.cpp:937
clang
The JSON file list parser is used to communicate input to InstallAPI.
Definition CalledOnceCheck.h:17
clang::isa
bool isa(CodeGen::Address addr)
Definition Address.h:330
clang::cast
U cast(CodeGen::Address addr)
Definition Address.h:327
llvm
Diagnostic wrappers for TextAPI types for error reporting.
Definition Dominators.h:30
cir::MissingFeatures::msvcBuiltins
static bool msvcBuiltins()
Definition MissingFeatures.h:314
cir::MissingFeatures::handleBuiltinICEArguments
static bool handleBuiltinICEArguments()
Definition MissingFeatures.h:286
cir::MissingFeatures::aarch64SIMDIntrinsics
static bool aarch64SIMDIntrinsics()
Definition MissingFeatures.h:194
cir::MissingFeatures::aarch64SVEIntrinsics
static bool aarch64SVEIntrinsics()
Definition MissingFeatures.h:196
cir::MissingFeatures::emitConstrainedFPCall
static bool emitConstrainedFPCall()
Definition MissingFeatures.h:263
cir::MissingFeatures::aarch64SMEIntrinsics
static bool aarch64SMEIntrinsics()
Definition MissingFeatures.h:195
cir::MissingFeatures::aarch64TblBuiltinExpr
static bool aarch64TblBuiltinExpr()
Definition MissingFeatures.h:197
Generated on for clang by doxygen 1.14.0