clang 23.0.0git
CIRGenBuiltinAArch64.cpp
Go to the documentation of this file.
1//===---- CIRGenBuiltinAArch64.cpp - Emit CIR for AArch64 builtins --------===//
2// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
3// See https://llvm.org/LICENSE.txt for license information.
4// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5//
6//===----------------------------------------------------------------------===//
7//
8// This contains code to emit ARM64 Builtin calls as CIR or a function call
9// to be later resolved.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CIRGenBuilder.h"
14#include "CIRGenFunction.h"
15#include "clang/Basic/AArch64CodeGenUtils.h"
16#include "clang/Basic/TargetBuiltins.h"
17#include "clang/CIR/Dialect/IR/CIRTypes.h"
18#include "clang/CIR/MissingFeatures.h"
19
20// TODO(cir): once all builtins are covered, decide whether we still
21// need to use LLVM intrinsics or if there's a better approach to follow. Right
22// now the intrinsics are reused to make it convenient to encode all thousands
23// of them and passing down to LLVM lowering.
24#include "llvm/IR/Intrinsics.h"
25#include "llvm/IR/IntrinsicsAArch64.h"
26
27#include "mlir/IR/BuiltinTypes.h"
28#include "mlir/IR/Value.h"
29#include "clang/AST/GlobalDecl.h"
30#include "clang/Basic/Builtins.h"
31
32using namespace clang;
33using namespace clang::CIRGen;
34using namespace llvm;
35using namespace clang::aarch64;
36
37// Generate vscale * scalingFactor
38static mlir::Value genVscaleTimesFactor(mlir::Location loc,
39 CIRGenBuilderTy builder,
40 mlir::Type cirTy,
41 int32_t scalingFactor) {
42 mlir::Value vscale = builder.emitIntrinsicCallOp(loc, "vscale", cirTy);
43 return builder.createNUWAMul(loc, vscale,
44 builder.getUInt64(scalingFactor, loc));
45}
46
47#define SVEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
48 {#NameBase, SVE::BI__builtin_sve_##NameBase, Intrinsic::LLVMIntrinsic, 0, \
49 TypeModifier}
50
51#define SVEMAP2(NameBase, TypeModifier) \
52 {#NameBase, SVE::BI__builtin_sve_##NameBase, 0, 0, TypeModifier}
53static const ARMVectorIntrinsicInfo aarch64SVEIntrinsicMap[] = {
54#define GET_SVE_LLVM_INTRINSIC_MAP
55#include "clang/Basic/arm_sve_builtin_cg.inc"
56#undef GET_SVE_LLVM_INTRINSIC_MAP
57};
58
59static bool aarch64SIMDIntrinsicsProvenSorted = false;
60static bool aarch64SISDIntrinsicsProvenSorted = false;
61static bool aarch64SVEIntrinsicsProvenSorted = false;
62
63// Check if Builtin `builtinId` is present in `intrinsicMap`. If yes, returns
64// the corresponding info struct.
65static const ARMVectorIntrinsicInfo *
66findARMVectorIntrinsicInMap(ArrayRef<ARMVectorIntrinsicInfo> intrinsicMap,
67 unsigned builtinID, bool &mapProvenSorted) {
68
69#ifndef NDEBUG
70 if (!mapProvenSorted) {
71 assert(llvm::is_sorted(intrinsicMap));
72 mapProvenSorted = true;
73 }
74#endif
75
76 const ARMVectorIntrinsicInfo *info =
77 llvm::lower_bound(intrinsicMap, builtinID);
78
79 if (info != intrinsicMap.end() && info->BuiltinID == builtinID)
80 return info;
81
82 return nullptr;
83}
84
85//===----------------------------------------------------------------------===//
86// Generic helpers
87//===----------------------------------------------------------------------===//
88static llvm::StringRef getLLVMIntrNameNoPrefix(llvm::Intrinsic::ID intrID) {
89 llvm::StringRef llvmIntrName = llvm::Intrinsic::getBaseName(intrID);
90 assert(llvmIntrName.starts_with("llvm.") && "Not an LLVM intrinsic!");
91 return llvmIntrName.drop_front(/*strlen("llvm.")=*/5);
92}
93
94//===----------------------------------------------------------------------===//
95// NEON helpers
96//===----------------------------------------------------------------------===//
97/// Return true if BuiltinID is an overloaded Neon intrinsic with an extra
98/// argument that specifies the vector type. The additional argument is meant
99/// for Sema checking (see `CheckNeonBuiltinFunctionCall`) and this function
100/// should be kept consistent with the logic in Sema.
101/// TODO: Make this return false for SISD builtins.
102/// TODO(cir): Share this with ARM.cpp
103static bool hasExtraNeonArgument(unsigned builtinID) {
104 // Required by the headers included below, but not in this particular
105 // function.
106 [[maybe_unused]] int PtrArgNum = -1;
107 [[maybe_unused]] bool HasConstPtr = false;
108
109 // The mask encodes the type. We don't care about the actual value. Instead,
110 // we just check whether its been set.
111 uint64_t mask = 0;
112 switch (builtinID) {
113#define GET_NEON_OVERLOAD_CHECK
114#include "clang/Basic/arm_fp16.inc"
115#include "clang/Basic/arm_neon.inc"
116#undef GET_NEON_OVERLOAD_CHECK
117 // Non-neon builtins for controling VFP that take extra argument for
118 // discriminating the type.
119 case ARM::BI__builtin_arm_vcvtr_f:
120 case ARM::BI__builtin_arm_vcvtr_d:
121 mask = 1;
122 }
123 switch (builtinID) {
124 default:
125 break;
126 }
127
128 return mask != 0;
129}
130
131static cir::VectorType getFloatNeonType(CIRGenFunction &cgf,
132 NeonTypeFlags intTypeFlags) {
133 int isQuad = intTypeFlags.isQuad();
134 switch (intTypeFlags.getEltType()) {
135 case NeonTypeFlags::Int16:
136 return cir::VectorType::get(cgf.fP16Ty, (4 << isQuad));
137 case NeonTypeFlags::Int32:
138 return cir::VectorType::get(cgf.floatTy, (2 << isQuad));
139 case NeonTypeFlags::Int64:
140 return cir::VectorType::get(cgf.doubleTy, (1 << isQuad));
141 default:
142 llvm_unreachable("Type can't be converted to floating-point!");
143 }
144}
145
146static int64_t getIntValueFromConstOp(mlir::Value val) {
147 return val.getDefiningOp<cir::ConstantOp>().getIntValue().getSExtValue();
148}
149
150/// Build a constant shift amount vector of `vecTy` to shift a vector
151/// Here `shiftVal` is a constant integer that will be broadcast into a
152/// a const vector of `vecTy` which is the return value of this function
153/// If `neg` is true, the shift amount is negated before splatting (used
154/// when encoding a right shift as a left shift by a negative amount for
155/// intrinsics like aarch64.neon.{s,u}rshl).
156static mlir::Value emitNeonShiftVector(CIRGenBuilderTy &builder,
157 mlir::Value shiftVal,
158 cir::VectorType vecTy,
159 mlir::Location loc, bool neg) {
160 if (neg) {
161 int64_t shiftAmt = -getIntValueFromConstOp(shiftVal);
162 shiftVal = builder.getConstantInt(loc, vecTy.getElementType(), shiftAmt);
163 }
164 mlir::Type eltTy = vecTy.getElementType();
165 if (shiftVal.getType() != eltTy) {
166 shiftVal = builder.createIntCast(shiftVal, eltTy);
167 }
168 return cir::VecSplatOp::create(builder, loc, vecTy, shiftVal);
169}
170
171// TODO(cir): Remove `cgm` from the list of arguments once all NYI(s) are gone.
172template <typename Operation>
173static mlir::Value
174emitNeonCallToOp(CIRGenModule &cgm, CIRGenBuilderTy &builder,
175 llvm::SmallVector<mlir::Type> argTypes,
176 llvm::SmallVectorImpl<mlir::Value> &args,
177 std::optional<llvm::StringRef> intrinsicName,
178 mlir::Type funcResTy, mlir::Location loc,
179 bool isConstrainedFPIntrinsic = false, unsigned shift = 0,
180 bool rightshift = false) {
181 // TODO(cir): Consider removing the following unreachable when we have
182 // emitConstrainedFPCall feature implemented
183 assert(!cir::MissingFeatures::emitConstrainedFPCall());
184 if (isConstrainedFPIntrinsic)
185 cgm.errorNYI(loc, std::string("constrained FP intrinsic"));
186
187 for (unsigned j = 0; j < argTypes.size(); ++j) {
188 if (isConstrainedFPIntrinsic) {
189 assert(!cir::MissingFeatures::emitConstrainedFPCall());
190 }
191 if (shift > 0 && shift == j) {
192 args[j] = emitNeonShiftVector(builder, args[j],
193 mlir::cast<cir::VectorType>(argTypes[j]),
194 loc, rightshift);
195 } else {
196 args[j] = builder.createBitcast(args[j], argTypes[j]);
197 }
198 }
199 if (isConstrainedFPIntrinsic) {
200 assert(!cir::MissingFeatures::emitConstrainedFPCall());
201 return nullptr;
202 }
203 if constexpr (std::is_same_v<Operation, cir::LLVMIntrinsicCallOp>) {
204 return Operation::create(builder, loc,
205 builder.getStringAttr(intrinsicName.value()),
206 funcResTy, args)
207 .getResult();
208 } else {
209 return Operation::create(builder, loc, funcResTy, args).getResult();
210 }
211}
212
213// TODO(cir): Remove `cgm` from the list of arguments once all NYI(s) are gone.
214static mlir::Value emitNeonCall(CIRGenModule &cgm, CIRGenBuilderTy &builder,
215 llvm::SmallVector<mlir::Type> argTypes,
216 llvm::SmallVectorImpl<mlir::Value> &args,
217 llvm::StringRef intrinsicName,
218 mlir::Type funcResTy, mlir::Location loc,
219 bool isConstrainedFPIntrinsic = false,
220 unsigned shift = 0, bool rightshift = false) {
221 return emitNeonCallToOp<cir::LLVMIntrinsicCallOp>(
222 cgm, builder, std::move(argTypes), args, intrinsicName, funcResTy, loc,
223 isConstrainedFPIntrinsic, shift, rightshift);
224}
225
226// Computes the input vector type for a NEON pairwise widening operation (e.g.
227// vpaddl/vpadal). Given a result vector type, it derives the corresponding
228// input type by halving the element bit width and doubling the number of lanes,
229// while setting the signedness based on usgn.
230static cir::VectorType getNeonPairwiseWidenInputType(cir::VectorType resType,
231 bool usgn) {
232 mlir::Type elemTy = resType.getElementType();
233 uint64_t resLanes = resType.getSize();
234 auto intTy = mlir::dyn_cast<cir::IntType>(elemTy);
235 assert(intTy && "vpaddl result type must be an integer vector");
236
237 unsigned resWidth = intTy.getWidth();
238 assert((resWidth == 16 || resWidth == 32 || resWidth == 64) &&
239 "unexpected vpaddl result element width");
240
241 unsigned argWidth = resWidth / 2;
242 unsigned argLanes = resLanes * 2;
243 cir::VectorType result = cir::VectorType::get(
244 cir::IntType::get(resType.getContext(), argWidth, /* is_signed */ !usgn),
245 argLanes);
246 return result;
247}
248
249static mlir::Value emitCommonNeonSISDBuiltinExpr(
250 CIRGenFunction &cgf, const ARMVectorIntrinsicInfo &info,
251 llvm::SmallVectorImpl<mlir::Value> &ops, const CallExpr *expr) {
252 assert(info.LLVMIntrinsic && "Generic code assumes a valid intrinsic");
253
254 switch (info.BuiltinID) {
255 case NEON::BI__builtin_neon_vcled_s64:
256 case NEON::BI__builtin_neon_vcled_u64:
257 case NEON::BI__builtin_neon_vcles_f32:
258 case NEON::BI__builtin_neon_vcled_f64:
259 case NEON::BI__builtin_neon_vcltd_s64:
260 case NEON::BI__builtin_neon_vcltd_u64:
261 case NEON::BI__builtin_neon_vclts_f32:
262 case NEON::BI__builtin_neon_vcltd_f64:
263 case NEON::BI__builtin_neon_vcales_f32:
264 case NEON::BI__builtin_neon_vcaled_f64:
265 case NEON::BI__builtin_neon_vcalts_f32:
266 case NEON::BI__builtin_neon_vcaltd_f64:
267 cgf.cgm.errorNYI(expr->getSourceRange(),
268 std::string("unimplemented AArch64 builtin call: ") +
269 cgf.getContext().BuiltinInfo.getName(info.BuiltinID));
270 break;
271 }
272
273 llvm::StringRef llvmIntrName = getLLVMIntrNameNoPrefix(
274 static_cast<llvm::Intrinsic::ID>(info.LLVMIntrinsic));
275 mlir::Location loc = cgf.getLoc(expr->getExprLoc());
276
277 // The switch stmt is intended to help catch NYI cases and will be removed
278 // once the CIR implementation is complete. Avoid adding specialized
279 // code in cases - that should only be required for a handful of examples.
280 switch (info.BuiltinID) {
281 default:
282 cgf.cgm.errorNYI(expr->getSourceRange(),
283 std::string("unimplemented AArch64 builtin call: ") +
284 cgf.getContext().BuiltinInfo.getName(info.BuiltinID));
285 break;
286 case NEON::BI__builtin_neon_vminv_s8:
287 case NEON::BI__builtin_neon_vminvq_s8:
288 case NEON::BI__builtin_neon_vminv_s16:
289 case NEON::BI__builtin_neon_vminvq_s16:
290 case NEON::BI__builtin_neon_vminv_s32:
291 case NEON::BI__builtin_neon_vminvq_s32:
292 case NEON::BI__builtin_neon_vminv_u8:
293 case NEON::BI__builtin_neon_vminvq_u8:
294 case NEON::BI__builtin_neon_vminv_u16:
295 case NEON::BI__builtin_neon_vminvq_u16:
296 case NEON::BI__builtin_neon_vminv_u32:
297 case NEON::BI__builtin_neon_vminvq_u32:
298 case NEON::BI__builtin_neon_vminv_f32:
299 case NEON::BI__builtin_neon_vminvq_f32:
300 case NEON::BI__builtin_neon_vminvq_f64:
301 case NEON::BI__builtin_neon_vminnmv_f32:
302 case NEON::BI__builtin_neon_vminnmvq_f32:
303 case NEON::BI__builtin_neon_vminnmvq_f64:
304 case NEON::BI__builtin_neon_vabdd_f64:
305 case NEON::BI__builtin_neon_vabds_f32:
306 case NEON::BI__builtin_neon_vshld_s64:
307 case NEON::BI__builtin_neon_vshld_u64:
308 case NEON::BI__builtin_neon_vpmins_f32:
309 case NEON::BI__builtin_neon_vpminqd_f64:
310 case NEON::BI__builtin_neon_vpminnms_f32:
311 case NEON::BI__builtin_neon_vpminnmqd_f64:
312 case NEON::BI__builtin_neon_vcvts_n_f32_s32:
313 case NEON::BI__builtin_neon_vcvts_n_f32_u32:
314 case NEON::BI__builtin_neon_vcvts_n_s32_f32:
315 case NEON::BI__builtin_neon_vcvts_n_u32_f32:
316 case NEON::BI__builtin_neon_vcvtd_n_f64_s64:
317 case NEON::BI__builtin_neon_vcvtd_n_f64_u64:
318 case NEON::BI__builtin_neon_vcvtd_n_s64_f64:
319 case NEON::BI__builtin_neon_vcvtd_n_u64_f64:
320 case NEON::BI__builtin_neon_vaddlv_s32:
321 case NEON::BI__builtin_neon_vaddlv_u32:
322 case NEON::BI__builtin_neon_vaddlvq_s32:
323 case NEON::BI__builtin_neon_vaddlvq_u32:
324 case NEON::BI__builtin_neon_vaddv_s8:
325 case NEON::BI__builtin_neon_vaddv_s16:
326 case NEON::BI__builtin_neon_vaddv_s32:
327 case NEON::BI__builtin_neon_vaddv_u8:
328 case NEON::BI__builtin_neon_vaddv_u16:
329 case NEON::BI__builtin_neon_vaddv_u32:
330 case NEON::BI__builtin_neon_vaddv_f32:
331 case NEON::BI__builtin_neon_vaddvq_s8:
332 case NEON::BI__builtin_neon_vaddvq_s16:
333 case NEON::BI__builtin_neon_vaddvq_s32:
334 case NEON::BI__builtin_neon_vaddvq_s64:
335 case NEON::BI__builtin_neon_vaddvq_u8:
336 case NEON::BI__builtin_neon_vaddvq_u16:
337 case NEON::BI__builtin_neon_vaddvq_u32:
338 case NEON::BI__builtin_neon_vaddvq_u64:
339 case NEON::BI__builtin_neon_vaddvq_f32:
340 case NEON::BI__builtin_neon_vaddvq_f64:
341 case NEON::BI__builtin_neon_vabdh_f16:
342 case NEON::BI__builtin_neon_vrecpeh_f16:
343 case NEON::BI__builtin_neon_vrecpxh_f16:
344 case NEON::BI__builtin_neon_vrsqrteh_f16:
345 case NEON::BI__builtin_neon_vrsqrtsh_f16:
346 return emitNeonCall(cgf.cgm, cgf.getBuilder(),
347 {cgf.convertType(expr->getArg(0)->getType())}, ops,
348 llvmIntrName, cgf.convertType(expr->getType()), loc);
349 }
350
351 return nullptr;
352}
353
354//===----------------------------------------------------------------------===//
355// Emit-helpers
356//===----------------------------------------------------------------------===//
357static mlir::Value
358emitAArch64CompareBuiltinExpr(CIRGenFunction &cgf, CIRGenBuilderTy &builder,
359 mlir::Location loc, mlir::Value src,
360 mlir::Type retTy, const cir::CmpOpKind kind) {
361
362 bool scalarCmp = !isa<cir::VectorType>(src.getType());
363 if (!scalarCmp) {
364 assert(!cast<cir::VectorType>(retTy).getIsScalable() &&
365 "This is only intended for fixed-width vectors");
366 // Vector types are cast to i8 vectors. Recover original type.
367 src = builder.createBitcast(src, retTy);
368 }
369
370 mlir::Value zero = builder.getNullValue(src.getType(), loc);
371
372 if (!scalarCmp)
373 return builder.createVecCompare(loc, kind, src, zero);
374
375 // For scalars, cast !cir.bool to !cir.int<s, 1> so that the compare
376 // result is sign- rather zero-extended when casting to the output
377 // retType.
378 mlir::Value cmp = builder.createCast(
379 loc, cir::CastKind::bool_to_int,
380 builder.createCompare(loc, kind, src, zero), builder.getSIntNTy(1));
381
382 return builder.createCast(loc, cir::CastKind::integral, cmp, retTy);
383}
384
385// TODO(cir): Remove `loc` from the list of arguments once all NYIs are gone.
386static cir::VectorType getNeonType(CIRGenFunction *cgf, NeonTypeFlags typeFlags,
387 mlir::Location loc,
388 bool hasLegalHalfType = true,
389 bool v1Ty = false,
390 bool allowBFloatArgsAndRet = true) {
391 int isQuad = typeFlags.isQuad();
392 switch (typeFlags.getEltType()) {
393 case NeonTypeFlags::Int8:
394 case NeonTypeFlags::Poly8:
395 return cir::VectorType::get(typeFlags.isUnsigned() ? cgf->uInt8Ty
396 : cgf->sInt8Ty,
397 v1Ty ? 1 : (8 << isQuad));
398 case NeonTypeFlags::MFloat8:
399 return cir::VectorType::get(cgf->uInt8Ty, v1Ty ? 1 : (8 << isQuad));
400 case NeonTypeFlags::Int16:
401 case NeonTypeFlags::Poly16:
402 return cir::VectorType::get(typeFlags.isUnsigned() ? cgf->uInt16Ty
403 : cgf->sInt16Ty,
404 v1Ty ? 1 : (4 << isQuad));
405 case NeonTypeFlags::BFloat16:
406 if (allowBFloatArgsAndRet)
407 return cir::VectorType::get(cgf->getCIRGenModule().bFloat16Ty,
408 v1Ty ? 1 : (4 << isQuad));
409 return cir::VectorType::get(cgf->uInt16Ty, v1Ty ? 1 : (4 << isQuad));
410 case NeonTypeFlags::Float16:
411 if (hasLegalHalfType)
412 return cir::VectorType::get(cgf->getCIRGenModule().fP16Ty,
413 v1Ty ? 1 : (4 << isQuad));
414 return cir::VectorType::get(cgf->uInt16Ty, v1Ty ? 1 : (4 << isQuad));
415 case NeonTypeFlags::Int32:
416 return cir::VectorType::get(typeFlags.isUnsigned() ? cgf->uInt32Ty
417 : cgf->sInt32Ty,
418 v1Ty ? 1 : (2 << isQuad));
419 case NeonTypeFlags::Int64:
420 case NeonTypeFlags::Poly64:
421 return cir::VectorType::get(typeFlags.isUnsigned() ? cgf->uInt64Ty
422 : cgf->sInt64Ty,
423 v1Ty ? 1 : (1 << isQuad));
424 case NeonTypeFlags::Poly128:
425 // FIXME: i128 and f128 doesn't get fully support in Clang and llvm.
426 // There is a lot of i128 and f128 API missing.
427 // so we use v16i8 to represent poly128 and get pattern matched.
428 cgf->getCIRGenModule().errorNYI(loc, std::string("NEON type: Poly128"));
429 [[fallthrough]];
430 case NeonTypeFlags::Float32:
431 return cir::VectorType::get(cgf->getCIRGenModule().floatTy,
432 v1Ty ? 1 : (2 << isQuad));
433 case NeonTypeFlags::Float64:
434 return cir::VectorType::get(cgf->getCIRGenModule().doubleTy,
435 v1Ty ? 1 : (1 << isQuad));
436 }
437 llvm_unreachable("Unknown vector element type!");
438}
439
440static mlir::Value emitNeonSplat(CIRGenBuilderTy &builder, mlir::Location loc,
441 mlir::Value v, mlir::Value lane,
442 unsigned int resEltCnt) {
443 assert(isa<cir::ConstantOp>(lane.getDefiningOp()) &&
444 "lane number is not a constant!");
445 int64_t laneCst = getIntValueFromConstOp(lane);
446 llvm::SmallVector<int64_t, 4> shuffleMask(resEltCnt, laneCst);
447 return builder.createVecShuffle(loc, v, shuffleMask);
448}
449
450/// Flip the signedness of `vecTy`'s element type, keeping the width and
451/// number of lanes the same. Used when a NEON intrinsic takes a shift
452/// amount vector that must be signed (e.g. aarch64.neon.urshl takes a
453/// signed amount even though the data vector is unsigned).
454static cir::VectorType getSignChangedVectorType(CIRGenBuilderTy &builder,
455 cir::VectorType vecTy) {
456 auto elemTy = mlir::cast<cir::IntType>(vecTy.getElementType());
457 elemTy = elemTy.isSigned() ? builder.getUIntNTy(elemTy.getWidth())
458 : builder.getSIntNTy(elemTy.getWidth());
459 return cir::VectorType::get(elemTy, vecTy.getSize());
460}
461
462static mlir::Value emitCommonNeonShift(CIRGenBuilderTy &builder,
463 mlir::Location loc,
464 cir::VectorType resTy,
465 mlir::Value shifTgt,
466 mlir::Value shiftAmt, bool shiftLeft) {
467 shiftAmt = emitNeonShiftVector(builder, shiftAmt, resTy, loc, /*neg=*/false);
468 return cir::ShiftOp::create(builder, loc, resTy,
469 builder.createBitcast(shifTgt, resTy), shiftAmt,
470 shiftLeft);
471}
472
473// Right-shift a vector by a constant.
474static mlir::Value emitNeonRShiftImm(CIRGenFunction &cgf, mlir::Value shiftVec,
475 mlir::Value shiftVal,
476 cir::VectorType vecTy, bool usgn,
477 mlir::Location loc) {
478 CIRGenBuilderTy &builder = cgf.getBuilder();
479 int64_t shiftAmt = getIntValueFromConstOp(shiftVal);
480 int eltSize =
481 cgf.cgm.getDataLayout().getTypeSizeInBits(vecTy.getElementType());
482
483 shiftVec = builder.createBitcast(shiftVec, vecTy);
484 // lshr/ashr are undefined when the shift amount is equal to the vector
485 // element size.
486 if (shiftAmt == eltSize) {
487 if (usgn) {
488 // Right-shifting an unsigned value by its size yields 0.
489 return builder.getZero(loc, vecTy);
490 }
491 // Right-shifting a signed value by its size is equivalent
492 // to a shift of size-1.
493 --shiftAmt;
494 shiftVal = builder.getConstInt(loc, vecTy.getElementType(), shiftAmt);
495 }
496 return emitCommonNeonShift(builder, loc, vecTy, shiftVec, shiftVal,
497 /*shiftLeft=*/false);
498}
499
500static cir::VectorType getIntVecFromVecTy(CIRGenBuilderTy &builder,
501 cir::VectorType vecTy) {
502 if (!cir::isAnyFloatingPointType(vecTy.getElementType()))
503 return vecTy;
504
505 if (mlir::isa<cir::SingleType>(vecTy.getElementType()))
506 return cir::VectorType::get(builder.getSInt32Ty(), vecTy.getSize());
507 if (mlir::isa<cir::DoubleType>(vecTy.getElementType()))
508 return cir::VectorType::get(builder.getSInt64Ty(), vecTy.getSize());
509 llvm_unreachable(
510 "Unsupported element type in getVecOfIntTypeWithSameEltWidth");
511}
512
513static mlir::Value emitCommonNeonBuiltinExpr(
514 CIRGenFunction &cgf, unsigned builtinID, unsigned llvmIntrinsic,
515 unsigned altLLVMIntrinsic, const char *nameHint, unsigned modifier,
516 const CallExpr *expr, llvm::SmallVectorImpl<mlir::Value> &ops) {
517 mlir::Location loc = cgf.getLoc(expr->getExprLoc());
518 clang::ASTContext &ctx = cgf.getContext();
519
520 // Extract the trailing immediate argument that encodes the type discriminator
521 // for this overloaded intrinsic.
522 // TODO: Move to the parent code that takes care of argument processing.
523 const clang::Expr *arg = expr->getArg(expr->getNumArgs() - 1);
524 std::optional<llvm::APSInt> neonTypeConst = arg->getIntegerConstantExpr(ctx);
525 if (!neonTypeConst)
526 return nullptr;
527
528 // Determine the type of this overloaded NEON intrinsic.
529 NeonTypeFlags neonType(neonTypeConst->getZExtValue());
530 const bool isUnsigned = neonType.isUnsigned();
531 const bool hasLegalHalfType = cgf.getTarget().hasFastHalfType();
532 const bool usgn = neonType.isUnsigned();
533
534 // The value of allowBFloatArgsAndRet is true for AArch64, but it should
535 // come from ABI info.
536 // TODO(cir): Use ABInfo to extract this information
537 const bool allowBFloatArgsAndRet = cgf.getTarget().hasFastHalfType();
538 // FIXME
539 // getTargetHooks().getABIInfo().allowBFloatArgsAndRet();
540
541 cir::VectorType vTy = getNeonType(&cgf, neonType, loc, hasLegalHalfType,
542 false, allowBFloatArgsAndRet);
543 cir::VectorType ty = vTy;
544 if (!ty)
545 return nullptr;
546
547 switch (builtinID) {
548 case NEON::BI__builtin_neon_splat_lane_v:
549 case NEON::BI__builtin_neon_splat_laneq_v:
550 case NEON::BI__builtin_neon_splatq_lane_v:
551 case NEON::BI__builtin_neon_splatq_laneq_v: {
552 uint64_t numElements = vTy.getSize();
553 if (builtinID == NEON::BI__builtin_neon_splatq_lane_v)
554 numElements *= 2;
555 if (builtinID == NEON::BI__builtin_neon_splat_laneq_v)
556 numElements /= 2;
557 ops[0] = cgf.getBuilder().createBitcast(loc, ops[0], vTy);
558 return emitNeonSplat(cgf.getBuilder(), loc, ops[0], ops[1], numElements);
559 }
560 case NEON::BI__builtin_neon_vpadd_v:
561 case NEON::BI__builtin_neon_vpaddq_v:
562 case NEON::BI__builtin_neon_vabs_v:
563 case NEON::BI__builtin_neon_vabsq_v:
564 case NEON::BI__builtin_neon_vadd_v:
565 case NEON::BI__builtin_neon_vaddq_v:
566 case NEON::BI__builtin_neon_vaddhn_v:
567 case NEON::BI__builtin_neon_vcale_v:
568 case NEON::BI__builtin_neon_vcaleq_v:
569 case NEON::BI__builtin_neon_vcalt_v:
570 case NEON::BI__builtin_neon_vcaltq_v:
571 case NEON::BI__builtin_neon_vcage_v:
572 case NEON::BI__builtin_neon_vcageq_v:
573 case NEON::BI__builtin_neon_vcagt_v:
574 case NEON::BI__builtin_neon_vcagtq_v:
575 cgf.cgm.errorNYI(expr->getSourceRange(),
576 std::string("unimplemented AArch64 builtin call: ") +
577 ctx.BuiltinInfo.getName(builtinID));
578 return mlir::Value{};
579 case NEON::BI__builtin_neon_vceqz_v:
580 case NEON::BI__builtin_neon_vceqzq_v:
581 return emitAArch64CompareBuiltinExpr(cgf, cgf.getBuilder(), loc, ops[0],
582 vTy, cir::CmpOpKind::eq);
583 case NEON::BI__builtin_neon_vcgez_v:
584 case NEON::BI__builtin_neon_vcgezq_v:
585 case NEON::BI__builtin_neon_vclez_v:
586 case NEON::BI__builtin_neon_vclezq_v:
587 case NEON::BI__builtin_neon_vcgtz_v:
588 case NEON::BI__builtin_neon_vcgtzq_v:
589 case NEON::BI__builtin_neon_vcltz_v:
590 case NEON::BI__builtin_neon_vcltzq_v:
591 case NEON::BI__builtin_neon_vclz_v:
592 case NEON::BI__builtin_neon_vclzq_v:
593 case NEON::BI__builtin_neon_vcvt_f32_v:
594 case NEON::BI__builtin_neon_vcvtq_f32_v:
595 case NEON::BI__builtin_neon_vcvt_f16_s16:
596 case NEON::BI__builtin_neon_vcvt_f16_u16:
597 case NEON::BI__builtin_neon_vcvtq_f16_s16:
598 case NEON::BI__builtin_neon_vcvtq_f16_u16:
599 case NEON::BI__builtin_neon_vcvt_n_f16_s16:
600 case NEON::BI__builtin_neon_vcvt_n_f16_u16:
601 case NEON::BI__builtin_neon_vcvtq_n_f16_s16:
602 case NEON::BI__builtin_neon_vcvtq_n_f16_u16:
603 cgf.cgm.errorNYI(expr->getSourceRange(),
604 std::string("unimplemented AArch64 builtin call: ") +
605 ctx.BuiltinInfo.getName(builtinID));
606 return mlir::Value{};
607 case NEON::BI__builtin_neon_vcvt_n_f32_v:
608 case NEON::BI__builtin_neon_vcvt_n_f64_v:
609 case NEON::BI__builtin_neon_vcvtq_n_f32_v:
610 case NEON::BI__builtin_neon_vcvtq_n_f64_v: {
611 // The constant argument to an _n_ intrinsic always is Int32Ty.
612 mlir::Type cstIntTy = cgf.sInt32Ty;
613 llvm::StringRef llvmIntrName =
614 getLLVMIntrNameNoPrefix(static_cast<llvm::Intrinsic::ID>(
615 usgn ? llvmIntrinsic : altLLVMIntrinsic));
616 return emitNeonCall(cgf.getCIRGenModule(), cgf.getBuilder(),
617 /*argTypes=*/{vTy, cstIntTy}, ops, llvmIntrName,
618 /*funcResTy=*/getFloatNeonType(cgf, neonType), loc);
619 }
620 case NEON::BI__builtin_neon_vcvt_n_s16_f16:
621 case NEON::BI__builtin_neon_vcvt_n_s32_v:
622 case NEON::BI__builtin_neon_vcvt_n_u16_f16:
623 case NEON::BI__builtin_neon_vcvt_n_u32_v:
624 case NEON::BI__builtin_neon_vcvt_n_s64_v:
625 case NEON::BI__builtin_neon_vcvt_n_u64_v:
626 case NEON::BI__builtin_neon_vcvtq_n_s16_f16:
627 case NEON::BI__builtin_neon_vcvtq_n_s32_v:
628 case NEON::BI__builtin_neon_vcvtq_n_u16_f16:
629 case NEON::BI__builtin_neon_vcvtq_n_u32_v:
630 case NEON::BI__builtin_neon_vcvtq_n_s64_v:
631 case NEON::BI__builtin_neon_vcvtq_n_u64_v: {
632 // The constant argument to an _n_ intrinsic always is Int32Ty.
633 mlir::Type cstIntTy = cgf.sInt32Ty;
634 llvm::StringRef llvmIntrName = getLLVMIntrNameNoPrefix(
635 static_cast<llvm::Intrinsic::ID>(llvmIntrinsic));
636 return emitNeonCall(
637 cgf.getCIRGenModule(), cgf.getBuilder(),
638 /*argTypes=*/{getFloatNeonType(cgf, neonType), cstIntTy}, ops,
639 llvmIntrName,
640 /*funcResTy=*/vTy, loc);
641 }
642 case NEON::BI__builtin_neon_vcvt_s32_v:
643 case NEON::BI__builtin_neon_vcvt_u32_v:
644 case NEON::BI__builtin_neon_vcvt_s64_v:
645 case NEON::BI__builtin_neon_vcvt_u64_v:
646 case NEON::BI__builtin_neon_vcvt_s16_f16:
647 case NEON::BI__builtin_neon_vcvt_u16_f16:
648 case NEON::BI__builtin_neon_vcvtq_s32_v:
649 case NEON::BI__builtin_neon_vcvtq_u32_v:
650 case NEON::BI__builtin_neon_vcvtq_s64_v:
651 case NEON::BI__builtin_neon_vcvtq_u64_v:
652 case NEON::BI__builtin_neon_vcvtq_s16_f16:
653 case NEON::BI__builtin_neon_vcvtq_u16_f16:
654 case NEON::BI__builtin_neon_vcvta_s16_f16:
655 case NEON::BI__builtin_neon_vcvta_s32_v:
656 case NEON::BI__builtin_neon_vcvta_s64_v:
657 case NEON::BI__builtin_neon_vcvta_u16_f16:
658 case NEON::BI__builtin_neon_vcvta_u32_v:
659 case NEON::BI__builtin_neon_vcvta_u64_v:
660 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
661 case NEON::BI__builtin_neon_vcvtaq_s32_v:
662 case NEON::BI__builtin_neon_vcvtaq_s64_v:
663 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
664 case NEON::BI__builtin_neon_vcvtaq_u32_v:
665 case NEON::BI__builtin_neon_vcvtaq_u64_v:
666 case NEON::BI__builtin_neon_vcvtn_s16_f16:
667 case NEON::BI__builtin_neon_vcvtn_s32_v:
668 case NEON::BI__builtin_neon_vcvtn_s64_v:
669 case NEON::BI__builtin_neon_vcvtn_u16_f16:
670 case NEON::BI__builtin_neon_vcvtn_u32_v:
671 case NEON::BI__builtin_neon_vcvtn_u64_v:
672 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
673 case NEON::BI__builtin_neon_vcvtnq_s32_v:
674 case NEON::BI__builtin_neon_vcvtnq_s64_v:
675 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
676 case NEON::BI__builtin_neon_vcvtnq_u32_v:
677 case NEON::BI__builtin_neon_vcvtnq_u64_v:
678 case NEON::BI__builtin_neon_vcvtp_s16_f16:
679 case NEON::BI__builtin_neon_vcvtp_s32_v:
680 case NEON::BI__builtin_neon_vcvtp_s64_v:
681 case NEON::BI__builtin_neon_vcvtp_u16_f16:
682 case NEON::BI__builtin_neon_vcvtp_u32_v:
683 case NEON::BI__builtin_neon_vcvtp_u64_v:
684 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
685 case NEON::BI__builtin_neon_vcvtpq_s32_v:
686 case NEON::BI__builtin_neon_vcvtpq_s64_v:
687 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
688 case NEON::BI__builtin_neon_vcvtpq_u32_v:
689 case NEON::BI__builtin_neon_vcvtpq_u64_v:
690 case NEON::BI__builtin_neon_vcvtm_s16_f16:
691 case NEON::BI__builtin_neon_vcvtm_s32_v:
692 case NEON::BI__builtin_neon_vcvtm_s64_v:
693 case NEON::BI__builtin_neon_vcvtm_u16_f16:
694 case NEON::BI__builtin_neon_vcvtm_u32_v:
695 case NEON::BI__builtin_neon_vcvtm_u64_v:
696 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
697 case NEON::BI__builtin_neon_vcvtmq_s32_v:
698 case NEON::BI__builtin_neon_vcvtmq_s64_v:
699 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
700 case NEON::BI__builtin_neon_vcvtmq_u32_v:
701 case NEON::BI__builtin_neon_vcvtmq_u64_v:
702 case NEON::BI__builtin_neon_vcvtx_f32_v:
703 case NEON::BI__builtin_neon_vext_v:
704 case NEON::BI__builtin_neon_vextq_v:
705 case NEON::BI__builtin_neon_vfma_v:
706 case NEON::BI__builtin_neon_vfmaq_v:
707 case NEON::BI__builtin_neon_vld1_v:
708 case NEON::BI__builtin_neon_vld1q_v:
709 case NEON::BI__builtin_neon_vld1_x2_v:
710 case NEON::BI__builtin_neon_vld1q_x2_v:
711 case NEON::BI__builtin_neon_vld1_x3_v:
712 case NEON::BI__builtin_neon_vld1q_x3_v:
713 case NEON::BI__builtin_neon_vld1_x4_v:
714 case NEON::BI__builtin_neon_vld1q_x4_v:
715 case NEON::BI__builtin_neon_vld2_v:
716 case NEON::BI__builtin_neon_vld2q_v:
717 case NEON::BI__builtin_neon_vld3_v:
718 case NEON::BI__builtin_neon_vld3q_v:
719 case NEON::BI__builtin_neon_vld4_v:
720 case NEON::BI__builtin_neon_vld4q_v:
721 case NEON::BI__builtin_neon_vld2_dup_v:
722 case NEON::BI__builtin_neon_vld2q_dup_v:
723 case NEON::BI__builtin_neon_vld3_dup_v:
724 case NEON::BI__builtin_neon_vld3q_dup_v:
725 case NEON::BI__builtin_neon_vld4_dup_v:
726 case NEON::BI__builtin_neon_vld4q_dup_v:
727 case NEON::BI__builtin_neon_vld1_dup_v:
728 case NEON::BI__builtin_neon_vld1q_dup_v:
729 case NEON::BI__builtin_neon_vld2_lane_v:
730 case NEON::BI__builtin_neon_vld2q_lane_v:
731 case NEON::BI__builtin_neon_vld3_lane_v:
732 case NEON::BI__builtin_neon_vld3q_lane_v:
733 case NEON::BI__builtin_neon_vld4_lane_v:
734 case NEON::BI__builtin_neon_vld4q_lane_v:
735 case NEON::BI__builtin_neon_vmovl_v:
736 case NEON::BI__builtin_neon_vmovn_v:
737 case NEON::BI__builtin_neon_vmull_v:
738 case NEON::BI__builtin_neon_vpadal_v:
739 case NEON::BI__builtin_neon_vpadalq_v:
740 cgf.cgm.errorNYI(expr->getSourceRange(),
741 std::string("Reached code-path for ARM builtin call ") +
742 ctx.BuiltinInfo.getName(builtinID) +
743 "(ARM builtins are not supported ATM)");
744 return mlir::Value{};
745 case NEON::BI__builtin_neon_vpaddl_v:
746 case NEON::BI__builtin_neon_vpaddlq_v: {
747 llvm::StringRef llvmIntrName =
748 getLLVMIntrNameNoPrefix(static_cast<llvm::Intrinsic::ID>(
749 usgn ? llvmIntrinsic : altLLVMIntrinsic));
750 return emitNeonCall(cgf.getCIRGenModule(), cgf.getBuilder(),
751 /*argTypes=*/{getNeonPairwiseWidenInputType(vTy, usgn)},
752 ops, llvmIntrName,
753 /*funcResTy=*/vTy, loc);
754 }
755 case NEON::BI__builtin_neon_vqdmlal_v:
756 case NEON::BI__builtin_neon_vqdmlsl_v:
757 case NEON::BI__builtin_neon_vqdmulhq_lane_v:
758 case NEON::BI__builtin_neon_vqdmulh_lane_v:
759 case NEON::BI__builtin_neon_vqrdmulhq_lane_v:
760 case NEON::BI__builtin_neon_vqrdmulh_lane_v:
761 case NEON::BI__builtin_neon_vqdmulhq_laneq_v:
762 case NEON::BI__builtin_neon_vqdmulh_laneq_v:
763 case NEON::BI__builtin_neon_vqrdmulhq_laneq_v:
764 case NEON::BI__builtin_neon_vqrdmulh_laneq_v:
765 case NEON::BI__builtin_neon_vqshl_n_v:
766 case NEON::BI__builtin_neon_vqshlq_n_v:
767 case NEON::BI__builtin_neon_vqshlu_n_v:
768 case NEON::BI__builtin_neon_vqshluq_n_v:
769 case NEON::BI__builtin_neon_vrecpe_v:
770 case NEON::BI__builtin_neon_vrecpeq_v:
771 case NEON::BI__builtin_neon_vrsqrte_v:
772 case NEON::BI__builtin_neon_vrsqrteq_v:
773 case NEON::BI__builtin_neon_vrndi_v:
774 case NEON::BI__builtin_neon_vrndiq_v:
775 cgf.cgm.errorNYI(expr->getSourceRange(),
776 std::string("unimplemented AArch64 builtin call: ") +
777 ctx.BuiltinInfo.getName(builtinID));
778 return mlir::Value{};
779 case NEON::BI__builtin_neon_vrshr_n_v:
780 case NEON::BI__builtin_neon_vrshrq_n_v: {
781 llvm::StringRef intrName =
782 usgn ? "aarch64.neon.urshl" : "aarch64.neon.srshl";
783 return emitNeonCall(
784 cgf.cgm, cgf.getBuilder(),
785 {ty, usgn ? getSignChangedVectorType(cgf.getBuilder(), ty) : ty}, ops,
786 intrName, ty, loc, /*isConstrainedFPIntrinsic=*/false,
787 /*shift=*/1,
788 /*rightshift=*/true);
789 }
790 case NEON::BI__builtin_neon_vsha512hq_u64:
791 case NEON::BI__builtin_neon_vsha512h2q_u64:
792 case NEON::BI__builtin_neon_vsha512su0q_u64:
793 case NEON::BI__builtin_neon_vsha512su1q_u64:
794 cgf.cgm.errorNYI(expr->getSourceRange(),
795 std::string("unimplemented AArch64 builtin call: ") +
796 ctx.BuiltinInfo.getName(builtinID));
797 return mlir::Value{};
798 case NEON::BI__builtin_neon_vshl_n_v:
799 case NEON::BI__builtin_neon_vshlq_n_v:
800 return emitCommonNeonShift(cgf.getBuilder(), loc, vTy, ops[0], ops[1],
801 /*shiftLeft=*/true);
802 case NEON::BI__builtin_neon_vshll_n_v:
803 case NEON::BI__builtin_neon_vshrn_n_v:
804 cgf.cgm.errorNYI(expr->getSourceRange(),
805 std::string("unimplemented AArch64 builtin call: ") +
806 ctx.BuiltinInfo.getName(builtinID));
807 return mlir::Value{};
808 case NEON::BI__builtin_neon_vshr_n_v:
809 case NEON::BI__builtin_neon_vshrq_n_v:
810 return emitNeonRShiftImm(cgf, ops[0], ops[1], vTy, isUnsigned, loc);
811 case NEON::BI__builtin_neon_vst1_v:
812 case NEON::BI__builtin_neon_vst1q_v:
813 case NEON::BI__builtin_neon_vst2_v:
814 case NEON::BI__builtin_neon_vst2q_v:
815 case NEON::BI__builtin_neon_vst3_v:
816 case NEON::BI__builtin_neon_vst3q_v:
817 case NEON::BI__builtin_neon_vst4_v:
818 case NEON::BI__builtin_neon_vst4q_v:
819 case NEON::BI__builtin_neon_vst2_lane_v:
820 case NEON::BI__builtin_neon_vst2q_lane_v:
821 case NEON::BI__builtin_neon_vst3_lane_v:
822 case NEON::BI__builtin_neon_vst3q_lane_v:
823 case NEON::BI__builtin_neon_vst4_lane_v:
824 case NEON::BI__builtin_neon_vst4q_lane_v:
825 case NEON::BI__builtin_neon_vsm3partw1q_u32:
826 case NEON::BI__builtin_neon_vsm3partw2q_u32:
827 case NEON::BI__builtin_neon_vsm3ss1q_u32:
828 case NEON::BI__builtin_neon_vsm4ekeyq_u32:
829 case NEON::BI__builtin_neon_vsm4eq_u32:
830 case NEON::BI__builtin_neon_vsm3tt1aq_u32:
831 case NEON::BI__builtin_neon_vsm3tt1bq_u32:
832 case NEON::BI__builtin_neon_vsm3tt2aq_u32:
833 case NEON::BI__builtin_neon_vsm3tt2bq_u32:
834 case NEON::BI__builtin_neon_vst1_x2_v:
835 case NEON::BI__builtin_neon_vst1q_x2_v:
836 case NEON::BI__builtin_neon_vst1_x3_v:
837 case NEON::BI__builtin_neon_vst1q_x3_v:
838 case NEON::BI__builtin_neon_vst1_x4_v:
839 case NEON::BI__builtin_neon_vst1q_x4_v:
840 case NEON::BI__builtin_neon_vsubhn_v:
841 case NEON::BI__builtin_neon_vtrn_v:
842 case NEON::BI__builtin_neon_vtrnq_v:
843 case NEON::BI__builtin_neon_vtst_v:
844 case NEON::BI__builtin_neon_vtstq_v:
845 case NEON::BI__builtin_neon_vuzp_v:
846 case NEON::BI__builtin_neon_vuzpq_v:
847 case NEON::BI__builtin_neon_vxarq_u64:
848 case NEON::BI__builtin_neon_vzip_v:
849 case NEON::BI__builtin_neon_vzipq_v:
850 case NEON::BI__builtin_neon_vdot_s32:
851 case NEON::BI__builtin_neon_vdot_u32:
852 case NEON::BI__builtin_neon_vdotq_s32:
853 case NEON::BI__builtin_neon_vdotq_u32:
854 case NEON::BI__builtin_neon_vfmlal_low_f16:
855 case NEON::BI__builtin_neon_vfmlalq_low_f16:
856 case NEON::BI__builtin_neon_vfmlsl_low_f16:
857 case NEON::BI__builtin_neon_vfmlslq_low_f16:
858 case NEON::BI__builtin_neon_vfmlal_high_f16:
859 case NEON::BI__builtin_neon_vfmlalq_high_f16:
860 case NEON::BI__builtin_neon_vfmlsl_high_f16:
861 case NEON::BI__builtin_neon_vfmlslq_high_f16:
862 case NEON::BI__builtin_neon_vmmlaq_s32:
863 case NEON::BI__builtin_neon_vmmlaq_u32:
864 cgf.cgm.errorNYI(expr->getSourceRange(),
865 std::string("unimplemented AArch64 builtin call: ") +
866 ctx.BuiltinInfo.getName(builtinID));
867 return mlir::Value{};
868 case NEON::BI__builtin_neon_vmul_v:
869 case NEON::BI__builtin_neon_vmulq_v:
870 return cgf.getBuilder().emitIntrinsicCallOp(loc, "aarch64.neon.pmul", vTy,
871 ops);
872 case NEON::BI__builtin_neon_vusmmlaq_s32:
873 case NEON::BI__builtin_neon_vusdot_s32:
874 case NEON::BI__builtin_neon_vusdotq_s32:
875 case NEON::BI__builtin_neon_vbfdot_f32:
876 case NEON::BI__builtin_neon_vbfdotq_f32:
877 case NEON::BI__builtin_neon___a32_vcvt_bf16_f32:
878 default:
879 cgf.cgm.errorNYI(expr->getSourceRange(),
880 std::string("unimplemented AArch64 builtin call: ") +
881 ctx.BuiltinInfo.getName(builtinID));
882 return mlir::Value{};
883
884 cgf.cgm.errorNYI(expr->getSourceRange(),
885 std::string("unimplemented AArch64 builtin call: ") +
886 ctx.BuiltinInfo.getName(builtinID));
887 return mlir::Value{};
888 }
889}
890
891// Emit an intrinsic where all operands are of the same type as the result.
892// Depending on mode, this may be a constrained floating-point intrinsic.
893static mlir::Value
894emitCallMaybeConstrainedBuiltin(CIRGenBuilderTy &builder, mlir::Location loc,
895 StringRef intrName, mlir::Type retTy,
896 llvm::SmallVector<mlir::Value> &ops) {
897 assert(!cir::MissingFeatures::emitConstrainedFPCall());
898
899 return builder.emitIntrinsicCallOp(loc, intrName, retTy, ops);
900}
901
902bool CIRGenFunction::getAArch64SVEProcessedOperands(
903 unsigned builtinID, const CallExpr *expr, SmallVectorImpl<mlir::Value> &ops,
904 SVETypeFlags typeFlags) {
905 // Find out if any arguments are required to be integer constant expressions.
906 unsigned iceArguments = 0;
907 ASTContext::GetBuiltinTypeError error;
908 getContext().GetBuiltinType(builtinID, error, &iceArguments);
909 assert(error == ASTContext::GE_None && "Should not codegen an error");
910
911 for (unsigned i = 0, e = expr->getNumArgs(); i != e; i++) {
912 bool isIce = iceArguments & (1 << i);
913 mlir::Value arg = emitScalarExpr(expr->getArg(i));
914
915 if (isIce) {
916 cgm.errorNYI(expr->getSourceRange(),
917 std::string("unimplemented AArch64 builtin call: ") +
918 getContext().BuiltinInfo.getName(builtinID));
919 }
920
921 // FIXME: Handle types like svint16x2_t, which are currently incorrectly
922 // converted to i32. These should be treated as structs and unpacked.
923
924 ops.push_back(arg);
925 }
926 return true;
927}
928
929// Reinterpret the input predicate so that it can be used to correctly isolate
930// the elements of the specified datatype.
931mlir::Value CIRGenFunction::emitSVEPredicateCast(mlir::Value pred,
932 unsigned minNumElts,
933 mlir::Location loc) {
934
935 // TODO: Handle "aarch64.svcount" once we get round to supporting SME.
936
937 auto retTy = cir::VectorType::get(builder.getUIntNTy(1), minNumElts,
938 /*is_scalable=*/true);
939 if (pred.getType() == retTy)
940 return pred;
941
942 llvm::Intrinsic::ID intID;
943 switch (minNumElts) {
944 default:
945 llvm_unreachable("unsupported element count!");
946 case 1:
947 case 2:
948 case 4:
949 case 8:
950 intID = Intrinsic::aarch64_sve_convert_from_svbool;
951 break;
952 case 16:
953 intID = Intrinsic::aarch64_sve_convert_to_svbool;
954 break;
955 }
956
957 llvm::StringRef llvmIntrName = getLLVMIntrNameNoPrefix(intID);
958 auto call = builder.emitIntrinsicCallOp(loc, llvmIntrName, retTy,
959 mlir::ValueRange{pred});
960 assert(call.getType() == retTy && "Unexpected return type!");
961 return call;
962}
963
964//===----------------------------------------------------------------------===//
965// SVE helpers
966//===----------------------------------------------------------------------===//
967// Get the minimum number of elements in an SVE vector for the given element
968// type. The actual number of elements in the vector would be an integer (power
969// of two) multiple of this value.
970static unsigned getSVEMinEltCount(clang::SVETypeFlags::EltType sveType) {
971 switch (sveType) {
972 default:
973 llvm_unreachable("Invalid SVETypeFlag!");
974
975 case SVETypeFlags::EltTyInt8:
976 return 16;
977 case SVETypeFlags::EltTyInt16:
978 return 8;
979 case SVETypeFlags::EltTyInt32:
980 return 4;
981 case SVETypeFlags::EltTyInt64:
982 return 2;
983
984 case SVETypeFlags::EltTyMFloat8:
985 return 16;
986 case SVETypeFlags::EltTyFloat16:
987 case SVETypeFlags::EltTyBFloat16:
988 return 8;
989 case SVETypeFlags::EltTyFloat32:
990 return 4;
991 case SVETypeFlags::EltTyFloat64:
992 return 2;
993
994 case SVETypeFlags::EltTyBool8:
995 return 16;
996 case SVETypeFlags::EltTyBool16:
997 return 8;
998 case SVETypeFlags::EltTyBool32:
999 return 4;
1000 case SVETypeFlags::EltTyBool64:
1001 return 2;
1002 }
1003}
1004
1005// TODO(cir): Share with OGCG
1006constexpr unsigned sveBitsPerBlock = 128;
1007
1008static cir::VectorType getSVEVectorForElementType(CIRGenModule &cgm,
1009 mlir::Type eltTy) {
1010 unsigned numElts =
1011 sveBitsPerBlock / cgm.getDataLayout().getTypeSizeInBits(eltTy);
1012 return cir::VectorType::get(eltTy, numElts, /*is_scalable=*/true);
1013}
1014
1015//===----------------------------------------------------------------------===//
1016// SVE helpers
1017//===----------------------------------------------------------------------===//
1018std::optional<mlir::Value>
1019CIRGenFunction::emitAArch64SVEBuiltinExpr(unsigned builtinID,
1020 const CallExpr *expr) {
1021 mlir::Type ty = convertType(expr->getType());
1022
1023 if (builtinID >= SVE::BI__builtin_sve_reinterpret_s8_s8 &&
1024 builtinID <= SVE::BI__builtin_sve_reinterpret_f64_f64_x4) {
1025 cgm.errorNYI(expr->getSourceRange(),
1026 std::string("unimplemented AArch64 builtin call: ") +
1027 getContext().BuiltinInfo.getName(builtinID));
1028 return mlir::Value{};
1029 }
1030
1031 assert(!cir::MissingFeatures::aarch64SVEIntrinsics());
1032
1033 auto *builtinIntrInfo = findARMVectorIntrinsicInMap(
1034 aarch64SVEIntrinsicMap, builtinID, aarch64SVEIntrinsicsProvenSorted);
1035
1036 // The operands of the builtin call
1037 llvm::SmallVector<mlir::Value> ops;
1038
1039 SVETypeFlags typeFlags(builtinIntrInfo->TypeModifier);
1040 if (!CIRGenFunction::getAArch64SVEProcessedOperands(builtinID, expr, ops,
1041 typeFlags))
1042 return mlir::Value{};
1043
1044 if (typeFlags.isLoad() || typeFlags.isStore() || typeFlags.isGatherLoad() ||
1045 typeFlags.isScatterStore() || typeFlags.isPrefetch() ||
1046 typeFlags.isGatherPrefetch() || typeFlags.isStructLoad() ||
1047 typeFlags.isStructStore() || typeFlags.isTupleSet() ||
1048 typeFlags.isTupleGet() || typeFlags.isTupleCreate() ||
1049 typeFlags.isUndef())
1050 cgm.errorNYI(expr->getSourceRange(),
1051 std::string("unimplemented AArch64 builtin call: ") +
1052 getContext().BuiltinInfo.getName(builtinID));
1053
1054 mlir::Location loc = getLoc(expr->getExprLoc());
1055
1056 // Handle built-ins for which there is a corresponding LLVM Intrinsic.
1057 // -------------------------------------------------------------------
1058 if (builtinIntrInfo->LLVMIntrinsic != 0) {
1059 // Emit set FPMR for intrinsics that require it.
1060 if (typeFlags.setsFPMR())
1061 cgm.errorNYI(expr->getSourceRange(),
1062 std::string("unimplemented AArch64 builtin call: ") +
1063 getContext().BuiltinInfo.getName(builtinID));
1064
1065 // Zero-ing predication
1066 if (typeFlags.getMergeType() == SVETypeFlags::MergeZeroExp) {
1067 auto null = builder.getNullValue(convertType(expr->getType()),
1068 getLoc(expr->getExprLoc()));
1069 ops.insert(ops.begin(), null);
1070 }
1071
1072 if (typeFlags.getMergeType() == SVETypeFlags::MergeAnyExp)
1073 ops.insert(ops.begin(),
1074 builder.getConstant(loc, cir::UndefAttr::get(ty)));
1075
1076 // Some ACLE builtins leave out the argument to specify the predicate
1077 // pattern, which is expected to be expanded to an SV_ALL pattern.
1078 if (typeFlags.isAppendSVALL())
1079 cgm.errorNYI(expr->getSourceRange(),
1080 std::string("unimplemented AArch64 builtin call: ") +
1081 getContext().BuiltinInfo.getName(builtinID));
1082 if (typeFlags.isInsertOp1SVALL())
1083 cgm.errorNYI(expr->getSourceRange(),
1084 std::string("unimplemented AArch64 builtin call: ") +
1085 getContext().BuiltinInfo.getName(builtinID));
1086
1087 // Predicates must match the main datatype.
1088 for (mlir::Value &op : ops)
1089 if (auto predTy = dyn_cast<cir::VectorType>(op.getType()))
1090 if (auto cirInt = dyn_cast<cir::IntType>(predTy.getElementType()))
1091 if (cirInt.getWidth() == 1)
1092 op = emitSVEPredicateCast(
1093 op, getSVEMinEltCount(typeFlags.getEltType()), loc);
1094
1095 // Splat scalar operand to vector (intrinsics with _n infix)
1096 if (typeFlags.hasSplatOperand()) {
1097 unsigned opNo = typeFlags.getSplatOperand();
1098 ops[opNo] = cir::VecSplatOp::create(
1099 builder, loc, getSVEVectorForElementType(cgm, ops[opNo].getType()),
1100 ops[opNo]);
1101 }
1102
1103 if (typeFlags.isReverseCompare())
1104 cgm.errorNYI(expr->getSourceRange(),
1105 std::string("unimplemented AArch64 builtin call: ") +
1106 getContext().BuiltinInfo.getName(builtinID));
1107 if (typeFlags.isReverseUSDOT())
1108 cgm.errorNYI(expr->getSourceRange(),
1109 std::string("unimplemented AArch64 builtin call: ") +
1110 getContext().BuiltinInfo.getName(builtinID));
1111 if (typeFlags.isReverseMergeAnyBinOp() &&
1112 typeFlags.getMergeType() == SVETypeFlags::MergeAny)
1113 cgm.errorNYI(expr->getSourceRange(),
1114 std::string("unimplemented AArch64 builtin call: ") +
1115 getContext().BuiltinInfo.getName(builtinID));
1116 if (typeFlags.isReverseMergeAnyAccOp() &&
1117 typeFlags.getMergeType() == SVETypeFlags::MergeAny)
1118 cgm.errorNYI(expr->getSourceRange(),
1119 std::string("unimplemented AArch64 builtin call: ") +
1120 getContext().BuiltinInfo.getName(builtinID));
1121
1122 // Predicated intrinsics with _z suffix.
1123 if (typeFlags.getMergeType() == SVETypeFlags::MergeZero) {
1124 cgm.errorNYI(expr->getSourceRange(),
1125 std::string("unimplemented AArch64 builtin call: ") +
1126 getContext().BuiltinInfo.getName(builtinID));
1127 }
1128
1129 llvm::StringRef llvmIntrName = getLLVMIntrNameNoPrefix(
1130 static_cast<llvm::Intrinsic::ID>(builtinIntrInfo->LLVMIntrinsic));
1131 auto retTy = convertType(expr->getType());
1132
1133 auto call = builder.emitIntrinsicCallOp(loc, llvmIntrName, retTy,
1134 mlir::ValueRange{ops});
1135 if (call.getType() == retTy)
1136 return call;
1137
1138 // Predicate results must be converted to svbool_t.
1139 if (isa<mlir::VectorType>(retTy) &&
1140 cast<mlir::VectorType>(retTy).isScalable())
1141 cgm.errorNYI(expr->getSourceRange(),
1142 std::string("unimplemented AArch64 builtin call: ") +
1143 getContext().BuiltinInfo.getName(builtinID));
1144 // TODO Handle struct types, e.g. svint8x2_t (update the converter first).
1145
1146 llvm_unreachable("unsupported element count!");
1147 }
1148
1149 // Handle the remaining built-ins.
1150 // -------------------------------
1151 switch (builtinID) {
1152 default:
1153 return std::nullopt;
1154
1155 case SVE::BI__builtin_sve_svreinterpret_b:
1156 case SVE::BI__builtin_sve_svreinterpret_c:
1157 case SVE::BI__builtin_sve_svpsel_lane_b8:
1158 case SVE::BI__builtin_sve_svpsel_lane_b16:
1159 case SVE::BI__builtin_sve_svpsel_lane_b32:
1160 case SVE::BI__builtin_sve_svpsel_lane_b64:
1161 case SVE::BI__builtin_sve_svpsel_lane_c8:
1162 case SVE::BI__builtin_sve_svpsel_lane_c16:
1163 case SVE::BI__builtin_sve_svpsel_lane_c32:
1164 case SVE::BI__builtin_sve_svpsel_lane_c64:
1165 case SVE::BI__builtin_sve_svmov_b_z:
1166 case SVE::BI__builtin_sve_svnot_b_z:
1167 case SVE::BI__builtin_sve_svmovlb_u16:
1168 case SVE::BI__builtin_sve_svmovlb_u32:
1169 case SVE::BI__builtin_sve_svmovlb_u64:
1170 case SVE::BI__builtin_sve_svmovlb_s16:
1171 case SVE::BI__builtin_sve_svmovlb_s32:
1172 case SVE::BI__builtin_sve_svmovlb_s64:
1173 case SVE::BI__builtin_sve_svmovlt_u16:
1174 case SVE::BI__builtin_sve_svmovlt_u32:
1175 case SVE::BI__builtin_sve_svmovlt_u64:
1176 case SVE::BI__builtin_sve_svmovlt_s16:
1177 case SVE::BI__builtin_sve_svmovlt_s32:
1178 case SVE::BI__builtin_sve_svmovlt_s64:
1179 case SVE::BI__builtin_sve_svpmullt_u16:
1180 case SVE::BI__builtin_sve_svpmullt_u64:
1181 case SVE::BI__builtin_sve_svpmullt_n_u16:
1182 case SVE::BI__builtin_sve_svpmullt_n_u64:
1183 case SVE::BI__builtin_sve_svpmullb_u16:
1184 case SVE::BI__builtin_sve_svpmullb_u64:
1185 case SVE::BI__builtin_sve_svpmullb_n_u16:
1186 case SVE::BI__builtin_sve_svpmullb_n_u64:
1187
1188 case SVE::BI__builtin_sve_svdup_n_b8:
1189 case SVE::BI__builtin_sve_svdup_n_b16:
1190 case SVE::BI__builtin_sve_svdup_n_b32:
1191 case SVE::BI__builtin_sve_svdup_n_b64:
1192
1193 case SVE::BI__builtin_sve_svdupq_n_b8:
1194 case SVE::BI__builtin_sve_svdupq_n_b16:
1195 case SVE::BI__builtin_sve_svdupq_n_b32:
1196 case SVE::BI__builtin_sve_svdupq_n_b64:
1197 case SVE::BI__builtin_sve_svdupq_n_u8:
1198 case SVE::BI__builtin_sve_svdupq_n_s8:
1199 case SVE::BI__builtin_sve_svdupq_n_u64:
1200 case SVE::BI__builtin_sve_svdupq_n_f64:
1201 case SVE::BI__builtin_sve_svdupq_n_s64:
1202 case SVE::BI__builtin_sve_svdupq_n_u16:
1203 case SVE::BI__builtin_sve_svdupq_n_f16:
1204 case SVE::BI__builtin_sve_svdupq_n_bf16:
1205 case SVE::BI__builtin_sve_svdupq_n_s16:
1206 case SVE::BI__builtin_sve_svdupq_n_u32:
1207 case SVE::BI__builtin_sve_svdupq_n_f32:
1208 case SVE::BI__builtin_sve_svdupq_n_s32:
1209 case SVE::BI__builtin_sve_svpfalse_b:
1210 case SVE::BI__builtin_sve_svpfalse_c:
1211 cgm.errorNYI(expr->getSourceRange(),
1212 std::string("unimplemented AArch64 builtin call: ") +
1213 getContext().BuiltinInfo.getName(builtinID));
1214 return mlir::Value{};
1215
1216 case SVE::BI__builtin_sve_svlen_u8:
1217 case SVE::BI__builtin_sve_svlen_s8:
1218 return genVscaleTimesFactor(loc, builder, convertType(expr->getType()), 16);
1219
1220 case SVE::BI__builtin_sve_svlen_u16:
1221 case SVE::BI__builtin_sve_svlen_s16:
1222 case SVE::BI__builtin_sve_svlen_f16:
1223 case SVE::BI__builtin_sve_svlen_bf16:
1224 return genVscaleTimesFactor(loc, builder, convertType(expr->getType()), 8);
1225
1226 case SVE::BI__builtin_sve_svlen_u32:
1227 case SVE::BI__builtin_sve_svlen_s32:
1228 case SVE::BI__builtin_sve_svlen_f32:
1229 return genVscaleTimesFactor(loc, builder, convertType(expr->getType()), 4);
1230
1231 case SVE::BI__builtin_sve_svlen_u64:
1232 case SVE::BI__builtin_sve_svlen_s64:
1233 case SVE::BI__builtin_sve_svlen_f64:
1234 return genVscaleTimesFactor(loc, builder, convertType(expr->getType()), 2);
1235
1236 case SVE::BI__builtin_sve_svtbl2_u8:
1237 case SVE::BI__builtin_sve_svtbl2_s8:
1238 case SVE::BI__builtin_sve_svtbl2_u16:
1239 case SVE::BI__builtin_sve_svtbl2_s16:
1240 case SVE::BI__builtin_sve_svtbl2_u32:
1241 case SVE::BI__builtin_sve_svtbl2_s32:
1242 case SVE::BI__builtin_sve_svtbl2_u64:
1243 case SVE::BI__builtin_sve_svtbl2_s64:
1244 case SVE::BI__builtin_sve_svtbl2_f16:
1245 case SVE::BI__builtin_sve_svtbl2_bf16:
1246 case SVE::BI__builtin_sve_svtbl2_f32:
1247 case SVE::BI__builtin_sve_svtbl2_f64:
1248 case SVE::BI__builtin_sve_svset_neonq_s8:
1249 case SVE::BI__builtin_sve_svset_neonq_s16:
1250 case SVE::BI__builtin_sve_svset_neonq_s32:
1251 case SVE::BI__builtin_sve_svset_neonq_s64:
1252 case SVE::BI__builtin_sve_svset_neonq_u8:
1253 case SVE::BI__builtin_sve_svset_neonq_u16:
1254 case SVE::BI__builtin_sve_svset_neonq_u32:
1255 case SVE::BI__builtin_sve_svset_neonq_u64:
1256 case SVE::BI__builtin_sve_svset_neonq_f16:
1257 case SVE::BI__builtin_sve_svset_neonq_f32:
1258 case SVE::BI__builtin_sve_svset_neonq_f64:
1259 case SVE::BI__builtin_sve_svset_neonq_bf16:
1260 case SVE::BI__builtin_sve_svget_neonq_s8:
1261 case SVE::BI__builtin_sve_svget_neonq_s16:
1262 case SVE::BI__builtin_sve_svget_neonq_s32:
1263 case SVE::BI__builtin_sve_svget_neonq_s64:
1264 case SVE::BI__builtin_sve_svget_neonq_u8:
1265 case SVE::BI__builtin_sve_svget_neonq_u16:
1266 case SVE::BI__builtin_sve_svget_neonq_u32:
1267 case SVE::BI__builtin_sve_svget_neonq_u64:
1268 case SVE::BI__builtin_sve_svget_neonq_f16:
1269 case SVE::BI__builtin_sve_svget_neonq_f32:
1270 case SVE::BI__builtin_sve_svget_neonq_f64:
1271 case SVE::BI__builtin_sve_svget_neonq_bf16:
1272 case SVE::BI__builtin_sve_svdup_neonq_s8:
1273 case SVE::BI__builtin_sve_svdup_neonq_s16:
1274 case SVE::BI__builtin_sve_svdup_neonq_s32:
1275 case SVE::BI__builtin_sve_svdup_neonq_s64:
1276 case SVE::BI__builtin_sve_svdup_neonq_u8:
1277 case SVE::BI__builtin_sve_svdup_neonq_u16:
1278 case SVE::BI__builtin_sve_svdup_neonq_u32:
1279 case SVE::BI__builtin_sve_svdup_neonq_u64:
1280 case SVE::BI__builtin_sve_svdup_neonq_f16:
1281 case SVE::BI__builtin_sve_svdup_neonq_f32:
1282 case SVE::BI__builtin_sve_svdup_neonq_f64:
1283 case SVE::BI__builtin_sve_svdup_neonq_bf16:
1284 cgm.errorNYI(expr->getSourceRange(),
1285 std::string("unimplemented AArch64 builtin call: ") +
1286 getContext().BuiltinInfo.getName(builtinID));
1287 return mlir::Value{};
1288 }
1289
1290 // Unreachable: All cases in the switch above return.
1291}
1292
1293std::optional<mlir::Value>
1294CIRGenFunction::emitAArch64SMEBuiltinExpr(unsigned builtinID,
1295 const CallExpr *expr) {
1296 assert(!cir::MissingFeatures::aarch64SMEIntrinsics());
1297
1298 cgm.errorNYI(expr->getSourceRange(),
1299 std::string("unimplemented AArch64 builtin call: ") +
1300 getContext().BuiltinInfo.getName(builtinID));
1301 return mlir::Value{};
1302}
1303
1304// Some intrinsics are equivalent for codegen.
1305static const std::pair<unsigned, unsigned> neonEquivalentIntrinsicMap[] = {
1306 {
1307 NEON::BI__builtin_neon_splat_lane_bf16,
1308 NEON::BI__builtin_neon_splat_lane_v,
1309 },
1310 {
1311 NEON::BI__builtin_neon_splat_laneq_bf16,
1312 NEON::BI__builtin_neon_splat_laneq_v,
1313 },
1314 {
1315 NEON::BI__builtin_neon_splatq_lane_bf16,
1316 NEON::BI__builtin_neon_splatq_lane_v,
1317 },
1318 {
1319 NEON::BI__builtin_neon_splatq_laneq_bf16,
1320 NEON::BI__builtin_neon_splatq_laneq_v,
1321 },
1322 {
1323 NEON::BI__builtin_neon_vabd_f16,
1324 NEON::BI__builtin_neon_vabd_v,
1325 },
1326 {
1327 NEON::BI__builtin_neon_vabdq_f16,
1328 NEON::BI__builtin_neon_vabdq_v,
1329 },
1330 {
1331 NEON::BI__builtin_neon_vabs_f16,
1332 NEON::BI__builtin_neon_vabs_v,
1333 },
1334 {
1335 NEON::BI__builtin_neon_vabsq_f16,
1336 NEON::BI__builtin_neon_vabsq_v,
1337 },
1338 {
1339 NEON::BI__builtin_neon_vcage_f16,
1340 NEON::BI__builtin_neon_vcage_v,
1341 },
1342 {
1343 NEON::BI__builtin_neon_vcageq_f16,
1344 NEON::BI__builtin_neon_vcageq_v,
1345 },
1346 {
1347 NEON::BI__builtin_neon_vcagt_f16,
1348 NEON::BI__builtin_neon_vcagt_v,
1349 },
1350 {
1351 NEON::BI__builtin_neon_vcagtq_f16,
1352 NEON::BI__builtin_neon_vcagtq_v,
1353 },
1354 {
1355 NEON::BI__builtin_neon_vcale_f16,
1356 NEON::BI__builtin_neon_vcale_v,
1357 },
1358 {
1359 NEON::BI__builtin_neon_vcaleq_f16,
1360 NEON::BI__builtin_neon_vcaleq_v,
1361 },
1362 {
1363 NEON::BI__builtin_neon_vcalt_f16,
1364 NEON::BI__builtin_neon_vcalt_v,
1365 },
1366 {
1367 NEON::BI__builtin_neon_vcaltq_f16,
1368 NEON::BI__builtin_neon_vcaltq_v,
1369 },
1370 {
1371 NEON::BI__builtin_neon_vceqz_f16,
1372 NEON::BI__builtin_neon_vceqz_v,
1373 },
1374 {
1375 NEON::BI__builtin_neon_vceqzq_f16,
1376 NEON::BI__builtin_neon_vceqzq_v,
1377 },
1378 {
1379 NEON::BI__builtin_neon_vcgez_f16,
1380 NEON::BI__builtin_neon_vcgez_v,
1381 },
1382 {
1383 NEON::BI__builtin_neon_vcgezq_f16,
1384 NEON::BI__builtin_neon_vcgezq_v,
1385 },
1386 {
1387 NEON::BI__builtin_neon_vcgtz_f16,
1388 NEON::BI__builtin_neon_vcgtz_v,
1389 },
1390 {
1391 NEON::BI__builtin_neon_vcgtzq_f16,
1392 NEON::BI__builtin_neon_vcgtzq_v,
1393 },
1394 {
1395 NEON::BI__builtin_neon_vclez_f16,
1396 NEON::BI__builtin_neon_vclez_v,
1397 },
1398 {
1399 NEON::BI__builtin_neon_vclezq_f16,
1400 NEON::BI__builtin_neon_vclezq_v,
1401 },
1402 {
1403 NEON::BI__builtin_neon_vcltz_f16,
1404 NEON::BI__builtin_neon_vcltz_v,
1405 },
1406 {
1407 NEON::BI__builtin_neon_vcltzq_f16,
1408 NEON::BI__builtin_neon_vcltzq_v,
1409 },
1410 {
1411 NEON::BI__builtin_neon_vfma_f16,
1412 NEON::BI__builtin_neon_vfma_v,
1413 },
1414 {
1415 NEON::BI__builtin_neon_vfma_lane_f16,
1416 NEON::BI__builtin_neon_vfma_lane_v,
1417 },
1418 {
1419 NEON::BI__builtin_neon_vfma_laneq_f16,
1420 NEON::BI__builtin_neon_vfma_laneq_v,
1421 },
1422 {
1423 NEON::BI__builtin_neon_vfmaq_f16,
1424 NEON::BI__builtin_neon_vfmaq_v,
1425 },
1426 {
1427 NEON::BI__builtin_neon_vfmaq_lane_f16,
1428 NEON::BI__builtin_neon_vfmaq_lane_v,
1429 },
1430 {
1431 NEON::BI__builtin_neon_vfmaq_laneq_f16,
1432 NEON::BI__builtin_neon_vfmaq_laneq_v,
1433 },
1434 {NEON::BI__builtin_neon_vld1_bf16_x2, NEON::BI__builtin_neon_vld1_x2_v},
1435 {NEON::BI__builtin_neon_vld1_bf16_x3, NEON::BI__builtin_neon_vld1_x3_v},
1436 {NEON::BI__builtin_neon_vld1_bf16_x4, NEON::BI__builtin_neon_vld1_x4_v},
1437 {NEON::BI__builtin_neon_vld1_bf16, NEON::BI__builtin_neon_vld1_v},
1438 {NEON::BI__builtin_neon_vld1_dup_bf16, NEON::BI__builtin_neon_vld1_dup_v},
1439 {NEON::BI__builtin_neon_vld1_lane_bf16, NEON::BI__builtin_neon_vld1_lane_v},
1440 {NEON::BI__builtin_neon_vld1q_bf16_x2, NEON::BI__builtin_neon_vld1q_x2_v},
1441 {NEON::BI__builtin_neon_vld1q_bf16_x3, NEON::BI__builtin_neon_vld1q_x3_v},
1442 {NEON::BI__builtin_neon_vld1q_bf16_x4, NEON::BI__builtin_neon_vld1q_x4_v},
1443 {NEON::BI__builtin_neon_vld1q_bf16, NEON::BI__builtin_neon_vld1q_v},
1444 {NEON::BI__builtin_neon_vld1q_dup_bf16, NEON::BI__builtin_neon_vld1q_dup_v},
1445 {NEON::BI__builtin_neon_vld1q_lane_bf16,
1446 NEON::BI__builtin_neon_vld1q_lane_v},
1447 {NEON::BI__builtin_neon_vld2_bf16, NEON::BI__builtin_neon_vld2_v},
1448 {NEON::BI__builtin_neon_vld2_dup_bf16, NEON::BI__builtin_neon_vld2_dup_v},
1449 {NEON::BI__builtin_neon_vld2_lane_bf16, NEON::BI__builtin_neon_vld2_lane_v},
1450 {NEON::BI__builtin_neon_vld2q_bf16, NEON::BI__builtin_neon_vld2q_v},
1451 {NEON::BI__builtin_neon_vld2q_dup_bf16, NEON::BI__builtin_neon_vld2q_dup_v},
1452 {NEON::BI__builtin_neon_vld2q_lane_bf16,
1453 NEON::BI__builtin_neon_vld2q_lane_v},
1454 {NEON::BI__builtin_neon_vld3_bf16, NEON::BI__builtin_neon_vld3_v},
1455 {NEON::BI__builtin_neon_vld3_dup_bf16, NEON::BI__builtin_neon_vld3_dup_v},
1456 {NEON::BI__builtin_neon_vld3_lane_bf16, NEON::BI__builtin_neon_vld3_lane_v},
1457 {NEON::BI__builtin_neon_vld3q_bf16, NEON::BI__builtin_neon_vld3q_v},
1458 {NEON::BI__builtin_neon_vld3q_dup_bf16, NEON::BI__builtin_neon_vld3q_dup_v},
1459 {NEON::BI__builtin_neon_vld3q_lane_bf16,
1460 NEON::BI__builtin_neon_vld3q_lane_v},
1461 {NEON::BI__builtin_neon_vld4_bf16, NEON::BI__builtin_neon_vld4_v},
1462 {NEON::BI__builtin_neon_vld4_dup_bf16, NEON::BI__builtin_neon_vld4_dup_v},
1463 {NEON::BI__builtin_neon_vld4_lane_bf16, NEON::BI__builtin_neon_vld4_lane_v},
1464 {NEON::BI__builtin_neon_vld4q_bf16, NEON::BI__builtin_neon_vld4q_v},
1465 {NEON::BI__builtin_neon_vld4q_dup_bf16, NEON::BI__builtin_neon_vld4q_dup_v},
1466 {NEON::BI__builtin_neon_vld4q_lane_bf16,
1467 NEON::BI__builtin_neon_vld4q_lane_v},
1468 {
1469 NEON::BI__builtin_neon_vmax_f16,
1470 NEON::BI__builtin_neon_vmax_v,
1471 },
1472 {
1473 NEON::BI__builtin_neon_vmaxnm_f16,
1474 NEON::BI__builtin_neon_vmaxnm_v,
1475 },
1476 {
1477 NEON::BI__builtin_neon_vmaxnmq_f16,
1478 NEON::BI__builtin_neon_vmaxnmq_v,
1479 },
1480 {
1481 NEON::BI__builtin_neon_vmaxq_f16,
1482 NEON::BI__builtin_neon_vmaxq_v,
1483 },
1484 {
1485 NEON::BI__builtin_neon_vmin_f16,
1486 NEON::BI__builtin_neon_vmin_v,
1487 },
1488 {
1489 NEON::BI__builtin_neon_vminnm_f16,
1490 NEON::BI__builtin_neon_vminnm_v,
1491 },
1492 {
1493 NEON::BI__builtin_neon_vminnmq_f16,
1494 NEON::BI__builtin_neon_vminnmq_v,
1495 },
1496 {
1497 NEON::BI__builtin_neon_vminq_f16,
1498 NEON::BI__builtin_neon_vminq_v,
1499 },
1500 {
1501 NEON::BI__builtin_neon_vmulx_f16,
1502 NEON::BI__builtin_neon_vmulx_v,
1503 },
1504 {
1505 NEON::BI__builtin_neon_vmulxq_f16,
1506 NEON::BI__builtin_neon_vmulxq_v,
1507 },
1508 {
1509 NEON::BI__builtin_neon_vpadd_f16,
1510 NEON::BI__builtin_neon_vpadd_v,
1511 },
1512 {
1513 NEON::BI__builtin_neon_vpaddq_f16,
1514 NEON::BI__builtin_neon_vpaddq_v,
1515 },
1516 {
1517 NEON::BI__builtin_neon_vpmax_f16,
1518 NEON::BI__builtin_neon_vpmax_v,
1519 },
1520 {
1521 NEON::BI__builtin_neon_vpmaxnm_f16,
1522 NEON::BI__builtin_neon_vpmaxnm_v,
1523 },
1524 {
1525 NEON::BI__builtin_neon_vpmaxnmq_f16,
1526 NEON::BI__builtin_neon_vpmaxnmq_v,
1527 },
1528 {
1529 NEON::BI__builtin_neon_vpmaxq_f16,
1530 NEON::BI__builtin_neon_vpmaxq_v,
1531 },
1532 {
1533 NEON::BI__builtin_neon_vpmin_f16,
1534 NEON::BI__builtin_neon_vpmin_v,
1535 },
1536 {
1537 NEON::BI__builtin_neon_vpminnm_f16,
1538 NEON::BI__builtin_neon_vpminnm_v,
1539 },
1540 {
1541 NEON::BI__builtin_neon_vpminnmq_f16,
1542 NEON::BI__builtin_neon_vpminnmq_v,
1543 },
1544 {
1545 NEON::BI__builtin_neon_vpminq_f16,
1546 NEON::BI__builtin_neon_vpminq_v,
1547 },
1548 {
1549 NEON::BI__builtin_neon_vrecpe_f16,
1550 NEON::BI__builtin_neon_vrecpe_v,
1551 },
1552 {
1553 NEON::BI__builtin_neon_vrecpeq_f16,
1554 NEON::BI__builtin_neon_vrecpeq_v,
1555 },
1556 {
1557 NEON::BI__builtin_neon_vrecps_f16,
1558 NEON::BI__builtin_neon_vrecps_v,
1559 },
1560 {
1561 NEON::BI__builtin_neon_vrecpsq_f16,
1562 NEON::BI__builtin_neon_vrecpsq_v,
1563 },
1564 {
1565 NEON::BI__builtin_neon_vrnd_f16,
1566 NEON::BI__builtin_neon_vrnd_v,
1567 },
1568 {
1569 NEON::BI__builtin_neon_vrnda_f16,
1570 NEON::BI__builtin_neon_vrnda_v,
1571 },
1572 {
1573 NEON::BI__builtin_neon_vrndaq_f16,
1574 NEON::BI__builtin_neon_vrndaq_v,
1575 },
1576 {
1577 NEON::BI__builtin_neon_vrndi_f16,
1578 NEON::BI__builtin_neon_vrndi_v,
1579 },
1580 {
1581 NEON::BI__builtin_neon_vrndiq_f16,
1582 NEON::BI__builtin_neon_vrndiq_v,
1583 },
1584 {
1585 NEON::BI__builtin_neon_vrndm_f16,
1586 NEON::BI__builtin_neon_vrndm_v,
1587 },
1588 {
1589 NEON::BI__builtin_neon_vrndmq_f16,
1590 NEON::BI__builtin_neon_vrndmq_v,
1591 },
1592 {
1593 NEON::BI__builtin_neon_vrndn_f16,
1594 NEON::BI__builtin_neon_vrndn_v,
1595 },
1596 {
1597 NEON::BI__builtin_neon_vrndnq_f16,
1598 NEON::BI__builtin_neon_vrndnq_v,
1599 },
1600 {
1601 NEON::BI__builtin_neon_vrndp_f16,
1602 NEON::BI__builtin_neon_vrndp_v,
1603 },
1604 {
1605 NEON::BI__builtin_neon_vrndpq_f16,
1606 NEON::BI__builtin_neon_vrndpq_v,
1607 },
1608 {
1609 NEON::BI__builtin_neon_vrndq_f16,
1610 NEON::BI__builtin_neon_vrndq_v,
1611 },
1612 {
1613 NEON::BI__builtin_neon_vrndx_f16,
1614 NEON::BI__builtin_neon_vrndx_v,
1615 },
1616 {
1617 NEON::BI__builtin_neon_vrndxq_f16,
1618 NEON::BI__builtin_neon_vrndxq_v,
1619 },
1620 {
1621 NEON::BI__builtin_neon_vrsqrte_f16,
1622 NEON::BI__builtin_neon_vrsqrte_v,
1623 },
1624 {
1625 NEON::BI__builtin_neon_vrsqrteq_f16,
1626 NEON::BI__builtin_neon_vrsqrteq_v,
1627 },
1628 {
1629 NEON::BI__builtin_neon_vrsqrts_f16,
1630 NEON::BI__builtin_neon_vrsqrts_v,
1631 },
1632 {
1633 NEON::BI__builtin_neon_vrsqrtsq_f16,
1634 NEON::BI__builtin_neon_vrsqrtsq_v,
1635 },
1636 {
1637 NEON::BI__builtin_neon_vsqrt_f16,
1638 NEON::BI__builtin_neon_vsqrt_v,
1639 },
1640 {
1641 NEON::BI__builtin_neon_vsqrtq_f16,
1642 NEON::BI__builtin_neon_vsqrtq_v,
1643 },
1644 {NEON::BI__builtin_neon_vst1_bf16_x2, NEON::BI__builtin_neon_vst1_x2_v},
1645 {NEON::BI__builtin_neon_vst1_bf16_x3, NEON::BI__builtin_neon_vst1_x3_v},
1646 {NEON::BI__builtin_neon_vst1_bf16_x4, NEON::BI__builtin_neon_vst1_x4_v},
1647 {NEON::BI__builtin_neon_vst1_bf16, NEON::BI__builtin_neon_vst1_v},
1648 {NEON::BI__builtin_neon_vst1_lane_bf16, NEON::BI__builtin_neon_vst1_lane_v},
1649 {NEON::BI__builtin_neon_vst1q_bf16_x2, NEON::BI__builtin_neon_vst1q_x2_v},
1650 {NEON::BI__builtin_neon_vst1q_bf16_x3, NEON::BI__builtin_neon_vst1q_x3_v},
1651 {NEON::BI__builtin_neon_vst1q_bf16_x4, NEON::BI__builtin_neon_vst1q_x4_v},
1652 {NEON::BI__builtin_neon_vst1q_bf16, NEON::BI__builtin_neon_vst1q_v},
1653 {NEON::BI__builtin_neon_vst1q_lane_bf16,
1654 NEON::BI__builtin_neon_vst1q_lane_v},
1655 {NEON::BI__builtin_neon_vst2_bf16, NEON::BI__builtin_neon_vst2_v},
1656 {NEON::BI__builtin_neon_vst2_lane_bf16, NEON::BI__builtin_neon_vst2_lane_v},
1657 {NEON::BI__builtin_neon_vst2q_bf16, NEON::BI__builtin_neon_vst2q_v},
1658 {NEON::BI__builtin_neon_vst2q_lane_bf16,
1659 NEON::BI__builtin_neon_vst2q_lane_v},
1660 {NEON::BI__builtin_neon_vst3_bf16, NEON::BI__builtin_neon_vst3_v},
1661 {NEON::BI__builtin_neon_vst3_lane_bf16, NEON::BI__builtin_neon_vst3_lane_v},
1662 {NEON::BI__builtin_neon_vst3q_bf16, NEON::BI__builtin_neon_vst3q_v},
1663 {NEON::BI__builtin_neon_vst3q_lane_bf16,
1664 NEON::BI__builtin_neon_vst3q_lane_v},
1665 {NEON::BI__builtin_neon_vst4_bf16, NEON::BI__builtin_neon_vst4_v},
1666 {NEON::BI__builtin_neon_vst4_lane_bf16, NEON::BI__builtin_neon_vst4_lane_v},
1667 {NEON::BI__builtin_neon_vst4q_bf16, NEON::BI__builtin_neon_vst4q_v},
1668 {NEON::BI__builtin_neon_vst4q_lane_bf16,
1669 NEON::BI__builtin_neon_vst4q_lane_v},
1670 // The mangling rules cause us to have one ID for each type for
1671 // vldap1(q)_lane and vstl1(q)_lane, but codegen is equivalent for all of
1672 // them. Choose an arbitrary one to be handled as tha canonical variation.
1673 {NEON::BI__builtin_neon_vldap1_lane_u64,
1674 NEON::BI__builtin_neon_vldap1_lane_s64},
1675 {NEON::BI__builtin_neon_vldap1_lane_f64,
1676 NEON::BI__builtin_neon_vldap1_lane_s64},
1677 {NEON::BI__builtin_neon_vldap1_lane_p64,
1678 NEON::BI__builtin_neon_vldap1_lane_s64},
1679 {NEON::BI__builtin_neon_vldap1q_lane_u64,
1680 NEON::BI__builtin_neon_vldap1q_lane_s64},
1681 {NEON::BI__builtin_neon_vldap1q_lane_f64,
1682 NEON::BI__builtin_neon_vldap1q_lane_s64},
1683 {NEON::BI__builtin_neon_vldap1q_lane_p64,
1684 NEON::BI__builtin_neon_vldap1q_lane_s64},
1685 {NEON::BI__builtin_neon_vstl1_lane_u64,
1686 NEON::BI__builtin_neon_vstl1_lane_s64},
1687 {NEON::BI__builtin_neon_vstl1_lane_f64,
1688 NEON::BI__builtin_neon_vstl1_lane_s64},
1689 {NEON::BI__builtin_neon_vstl1_lane_p64,
1690 NEON::BI__builtin_neon_vstl1_lane_s64},
1691 {NEON::BI__builtin_neon_vstl1q_lane_u64,
1692 NEON::BI__builtin_neon_vstl1q_lane_s64},
1693 {NEON::BI__builtin_neon_vstl1q_lane_f64,
1694 NEON::BI__builtin_neon_vstl1q_lane_s64},
1695 {NEON::BI__builtin_neon_vstl1q_lane_p64,
1696 NEON::BI__builtin_neon_vstl1q_lane_s64},
1697};
1698
1699std::optional<mlir::Value>
1700CIRGenFunction::emitAArch64BuiltinExpr(unsigned builtinID, const CallExpr *expr,
1701 ReturnValueSlot returnValue,
1702 llvm::Triple::ArchType arch) {
1703 if (builtinID >= clang::AArch64::FirstSVEBuiltin &&
1704 builtinID <= clang::AArch64::LastSVEBuiltin)
1705 return emitAArch64SVEBuiltinExpr(builtinID, expr);
1706
1707 if (builtinID >= clang::AArch64::FirstSMEBuiltin &&
1708 builtinID <= clang::AArch64::LastSMEBuiltin)
1709 return emitAArch64SMEBuiltinExpr(builtinID, expr);
1710
1711 if (builtinID == Builtin::BI__builtin_cpu_supports) {
1712 cgm.errorNYI(expr->getSourceRange(),
1713 std::string("unimplemented AArch64 builtin call: ") +
1714 getContext().BuiltinInfo.getName(builtinID));
1715 return mlir::Value{};
1716 }
1717
1718 switch (builtinID) {
1719 default:
1720 break;
1721 case clang::AArch64::BI__builtin_arm_nop:
1722 case clang::AArch64::BI__builtin_arm_yield:
1723 case clang::AArch64::BI__yield:
1724 case clang::AArch64::BI__builtin_arm_wfe:
1725 case clang::AArch64::BI__wfe:
1726 case clang::AArch64::BI__builtin_arm_wfi:
1727 case clang::AArch64::BI__wfi:
1728 case clang::AArch64::BI__builtin_arm_sev:
1729 case clang::AArch64::BI__sev:
1730 case clang::AArch64::BI__builtin_arm_sevl:
1731 case clang::AArch64::BI__sevl:
1732 cgm.errorNYI(expr->getSourceRange(),
1733 std::string("unimplemented AArch64 builtin call: ") +
1734 getContext().BuiltinInfo.getName(builtinID));
1735 return mlir::Value{};
1736 }
1737
1738 if (builtinID == clang::AArch64::BI__builtin_arm_trap) {
1739 cgm.errorNYI(expr->getSourceRange(),
1740 std::string("unimplemented AArch64 builtin call: ") +
1741 getContext().BuiltinInfo.getName(builtinID));
1742 return mlir::Value{};
1743 }
1744
1745 if (builtinID == clang::AArch64::BI__builtin_arm_get_sme_state) {
1746 cgm.errorNYI(expr->getSourceRange(),
1747 std::string("unimplemented AArch64 builtin call: ") +
1748 getContext().BuiltinInfo.getName(builtinID));
1749 return mlir::Value{};
1750 }
1751
1752 if (builtinID == clang::AArch64::BI__builtin_arm_rbit) {
1753 cgm.errorNYI(expr->getSourceRange(),
1754 std::string("unimplemented AArch64 builtin call: ") +
1755 getContext().BuiltinInfo.getName(builtinID));
1756 return mlir::Value{};
1757 }
1758 if (builtinID == clang::AArch64::BI__builtin_arm_rbit64) {
1759 cgm.errorNYI(expr->getSourceRange(),
1760 std::string("unimplemented AArch64 builtin call: ") +
1761 getContext().BuiltinInfo.getName(builtinID));
1762 return mlir::Value{};
1763 }
1764
1765 if (builtinID == clang::AArch64::BI__builtin_arm_clz ||
1766 builtinID == clang::AArch64::BI__builtin_arm_clz64) {
1767 cgm.errorNYI(expr->getSourceRange(),
1768 std::string("unimplemented AArch64 builtin call: ") +
1769 getContext().BuiltinInfo.getName(builtinID));
1770 return mlir::Value{};
1771 }
1772
1773 if (builtinID == clang::AArch64::BI__builtin_arm_cls) {
1774 cgm.errorNYI(expr->getSourceRange(),
1775 std::string("unimplemented AArch64 builtin call: ") +
1776 getContext().BuiltinInfo.getName(builtinID));
1777 return mlir::Value{};
1778 }
1779 if (builtinID == clang::AArch64::BI__builtin_arm_cls64) {
1780 cgm.errorNYI(expr->getSourceRange(),
1781 std::string("unimplemented AArch64 builtin call: ") +
1782 getContext().BuiltinInfo.getName(builtinID));
1783 return mlir::Value{};
1784 }
1785
1786 if (builtinID == clang::AArch64::BI__builtin_arm_rint32zf ||
1787 builtinID == clang::AArch64::BI__builtin_arm_rint32z) {
1788 cgm.errorNYI(expr->getSourceRange(),
1789 std::string("unimplemented AArch64 builtin call: ") +
1790 getContext().BuiltinInfo.getName(builtinID));
1791 return mlir::Value{};
1792 }
1793
1794 if (builtinID == clang::AArch64::BI__builtin_arm_rint64zf ||
1795 builtinID == clang::AArch64::BI__builtin_arm_rint64z) {
1796 cgm.errorNYI(expr->getSourceRange(),
1797 std::string("unimplemented AArch64 builtin call: ") +
1798 getContext().BuiltinInfo.getName(builtinID));
1799 return mlir::Value{};
1800 }
1801
1802 if (builtinID == clang::AArch64::BI__builtin_arm_rint32xf ||
1803 builtinID == clang::AArch64::BI__builtin_arm_rint32x) {
1804 cgm.errorNYI(expr->getSourceRange(),
1805 std::string("unimplemented AArch64 builtin call: ") +
1806 getContext().BuiltinInfo.getName(builtinID));
1807 return mlir::Value{};
1808 }
1809
1810 if (builtinID == clang::AArch64::BI__builtin_arm_rint64xf ||
1811 builtinID == clang::AArch64::BI__builtin_arm_rint64x) {
1812 cgm.errorNYI(expr->getSourceRange(),
1813 std::string("unimplemented AArch64 builtin call: ") +
1814 getContext().BuiltinInfo.getName(builtinID));
1815 return mlir::Value{};
1816 }
1817
1818 if (builtinID == clang::AArch64::BI__builtin_arm_jcvt) {
1819 cgm.errorNYI(expr->getSourceRange(),
1820 std::string("unimplemented AArch64 builtin call: ") +
1821 getContext().BuiltinInfo.getName(builtinID));
1822 return mlir::Value{};
1823 }
1824
1825 if (builtinID == clang::AArch64::BI__builtin_arm_ld64b ||
1826 builtinID == clang::AArch64::BI__builtin_arm_st64b ||
1827 builtinID == clang::AArch64::BI__builtin_arm_st64bv ||
1828 builtinID == clang::AArch64::BI__builtin_arm_st64bv0) {
1829 cgm.errorNYI(expr->getSourceRange(),
1830 std::string("unimplemented AArch64 builtin call: ") +
1831 getContext().BuiltinInfo.getName(builtinID));
1832 return mlir::Value{};
1833 }
1834
1835 if (builtinID == clang::AArch64::BI__builtin_arm_rndr ||
1836 builtinID == clang::AArch64::BI__builtin_arm_rndrrs) {
1837 cgm.errorNYI(expr->getSourceRange(),
1838 std::string("unimplemented AArch64 builtin call: ") +
1839 getContext().BuiltinInfo.getName(builtinID));
1840 return mlir::Value{};
1841 }
1842
1843 if (builtinID == clang::AArch64::BI__clear_cache) {
1844 cgm.errorNYI(expr->getSourceRange(),
1845 std::string("unimplemented AArch64 builtin call: ") +
1846 getContext().BuiltinInfo.getName(builtinID));
1847 return mlir::Value{};
1848 }
1849
1850 if ((builtinID == clang::AArch64::BI__builtin_arm_ldrex ||
1851 builtinID == clang::AArch64::BI__builtin_arm_ldaex) &&
1852 getContext().getTypeSize(expr->getType()) == 128) {
1853 cgm.errorNYI(expr->getSourceRange(),
1854 std::string("unimplemented AArch64 builtin call: ") +
1855 getContext().BuiltinInfo.getName(builtinID));
1856 return mlir::Value{};
1857 }
1858 if (builtinID == clang::AArch64::BI__builtin_arm_ldrex ||
1859 builtinID == clang::AArch64::BI__builtin_arm_ldaex) {
1860 cgm.errorNYI(expr->getSourceRange(),
1861 std::string("unimplemented AArch64 builtin call: ") +
1862 getContext().BuiltinInfo.getName(builtinID));
1863 return mlir::Value{};
1864 }
1865
1866 if ((builtinID == clang::AArch64::BI__builtin_arm_strex ||
1867 builtinID == clang::AArch64::BI__builtin_arm_stlex) &&
1868 getContext().getTypeSize(expr->getArg(0)->getType()) == 128) {
1869 cgm.errorNYI(expr->getSourceRange(),
1870 std::string("unimplemented AArch64 builtin call: ") +
1871 getContext().BuiltinInfo.getName(builtinID));
1872 return mlir::Value{};
1873 }
1874
1875 if (builtinID == clang::AArch64::BI__builtin_arm_strex ||
1876 builtinID == clang::AArch64::BI__builtin_arm_stlex) {
1877 cgm.errorNYI(expr->getSourceRange(),
1878 std::string("unimplemented AArch64 builtin call: ") +
1879 getContext().BuiltinInfo.getName(builtinID));
1880 return mlir::Value{};
1881 }
1882
1883 if (builtinID == clang::AArch64::BI__getReg) {
1884 cgm.errorNYI(expr->getSourceRange(),
1885 std::string("unimplemented AArch64 builtin call: ") +
1886 getContext().BuiltinInfo.getName(builtinID));
1887 return mlir::Value{};
1888 }
1889
1890 if (builtinID == clang::AArch64::BI__break) {
1891 cgm.errorNYI(expr->getSourceRange(),
1892 std::string("unimplemented AArch64 builtin call: ") +
1893 getContext().BuiltinInfo.getName(builtinID));
1894 return mlir::Value{};
1895 }
1896
1897 if (builtinID == clang::AArch64::BI__builtin_arm_clrex) {
1898 cgm.errorNYI(expr->getSourceRange(),
1899 std::string("unimplemented AArch64 builtin call: ") +
1900 getContext().BuiltinInfo.getName(builtinID));
1901 return mlir::Value{};
1902 }
1903
1904 if (builtinID == clang::AArch64::BI_ReadWriteBarrier) {
1905 cgm.errorNYI(expr->getSourceRange(),
1906 std::string("unimplemented AArch64 builtin call: ") +
1907 getContext().BuiltinInfo.getName(builtinID));
1908 return mlir::Value{};
1909 }
1910
1911 // CRC32
1912 Intrinsic::ID crcIntrinsicID = Intrinsic::not_intrinsic;
1913 switch (builtinID) {
1914 case clang::AArch64::BI__builtin_arm_crc32b:
1915 crcIntrinsicID = Intrinsic::aarch64_crc32b;
1916 break;
1917 case clang::AArch64::BI__builtin_arm_crc32cb:
1918 crcIntrinsicID = Intrinsic::aarch64_crc32cb;
1919 break;
1920 case clang::AArch64::BI__builtin_arm_crc32h:
1921 crcIntrinsicID = Intrinsic::aarch64_crc32h;
1922 break;
1923 case clang::AArch64::BI__builtin_arm_crc32ch:
1924 crcIntrinsicID = Intrinsic::aarch64_crc32ch;
1925 break;
1926 case clang::AArch64::BI__builtin_arm_crc32w:
1927 crcIntrinsicID = Intrinsic::aarch64_crc32w;
1928 break;
1929 case clang::AArch64::BI__builtin_arm_crc32cw:
1930 crcIntrinsicID = Intrinsic::aarch64_crc32cw;
1931 break;
1932 case clang::AArch64::BI__builtin_arm_crc32d:
1933 crcIntrinsicID = Intrinsic::aarch64_crc32x;
1934 break;
1935 case clang::AArch64::BI__builtin_arm_crc32cd:
1936 crcIntrinsicID = Intrinsic::aarch64_crc32cx;
1937 break;
1938 }
1939
1940 if (crcIntrinsicID != Intrinsic::not_intrinsic) {
1941 cgm.errorNYI(expr->getSourceRange(),
1942 std::string("unimplemented AArch64 builtin call: ") +
1943 getContext().BuiltinInfo.getName(builtinID));
1944 return mlir::Value{};
1945 }
1946
1947 // Memory Operations (MOPS)
1948 if (builtinID == AArch64::BI__builtin_arm_mops_memset_tag) {
1949 cgm.errorNYI(expr->getSourceRange(),
1950 std::string("unimplemented AArch64 builtin call: ") +
1951 getContext().BuiltinInfo.getName(builtinID));
1952 return mlir::Value{};
1953 }
1954
1955 // Memory Tagging Extensions (MTE) Intrinsics
1956 Intrinsic::ID mteIntrinsicID = Intrinsic::not_intrinsic;
1957 switch (builtinID) {
1958 case clang::AArch64::BI__builtin_arm_irg:
1959 mteIntrinsicID = Intrinsic::aarch64_irg;
1960 break;
1961 case clang::AArch64::BI__builtin_arm_addg:
1962 mteIntrinsicID = Intrinsic::aarch64_addg;
1963 break;
1964 case clang::AArch64::BI__builtin_arm_gmi:
1965 mteIntrinsicID = Intrinsic::aarch64_gmi;
1966 break;
1967 case clang::AArch64::BI__builtin_arm_ldg:
1968 mteIntrinsicID = Intrinsic::aarch64_ldg;
1969 break;
1970 case clang::AArch64::BI__builtin_arm_stg:
1971 mteIntrinsicID = Intrinsic::aarch64_stg;
1972 break;
1973 case clang::AArch64::BI__builtin_arm_subp:
1974 mteIntrinsicID = Intrinsic::aarch64_subp;
1975 break;
1976 }
1977
1978 if (mteIntrinsicID != Intrinsic::not_intrinsic) {
1979 cgm.errorNYI(expr->getSourceRange(),
1980 std::string("unimplemented AArch64 builtin call: ") +
1981 getContext().BuiltinInfo.getName(builtinID));
1982 return mlir::Value{};
1983 }
1984
1985 if (builtinID == clang::AArch64::BI__builtin_arm_rsr ||
1986 builtinID == clang::AArch64::BI__builtin_arm_rsr64 ||
1987 builtinID == clang::AArch64::BI__builtin_arm_rsr128 ||
1988 builtinID == clang::AArch64::BI__builtin_arm_rsrp ||
1989 builtinID == clang::AArch64::BI__builtin_arm_wsr ||
1990 builtinID == clang::AArch64::BI__builtin_arm_wsr64 ||
1991 builtinID == clang::AArch64::BI__builtin_arm_wsr128 ||
1992 builtinID == clang::AArch64::BI__builtin_arm_wsrp) {
1993 cgm.errorNYI(expr->getSourceRange(),
1994 std::string("unimplemented AArch64 builtin call: ") +
1995 getContext().BuiltinInfo.getName(builtinID));
1996 return mlir::Value{};
1997 }
1998
1999 if (builtinID == clang::AArch64::BI_ReadStatusReg ||
2000 builtinID == clang::AArch64::BI_WriteStatusReg ||
2001 builtinID == clang::AArch64::BI__sys) {
2002 cgm.errorNYI(expr->getSourceRange(),
2003 std::string("unimplemented AArch64 builtin call: ") +
2004 getContext().BuiltinInfo.getName(builtinID));
2005 return mlir::Value{};
2006 }
2007
2008 if (builtinID == clang::AArch64::BI_AddressOfReturnAddress) {
2009 cgm.errorNYI(expr->getSourceRange(),
2010 std::string("unimplemented AArch64 builtin call: ") +
2011 getContext().BuiltinInfo.getName(builtinID));
2012 return mlir::Value{};
2013 }
2014
2015 if (builtinID == clang::AArch64::BI__builtin_sponentry) {
2016 cgm.errorNYI(expr->getSourceRange(),
2017 std::string("unimplemented AArch64 builtin call: ") +
2018 getContext().BuiltinInfo.getName(builtinID));
2019 return mlir::Value{};
2020 }
2021
2022 if (builtinID == clang::AArch64::BI__mulh ||
2023 builtinID == clang::AArch64::BI__umulh) {
2024 cgm.errorNYI(expr->getSourceRange(),
2025 std::string("unimplemented AArch64 builtin call: ") +
2026 getContext().BuiltinInfo.getName(builtinID));
2027 return mlir::Value{};
2028 }
2029
2030 if (builtinID == AArch64::BI__writex18byte ||
2031 builtinID == AArch64::BI__writex18word ||
2032 builtinID == AArch64::BI__writex18dword ||
2033 builtinID == AArch64::BI__writex18qword) {
2034 cgm.errorNYI(expr->getSourceRange(),
2035 std::string("unimplemented AArch64 builtin call: ") +
2036 getContext().BuiltinInfo.getName(builtinID));
2037 return mlir::Value{};
2038 }
2039
2040 if (builtinID == AArch64::BI__readx18byte ||
2041 builtinID == AArch64::BI__readx18word ||
2042 builtinID == AArch64::BI__readx18dword ||
2043 builtinID == AArch64::BI__readx18qword) {
2044 cgm.errorNYI(expr->getSourceRange(),
2045 std::string("unimplemented AArch64 builtin call: ") +
2046 getContext().BuiltinInfo.getName(builtinID));
2047 return mlir::Value{};
2048 }
2049
2050 if (builtinID == AArch64::BI__addx18byte ||
2051 builtinID == AArch64::BI__addx18word ||
2052 builtinID == AArch64::BI__addx18dword ||
2053 builtinID == AArch64::BI__addx18qword ||
2054 builtinID == AArch64::BI__incx18byte ||
2055 builtinID == AArch64::BI__incx18word ||
2056 builtinID == AArch64::BI__incx18dword ||
2057 builtinID == AArch64::BI__incx18qword) {
2058 cgm.errorNYI(expr->getSourceRange(),
2059 std::string("unimplemented AArch64 builtin call: ") +
2060 getContext().BuiltinInfo.getName(builtinID));
2061 return mlir::Value{};
2062 }
2063
2064 if (builtinID == AArch64::BI_CopyDoubleFromInt64 ||
2065 builtinID == AArch64::BI_CopyFloatFromInt32 ||
2066 builtinID == AArch64::BI_CopyInt32FromFloat ||
2067 builtinID == AArch64::BI_CopyInt64FromDouble) {
2068 cgm.errorNYI(expr->getSourceRange(),
2069 std::string("unimplemented AArch64 builtin call: ") +
2070 getContext().BuiltinInfo.getName(builtinID));
2071 return mlir::Value{};
2072 }
2073
2074 if (builtinID == AArch64::BI_CountLeadingOnes ||
2075 builtinID == AArch64::BI_CountLeadingOnes64 ||
2076 builtinID == AArch64::BI_CountLeadingZeros ||
2077 builtinID == AArch64::BI_CountLeadingZeros64) {
2078 cgm.errorNYI(expr->getSourceRange(),
2079 std::string("unimplemented AArch64 builtin call: ") +
2080 getContext().BuiltinInfo.getName(builtinID));
2081 return mlir::Value{};
2082 }
2083
2084 if (builtinID == AArch64::BI_CountLeadingSigns ||
2085 builtinID == AArch64::BI_CountLeadingSigns64) {
2086 cgm.errorNYI(expr->getSourceRange(),
2087 std::string("unimplemented AArch64 builtin call: ") +
2088 getContext().BuiltinInfo.getName(builtinID));
2089 return mlir::Value{};
2090 }
2091
2092 if (builtinID == AArch64::BI_CountOneBits ||
2093 builtinID == AArch64::BI_CountOneBits64) {
2094 cgm.errorNYI(expr->getSourceRange(),
2095 std::string("unimplemented AArch64 builtin call: ") +
2096 getContext().BuiltinInfo.getName(builtinID));
2097 return mlir::Value{};
2098 }
2099
2100 if (builtinID == AArch64::BI__prefetch) {
2101 cgm.errorNYI(expr->getSourceRange(),
2102 std::string("unimplemented AArch64 builtin call: ") +
2103 getContext().BuiltinInfo.getName(builtinID));
2104 return mlir::Value{};
2105 }
2106
2107 if (builtinID == AArch64::BI__hlt) {
2108 cgm.errorNYI(expr->getSourceRange(),
2109 std::string("unimplemented AArch64 builtin call: ") +
2110 getContext().BuiltinInfo.getName(builtinID));
2111 return mlir::Value{};
2112 }
2113
2114 if (builtinID == NEON::BI__builtin_neon_vcvth_bf16_f32) {
2115 cgm.errorNYI(expr->getSourceRange(),
2116 std::string("unimplemented AArch64 builtin call: ") +
2117 getContext().BuiltinInfo.getName(builtinID));
2118 return mlir::Value{};
2119 }
2120
2121 // Handle MSVC intrinsics before argument evaluation to prevent double
2122 // evaluation.
2123 assert(!cir::MissingFeatures::msvcBuiltins());
2124
2125 // Some intrinsics are equivalent - if they are use the base intrinsic ID.
2126 auto it = llvm::find_if(neonEquivalentIntrinsicMap, [builtinID](auto &p) {
2127 return p.first == builtinID;
2128 });
2129 if (it != end(neonEquivalentIntrinsicMap))
2130 builtinID = it->second;
2131
2132 // Find out if any arguments are required to be integer constant
2133 // expressions.
2134 assert(!cir::MissingFeatures::handleBuiltinICEArguments());
2135 unsigned iceArguments = 0;
2136 ASTContext::GetBuiltinTypeError error;
2137 getContext().GetBuiltinType(builtinID, error, &iceArguments);
2138 assert(error == ASTContext::GE_None && "Should not codegen an error");
2139 llvm::SmallVector<mlir::Value> ops;
2140
2141 // Skip extra arguments used to discriminate vector types and that are
2142 // intended for Sema checking.
2143 bool hasExtraArg = hasExtraNeonArgument(builtinID);
2144 unsigned numArgs = expr->getNumArgs() - (hasExtraArg ? 1 : 0);
2145 for (unsigned i = 0, e = numArgs; i != e; i++) {
2146 if (i == 0) {
2147 switch (builtinID) {
2148 case NEON::BI__builtin_neon_vld1_v:
2149 case NEON::BI__builtin_neon_vld1q_v:
2150 case NEON::BI__builtin_neon_vld1_dup_v:
2151 case NEON::BI__builtin_neon_vld1q_dup_v:
2152 case NEON::BI__builtin_neon_vld1_lane_v:
2153 case NEON::BI__builtin_neon_vld1q_lane_v:
2154 case NEON::BI__builtin_neon_vst1_v:
2155 case NEON::BI__builtin_neon_vst1q_v:
2156 case NEON::BI__builtin_neon_vst1_lane_v:
2157 case NEON::BI__builtin_neon_vst1q_lane_v:
2158 case NEON::BI__builtin_neon_vldap1_lane_s64:
2159 case NEON::BI__builtin_neon_vldap1q_lane_s64:
2160 case NEON::BI__builtin_neon_vstl1_lane_s64:
2161 case NEON::BI__builtin_neon_vstl1q_lane_s64:
2162 // Get the alignment for the argument in addition to the value;
2163 // we'll use it later.
2164 cgm.errorNYI(
2165 expr->getSourceRange(),
2166 std::string("unimplemented AArch64 builtin argument handling ") +
2167 getContext().BuiltinInfo.getName(builtinID));
2168 }
2169 }
2170 ops.push_back(
2171 emitScalarOrConstFoldImmArg(iceArguments, i, expr->getArg(i)));
2172 }
2173
2174 const ARMVectorIntrinsicInfo *builtin = findARMVectorIntrinsicInMap(
2175 AArch64SISDIntrinsicMap, builtinID, aarch64SISDIntrinsicsProvenSorted);
2176 if (builtin)
2177 return emitCommonNeonSISDBuiltinExpr(*this, *builtin, ops, expr);
2178
2179 // Not all intrinsics handled by the common case work for AArch64 yet, so only
2180 // defer to common code if it's been added to our special map.
2181 assert(!cir::MissingFeatures::aarch64SIMDIntrinsics());
2182
2183 assert(!cir::MissingFeatures::aarch64TblBuiltinExpr());
2184
2185 const Expr *arg = expr->getArg(expr->getNumArgs() - 1);
2186 NeonTypeFlags type(0);
2187 // A trailing constant integer is used for discriminating overloaded builtin
2188 // calls. Use it to determine the type of this overloaded NEON intrinsic.
2189 if (std::optional<llvm::APSInt> result =
2190 arg->getIntegerConstantExpr(getContext()))
2191 type = NeonTypeFlags(result->getZExtValue());
2192
2193 bool usgn = type.isUnsigned();
2194
2195 mlir::Location loc = getLoc(expr->getExprLoc());
2196
2197 // Not all intrinsics handled by the common case work for AArch64 yet, so only
2198 // defer to common code if it's been added to our special map.
2199 builtin = findARMVectorIntrinsicInMap(AArch64SIMDIntrinsicMap, builtinID,
2200 aarch64SIMDIntrinsicsProvenSorted);
2201 if (builtin)
2202 return emitCommonNeonBuiltinExpr(
2203 *this, builtin->BuiltinID, builtin->LLVMIntrinsic,
2204 builtin->AltLLVMIntrinsic, builtin->NameHint, builtin->TypeModifier,
2205 expr, ops);
2206
2207 // Handle non-overloaded intrinsics first.
2208 switch (builtinID) {
2209 default:
2210 break;
2211 case NEON::BI__builtin_neon_vabsh_f16: {
2212 return cir::FAbsOp::create(builder, loc, ops);
2213 }
2214 case NEON::BI__builtin_neon_vaddq_p128:
2215 case NEON::BI__builtin_neon_vldrq_p128:
2216 case NEON::BI__builtin_neon_vstrq_p128:
2217 case NEON::BI__builtin_neon_vcvts_f32_u32:
2218 case NEON::BI__builtin_neon_vcvtd_f64_u64:
2219 case NEON::BI__builtin_neon_vcvts_f32_s32:
2220 case NEON::BI__builtin_neon_vcvtd_f64_s64:
2221 case NEON::BI__builtin_neon_vcvth_f16_u16:
2222 case NEON::BI__builtin_neon_vcvth_f16_u32:
2223 case NEON::BI__builtin_neon_vcvth_f16_u64:
2224 case NEON::BI__builtin_neon_vcvth_f16_s16:
2225 case NEON::BI__builtin_neon_vcvth_f16_s32:
2226 case NEON::BI__builtin_neon_vcvth_f16_s64:
2227 case NEON::BI__builtin_neon_vcvtah_u16_f16:
2228 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
2229 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
2230 case NEON::BI__builtin_neon_vcvtph_u16_f16:
2231 case NEON::BI__builtin_neon_vcvth_u16_f16:
2232 case NEON::BI__builtin_neon_vcvtah_s16_f16:
2233 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
2234 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
2235 case NEON::BI__builtin_neon_vcvtph_s16_f16:
2236 case NEON::BI__builtin_neon_vcvth_s16_f16:
2237 case NEON::BI__builtin_neon_vcaleh_f16:
2238 case NEON::BI__builtin_neon_vcalth_f16:
2239 case NEON::BI__builtin_neon_vcageh_f16:
2240 case NEON::BI__builtin_neon_vcagth_f16:
2241 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
2242 case NEON::BI__builtin_neon_vcvth_n_u16_f16:
2243 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
2244 case NEON::BI__builtin_neon_vcvth_n_f16_u16:
2245 case NEON::BI__builtin_neon_vpaddd_s64:
2246 case NEON::BI__builtin_neon_vpaddd_f64:
2247 case NEON::BI__builtin_neon_vpadds_f32:
2248 cgm.errorNYI(expr->getSourceRange(),
2249 std::string("unimplemented AArch64 builtin call: ") +
2250 getContext().BuiltinInfo.getName(builtinID));
2251 return mlir::Value{};
2252 case NEON::BI__builtin_neon_vceqzd_s64:
2253 case NEON::BI__builtin_neon_vceqzd_f64:
2254 case NEON::BI__builtin_neon_vceqzs_f32:
2255 case NEON::BI__builtin_neon_vceqzh_f16:
2256 return emitAArch64CompareBuiltinExpr(
2257 *this, builder, loc, ops[0],
2258 convertType(expr->getCallReturnType(getContext())), cir::CmpOpKind::eq);
2259 case NEON::BI__builtin_neon_vcgezd_s64:
2260 case NEON::BI__builtin_neon_vcgezd_f64:
2261 case NEON::BI__builtin_neon_vcgezs_f32:
2262 case NEON::BI__builtin_neon_vcgezh_f16:
2263 case NEON::BI__builtin_neon_vclezd_s64:
2264 case NEON::BI__builtin_neon_vclezd_f64:
2265 case NEON::BI__builtin_neon_vclezs_f32:
2266 case NEON::BI__builtin_neon_vclezh_f16:
2267 case NEON::BI__builtin_neon_vcgtzd_s64:
2268 case NEON::BI__builtin_neon_vcgtzd_f64:
2269 case NEON::BI__builtin_neon_vcgtzs_f32:
2270 case NEON::BI__builtin_neon_vcgtzh_f16:
2271 case NEON::BI__builtin_neon_vcltzd_s64:
2272 case NEON::BI__builtin_neon_vcltzd_f64:
2273 case NEON::BI__builtin_neon_vcltzs_f32:
2274 case NEON::BI__builtin_neon_vcltzh_f16:
2275 case NEON::BI__builtin_neon_vceqzd_u64: {
2276 return emitAArch64CompareBuiltinExpr(
2277 *this, builder, loc, ops[0],
2278 convertType(expr->getCallReturnType(getContext())), cir::CmpOpKind::eq);
2279 }
2280 case NEON::BI__builtin_neon_vceqd_f64:
2281 case NEON::BI__builtin_neon_vcled_f64:
2282 case NEON::BI__builtin_neon_vcltd_f64:
2283 case NEON::BI__builtin_neon_vcged_f64:
2284 case NEON::BI__builtin_neon_vcgtd_f64:
2285 case NEON::BI__builtin_neon_vceqs_f32:
2286 case NEON::BI__builtin_neon_vcles_f32:
2287 case NEON::BI__builtin_neon_vclts_f32:
2288 case NEON::BI__builtin_neon_vcges_f32:
2289 case NEON::BI__builtin_neon_vcgts_f32:
2290 case NEON::BI__builtin_neon_vceqh_f16:
2291 case NEON::BI__builtin_neon_vcleh_f16:
2292 case NEON::BI__builtin_neon_vclth_f16:
2293 case NEON::BI__builtin_neon_vcgeh_f16:
2294 case NEON::BI__builtin_neon_vcgth_f16:
2295 case NEON::BI__builtin_neon_vceqd_s64:
2296 case NEON::BI__builtin_neon_vceqd_u64:
2297 case NEON::BI__builtin_neon_vcgtd_s64:
2298 case NEON::BI__builtin_neon_vcgtd_u64:
2299 case NEON::BI__builtin_neon_vcltd_s64:
2300 case NEON::BI__builtin_neon_vcltd_u64:
2301 case NEON::BI__builtin_neon_vcged_u64:
2302 case NEON::BI__builtin_neon_vcged_s64:
2303 case NEON::BI__builtin_neon_vcled_u64:
2304 case NEON::BI__builtin_neon_vcled_s64:
2305 cgm.errorNYI(expr->getSourceRange(),
2306 std::string("unimplemented AArch64 builtin call: ") +
2307 getContext().BuiltinInfo.getName(builtinID));
2308 return mlir::Value{};
2309 case NEON::BI__builtin_neon_vnegd_s64: {
2310 return builder.createNeg(ops[0]);
2311 }
2312 case NEON::BI__builtin_neon_vnegh_f16: {
2313 return builder.createFNeg(ops[0]);
2314 }
2315 case NEON::BI__builtin_neon_vtstd_s64:
2316 case NEON::BI__builtin_neon_vtstd_u64:
2317 case NEON::BI__builtin_neon_vset_lane_i8:
2318 case NEON::BI__builtin_neon_vset_lane_i16:
2319 case NEON::BI__builtin_neon_vset_lane_i32:
2320 case NEON::BI__builtin_neon_vset_lane_i64:
2321 case NEON::BI__builtin_neon_vset_lane_bf16:
2322 case NEON::BI__builtin_neon_vset_lane_f32:
2323 case NEON::BI__builtin_neon_vsetq_lane_i8:
2324 case NEON::BI__builtin_neon_vsetq_lane_i16:
2325 case NEON::BI__builtin_neon_vsetq_lane_i32:
2326 case NEON::BI__builtin_neon_vsetq_lane_i64:
2327 case NEON::BI__builtin_neon_vsetq_lane_bf16:
2328 case NEON::BI__builtin_neon_vsetq_lane_f32:
2329 case NEON::BI__builtin_neon_vset_lane_f64:
2330 case NEON::BI__builtin_neon_vset_lane_mf8:
2331 case NEON::BI__builtin_neon_vsetq_lane_mf8:
2332 case NEON::BI__builtin_neon_vsetq_lane_f64:
2333 case NEON::BI__builtin_neon_vget_lane_i8:
2334 case NEON::BI__builtin_neon_vdupb_lane_i8:
2335 case NEON::BI__builtin_neon_vgetq_lane_i8:
2336 case NEON::BI__builtin_neon_vdupb_laneq_i8:
2337 case NEON::BI__builtin_neon_vget_lane_mf8:
2338 case NEON::BI__builtin_neon_vdupb_lane_mf8:
2339 case NEON::BI__builtin_neon_vgetq_lane_mf8:
2340 case NEON::BI__builtin_neon_vdupb_laneq_mf8:
2341 case NEON::BI__builtin_neon_vget_lane_i16:
2342 case NEON::BI__builtin_neon_vduph_lane_i16:
2343 case NEON::BI__builtin_neon_vgetq_lane_i16:
2344 case NEON::BI__builtin_neon_vduph_laneq_i16:
2345 case NEON::BI__builtin_neon_vget_lane_i32:
2346 case NEON::BI__builtin_neon_vdups_lane_i32:
2347 case NEON::BI__builtin_neon_vdups_lane_f32:
2348 case NEON::BI__builtin_neon_vgetq_lane_i32:
2349 case NEON::BI__builtin_neon_vdups_laneq_i32:
2350 case NEON::BI__builtin_neon_vget_lane_i64:
2351 case NEON::BI__builtin_neon_vdupd_lane_i64:
2352 case NEON::BI__builtin_neon_vdupd_lane_f64:
2353 case NEON::BI__builtin_neon_vgetq_lane_i64:
2354 case NEON::BI__builtin_neon_vdupd_laneq_i64:
2355 case NEON::BI__builtin_neon_vget_lane_f32:
2356 case NEON::BI__builtin_neon_vget_lane_f64:
2357 case NEON::BI__builtin_neon_vgetq_lane_f32:
2358 case NEON::BI__builtin_neon_vdups_laneq_f32:
2359 case NEON::BI__builtin_neon_vgetq_lane_f64:
2360 case NEON::BI__builtin_neon_vdupd_laneq_f64:
2361 cgm.errorNYI(expr->getSourceRange(),
2362 std::string("unimplemented AArch64 builtin call: ") +
2363 getContext().BuiltinInfo.getName(builtinID));
2364 return mlir::Value{};
2365 case NEON::BI__builtin_neon_vaddh_f16:
2366 return builder.createFAdd(loc, ops[0], ops[1]);
2367 case NEON::BI__builtin_neon_vsubh_f16:
2368 return builder.createFSub(loc, ops[0], ops[1]);
2369 case NEON::BI__builtin_neon_vmulh_f16:
2370 return builder.createFMul(loc, ops[0], ops[1]);
2371 case NEON::BI__builtin_neon_vdivh_f16:
2372 return builder.createFDiv(loc, ops[0], ops[1]);
2373 case NEON::BI__builtin_neon_vfmah_f16:
2374 // NEON intrinsic puts accumulator first, unlike the LLVM fma.
2375 std::rotate(ops.begin(), ops.begin() + 1, ops.end());
2376 return emitCallMaybeConstrainedBuiltin(builder, loc, "fma",
2377 convertType(expr->getType()), ops);
2378 break;
2379 case NEON::BI__builtin_neon_vfmsh_f16:
2380 // NEON intrinsic puts accumulator first, unlike the LLVM fma.
2381 std::rotate(ops.begin(), ops.begin() + 1, ops.end());
2382 ops[0] = builder.createFNeg(ops[0]);
2383 return emitCallMaybeConstrainedBuiltin(builder, loc, "fma",
2384 convertType(expr->getType()), ops);
2385 case NEON::BI__builtin_neon_vaddd_s64:
2386 case NEON::BI__builtin_neon_vaddd_u64:
2387 case NEON::BI__builtin_neon_vsubd_s64:
2388 case NEON::BI__builtin_neon_vsubd_u64:
2389 case NEON::BI__builtin_neon_vqdmlalh_s16:
2390 case NEON::BI__builtin_neon_vqdmlslh_s16:
2391 cgm.errorNYI(expr->getSourceRange(),
2392 std::string("unimplemented AArch64 builtin call: ") +
2393 getContext().BuiltinInfo.getName(builtinID));
2394 return mlir::Value{};
2395 case NEON::BI__builtin_neon_vqshlud_n_s64: {
2396 cir::IntType int64Type = builder.getSInt64Ty();
2397 ops[1] = builder.getSInt64(getZExtIntValueFromConstOp(ops[1]), loc);
2398 return emitNeonCall(cgm, builder, {int64Type, int64Type}, ops,
2399 "aarch64.neon.sqshlu", convertType(expr->getType()),
2400 loc);
2401 }
2402 case NEON::BI__builtin_neon_vqshld_n_u64:
2403 case NEON::BI__builtin_neon_vqshld_n_s64: {
2404 cir::IntType int64Type = builtinID == NEON::BI__builtin_neon_vqshld_n_u64
2405 ? builder.getUInt64Ty()
2406 : builder.getSInt64Ty();
2407 llvm::StringRef intrinsicName =
2408 builtinID == NEON::BI__builtin_neon_vqshld_n_u64 ? "aarch64.neon.uqshl"
2409 : "aarch64.neon.sqshl";
2410 ops[1] = builder.getSInt64(getZExtIntValueFromConstOp(ops[1]), loc);
2411 return emitNeonCall(cgm, builder, {int64Type, int64Type}, ops,
2412 intrinsicName, convertType(expr->getType()), loc);
2413 }
2414 case NEON::BI__builtin_neon_vrshrd_n_u64:
2415 case NEON::BI__builtin_neon_vrshrd_n_s64: {
2416 llvm::StringRef intrName = builtinID == NEON::BI__builtin_neon_vrshrd_n_s64
2417 ? "aarch64.neon.srshl"
2418 : "aarch64.neon.urshl";
2419 cir::IntType int64Ty = builtinID == NEON::BI__builtin_neon_vqshld_n_u64
2420 ? builder.getUInt64Ty()
2421 : builder.getSInt64Ty();
2422 int64_t sv = -cast<cir::IntAttr>(
2423 cast<cir::ConstantOp>(ops[1].getDefiningOp()).getValue())
2424 .getSInt();
2425 ops[1] = builder.getSInt64(sv, loc);
2426 return emitNeonCall(cgm, builder, {int64Ty, builder.getSInt64Ty()}, ops,
2427 intrName, int64Ty, loc);
2428 }
2429 case NEON::BI__builtin_neon_vrsrad_n_u64:
2430 case NEON::BI__builtin_neon_vrsrad_n_s64: {
2431 cir::IntType int64Type = builtinID == NEON::BI__builtin_neon_vrsrad_n_u64
2432 ? builder.getUInt64Ty()
2433 : builder.getSInt64Ty();
2434 ops[2] = builder.createNeg(ops[2]);
2435 const StringRef intrName = builtinID == NEON::BI__builtin_neon_vrsrad_n_u64
2436 ? "aarch64.neon.urshl"
2437 : "aarch64.neon.srshl";
2438
2439 llvm::SmallVector<mlir::Value, 2> args = {
2440 ops[1], builder.createIntCast(ops[2], builder.getSInt64Ty())};
2441 ops[1] = builder.emitIntrinsicCallOp(loc, intrName, int64Type, args);
2442 return builder.createAdd(loc, ops[0],
2443 builder.createBitcast(ops[1], int64Type));
2444 }
2445 case NEON::BI__builtin_neon_vshld_n_s64:
2446 case NEON::BI__builtin_neon_vshld_n_u64: {
2447 auto loc = getLoc(expr->getExprLoc());
2448 std::optional<llvm::APSInt> amt =
2449 expr->getArg(1)->getIntegerConstantExpr(getContext());
2450 assert(amt && "Expected argument to be a constant");
2451 return builder.createShiftLeft(loc, ops[0], amt->getZExtValue());
2452 }
2453 case NEON::BI__builtin_neon_vshrd_n_s64: {
2454 std::optional<llvm::APSInt> amt =
2455 expr->getArg(1)->getIntegerConstantExpr(getContext());
2456 assert(amt && "Expected argument to be a constant");
2457 return builder.createShiftRight(
2458 loc, ops[0], std::min(static_cast<uint64_t>(63), amt->getZExtValue()));
2459 }
2460 case NEON::BI__builtin_neon_vshrd_n_u64: {
2461 std::optional<llvm::APSInt> amt =
2462 expr->getArg(1)->getIntegerConstantExpr(getContext());
2463 assert(amt && "Expected argument to be a constant");
2464 uint64_t shiftAmt = amt->getZExtValue();
2465 // Right-shifting an unsigned value by its size yields 0.
2466 if (shiftAmt == 64)
2467 return builder.getConstInt(loc, builder.getUInt64Ty(), 0);
2468 return builder.createShiftRight(loc, ops[0], shiftAmt);
2469 }
2470 case NEON::BI__builtin_neon_vsrad_n_s64:
2471 case NEON::BI__builtin_neon_vsrad_n_u64:
2472 case NEON::BI__builtin_neon_vqdmlalh_lane_s16:
2473 case NEON::BI__builtin_neon_vqdmlalh_laneq_s16:
2474 case NEON::BI__builtin_neon_vqdmlslh_lane_s16:
2475 case NEON::BI__builtin_neon_vqdmlslh_laneq_s16:
2476 case NEON::BI__builtin_neon_vqdmlals_s32:
2477 case NEON::BI__builtin_neon_vqdmlsls_s32:
2478 case NEON::BI__builtin_neon_vqdmlals_lane_s32:
2479 case NEON::BI__builtin_neon_vqdmlals_laneq_s32:
2480 case NEON::BI__builtin_neon_vqdmlsls_lane_s32:
2481 case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: {
2482 cgm.errorNYI(expr->getSourceRange(),
2483 std::string("unimplemented AArch64 builtin call: ") +
2484 getContext().BuiltinInfo.getName(builtinID));
2485 return mlir::Value{};
2486 }
2487 case NEON::BI__builtin_neon_vget_lane_bf16:
2488 case NEON::BI__builtin_neon_vduph_lane_bf16:
2489 case NEON::BI__builtin_neon_vduph_lane_f16:
2490 case NEON::BI__builtin_neon_vgetq_lane_bf16:
2491 case NEON::BI__builtin_neon_vduph_laneq_bf16:
2492 case NEON::BI__builtin_neon_vduph_laneq_f16: {
2493 return cir::VecExtractOp::create(builder, loc, ops[0], ops[1]);
2494 }
2495 case NEON::BI__builtin_neon_vcvt_bf16_f32:
2496 case NEON::BI__builtin_neon_vcvtq_low_bf16_f32:
2497 case NEON::BI__builtin_neon_vcvtq_high_bf16_f32:
2498 case clang::AArch64::BI_InterlockedAdd:
2499 case clang::AArch64::BI_InterlockedAdd_acq:
2500 case clang::AArch64::BI_InterlockedAdd_rel:
2501 case clang::AArch64::BI_InterlockedAdd_nf:
2502 case clang::AArch64::BI_InterlockedAdd64:
2503 case clang::AArch64::BI_InterlockedAdd64_acq:
2504 case clang::AArch64::BI_InterlockedAdd64_rel:
2505 case clang::AArch64::BI_InterlockedAdd64_nf:
2506 cgm.errorNYI(expr->getSourceRange(),
2507 std::string("unimplemented AArch64 builtin call: ") +
2508 getContext().BuiltinInfo.getName(builtinID));
2509 return mlir::Value{};
2510 }
2511
2512 cir::VectorType ty = getNeonType(this, type, loc);
2513 if (!ty)
2514 return nullptr;
2515
2516 llvm::StringRef intrName;
2517
2518 switch (builtinID) {
2519 default:
2520 return std::nullopt;
2521 case NEON::BI__builtin_neon_vbsl_v:
2522 case NEON::BI__builtin_neon_vbslq_v: {
2523
2524 cir::VectorType bitTy = getIntVecFromVecTy(builder, ty);
2525 ops[0] = builder.createBitcast(ops[0], bitTy);
2526 ops[1] = builder.createBitcast(ops[1], bitTy);
2527 ops[2] = builder.createBitcast(ops[2], bitTy);
2528
2529 ops[1] = builder.createAnd(loc, ops[0], ops[1]);
2530 ops[2] = builder.createAnd(loc, builder.createNot(ops[0]), ops[2]);
2531 ops[0] = builder.createOr(loc, ops[1], ops[2]);
2532 return builder.createBitcast(ops[0], ty);
2533 }
2534 case NEON::BI__builtin_neon_vfma_lane_v:
2535 case NEON::BI__builtin_neon_vfmaq_lane_v:
2536 case NEON::BI__builtin_neon_vfma_laneq_v:
2537 case NEON::BI__builtin_neon_vfmaq_laneq_v:
2538 case NEON::BI__builtin_neon_vfmah_lane_f16:
2539 case NEON::BI__builtin_neon_vfmas_lane_f32:
2540 case NEON::BI__builtin_neon_vfmah_laneq_f16:
2541 case NEON::BI__builtin_neon_vfmas_laneq_f32:
2542 case NEON::BI__builtin_neon_vfmad_lane_f64:
2543 case NEON::BI__builtin_neon_vfmad_laneq_f64:
2544 cgm.errorNYI(expr->getSourceRange(),
2545 std::string("unimplemented AArch64 builtin call: ") +
2546 getContext().BuiltinInfo.getName(builtinID));
2547 return mlir::Value{};
2548 case NEON::BI__builtin_neon_vmull_v: {
2549 intrName = usgn ? "aarch64.neon.umull" : "aarch64.neon.smull";
2550 if (type.isPoly())
2551 intrName = "aarch64.neon.pmull";
2552 cir::VectorType argTy = builder.getExtendedOrTruncatedElementVectorType(
2553 ty, /*isExtended*/ false, !usgn);
2554 return emitNeonCall(cgm, builder, {argTy, argTy}, ops, intrName, ty, loc);
2555 }
2556 case NEON::BI__builtin_neon_vmax_v:
2557 case NEON::BI__builtin_neon_vmaxq_v:
2558 intrName = usgn ? "aarch64.neon.umax" : "aarch64.neon.smax";
2559 if (cir::isFPOrVectorOfFPType(ty))
2560 intrName = "aarch64.neon.fmax";
2561 return emitNeonCall(cgm, builder, {ty, ty}, ops, intrName, ty, loc);
2562 case NEON::BI__builtin_neon_vmaxh_f16:
2563 cgm.errorNYI(expr->getSourceRange(),
2564 std::string("unimplemented AArch64 builtin call: ") +
2565 getContext().BuiltinInfo.getName(builtinID));
2566 return mlir::Value{};
2567 case NEON::BI__builtin_neon_vmin_v:
2568 case NEON::BI__builtin_neon_vminq_v:
2569 intrName = usgn ? "aarch64.neon.umin" : "aarch64.neon.smin";
2570 if (cir::isFPOrVectorOfFPType(ty))
2571 intrName = "aarch64.neon.fmin";
2572 return emitNeonCall(cgm, builder, {ty, ty}, ops, intrName, ty, loc);
2573 case NEON::BI__builtin_neon_vminh_f16:
2574 cgm.errorNYI(expr->getSourceRange(),
2575 std::string("unimplemented AArch64 builtin call: ") +
2576 getContext().BuiltinInfo.getName(builtinID));
2577 return mlir::Value{};
2578 case NEON::BI__builtin_neon_vabd_v:
2579 case NEON::BI__builtin_neon_vabdq_v:
2580 intrName = usgn ? "aarch64.neon.uabd" : "aarch64.neon.sabd";
2581 if (cir::isFPOrVectorOfFPType(ty))
2582 intrName = "aarch64.neon.fabd";
2583 return emitNeonCall(cgm, builder, {ty, ty}, ops, intrName, ty, loc);
2584 case NEON::BI__builtin_neon_vpadal_v:
2585 case NEON::BI__builtin_neon_vpadalq_v: {
2586 intrName = usgn ? "aarch64.neon.uaddlp" : "aarch64.neon.saddlp";
2587 llvm::SmallVector<mlir::Value> inputs{ops[1]};
2588 mlir::Value pairwiseSum =
2589 emitNeonCall(cgm, builder, {getNeonPairwiseWidenInputType(ty, usgn)},
2590 inputs, intrName, ty, loc);
2591 mlir::Value accumValue = builder.createBitcast(loc, ops[0], ty);
2592 return cir::AddOp::create(builder, loc, ty, pairwiseSum, accumValue);
2593 }
2594 case NEON::BI__builtin_neon_vpmin_v:
2595 case NEON::BI__builtin_neon_vpminq_v:
2596 intrName = usgn ? "aarch64.neon.uminp" : "aarch64.neon.sminp";
2597 if (cir::isFPOrVectorOfFPType(ty))
2598 intrName = "aarch64.neon.fminp";
2599 return emitNeonCall(cgm, builder, {ty, ty}, ops, intrName, ty, loc);
2600 case NEON::BI__builtin_neon_vpmax_v:
2601 case NEON::BI__builtin_neon_vpmaxq_v:
2602 cgm.errorNYI(expr->getSourceRange(),
2603 std::string("unimplemented AArch64 builtin call: ") +
2604 getContext().BuiltinInfo.getName(builtinID));
2605 return mlir::Value{};
2606 case NEON::BI__builtin_neon_vminnm_v:
2607 case NEON::BI__builtin_neon_vminnmq_v:
2608 intrName = "aarch64.neon.fminnm";
2609 return emitNeonCall(cgm, builder, {ty, ty}, ops, intrName, ty, loc);
2610 case NEON::BI__builtin_neon_vminnmh_f16:
2611 cgm.errorNYI(expr->getSourceRange(),
2612 std::string("unimplemented AArch64 builtin call: ") +
2613 getContext().BuiltinInfo.getName(builtinID));
2614 return mlir::Value{};
2615 case NEON::BI__builtin_neon_vmaxnm_v:
2616 case NEON::BI__builtin_neon_vmaxnmq_v:
2617 intrName = "aarch64.neon.fmaxnm";
2618 return emitNeonCall(cgm, builder, {ty, ty}, ops, intrName, ty, loc);
2619 case NEON::BI__builtin_neon_vmaxnmh_f16:
2620 case NEON::BI__builtin_neon_vrecpss_f32:
2621 case NEON::BI__builtin_neon_vrecpsd_f64:
2622 case NEON::BI__builtin_neon_vrecpsh_f16:
2623 case NEON::BI__builtin_neon_vqshrun_n_v:
2624 case NEON::BI__builtin_neon_vqrshrun_n_v:
2625 case NEON::BI__builtin_neon_vqshrn_n_v:
2626 case NEON::BI__builtin_neon_vrshrn_n_v:
2627 case NEON::BI__builtin_neon_vqrshrn_n_v:
2628 case NEON::BI__builtin_neon_vrndah_f16:
2629 case NEON::BI__builtin_neon_vrnda_v:
2630 case NEON::BI__builtin_neon_vrndaq_v:
2631 case NEON::BI__builtin_neon_vrndih_f16:
2632 case NEON::BI__builtin_neon_vrndmh_f16:
2633 case NEON::BI__builtin_neon_vrndm_v:
2634 case NEON::BI__builtin_neon_vrndmq_v:
2635 case NEON::BI__builtin_neon_vrndnh_f16:
2636 case NEON::BI__builtin_neon_vrndn_v:
2637 case NEON::BI__builtin_neon_vrndnq_v:
2638 case NEON::BI__builtin_neon_vrndns_f32:
2639 case NEON::BI__builtin_neon_vrndph_f16:
2640 case NEON::BI__builtin_neon_vrndp_v:
2641 case NEON::BI__builtin_neon_vrndpq_v:
2642 case NEON::BI__builtin_neon_vrndxh_f16:
2643 case NEON::BI__builtin_neon_vrndx_v:
2644 case NEON::BI__builtin_neon_vrndxq_v:
2645 case NEON::BI__builtin_neon_vrndh_f16:
2646 case NEON::BI__builtin_neon_vrnd32x_f32:
2647 case NEON::BI__builtin_neon_vrnd32xq_f32:
2648 case NEON::BI__builtin_neon_vrnd32x_f64:
2649 case NEON::BI__builtin_neon_vrnd32xq_f64:
2650 case NEON::BI__builtin_neon_vrnd32z_f32:
2651 case NEON::BI__builtin_neon_vrnd32zq_f32:
2652 case NEON::BI__builtin_neon_vrnd32z_f64:
2653 case NEON::BI__builtin_neon_vrnd32zq_f64:
2654 case NEON::BI__builtin_neon_vrnd64x_f32:
2655 case NEON::BI__builtin_neon_vrnd64xq_f32:
2656 case NEON::BI__builtin_neon_vrnd64x_f64:
2657 case NEON::BI__builtin_neon_vrnd64xq_f64:
2658 case NEON::BI__builtin_neon_vrnd64z_f32:
2659 case NEON::BI__builtin_neon_vrnd64zq_f32:
2660 case NEON::BI__builtin_neon_vrnd64z_f64:
2661 case NEON::BI__builtin_neon_vrnd64zq_f64:
2662 case NEON::BI__builtin_neon_vrnd_v:
2663 case NEON::BI__builtin_neon_vrndq_v:
2664 cgm.errorNYI(expr->getSourceRange(),
2665 std::string("unimplemented AArch64 builtin call: ") +
2666 getContext().BuiltinInfo.getName(builtinID));
2667 return mlir::Value{};
2668 case NEON::BI__builtin_neon_vcvt_f64_v:
2669 case NEON::BI__builtin_neon_vcvtq_f64_v:
2670 ops[0] = builder.createBitcast(ops[0], ty);
2671 ty = getNeonType(
2672 this, NeonTypeFlags(NeonTypeFlags::Float64, false, type.isQuad()), loc);
2673 return builder.createCast(loc, cir::CastKind::int_to_float, ops[0], ty);
2674 case NEON::BI__builtin_neon_vcvt_f64_f32:
2675 case NEON::BI__builtin_neon_vcvt_f32_f64:
2676 case NEON::BI__builtin_neon_vcvt_s32_v:
2677 case NEON::BI__builtin_neon_vcvt_u32_v:
2678 case NEON::BI__builtin_neon_vcvt_s64_v:
2679 case NEON::BI__builtin_neon_vcvt_u64_v:
2680 case NEON::BI__builtin_neon_vcvt_s16_f16:
2681 case NEON::BI__builtin_neon_vcvt_u16_f16:
2682 case NEON::BI__builtin_neon_vcvtq_s32_v:
2683 case NEON::BI__builtin_neon_vcvtq_u32_v:
2684 case NEON::BI__builtin_neon_vcvtq_s64_v:
2685 case NEON::BI__builtin_neon_vcvtq_u64_v:
2686 case NEON::BI__builtin_neon_vcvtq_s16_f16:
2687 case NEON::BI__builtin_neon_vcvtq_u16_f16:
2688 case NEON::BI__builtin_neon_vcvta_s16_f16:
2689 case NEON::BI__builtin_neon_vcvta_u16_f16:
2690 case NEON::BI__builtin_neon_vcvta_s32_v:
2691 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
2692 case NEON::BI__builtin_neon_vcvtaq_s32_v:
2693 case NEON::BI__builtin_neon_vcvta_u32_v:
2694 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
2695 case NEON::BI__builtin_neon_vcvtaq_u32_v:
2696 case NEON::BI__builtin_neon_vcvta_s64_v:
2697 case NEON::BI__builtin_neon_vcvtaq_s64_v:
2698 case NEON::BI__builtin_neon_vcvta_u64_v:
2699 case NEON::BI__builtin_neon_vcvtaq_u64_v:
2700 case NEON::BI__builtin_neon_vcvtm_s16_f16:
2701 case NEON::BI__builtin_neon_vcvtm_s32_v:
2702 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
2703 case NEON::BI__builtin_neon_vcvtmq_s32_v:
2704 case NEON::BI__builtin_neon_vcvtm_u16_f16:
2705 case NEON::BI__builtin_neon_vcvtm_u32_v:
2706 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
2707 case NEON::BI__builtin_neon_vcvtmq_u32_v:
2708 case NEON::BI__builtin_neon_vcvtm_s64_v:
2709 case NEON::BI__builtin_neon_vcvtmq_s64_v:
2710 case NEON::BI__builtin_neon_vcvtm_u64_v:
2711 case NEON::BI__builtin_neon_vcvtmq_u64_v:
2712 case NEON::BI__builtin_neon_vcvtn_s16_f16:
2713 case NEON::BI__builtin_neon_vcvtn_s32_v:
2714 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
2715 case NEON::BI__builtin_neon_vcvtnq_s32_v:
2716 case NEON::BI__builtin_neon_vcvtn_u16_f16:
2717 case NEON::BI__builtin_neon_vcvtn_u32_v:
2718 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
2719 case NEON::BI__builtin_neon_vcvtnq_u32_v:
2720 case NEON::BI__builtin_neon_vcvtn_s64_v:
2721 case NEON::BI__builtin_neon_vcvtnq_s64_v:
2722 case NEON::BI__builtin_neon_vcvtn_u64_v:
2723 case NEON::BI__builtin_neon_vcvtnq_u64_v:
2724 case NEON::BI__builtin_neon_vcvtp_s16_f16:
2725 case NEON::BI__builtin_neon_vcvtp_s32_v:
2726 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
2727 case NEON::BI__builtin_neon_vcvtpq_s32_v:
2728 case NEON::BI__builtin_neon_vcvtp_u16_f16:
2729 case NEON::BI__builtin_neon_vcvtp_u32_v:
2730 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
2731 case NEON::BI__builtin_neon_vcvtpq_u32_v:
2732 case NEON::BI__builtin_neon_vcvtp_s64_v:
2733 case NEON::BI__builtin_neon_vcvtpq_s64_v:
2734 case NEON::BI__builtin_neon_vcvtp_u64_v:
2735 case NEON::BI__builtin_neon_vcvtpq_u64_v:
2736 case NEON::BI__builtin_neon_vmulx_v:
2737 case NEON::BI__builtin_neon_vmulxq_v:
2738 case NEON::BI__builtin_neon_vmulxh_lane_f16:
2739 case NEON::BI__builtin_neon_vmulxh_laneq_f16:
2740 case NEON::BI__builtin_neon_vmul_lane_v:
2741 case NEON::BI__builtin_neon_vmul_laneq_v:
2742 case NEON::BI__builtin_neon_vpmaxnm_v:
2743 case NEON::BI__builtin_neon_vpmaxnmq_v:
2744 cgm.errorNYI(expr->getSourceRange(),
2745 std::string("unimplemented AArch64 builtin call: ") +
2746 getContext().BuiltinInfo.getName(builtinID));
2747 return mlir::Value{};
2748 case NEON::BI__builtin_neon_vpminnm_v:
2749 case NEON::BI__builtin_neon_vpminnmq_v:
2750 intrName = "aarch64.neon.fminnmp";
2751 return emitNeonCall(cgm, builder, {ty, ty}, ops, intrName, ty, loc);
2752 case NEON::BI__builtin_neon_vsqrth_f16:
2753 cgm.errorNYI(expr->getSourceRange(),
2754 std::string("unimplemented AArch64 builtin call: ") +
2755 getContext().BuiltinInfo.getName(builtinID));
2756 return mlir::Value{};
2757 case NEON::BI__builtin_neon_vsqrt_v:
2758 case NEON::BI__builtin_neon_vsqrtq_v:
2759 assert(!cir::MissingFeatures::emitConstrainedFPCall());
2760 return emitNeonCall(cgm, builder, {ty}, ops, "sqrt", ty, loc);
2761 case NEON::BI__builtin_neon_vrbit_v:
2762 case NEON::BI__builtin_neon_vrbitq_v:
2763 case NEON::BI__builtin_neon_vmaxv_f16:
2764 case NEON::BI__builtin_neon_vmaxvq_f16:
2765 case NEON::BI__builtin_neon_vminv_f16:
2766 case NEON::BI__builtin_neon_vminvq_f16:
2767 case NEON::BI__builtin_neon_vmaxnmv_f16:
2768 case NEON::BI__builtin_neon_vmaxnmvq_f16:
2769 case NEON::BI__builtin_neon_vminnmv_f16:
2770 case NEON::BI__builtin_neon_vminnmvq_f16:
2771 case NEON::BI__builtin_neon_vmul_n_f64:
2772 cgm.errorNYI(expr->getSourceRange(),
2773 std::string("unimplemented AArch64 builtin call: ") +
2774 getContext().BuiltinInfo.getName(builtinID));
2775 return mlir::Value{};
2776 case NEON::BI__builtin_neon_vaddlv_u8:
2777 case NEON::BI__builtin_neon_vaddlvq_u8:
2778 case NEON::BI__builtin_neon_vaddlv_u16:
2779 case NEON::BI__builtin_neon_vaddlvq_u16:
2780 case NEON::BI__builtin_neon_vaddlv_s8:
2781 case NEON::BI__builtin_neon_vaddlvq_s8:
2782 case NEON::BI__builtin_neon_vaddlv_s16:
2783 case NEON::BI__builtin_neon_vaddlvq_s16: {
2784 mlir::Type argTy = convertType(expr->getArg(0)->getType());
2785 mlir::Type userRetTy = convertType(expr->getType());
2786 auto eltTy = mlir::cast<cir::IntType>(
2787 mlir::cast<cir::VectorType>(argTy).getElementType());
2788 bool isUnsigned = !eltTy.isSigned();
2789 // These builtins only use 8 and 16-bit element vectors; the intrinsic
2790 // always produces i32. The C result is i32 for 16-bit elements, but i16
2791 // for 8-bit elements, so we emit at i32 and narrow only in that case.
2792 bool needsTrunc = eltTy.getWidth() == 8;
2793 intrName = isUnsigned ? "aarch64.neon.uaddlv" : "aarch64.neon.saddlv";
2794 mlir::Type intrRetTy = userRetTy;
2795 if (needsTrunc)
2796 intrRetTy = isUnsigned ? builder.getUInt32Ty() : builder.getSInt32Ty();
2797 mlir::Value result =
2798 emitNeonCall(cgm, builder, {argTy}, ops, intrName, intrRetTy, loc);
2799 if (needsTrunc)
2800 result = builder.createIntCast(result, userRetTy);
2801 return result;
2802 }
2803 case NEON::BI__builtin_neon_vsri_n_v:
2804 case NEON::BI__builtin_neon_vsriq_n_v:
2805 case NEON::BI__builtin_neon_vsli_n_v:
2806 case NEON::BI__builtin_neon_vsliq_n_v:
2807 case NEON::BI__builtin_neon_vsra_n_v:
2808 case NEON::BI__builtin_neon_vsraq_n_v:
2809 cgm.errorNYI(expr->getSourceRange(),
2810 std::string("unimplemented AArch64 builtin call: ") +
2811 getContext().BuiltinInfo.getName(builtinID));
2812 return mlir::Value{};
2813 case NEON::BI__builtin_neon_vrsra_n_v:
2814 case NEON::BI__builtin_neon_vrsraq_n_v: {
2815 intrName = usgn ? "aarch64.neon.urshl" : "aarch64.neon.srshl";
2816 // The llvm intrinsic is expecting negative shift amount for right shift.
2817 // Thus we have to make shift amount vec type to be signed.
2818 cir::VectorType shiftAmtVecTy =
2819 usgn ? getSignChangedVectorType(builder, ty) : ty;
2820 llvm::SmallVector<mlir::Value, 2> tmpOps = {ops[1], ops[2]};
2821 mlir::Value tmp = emitNeonCall(cgm, builder, {ty, shiftAmtVecTy}, tmpOps,
2822 intrName, ty, loc,
2823 /*isConstrainedFPIntrinsic=*/false,
2824 /*shift=*/1, /*rightshift=*/true);
2825 ops[0] = builder.createBitcast(ops[0], ty);
2826 return builder.createAdd(loc, ops[0], tmp);
2827 }
2828 case NEON::BI__builtin_neon_vld1_v:
2829 case NEON::BI__builtin_neon_vld1q_v:
2830 case NEON::BI__builtin_neon_vst1_v:
2831 case NEON::BI__builtin_neon_vst1q_v:
2832 case NEON::BI__builtin_neon_vld1_lane_v:
2833 case NEON::BI__builtin_neon_vld1q_lane_v:
2834 case NEON::BI__builtin_neon_vldap1_lane_s64:
2835 case NEON::BI__builtin_neon_vldap1q_lane_s64:
2836 case NEON::BI__builtin_neon_vld1_dup_v:
2837 case NEON::BI__builtin_neon_vld1q_dup_v:
2838 case NEON::BI__builtin_neon_vst1_lane_v:
2839 case NEON::BI__builtin_neon_vst1q_lane_v:
2840 case NEON::BI__builtin_neon_vstl1_lane_s64:
2841 case NEON::BI__builtin_neon_vstl1q_lane_s64:
2842 case NEON::BI__builtin_neon_vld2_v:
2843 case NEON::BI__builtin_neon_vld2q_v:
2844 case NEON::BI__builtin_neon_vld3_v:
2845 case NEON::BI__builtin_neon_vld3q_v:
2846 case NEON::BI__builtin_neon_vld4_v:
2847 case NEON::BI__builtin_neon_vld4q_v:
2848 case NEON::BI__builtin_neon_vld2_dup_v:
2849 case NEON::BI__builtin_neon_vld2q_dup_v:
2850 case NEON::BI__builtin_neon_vld3_dup_v:
2851 case NEON::BI__builtin_neon_vld3q_dup_v:
2852 case NEON::BI__builtin_neon_vld4_dup_v:
2853 case NEON::BI__builtin_neon_vld4q_dup_v:
2854 case NEON::BI__builtin_neon_vld2_lane_v:
2855 case NEON::BI__builtin_neon_vld2q_lane_v:
2856 case NEON::BI__builtin_neon_vld3_lane_v:
2857 case NEON::BI__builtin_neon_vld3q_lane_v:
2858 case NEON::BI__builtin_neon_vld4_lane_v:
2859 case NEON::BI__builtin_neon_vld4q_lane_v:
2860 case NEON::BI__builtin_neon_vst2_v:
2861 case NEON::BI__builtin_neon_vst2q_v:
2862 case NEON::BI__builtin_neon_vst2_lane_v:
2863 case NEON::BI__builtin_neon_vst2q_lane_v:
2864 case NEON::BI__builtin_neon_vst3_v:
2865 case NEON::BI__builtin_neon_vst3q_v:
2866 case NEON::BI__builtin_neon_vst3_lane_v:
2867 case NEON::BI__builtin_neon_vst3q_lane_v:
2868 case NEON::BI__builtin_neon_vst4_v:
2869 case NEON::BI__builtin_neon_vst4q_v:
2870 case NEON::BI__builtin_neon_vst4_lane_v:
2871 case NEON::BI__builtin_neon_vst4q_lane_v:
2872 case NEON::BI__builtin_neon_vtrn_v:
2873 case NEON::BI__builtin_neon_vtrnq_v:
2874 case NEON::BI__builtin_neon_vuzp_v:
2875 case NEON::BI__builtin_neon_vuzpq_v:
2876 cgm.errorNYI(expr->getSourceRange(),
2877 std::string("unimplemented AArch64 builtin call: ") +
2878 getContext().BuiltinInfo.getName(builtinID));
2879 return mlir::Value{};
2880 case NEON::BI__builtin_neon_vzip_v:
2881 case NEON::BI__builtin_neon_vzipq_v: {
2882 ops[1] = builder.createBitcast(ops[1], ty);
2883 ops[2] = builder.createBitcast(ops[2], ty);
2884 // Adding a bitcast here as Ops[0] might be a void pointer.
2885 mlir::Value baseAddr =
2886 builder.createBitcast(ops[0], builder.getPointerTo(ty));
2887 mlir::Value sv;
2888 for (unsigned vi = 0; vi != 2; ++vi) {
2889 llvm::SmallVector<int64_t, 16> indices;
2890 for (unsigned i = 0, e = ty.getSize(); i != e; i += 2) {
2891 indices.push_back((i + vi * e) >> 1);
2892 indices.push_back(((i + vi * e) >> 1) + e);
2893 }
2894 cir::ConstantOp idx = builder.getConstInt(loc, builder.getSInt32Ty(), vi);
2895 mlir::Value addr = builder.createPtrStride(loc, baseAddr, idx);
2896 sv = builder.createVecShuffle(loc, ops[1], ops[2], indices);
2897 (void)builder.CIRBaseBuilderTy::createStore(loc, sv, addr);
2898 }
2899 return sv;
2900 }
2901 case NEON::BI__builtin_neon_vqtbl1q_v:
2902 case NEON::BI__builtin_neon_vqtbl2q_v:
2903 case NEON::BI__builtin_neon_vqtbl3q_v:
2904 case NEON::BI__builtin_neon_vqtbl4q_v:
2905 case NEON::BI__builtin_neon_vqtbx1q_v:
2906 case NEON::BI__builtin_neon_vqtbx2q_v:
2907 case NEON::BI__builtin_neon_vqtbx3q_v:
2908 case NEON::BI__builtin_neon_vqtbx4q_v:
2909 case NEON::BI__builtin_neon_vsqadd_v:
2910 case NEON::BI__builtin_neon_vsqaddq_v:
2911 case NEON::BI__builtin_neon_vuqadd_v:
2912 case NEON::BI__builtin_neon_vuqaddq_v:
2913 case NEON::BI__builtin_neon_vluti2_laneq_mf8:
2914 case NEON::BI__builtin_neon_vluti2_laneq_bf16:
2915 case NEON::BI__builtin_neon_vluti2_laneq_f16:
2916 case NEON::BI__builtin_neon_vluti2_laneq_p16:
2917 case NEON::BI__builtin_neon_vluti2_laneq_p8:
2918 case NEON::BI__builtin_neon_vluti2_laneq_s16:
2919 case NEON::BI__builtin_neon_vluti2_laneq_s8:
2920 case NEON::BI__builtin_neon_vluti2_laneq_u16:
2921 case NEON::BI__builtin_neon_vluti2_laneq_u8:
2922 case NEON::BI__builtin_neon_vluti2q_laneq_mf8:
2923 case NEON::BI__builtin_neon_vluti2q_laneq_bf16:
2924 case NEON::BI__builtin_neon_vluti2q_laneq_f16:
2925 case NEON::BI__builtin_neon_vluti2q_laneq_p16:
2926 case NEON::BI__builtin_neon_vluti2q_laneq_p8:
2927 case NEON::BI__builtin_neon_vluti2q_laneq_s16:
2928 case NEON::BI__builtin_neon_vluti2q_laneq_s8:
2929 case NEON::BI__builtin_neon_vluti2q_laneq_u16:
2930 case NEON::BI__builtin_neon_vluti2q_laneq_u8:
2931 case NEON::BI__builtin_neon_vluti2_lane_mf8:
2932 case NEON::BI__builtin_neon_vluti2_lane_bf16:
2933 case NEON::BI__builtin_neon_vluti2_lane_f16:
2934 case NEON::BI__builtin_neon_vluti2_lane_p16:
2935 case NEON::BI__builtin_neon_vluti2_lane_p8:
2936 case NEON::BI__builtin_neon_vluti2_lane_s16:
2937 case NEON::BI__builtin_neon_vluti2_lane_s8:
2938 case NEON::BI__builtin_neon_vluti2_lane_u16:
2939 case NEON::BI__builtin_neon_vluti2_lane_u8:
2940 case NEON::BI__builtin_neon_vluti2q_lane_mf8:
2941 case NEON::BI__builtin_neon_vluti2q_lane_bf16:
2942 case NEON::BI__builtin_neon_vluti2q_lane_f16:
2943 case NEON::BI__builtin_neon_vluti2q_lane_p16:
2944 case NEON::BI__builtin_neon_vluti2q_lane_p8:
2945 case NEON::BI__builtin_neon_vluti2q_lane_s16:
2946 case NEON::BI__builtin_neon_vluti2q_lane_s8:
2947 case NEON::BI__builtin_neon_vluti2q_lane_u16:
2948 case NEON::BI__builtin_neon_vluti2q_lane_u8:
2949 case NEON::BI__builtin_neon_vluti4q_lane_mf8:
2950 case NEON::BI__builtin_neon_vluti4q_lane_p8:
2951 case NEON::BI__builtin_neon_vluti4q_lane_s8:
2952 case NEON::BI__builtin_neon_vluti4q_lane_u8:
2953 case NEON::BI__builtin_neon_vluti4q_laneq_mf8:
2954 case NEON::BI__builtin_neon_vluti4q_laneq_p8:
2955 case NEON::BI__builtin_neon_vluti4q_laneq_s8:
2956 case NEON::BI__builtin_neon_vluti4q_laneq_u8:
2957 case NEON::BI__builtin_neon_vluti4q_lane_bf16_x2:
2958 case NEON::BI__builtin_neon_vluti4q_lane_f16_x2:
2959 case NEON::BI__builtin_neon_vluti4q_lane_p16_x2:
2960 case NEON::BI__builtin_neon_vluti4q_lane_s16_x2:
2961 case NEON::BI__builtin_neon_vluti4q_lane_u16_x2:
2962 case NEON::BI__builtin_neon_vluti4q_laneq_bf16_x2:
2963 case NEON::BI__builtin_neon_vluti4q_laneq_f16_x2:
2964 case NEON::BI__builtin_neon_vluti4q_laneq_p16_x2:
2965 case NEON::BI__builtin_neon_vluti4q_laneq_s16_x2:
2966 case NEON::BI__builtin_neon_vluti4q_laneq_u16_x2:
2967 case NEON::BI__builtin_neon_vmmlaq_f16_mf8_fpm:
2968 case NEON::BI__builtin_neon_vmmlaq_f32_mf8_fpm:
2969 case NEON::BI__builtin_neon_vcvt1_low_bf16_mf8_fpm:
2970 case NEON::BI__builtin_neon_vcvt1_bf16_mf8_fpm:
2971 case NEON::BI__builtin_neon_vcvt1_high_bf16_mf8_fpm:
2972 case NEON::BI__builtin_neon_vcvt2_low_bf16_mf8_fpm:
2973 case NEON::BI__builtin_neon_vcvt2_bf16_mf8_fpm:
2974 case NEON::BI__builtin_neon_vcvt2_high_bf16_mf8_fpm:
2975 case NEON::BI__builtin_neon_vcvt1_low_f16_mf8_fpm:
2976 case NEON::BI__builtin_neon_vcvt1_f16_mf8_fpm:
2977 case NEON::BI__builtin_neon_vcvt1_high_f16_mf8_fpm:
2978 case NEON::BI__builtin_neon_vcvt2_low_f16_mf8_fpm:
2979 case NEON::BI__builtin_neon_vcvt2_f16_mf8_fpm:
2980 case NEON::BI__builtin_neon_vcvt2_high_f16_mf8_fpm:
2981 case NEON::BI__builtin_neon_vcvt_mf8_f32_fpm:
2982 case NEON::BI__builtin_neon_vcvt_mf8_f16_fpm:
2983 case NEON::BI__builtin_neon_vcvtq_mf8_f16_fpm:
2984 case NEON::BI__builtin_neon_vcvt_high_mf8_f32_fpm:
2985 case NEON::BI__builtin_neon_vdot_f16_mf8_fpm:
2986 case NEON::BI__builtin_neon_vdotq_f16_mf8_fpm:
2987 case NEON::BI__builtin_neon_vdot_lane_f16_mf8_fpm:
2988 case NEON::BI__builtin_neon_vdotq_lane_f16_mf8_fpm:
2989 case NEON::BI__builtin_neon_vdot_laneq_f16_mf8_fpm:
2990 case NEON::BI__builtin_neon_vdotq_laneq_f16_mf8_fpm:
2991 case NEON::BI__builtin_neon_vdot_f32_mf8_fpm:
2992 case NEON::BI__builtin_neon_vdotq_f32_mf8_fpm:
2993 case NEON::BI__builtin_neon_vdot_lane_f32_mf8_fpm:
2994 case NEON::BI__builtin_neon_vdotq_lane_f32_mf8_fpm:
2995 case NEON::BI__builtin_neon_vdot_laneq_f32_mf8_fpm:
2996 case NEON::BI__builtin_neon_vdotq_laneq_f32_mf8_fpm:
2997 case NEON::BI__builtin_neon_vmlalbq_f16_mf8_fpm:
2998 case NEON::BI__builtin_neon_vmlaltq_f16_mf8_fpm:
2999 case NEON::BI__builtin_neon_vmlallbbq_f32_mf8_fpm:
3000 case NEON::BI__builtin_neon_vmlallbtq_f32_mf8_fpm:
3001 case NEON::BI__builtin_neon_vmlalltbq_f32_mf8_fpm:
3002 case NEON::BI__builtin_neon_vmlallttq_f32_mf8_fpm:
3003 case NEON::BI__builtin_neon_vmlalbq_lane_f16_mf8_fpm:
3004 case NEON::BI__builtin_neon_vmlalbq_laneq_f16_mf8_fpm:
3005 case NEON::BI__builtin_neon_vmlaltq_lane_f16_mf8_fpm:
3006 case NEON::BI__builtin_neon_vmlaltq_laneq_f16_mf8_fpm:
3007 case NEON::BI__builtin_neon_vmlallbbq_lane_f32_mf8_fpm:
3008 case NEON::BI__builtin_neon_vmlallbbq_laneq_f32_mf8_fpm:
3009 case NEON::BI__builtin_neon_vmlallbtq_lane_f32_mf8_fpm:
3010 case NEON::BI__builtin_neon_vmlallbtq_laneq_f32_mf8_fpm:
3011 case NEON::BI__builtin_neon_vmlalltbq_lane_f32_mf8_fpm:
3012 case NEON::BI__builtin_neon_vmlalltbq_laneq_f32_mf8_fpm:
3013 case NEON::BI__builtin_neon_vmlallttq_lane_f32_mf8_fpm:
3014 case NEON::BI__builtin_neon_vmlallttq_laneq_f32_mf8_fpm:
3015 case NEON::BI__builtin_neon_vamin_f16:
3016 case NEON::BI__builtin_neon_vaminq_f16:
3017 case NEON::BI__builtin_neon_vamin_f32:
3018 case NEON::BI__builtin_neon_vaminq_f32:
3019 case NEON::BI__builtin_neon_vaminq_f64:
3020 case NEON::BI__builtin_neon_vamax_f16:
3021 case NEON::BI__builtin_neon_vamaxq_f16:
3022 case NEON::BI__builtin_neon_vamax_f32:
3023 case NEON::BI__builtin_neon_vamaxq_f32:
3024 case NEON::BI__builtin_neon_vamaxq_f64:
3025 case NEON::BI__builtin_neon_vscale_f16:
3026 case NEON::BI__builtin_neon_vscaleq_f16:
3027 case NEON::BI__builtin_neon_vscale_f32:
3028 case NEON::BI__builtin_neon_vscaleq_f32:
3029 case NEON::BI__builtin_neon_vscaleq_f64:
3030 cgm.errorNYI(expr->getSourceRange(),
3031 std::string("unimplemented AArch64 builtin call: ") +
3032 getContext().BuiltinInfo.getName(builtinID));
3033 return mlir::Value{};
3034 }
3035
3036 // Unreachable: All cases in the switch above return.
3037}
AArch64CodeGenUtils.h
Utilities used for generating code for AArch64 that are shared between the classic and ClangIR code-g...
isUnsigned
static bool isUnsigned(SValBuilder &SVB, NonLoc Value)
Definition ArrayBoundChecker.cpp:307
Builtins.h
Defines enum values for all the target-independent builtin functions.
emitCommonNeonSISDBuiltinExpr
static mlir::Value emitCommonNeonSISDBuiltinExpr(CIRGenFunction &cgf, const ARMVectorIntrinsicInfo &info, llvm::SmallVectorImpl< mlir::Value > &ops, const CallExpr *expr)
Definition CIRGenBuiltinAArch64.cpp:249
hasExtraNeonArgument
static bool hasExtraNeonArgument(unsigned builtinID)
Return true if BuiltinID is an overloaded Neon intrinsic with an extra argument that specifies the ve...
Definition CIRGenBuiltinAArch64.cpp:103
aarch64SVEIntrinsicsProvenSorted
static bool aarch64SVEIntrinsicsProvenSorted
Definition CIRGenBuiltinAArch64.cpp:61
neonEquivalentIntrinsicMap
static const std::pair< unsigned, unsigned > neonEquivalentIntrinsicMap[]
Definition CIRGenBuiltinAArch64.cpp:1305
emitNeonSplat
static mlir::Value emitNeonSplat(CIRGenBuilderTy &builder, mlir::Location loc, mlir::Value v, mlir::Value lane, unsigned int resEltCnt)
Definition CIRGenBuiltinAArch64.cpp:440
emitNeonCallToOp
static mlir::Value emitNeonCallToOp(CIRGenModule &cgm, CIRGenBuilderTy &builder, llvm::SmallVector< mlir::Type > argTypes, llvm::SmallVectorImpl< mlir::Value > &args, std::optional< llvm::StringRef > intrinsicName, mlir::Type funcResTy, mlir::Location loc, bool isConstrainedFPIntrinsic=false, unsigned shift=0, bool rightshift=false)
Definition CIRGenBuiltinAArch64.cpp:174
aarch64SVEIntrinsicMap
static const ARMVectorIntrinsicInfo aarch64SVEIntrinsicMap[]
Definition CIRGenBuiltinAArch64.cpp:53
getSVEVectorForElementType
static cir::VectorType getSVEVectorForElementType(CIRGenModule &cgm, mlir::Type eltTy)
Definition CIRGenBuiltinAArch64.cpp:1008
getSVEMinEltCount
static unsigned getSVEMinEltCount(clang::SVETypeFlags::EltType sveType)
Definition CIRGenBuiltinAArch64.cpp:970
genVscaleTimesFactor
static mlir::Value genVscaleTimesFactor(mlir::Location loc, CIRGenBuilderTy builder, mlir::Type cirTy, int32_t scalingFactor)
Definition CIRGenBuiltinAArch64.cpp:38
getFloatNeonType
static cir::VectorType getFloatNeonType(CIRGenFunction &cgf, NeonTypeFlags intTypeFlags)
Definition CIRGenBuiltinAArch64.cpp:131
getLLVMIntrNameNoPrefix
static llvm::StringRef getLLVMIntrNameNoPrefix(llvm::Intrinsic::ID intrID)
Definition CIRGenBuiltinAArch64.cpp:88
getIntValueFromConstOp
static int64_t getIntValueFromConstOp(mlir::Value val)
Definition CIRGenBuiltinAArch64.cpp:146
emitCallMaybeConstrainedBuiltin
static mlir::Value emitCallMaybeConstrainedBuiltin(CIRGenBuilderTy &builder, mlir::Location loc, StringRef intrName, mlir::Type retTy, llvm::SmallVector< mlir::Value > &ops)
Definition CIRGenBuiltinAArch64.cpp:894
emitCommonNeonBuiltinExpr
static mlir::Value emitCommonNeonBuiltinExpr(CIRGenFunction &cgf, unsigned builtinID, unsigned llvmIntrinsic, unsigned altLLVMIntrinsic, const char *nameHint, unsigned modifier, const CallExpr *expr, llvm::SmallVectorImpl< mlir::Value > &ops)
Definition CIRGenBuiltinAArch64.cpp:513
emitNeonCall
static mlir::Value emitNeonCall(CIRGenModule &cgm, CIRGenBuilderTy &builder, llvm::SmallVector< mlir::Type > argTypes, llvm::SmallVectorImpl< mlir::Value > &args, llvm::StringRef intrinsicName, mlir::Type funcResTy, mlir::Location loc, bool isConstrainedFPIntrinsic=false, unsigned shift=0, bool rightshift=false)
Definition CIRGenBuiltinAArch64.cpp:214
getSignChangedVectorType
static cir::VectorType getSignChangedVectorType(CIRGenBuilderTy &builder, cir::VectorType vecTy)
Flip the signedness of vecTy's element type, keeping the width and number of lanes the same.
Definition CIRGenBuiltinAArch64.cpp:454
getNeonPairwiseWidenInputType
static cir::VectorType getNeonPairwiseWidenInputType(cir::VectorType resType, bool usgn)
Definition CIRGenBuiltinAArch64.cpp:230
findARMVectorIntrinsicInMap
static const ARMVectorIntrinsicInfo * findARMVectorIntrinsicInMap(ArrayRef< ARMVectorIntrinsicInfo > intrinsicMap, unsigned builtinID, bool &mapProvenSorted)
Definition CIRGenBuiltinAArch64.cpp:66
emitCommonNeonShift
static mlir::Value emitCommonNeonShift(CIRGenBuilderTy &builder, mlir::Location loc, cir::VectorType resTy, mlir::Value shifTgt, mlir::Value shiftAmt, bool shiftLeft)
Definition CIRGenBuiltinAArch64.cpp:462
aarch64SIMDIntrinsicsProvenSorted
static bool aarch64SIMDIntrinsicsProvenSorted
Definition CIRGenBuiltinAArch64.cpp:59
getIntVecFromVecTy
static cir::VectorType getIntVecFromVecTy(CIRGenBuilderTy &builder, cir::VectorType vecTy)
Definition CIRGenBuiltinAArch64.cpp:500
sveBitsPerBlock
constexpr unsigned sveBitsPerBlock
Definition CIRGenBuiltinAArch64.cpp:1006
emitNeonShiftVector
static mlir::Value emitNeonShiftVector(CIRGenBuilderTy &builder, mlir::Value shiftVal, cir::VectorType vecTy, mlir::Location loc, bool neg)
Build a constant shift amount vector of vecTy to shift a vector Here shiftVal is a constant integer t...
Definition CIRGenBuiltinAArch64.cpp:156
emitAArch64CompareBuiltinExpr
static mlir::Value emitAArch64CompareBuiltinExpr(CIRGenFunction &cgf, CIRGenBuilderTy &builder, mlir::Location loc, mlir::Value src, mlir::Type retTy, const cir::CmpOpKind kind)
Definition CIRGenBuiltinAArch64.cpp:358
emitNeonRShiftImm
static mlir::Value emitNeonRShiftImm(CIRGenFunction &cgf, mlir::Value shiftVec, mlir::Value shiftVal, cir::VectorType vecTy, bool usgn, mlir::Location loc)
Definition CIRGenBuiltinAArch64.cpp:474
getNeonType
static cir::VectorType getNeonType(CIRGenFunction *cgf, NeonTypeFlags typeFlags, mlir::Location loc, bool hasLegalHalfType=true, bool v1Ty=false, bool allowBFloatArgsAndRet=true)
Definition CIRGenBuiltinAArch64.cpp:386
aarch64SISDIntrinsicsProvenSorted
static bool aarch64SISDIntrinsicsProvenSorted
Definition CIRGenBuiltinAArch64.cpp:60
getType
TokenType getType() const
Returns the token's type, e.g.
Definition FormatToken.h:990
kind
*collection of selector each with an associated kind and an ordered *collection of selectors A selector has a kind
Definition OpenMPClause.h:9815
TargetBuiltins.h
Enumerates target-specific builtins in their own namespaces within namespace clang.
cir::CIRBaseBuilderTy::getNullValue
cir::ConstantOp getNullValue(mlir::Type ty, mlir::Location loc)
Definition CIRBaseBuilder.h:98
cir::CIRBaseBuilderTy::createCast
mlir::Value createCast(mlir::Location loc, cir::CastKind kind, mlir::Value src, mlir::Type newTy)
Definition CIRBaseBuilder.h:502
cir::CIRBaseBuilderTy::createNUWAMul
mlir::Value createNUWAMul(mlir::Location loc, mlir::Value lhs, mlir::Value rhs)
Definition CIRBaseBuilder.h:640
cir::CIRBaseBuilderTy::createVecCompare
cir::VecCmpOp createVecCompare(mlir::Location loc, cir::CmpOpKind kind, mlir::Value lhs, mlir::Value rhs)
Definition CIRBaseBuilder.h:704
cir::CIRBaseBuilderTy::createIntCast
mlir::Value createIntCast(mlir::Value src, mlir::Type newTy)
Definition CIRBaseBuilder.h:516
cir::CIRBaseBuilderTy::createBitcast
mlir::Value createBitcast(mlir::Value src, mlir::Type newTy)
Definition CIRBaseBuilder.h:536
cir::CIRBaseBuilderTy::createCompare
cir::CmpOp createCompare(mlir::Location loc, cir::CmpOpKind kind, mlir::Value lhs, mlir::Value rhs)
Definition CIRBaseBuilder.h:699
cir::CIRBaseBuilderTy::getConstantInt
cir::ConstantOp getConstantInt(mlir::Location loc, mlir::Type ty, int64_t value)
Definition CIRBaseBuilder.h:80
cir::CIRDataLayout::getTypeSizeInBits
llvm::TypeSize getTypeSizeInBits(mlir::Type ty) const
Definition CIRDataLayout.cpp:46
clang::ASTContext
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition ASTContext.h:227
clang::ASTContext::BuiltinInfo
Builtin::Context & BuiltinInfo
Definition ASTContext.h:807
clang::ASTContext::GetBuiltinType
QualType GetBuiltinType(unsigned ID, GetBuiltinTypeError &Error, unsigned *IntegerConstantArgs=nullptr) const
Return the type for the specified builtin.
Definition ASTContext.cpp:12907
clang::ASTContext::GE_None
@ GE_None
No error.
Definition ASTContext.h:2622
clang::Builtin::Context::getName
std::string getName(unsigned ID) const
Return the identifier name for the specified builtin, e.g.
Definition Builtins.cpp:80
clang::CIRGen::CIRGenBuilderTy::getUInt64
cir::ConstantOp getUInt64(uint64_t c, mlir::Location loc)
Definition CIRGenBuilder.h:460
clang::CIRGen::CIRGenBuilderTy::emitIntrinsicCallOp
mlir::Value emitIntrinsicCallOp(mlir::Location loc, const llvm::StringRef str, const mlir::Type &resTy, Operands &&...op)
Definition CIRGenBuilder.h:877
clang::CIRGen::CIRGenBuilderTy::getSIntNTy
cir::IntType getSIntNTy(int n)
Definition CIRGenBuilder.h:344
clang::CIRGen::CIRGenBuilderTy::createVecShuffle
cir::VecShuffleOp createVecShuffle(mlir::Location loc, mlir::Value vec1, mlir::Value vec2, llvm::ArrayRef< mlir::Attribute > maskAttrs)
Definition CIRGenBuilder.h:848
clang::CIRGen::CIRGenBuilderTy::getZero
cir::ConstantOp getZero(mlir::Location loc, mlir::Type ty)
Definition CIRGenBuilder.h:464
clang::CIRGen::CIRGenBuilderTy::getConstInt
cir::ConstantOp getConstInt(mlir::Location loc, llvm::APSInt intVal)
Definition CIRGenBuilder.cpp:64
clang::CIRGen::CIRGenBuilderTy::getUIntNTy
cir::IntType getUIntNTy(int n)
Definition CIRGenBuilder.h:329
clang::CIRGen::CIRGenFunction::convertType
mlir::Type convertType(clang::QualType t)
Definition CIRGenFunction.cpp:103
clang::CIRGen::CIRGenFunction::getTarget
const TargetInfo & getTarget() const
Definition CIRGenFunction.h:310
clang::CIRGen::CIRGenFunction::getLoc
mlir::Location getLoc(clang::SourceLocation srcLoc)
Helpers to convert Clang's SourceLocation to a MLIR Location.
Definition CIRGenFunction.cpp:107
clang::CIRGen::CIRGenFunction::getZExtIntValueFromConstOp
static int64_t getZExtIntValueFromConstOp(mlir::Value val)
Get zero-extended integer from a mlir::Value that is an int constant or a constant op.
Definition CIRGenFunction.h:287
clang::CIRGen::CIRGenFunction::emitSVEPredicateCast
mlir::Value emitSVEPredicateCast(mlir::Value pred, unsigned minNumElts, mlir::Location loc)
Definition CIRGenBuiltinAArch64.cpp:931
clang::CIRGen::CIRGenFunction::getAArch64SVEProcessedOperands
bool getAArch64SVEProcessedOperands(unsigned builtinID, const CallExpr *expr, SmallVectorImpl< mlir::Value > &ops, clang::SVETypeFlags typeFlags)
Definition CIRGenBuiltinAArch64.cpp:902
clang::CIRGen::CIRGenFunction::returnValue
Address returnValue
The temporary alloca to hold the return value.
Definition CIRGenFunction.h:90
clang::CIRGen::CIRGenFunction::emitAArch64BuiltinExpr
std::optional< mlir::Value > emitAArch64BuiltinExpr(unsigned builtinID, const CallExpr *expr, ReturnValueSlot returnValue, llvm::Triple::ArchType arch)
Definition CIRGenBuiltinAArch64.cpp:1700
clang::CIRGen::CIRGenFunction::emitAArch64SMEBuiltinExpr
std::optional< mlir::Value > emitAArch64SMEBuiltinExpr(unsigned builtinID, const CallExpr *expr)
Definition CIRGenBuiltinAArch64.cpp:1294
clang::CIRGen::CIRGenFunction::emitScalarExpr
mlir::Value emitScalarExpr(const clang::Expr *e, bool ignoreResultAssign=false)
Emit the computation of the specified expression of scalar type.
Definition CIRGenExprScalar.cpp:1748
clang::CIRGen::CIRGenFunction::getBuilder
CIRGenBuilderTy & getBuilder()
Definition CIRGenFunction.h:222
clang::CIRGen::CIRGenFunction::getContext
clang::ASTContext & getContext() const
Definition CIRGenFunction.h:220
clang::CIRGen::CIRGenFunction::emitAArch64SVEBuiltinExpr
std::optional< mlir::Value > emitAArch64SVEBuiltinExpr(unsigned builtinID, const CallExpr *expr)
Definition CIRGenBuiltinAArch64.cpp:1019
clang::CIRGen::CIRGenFunction::emitScalarOrConstFoldImmArg
mlir::Value emitScalarOrConstFoldImmArg(unsigned iceArguments, unsigned idx, const Expr *argExpr)
Definition CIRGenBuiltin.cpp:2651
clang::CIRGen::CIRGenModule
This class organizes the cross-function state that is used while generating CIR code.
Definition CIRGenModule.h:60
clang::CIRGen::CIRGenModule::errorNYI
DiagnosticBuilder errorNYI(SourceLocation, llvm::StringRef)
Helpers to emit "not yet implemented" error diagnostics.
Definition CIRGenModule.cpp:3570
clang::CIRGen::CIRGenModule::getDataLayout
const cir::CIRDataLayout getDataLayout() const
Definition CIRGenModule.h:177
clang::CIRGen::ReturnValueSlot
Contains the address where the return value of a function can be stored, and whether the address is v...
Definition CIRGenCall.h:260
clang::CallExpr
CallExpr - Represents a function call (C99 6.5.2.2, C++ [expr.call]).
Definition Expr.h:2946
clang::Expr
This represents one expression.
Definition Expr.h:112
clang::NeonTypeFlags
Flags to identify the types for overloaded Neon builtins.
Definition TargetBuiltins.h:253
clang::NeonTypeFlags::getEltType
EltType getEltType() const
Definition TargetBuiltins.h:286
clang::SVETypeFlags
Flags to identify the types for overloaded SVE builtins.
Definition TargetBuiltins.h:330
clang::SVETypeFlags::isReverseUSDOT
bool isReverseUSDOT() const
Definition TargetBuiltins.h:409
clang::SVETypeFlags::isGatherLoad
bool isGatherLoad() const
Definition TargetBuiltins.h:389
clang::SVETypeFlags::getEltType
EltType getEltType() const
Definition TargetBuiltins.h:367
clang::SVETypeFlags::isPrefetch
bool isPrefetch() const
Definition TargetBuiltins.h:404
clang::SVETypeFlags::isTupleSet
bool isTupleSet() const
Definition TargetBuiltins.h:415
clang::SVETypeFlags::isReverseMergeAnyAccOp
bool isReverseMergeAnyAccOp() const
Definition TargetBuiltins.h:411
clang::SVETypeFlags::isTupleGet
bool isTupleGet() const
Definition TargetBuiltins.h:414
clang::SVETypeFlags::isInsertOp1SVALL
bool isInsertOp1SVALL() const
Definition TargetBuiltins.h:407
clang::SVETypeFlags::isAppendSVALL
bool isAppendSVALL() const
Definition TargetBuiltins.h:406
clang::SVETypeFlags::isReverseMergeAnyBinOp
bool isReverseMergeAnyBinOp() const
Definition TargetBuiltins.h:410
clang::SVETypeFlags::isStructStore
bool isStructStore() const
Definition TargetBuiltins.h:392
clang::SVETypeFlags::isTupleCreate
bool isTupleCreate() const
Definition TargetBuiltins.h:413
clang::SVETypeFlags::isGatherPrefetch
bool isGatherPrefetch() const
Definition TargetBuiltins.h:408
clang::SVETypeFlags::hasSplatOperand
bool hasSplatOperand() const
Definition TargetBuiltins.h:383
clang::SVETypeFlags::getMergeType
MergeType getMergeType() const
Definition TargetBuiltins.h:375
clang::SVETypeFlags::isStructLoad
bool isStructLoad() const
Definition TargetBuiltins.h:391
clang::SVETypeFlags::getSplatOperand
unsigned getSplatOperand() const
Definition TargetBuiltins.h:379
clang::SVETypeFlags::isScatterStore
bool isScatterStore() const
Definition TargetBuiltins.h:390
clang::SVETypeFlags::isReverseCompare
bool isReverseCompare() const
Definition TargetBuiltins.h:405
clang::TargetInfo::hasFastHalfType
virtual bool hasFastHalfType() const
Determine whether the target has fast native support for operations on half types.
Definition TargetInfo.h:712
clang::aarch64::AArch64SISDIntrinsicMap
const ARMVectorIntrinsicInfo AArch64SISDIntrinsicMap[]
Definition AArch64CodeGenUtils.h:378
clang::aarch64::AArch64SIMDIntrinsicMap
const ARMVectorIntrinsicInfo AArch64SIMDIntrinsicMap[]
Definition AArch64CodeGenUtils.h:80
clang::ast_matchers::type
const internal::VariadicAllOfMatcher< Type > type
Matches Types in the clang AST.
Definition ASTMatchersInternal.cpp:809
clang::ast_matchers::expr
const internal::VariadicDynCastAllOfMatcher< Stmt, Expr > expr
Matches expressions.
Definition ASTMatchersInternal.cpp:937
clang
The JSON file list parser is used to communicate input to InstallAPI.
Definition CalledOnceCheck.h:17
clang::isa
bool isa(CodeGen::Address addr)
Definition Address.h:330
clang::cast
U cast(CodeGen::Address addr)
Definition Address.h:327
llvm
Diagnostic wrappers for TextAPI types for error reporting.
Definition Dominators.h:30
cir::MissingFeatures::msvcBuiltins
static bool msvcBuiltins()
Definition MissingFeatures.h:306
cir::MissingFeatures::handleBuiltinICEArguments
static bool handleBuiltinICEArguments()
Definition MissingFeatures.h:280
cir::MissingFeatures::aarch64SIMDIntrinsics
static bool aarch64SIMDIntrinsics()
Definition MissingFeatures.h:191
cir::MissingFeatures::aarch64SVEIntrinsics
static bool aarch64SVEIntrinsics()
Definition MissingFeatures.h:193
cir::MissingFeatures::emitConstrainedFPCall
static bool emitConstrainedFPCall()
Definition MissingFeatures.h:257
cir::MissingFeatures::aarch64SMEIntrinsics
static bool aarch64SMEIntrinsics()
Definition MissingFeatures.h:192
cir::MissingFeatures::aarch64TblBuiltinExpr
static bool aarch64TblBuiltinExpr()
Definition MissingFeatures.h:194
Generated on for clang by doxygen 1.14.0