CIRGenBuiltinAArch64.cpp

Go to the documentation of this file.

//===---- CIRGenBuiltinAArch64.cpp - Emit CIR for AArch64 builtins --------===//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This contains code to emit ARM64 Builtin calls as CIR or a function call
// to be later resolved.
//
//===----------------------------------------------------------------------===//
 
#include "CIRGenBuilder.h"
#include "CIRGenFunction.h"
#include "clang/Basic/AArch64CodeGenUtils.h"
#include "clang/Basic/TargetBuiltins.h"
#include "clang/CIR/Dialect/IR/CIRTypes.h"
#include "clang/CIR/MissingFeatures.h"
#include "clang/CodeGenUtils/TargetUtils.h"
 
// TODO(cir): once all builtins are covered, decide whether we still
// need to use LLVM intrinsics or if there's a better approach to follow. Right
// now the intrinsics are reused to make it convenient to encode all thousands
// of them and passing down to LLVM lowering.
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/IntrinsicsAArch64.h"
 
#include "mlir/IR/BuiltinTypes.h"
#include "mlir/IR/Value.h"
#include "clang/AST/GlobalDecl.h"
#include "clang/Basic/Builtins.h"
 
using namespace clang;
using namespace clang::CIRGen;
using namespace llvm;
using namespace clang::aarch64;
 
// Generate vscale * scalingFactor
static mlir::Value genVscaleTimesFactor(mlir::Location loc,
                                        CIRGenBuilderTy &builder,
                                        mlir::Type cirTy,
                                        int32_t scalingFactor) {
  mlir::Value vscale = builder.emitIntrinsicCallOp(loc, "vscale", cirTy);
  return builder.createNUWAMul(loc, vscale,
                               builder.getUInt64(scalingFactor, loc));
}
 
#define SVEMAP1(NameBase, LLVMIntrinsic, TypeModifier)                         \
  {SVE::BI__builtin_sve_##NameBase, Intrinsic::LLVMIntrinsic, TypeModifier}
 
#define SVEMAP2(NameBase, TypeModifier)                                        \
  {SVE::BI__builtin_sve_##NameBase, 0, TypeModifier}
static const AArch64SVEAndSMEVectorIntrinsicInfo aarch64SVEIntrinsicMap[] = {
#define GET_SVE_LLVM_INTRINSIC_MAP
#include "clang/Basic/arm_sve_builtin_cg.inc"
#undef GET_SVE_LLVM_INTRINSIC_MAP
};
 
static bool aarch64SIMDIntrinsicsProvenSorted = false;
static bool aarch64SISDIntrinsicsProvenSorted = false;
static bool aarch64SVEIntrinsicsProvenSorted = false;
 
// Check if Builtin `builtinId` is present in `intrinsicMap`. If yes, returns
// the corresponding info struct.
template <typename IntrinsicInfo>
static const IntrinsicInfo *
findARMVectorIntrinsicInMap(ArrayRef<IntrinsicInfo> intrinsicMap,
                            unsigned builtinID, bool &mapProvenSorted) {
 
#ifndef NDEBUG
  if (!mapProvenSorted) {
    assert(llvm::is_sorted(intrinsicMap));
    mapProvenSorted = true;
  }
#endif
 
  const IntrinsicInfo *info = llvm::lower_bound(intrinsicMap, builtinID);
 
  if (info != intrinsicMap.end() && info->BuiltinID == builtinID)
    return info;
 
  return nullptr;
}
 
//===----------------------------------------------------------------------===//
//  Generic helpers
//===----------------------------------------------------------------------===//
static llvm::StringRef getLLVMIntrNameNoPrefix(llvm::Intrinsic::ID intrID) {
  llvm::StringRef llvmIntrName = llvm::Intrinsic::getBaseName(intrID);
  assert(llvmIntrName.starts_with("llvm.") && "Not an LLVM intrinsic!");
  return llvmIntrName.drop_front(/*strlen("llvm.")=*/5);
}
 
static cir::VectorType getFloatNeonType(CIRGenFunction &cgf,
                                        NeonTypeFlags intTypeFlags) {
  int isQuad = intTypeFlags.isQuad();
  switch (intTypeFlags.getEltType()) {
  case NeonTypeFlags::Int16:
    return cir::VectorType::getFixed(cgf.fP16Ty, (4ULL << isQuad));
  case NeonTypeFlags::Int32:
    return cir::VectorType::getFixed(cgf.floatTy, (2ULL << isQuad));
  case NeonTypeFlags::Int64:
    return cir::VectorType::getFixed(cgf.doubleTy, (1ULL << isQuad));
  default:
    llvm_unreachable("Type can't be converted to floating-point!");
  }
}
 
static int64_t getIntValueFromConstOp(mlir::Value val) {
  return val.getDefiningOp<cir::ConstantOp>().getIntValue().getSExtValue();
}
 
/// Build a constant shift amount vector of `vecTy` to shift a vector
/// Here `shiftVal` is a constant integer that will be broadcast into a
/// a const vector of `vecTy` which is the return value of this function
/// If `neg` is true, the shift amount is negated before splatting (used
/// when encoding a right shift as a left shift by a negative amount for
/// intrinsics like aarch64.neon.{s,u}rshl).
static mlir::Value emitNeonShiftVector(CIRGenBuilderTy &builder,
                                       mlir::Value shiftVal,
                                       cir::VectorType vecTy,
                                       mlir::Location loc, bool neg) {
  if (neg) {
    int64_t shiftAmt = -getIntValueFromConstOp(shiftVal);
    shiftVal = builder.getConstantInt(loc, vecTy.getElementType(), shiftAmt);
  }
  mlir::Type eltTy = vecTy.getElementType();
  if (shiftVal.getType() != eltTy) {
    shiftVal = builder.createIntCast(shiftVal, eltTy);
  }
  return cir::VecSplatOp::create(builder, loc, vecTy, shiftVal);
}
 
// TODO(cir): Remove `cgm` from the list of arguments once all NYI(s) are gone.
template <typename Operation>
static mlir::Value
emitNeonCallToOp(CIRGenModule &cgm, CIRGenBuilderTy &builder,
                 llvm::SmallVector<mlir::Type> argTypes,
                 llvm::SmallVectorImpl<mlir::Value> &args,
                 std::optional<llvm::StringRef> intrinsicName,
                 mlir::Type funcResTy, mlir::Location loc,
                 bool isConstrainedFPIntrinsic = false, unsigned shift = 0,
                 bool rightshift = false) {
  // TODO(cir): Consider removing the following unreachable when we have
  // emitConstrainedFPCall feature implemented
  assert(!cir::MissingFeatures::emitConstrainedFPCall());
  if (isConstrainedFPIntrinsic)
    cgm.errorNYI(loc, std::string("constrained FP intrinsic"));
 
  for (unsigned j = 0; j < argTypes.size(); ++j) {
    if (isConstrainedFPIntrinsic) {
      assert(!cir::MissingFeatures::emitConstrainedFPCall());
    }
    if (shift > 0 && shift == j) {
      args[j] = emitNeonShiftVector(builder, args[j],
                                    mlir::cast<cir::VectorType>(argTypes[j]),
                                    loc, rightshift);
    } else {
      args[j] = builder.createBitcast(args[j], argTypes[j]);
    }
  }
  if (isConstrainedFPIntrinsic) {
    assert(!cir::MissingFeatures::emitConstrainedFPCall());
    return nullptr;
  }
  if constexpr (std::is_same_v<Operation, cir::LLVMIntrinsicCallOp>) {
    return Operation::create(builder, loc,
                             builder.getStringAttr(intrinsicName.value()),
                             funcResTy, args)
        .getResult();
  }
  if constexpr (std::is_same_v<Operation, cir::FMAOp>) {
    assert(args.size() == 3 && "fma expects three operands");
    return Operation::create(builder, loc, funcResTy, args[0], args[1], args[2],
                             builder.getConstrainedFPAttr())
        .getResult();
  }
  if constexpr (std::is_same_v<Operation, cir::SqrtOp>) {
    assert(args.size() == 1 && "sqrt expects one operand");
    return Operation::create(builder, loc, funcResTy, args[0],
                             builder.getConstrainedFPAttr())
        .getResult();
  }
  if constexpr (!std::is_same_v<Operation, cir::LLVMIntrinsicCallOp> &&
                !std::is_same_v<Operation, cir::FMAOp> &&
                !std::is_same_v<Operation, cir::SqrtOp>)
    return Operation::create(
               builder, loc, mlir::TypeRange{funcResTy}, args,
               cir::getDefaultProperties<Operation>(builder.getContext()))
        .getResult();
}
 
// TODO(cir): Remove `cgm` from the list of arguments once all NYI(s) are gone.
static mlir::Value emitNeonCall(CIRGenModule &cgm, CIRGenBuilderTy &builder,
                                llvm::SmallVector<mlir::Type> argTypes,
                                llvm::SmallVectorImpl<mlir::Value> &args,
                                llvm::StringRef intrinsicName,
                                mlir::Type funcResTy, mlir::Location loc,
                                bool isConstrainedFPIntrinsic = false,
                                unsigned shift = 0, bool rightshift = false) {
  return emitNeonCallToOp<cir::LLVMIntrinsicCallOp>(
      cgm, builder, std::move(argTypes), args, intrinsicName, funcResTy, loc,
      isConstrainedFPIntrinsic, shift, rightshift);
}
 
// Computes the input vector type for a NEON pairwise widening operation (e.g.
// vpaddl/vpadal). Given a result vector type, it derives the corresponding
// input type by halving the element bit width and doubling the number of lanes,
// while setting the signedness based on usgn.
static cir::VectorType getNeonPairwiseWidenInputType(cir::VectorType resType,
                                                     bool usgn) {
  mlir::Type elemTy = resType.getElementType();
  uint64_t resLanes = resType.getFixedSize();
  auto intTy = mlir::dyn_cast<cir::IntType>(elemTy);
  assert(intTy && "vpaddl result type must be an integer vector");
 
  unsigned resWidth = intTy.getWidth();
  assert((resWidth == 16 || resWidth == 32 || resWidth == 64) &&
         "unexpected vpaddl result element width");
 
  unsigned argWidth = resWidth / 2;
  unsigned argLanes = resLanes * 2;
  cir::VectorType result = cir::VectorType::getFixed(
      cir::IntType::get(resType.getContext(), argWidth, /* is_signed */ !usgn),
      argLanes);
  return result;
}
 
// Derive the LLVM intrinsic's per-operand argument types and its result
// type for use when emitting the intrinsic call.
//
// `modifier` is the TypeModifier bitmask from `ARMNeonVectorIntrinsicInfo`
// (callers pass `info.TypeModifier`; see AArch64CodeGenUtils.h). It encodes
// how the intrinsic's argument and return types relate to the builtin's
// scalar types. For SISD builtins the key flags are:
//   - VectorizeArgTypes: wrap each arg type into a fixed-width vector
//   - Use64BitVectors / Use128BitVectors: choose the vector width
//     (when neither is set the vector has 1 element)
//   - AddRetType / VectorizeRetType: analogous flags for the return type
//
// ARM.cpp lets LLVM resolve the intrinsic's signature (via
// `CGM.getIntrinsic`) and then walks the resolved Function* formal
// parameter types. CIR has no LLVMContext here, so we derive the same
// argument/result types directly from the Clang operand types.
static std::pair<mlir::Type, llvm::SmallVector<mlir::Type>>
deriveNeonSISDIntrinsicOperandTypes(CIRGenFunction &cgf, unsigned modifier,
                                    mlir::Type arg0Ty, mlir::Type resultTy,
                                    llvm::ArrayRef<mlir::Value> ops,
                                    unsigned iceArguments) {
  int vectorSize = 0;
  if (modifier & Use64BitVectors)
    vectorSize = 64;
  else if (modifier & Use128BitVectors)
    vectorSize = 128;
 
  auto wrapAsVector = [&](mlir::Type ty) -> cir::VectorType {
    unsigned bits = cgf.cgm.getDataLayout().getTypeSizeInBits(ty);
    unsigned elts = vectorSize ? vectorSize / bits : 1;
    return cir::VectorType::getFixed(ty, elts);
  };
 
  // Determine the vectorized data type.
  cir::VectorType vecArgTy;
  if (modifier & VectorizeArgTypes)
    vecArgTy = wrapAsVector(arg0Ty);
 
  // Determine the intrinsic result type: `VectorizeRetType` returns a
  // vector; otherwise, if data args are vectorized and `AddRetType` is
  // unset, use a vector return with the same shape as those args.
  mlir::Type funcResTy = resultTy;
  if (modifier & VectorizeRetType)
    funcResTy = wrapAsVector(resultTy);
  else if (vecArgTy && !(modifier & AddRetType))
    funcResTy = wrapAsVector(resultTy);
 
  // AdvSIMD_2Arg_Scalar_Narrow_Intrinsic in IntrinsicsAArch64.td models
  // operand 0 as LLVMExtendedType<0>: the result vector's lane count with
  // double-width elements. Reconstruct that LLVM operand type from the Clang
  // builtin's scalar argument type.
  if (modifier & ArgAsWidenedRetType) {
    auto resVecTy = mlir::dyn_cast<cir::VectorType>(funcResTy);
    assert(resVecTy &&
           "SISD LLVM argument reconstruction requires a vector result");
    vecArgTy = cir::VectorType::get(arg0Ty, resVecTy.getSize());
  }
 
  // True if `ty` is arg0's type, or an integer of the same width that only
  // differs in signedness. vsqadd/vuqadd mix the two on purpose: vsqaddb_u8
  // takes a uint8_t and an int8_t, but both become the same vector type.
  auto matchesArg0Ty = [&](mlir::Type ty) {
    if (ty == arg0Ty)
      return true;
    auto intTy = mlir::dyn_cast<cir::IntType>(ty);
    auto arg0IntTy = mlir::dyn_cast<cir::IntType>(arg0Ty);
    return intTy && arg0IntTy && intTy.getWidth() == arg0IntTy.getWidth();
  };
 
  // `vecArgTy` is populated by `VectorizeArgTypes` or
  // `ArgAsWidenedRetType`. When set, wrap every non-immediate data operand
  // that has the same scalar type as arg0. Checking the ICE bitmap prevents
  // an i32 immediate from being vectorized when it has the same type as a
  // data operand (e.g. vqshrns_n_s32). `VectorizeImmArg` marks the entries
  // whose trailing immediate must be widened as well.
  const bool widenTrailingImm = vecArgTy && ops.size() == 2 &&
                                (modifier & VectorizeImmArg) &&
                                (iceArguments & (1U << 1));
  llvm::SmallVector<mlir::Type> argTypes;
  argTypes.reserve(ops.size());
  for (unsigned i = 0, e = ops.size(); i != e; ++i) {
    bool isImmediate = iceArguments & (1U << i);
    bool asVector =
        vecArgTy && ((!isImmediate && matchesArg0Ty(ops[i].getType())) ||
                     (widenTrailingImm && i == 1));
    argTypes.push_back(asVector ? mlir::Type(vecArgTy) : ops[i].getType());
  }
 
  return {funcResTy, std::move(argTypes)};
}
 
// Source-operand vector type for a common NEON binary intrinsic: the
// double-element-width form of `vTy` when `ArgAsWidenedRetType` is set (e.g.
// vraddhn), otherwise `vTy`.
static cir::VectorType deriveNeonBinaryArgType(CIRGenBuilderTy &builder,
                                               unsigned modifier,
                                               cir::VectorType vTy) {
  if (modifier & ArgAsWidenedRetType)
    return builder.getExtendedOrTruncatedElementVectorType(vTy,
                                                           /*isExtended=*/true);
  return vTy;
}
 
/// Create a vector from an input scalar argument, usually for a NEON SISD
/// intrinsic call. Insert the argument into lane 0 of a poison vector.
static void vecExtendIntValue(CIRGenFunction &cgf, cir::VectorType argVTy,
                              mlir::Value &arg, mlir::Location loc) {
  CIRGenBuilderTy &builder = cgf.getBuilder();
  // TODO: Support floating-point scalar arguments when a SISD intrinsic
  // requires scalar-to-vector adaptation; current users are integer-only.
  cir::IntType eltTy = mlir::dyn_cast<cir::IntType>(argVTy.getElementType());
  assert(mlir::isa<cir::IntType>(arg.getType()) && eltTy);
  // Cast the scalar data operand to the vector element type before inserting
  // it into lane 0.
  arg = builder.createIntCast(arg, eltTy);
  mlir::Value zero = builder.getConstInt(loc, cgf.sizeTy, 0);
  mlir::Value poison = builder.getConstant(loc, cir::PoisonAttr::get(argVTy));
  arg = cir::VecInsertOp::create(builder, loc, poison, arg, zero);
}
 
static mlir::Value
emitCommonNeonSISDBuiltinExpr(CIRGenFunction &cgf,
                              const ARMNeonVectorIntrinsicInfo &info,
                              llvm::SmallVectorImpl<mlir::Value> &ops,
                              const CallExpr *expr, unsigned iceArguments) {
  assert(info.LLVMIntrinsic && "Generic code assumes a valid intrinsic");
 
  switch (info.BuiltinID) {
  case NEON::BI__builtin_neon_vcled_s64:
  case NEON::BI__builtin_neon_vcled_u64:
  case NEON::BI__builtin_neon_vcles_f32:
  case NEON::BI__builtin_neon_vcled_f64:
  case NEON::BI__builtin_neon_vcltd_s64:
  case NEON::BI__builtin_neon_vcltd_u64:
  case NEON::BI__builtin_neon_vclts_f32:
  case NEON::BI__builtin_neon_vcltd_f64:
  case NEON::BI__builtin_neon_vcales_f32:
  case NEON::BI__builtin_neon_vcaled_f64:
  case NEON::BI__builtin_neon_vcalts_f32:
  case NEON::BI__builtin_neon_vcaltd_f64:
    cgf.cgm.errorNYI(expr->getSourceRange(),
                     std::string("unimplemented AArch64 builtin call: ") +
                         cgf.getContext().BuiltinInfo.getName(info.BuiltinID));
    break;
  }
 
  unsigned intr = info.LLVMIntrinsic;
 
  // Use fptosi.sat/fptoui.sat unless under strict FP.
  assert(!cir::MissingFeatures::emitConstrainedFPCall());
  if (intr == Intrinsic::aarch64_neon_fcvtzs)
    intr = Intrinsic::fptosi_sat;
  else if (intr == Intrinsic::aarch64_neon_fcvtzu)
    intr = Intrinsic::fptoui_sat;
 
  llvm::StringRef llvmIntrName =
      getLLVMIntrNameNoPrefix(static_cast<llvm::Intrinsic::ID>(intr));
  mlir::Location loc = cgf.getLoc(expr->getExprLoc());
 
  // The switch stmt is intended to help catch NYI cases and will be removed
  // once the CIR implementation is complete. Avoid adding specialized
  // code in cases - that should only be required for a handful of examples.
  switch (info.BuiltinID) {
  default:
    cgf.cgm.errorNYI(expr->getSourceRange(),
                     std::string("unimplemented AArch64 builtin call: ") +
                         cgf.getContext().BuiltinInfo.getName(info.BuiltinID));
    break;
  case NEON::BI__builtin_neon_vminv_s8:
  case NEON::BI__builtin_neon_vminvq_s8:
  case NEON::BI__builtin_neon_vminv_s16:
  case NEON::BI__builtin_neon_vminvq_s16:
  case NEON::BI__builtin_neon_vminv_s32:
  case NEON::BI__builtin_neon_vminvq_s32:
  case NEON::BI__builtin_neon_vminv_u8:
  case NEON::BI__builtin_neon_vminvq_u8:
  case NEON::BI__builtin_neon_vminv_u16:
  case NEON::BI__builtin_neon_vminvq_u16:
  case NEON::BI__builtin_neon_vminv_u32:
  case NEON::BI__builtin_neon_vminvq_u32:
  case NEON::BI__builtin_neon_vminv_f32:
  case NEON::BI__builtin_neon_vminvq_f32:
  case NEON::BI__builtin_neon_vminvq_f64:
  case NEON::BI__builtin_neon_vminnmv_f32:
  case NEON::BI__builtin_neon_vminnmvq_f32:
  case NEON::BI__builtin_neon_vminnmvq_f64:
  case NEON::BI__builtin_neon_vabdd_f64:
  case NEON::BI__builtin_neon_vabds_f32:
  case NEON::BI__builtin_neon_vshld_s64:
  case NEON::BI__builtin_neon_vshld_u64:
  case NEON::BI__builtin_neon_vpmins_f32:
  case NEON::BI__builtin_neon_vpminqd_f64:
  case NEON::BI__builtin_neon_vpminnms_f32:
  case NEON::BI__builtin_neon_vpminnmqd_f64:
  case NEON::BI__builtin_neon_vcvtd_s32_f64:
  case NEON::BI__builtin_neon_vcvtd_s64_f64:
  case NEON::BI__builtin_neon_vcvtd_u64_f64:
  case NEON::BI__builtin_neon_vcvtd_u32_f64:
  case NEON::BI__builtin_neon_vcvts_n_f32_s32:
  case NEON::BI__builtin_neon_vcvts_n_f32_u32:
  case NEON::BI__builtin_neon_vcvts_n_s32_f32:
  case NEON::BI__builtin_neon_vcvts_n_u32_f32:
  case NEON::BI__builtin_neon_vcvtd_n_f64_s64:
  case NEON::BI__builtin_neon_vcvtd_n_f64_u64:
  case NEON::BI__builtin_neon_vcvtd_n_s64_f64:
  case NEON::BI__builtin_neon_vcvtd_n_u64_f64:
  case NEON::BI__builtin_neon_vcvts_s32_f32:
  case NEON::BI__builtin_neon_vcvts_s64_f32:
  case NEON::BI__builtin_neon_vcvts_u32_f32:
  case NEON::BI__builtin_neon_vcvts_u64_f32:
  case NEON::BI__builtin_neon_vcvtns_s32_f32:
  case NEON::BI__builtin_neon_vcvtns_s64_f32:
  case NEON::BI__builtin_neon_vcvtns_u32_f32:
  case NEON::BI__builtin_neon_vcvtns_u64_f32:
  case NEON::BI__builtin_neon_vcvtnd_s32_f64:
  case NEON::BI__builtin_neon_vcvtnd_s64_f64:
  case NEON::BI__builtin_neon_vcvtnd_u32_f64:
  case NEON::BI__builtin_neon_vcvtnd_u64_f64:
  case NEON::BI__builtin_neon_vcvtms_s32_f32:
  case NEON::BI__builtin_neon_vcvtms_s64_f32:
  case NEON::BI__builtin_neon_vcvtms_u32_f32:
  case NEON::BI__builtin_neon_vcvtms_u64_f32:
  case NEON::BI__builtin_neon_vcvtmd_s32_f64:
  case NEON::BI__builtin_neon_vcvtmd_u32_f64:
  case NEON::BI__builtin_neon_vcvtmd_s64_f64:
  case NEON::BI__builtin_neon_vcvtmd_u64_f64:
  case NEON::BI__builtin_neon_vcvtps_s32_f32:
  case NEON::BI__builtin_neon_vcvtps_s64_f32:
  case NEON::BI__builtin_neon_vcvtps_u32_f32:
  case NEON::BI__builtin_neon_vcvtps_u64_f32:
  case NEON::BI__builtin_neon_vcvtpd_s32_f64:
  case NEON::BI__builtin_neon_vcvtpd_u32_f64:
  case NEON::BI__builtin_neon_vcvtpd_s64_f64:
  case NEON::BI__builtin_neon_vcvtpd_u64_f64:
  case NEON::BI__builtin_neon_vcvtas_s32_f32:
  case NEON::BI__builtin_neon_vcvtas_s64_f32:
  case NEON::BI__builtin_neon_vcvtas_u32_f32:
  case NEON::BI__builtin_neon_vcvtas_u64_f32:
  case NEON::BI__builtin_neon_vcvtad_s32_f64:
  case NEON::BI__builtin_neon_vcvtad_u32_f64:
  case NEON::BI__builtin_neon_vcvtad_s64_f64:
  case NEON::BI__builtin_neon_vcvtad_u64_f64:
  case NEON::BI__builtin_neon_vcvtxd_f32_f64:
  case NEON::BI__builtin_neon_vaddlv_s32:
  case NEON::BI__builtin_neon_vaddlv_u32:
  case NEON::BI__builtin_neon_vaddlvq_s32:
  case NEON::BI__builtin_neon_vaddlvq_u32:
  case NEON::BI__builtin_neon_vaddv_s8:
  case NEON::BI__builtin_neon_vaddv_s16:
  case NEON::BI__builtin_neon_vaddv_s32:
  case NEON::BI__builtin_neon_vaddv_u8:
  case NEON::BI__builtin_neon_vaddv_u16:
  case NEON::BI__builtin_neon_vaddv_u32:
  case NEON::BI__builtin_neon_vaddv_f32:
  case NEON::BI__builtin_neon_vaddvq_s8:
  case NEON::BI__builtin_neon_vaddvq_s16:
  case NEON::BI__builtin_neon_vaddvq_s32:
  case NEON::BI__builtin_neon_vaddvq_s64:
  case NEON::BI__builtin_neon_vaddvq_u8:
  case NEON::BI__builtin_neon_vaddvq_u16:
  case NEON::BI__builtin_neon_vaddvq_u32:
  case NEON::BI__builtin_neon_vaddvq_u64:
  case NEON::BI__builtin_neon_vaddvq_f32:
  case NEON::BI__builtin_neon_vaddvq_f64:
  case NEON::BI__builtin_neon_vabdh_f16:
  case NEON::BI__builtin_neon_vrecpeh_f16:
  case NEON::BI__builtin_neon_vrecpxh_f16:
  case NEON::BI__builtin_neon_vrsqrteh_f16:
  case NEON::BI__builtin_neon_vrsqrtsh_f16:
  case NEON::BI__builtin_neon_vmaxv_s8:
  case NEON::BI__builtin_neon_vmaxvq_s8:
  case NEON::BI__builtin_neon_vmaxv_s16:
  case NEON::BI__builtin_neon_vmaxvq_s16:
  case NEON::BI__builtin_neon_vmaxv_s32:
  case NEON::BI__builtin_neon_vmaxvq_s32:
  case NEON::BI__builtin_neon_vmaxv_u8:
  case NEON::BI__builtin_neon_vmaxvq_u8:
  case NEON::BI__builtin_neon_vmaxv_u16:
  case NEON::BI__builtin_neon_vmaxvq_u16:
  case NEON::BI__builtin_neon_vmaxv_u32:
  case NEON::BI__builtin_neon_vmaxvq_u32:
  case NEON::BI__builtin_neon_vmaxv_f32:
  case NEON::BI__builtin_neon_vmaxvq_f32:
  case NEON::BI__builtin_neon_vmaxvq_f64:
  case NEON::BI__builtin_neon_vqrshrund_n_s64:
  case NEON::BI__builtin_neon_vqrshrnd_n_s64:
  case NEON::BI__builtin_neon_vqrshrnd_n_u64:
  case NEON::BI__builtin_neon_vqshrund_n_s64:
  case NEON::BI__builtin_neon_vqshrnd_n_s64:
  case NEON::BI__builtin_neon_vqshrnd_n_u64:
  case NEON::BI__builtin_neon_vmaxnmv_f32:
  case NEON::BI__builtin_neon_vmaxnmvq_f32:
  case NEON::BI__builtin_neon_vmaxnmvq_f64:
  case NEON::BI__builtin_neon_vsrid_n_s64:
  case NEON::BI__builtin_neon_vsrid_n_u64:
  case NEON::BI__builtin_neon_vslid_n_s64:
  case NEON::BI__builtin_neon_vslid_n_u64:
  case NEON::BI__builtin_neon_vpmaxs_f32:
  case NEON::BI__builtin_neon_vpmaxqd_f64:
  case NEON::BI__builtin_neon_vpmaxnms_f32:
  case NEON::BI__builtin_neon_vpmaxnmqd_f64:
  case NEON::BI__builtin_neon_vmulxh_f16:
  case NEON::BI__builtin_neon_vqshrunh_n_s16:
  case NEON::BI__builtin_neon_vqshruns_n_s32:
  case NEON::BI__builtin_neon_vqshrnh_n_s16:
  case NEON::BI__builtin_neon_vqshrns_n_s32:
  case NEON::BI__builtin_neon_vqshrnh_n_u16:
  case NEON::BI__builtin_neon_vqshrns_n_u32:
  case NEON::BI__builtin_neon_vqsubb_s8:
  case NEON::BI__builtin_neon_vqsubb_u8:
  case NEON::BI__builtin_neon_vqsubh_s16:
  case NEON::BI__builtin_neon_vqsubh_u16:
  case NEON::BI__builtin_neon_vqsubs_s32:
  case NEON::BI__builtin_neon_vqsubs_u32:
  case NEON::BI__builtin_neon_vqsubd_s64:
  case NEON::BI__builtin_neon_vqsubd_u64:
  case NEON::BI__builtin_neon_vqaddb_s8:
  case NEON::BI__builtin_neon_vqaddb_u8:
  case NEON::BI__builtin_neon_vqaddh_s16:
  case NEON::BI__builtin_neon_vqaddh_u16:
  case NEON::BI__builtin_neon_vqadds_s32:
  case NEON::BI__builtin_neon_vqadds_u32:
  case NEON::BI__builtin_neon_vqaddd_s64:
  case NEON::BI__builtin_neon_vqaddd_u64:
  case NEON::BI__builtin_neon_vsqaddb_u8:
  case NEON::BI__builtin_neon_vsqaddh_u16:
  case NEON::BI__builtin_neon_vsqadds_u32:
  case NEON::BI__builtin_neon_vsqaddd_u64:
  case NEON::BI__builtin_neon_vuqaddb_s8:
  case NEON::BI__builtin_neon_vuqaddh_s16:
  case NEON::BI__builtin_neon_vuqadds_s32:
  case NEON::BI__builtin_neon_vuqaddd_s64:
  case NEON::BI__builtin_neon_vqrshrns_n_s32:
  case NEON::BI__builtin_neon_vqrshrns_n_u32:
  case NEON::BI__builtin_neon_vqrshrnh_n_s16:
  case NEON::BI__builtin_neon_vqrshrnh_n_u16:
  case NEON::BI__builtin_neon_vqrshruns_n_s32:
  case NEON::BI__builtin_neon_vqrshrunh_n_s16:
  case NEON::BI__builtin_neon_vqshlb_s8:
  case NEON::BI__builtin_neon_vqshlb_u8:
  case NEON::BI__builtin_neon_vqshlh_s16:
  case NEON::BI__builtin_neon_vqshlh_u16:
  case NEON::BI__builtin_neon_vqshls_s32:
  case NEON::BI__builtin_neon_vqshls_u32:
  case NEON::BI__builtin_neon_vqshld_s64:
  case NEON::BI__builtin_neon_vqshld_u64:
  case NEON::BI__builtin_neon_vqshlb_n_s8:
  case NEON::BI__builtin_neon_vqshlb_n_u8:
  case NEON::BI__builtin_neon_vqshlh_n_s16:
  case NEON::BI__builtin_neon_vqshlh_n_u16:
  case NEON::BI__builtin_neon_vqshls_n_s32:
  case NEON::BI__builtin_neon_vqshls_n_u32:
  case NEON::BI__builtin_neon_vqshlub_n_s8:
  case NEON::BI__builtin_neon_vqshluh_n_s16:
  case NEON::BI__builtin_neon_vqshlus_n_s32:
    break;
  }
 
  CIRGenBuilderTy &builder = cgf.getBuilder();
  mlir::Type arg0Ty = cgf.convertType(expr->getArg(0)->getType());
  mlir::Type resultTy = cgf.convertType(expr->getType());
 
  // Derive per-operand argument types and the result type from the
  // TypeModifier flags. `emitNeonCall` takes care of per-operand
  // bitcasts to `argTypes`.
  auto [funcResTy, argTypes] = deriveNeonSISDIntrinsicOperandTypes(
      cgf, info.TypeModifier, arg0Ty, resultTy, ops, iceArguments);
 
  assert(argTypes.size() == ops.size());
  for (unsigned i = 0, e = ops.size(); i != e; ++i) {
    if (cgf.cgm.getDataLayout().getTypeSizeInBits(ops[i].getType()) ==
        cgf.cgm.getDataLayout().getTypeSizeInBits(argTypes[i]))
      continue;
 
    auto argVecTy = mlir::dyn_cast<cir::VectorType>(argTypes[i]);
    assert(argVecTy && !mlir::isa<cir::VectorType>(ops[i].getType()) &&
           "expecting vector LLVM intrinsic type and scalar Clang builtin "
           "type");
 
    vecExtendIntValue(cgf, argVecTy, ops[i], loc);
  }
 
  mlir::Value result = emitNeonCall(cgf.cgm, builder, std::move(argTypes), ops,
                                    llvmIntrName, funcResTy, loc);
 
  if (cgf.cgm.getDataLayout().getTypeSizeInBits(resultTy) <
      cgf.cgm.getDataLayout().getTypeSizeInBits(funcResTy)) {
 
    assert(mlir::isa<cir::VectorType>(result.getType()));
    return cir::VecExtractOp::create(builder, loc, result,
                                     builder.getConstInt(loc, cgf.sizeTy, 0));
  }
 
  return builder.createBitcast(loc, result, resultTy);
}
 
//===----------------------------------------------------------------------===//
//  Emit-helpers
//===----------------------------------------------------------------------===//
static mlir::Value
emitAArch64CompareBuiltinExpr(CIRGenFunction &cgf, CIRGenBuilderTy &builder,
                              mlir::Location loc, mlir::Value src,
                              mlir::Type retTy, const cir::CmpOpKind kind) {
 
  bool scalarCmp = !isa<cir::VectorType>(src.getType());
  if (!scalarCmp) {
    assert(!cast<cir::VectorType>(retTy).isScalable() &&
           "This is only intended for fixed-width vectors");
    // Vector types are cast to i8 vectors. Recover original type.
    src = builder.createBitcast(src, retTy);
  }
 
  mlir::Value zero = builder.getNullValue(src.getType(), loc);
 
  if (!scalarCmp)
    return builder.createVecCompare(loc, kind, src, zero);
 
  // For scalars, cast !cir.bool to !cir.int<s, 1> so that the compare
  // result is sign- rather zero-extended when casting to the output
  // retType.
  mlir::Value cmp = builder.createCast(
      loc, cir::CastKind::bool_to_int,
      builder.createCompare(loc, kind, src, zero), builder.getSIntNTy(1));
 
  return builder.createCast(loc, cir::CastKind::integral, cmp, retTy);
}
 
static cir::VectorType getNeonType(CIRGenFunction *cgf, NeonTypeFlags typeFlags,
                                   bool hasLegalHalfType = true,
                                   bool v1Ty = false,
                                   bool allowBFloatArgsAndRet = true) {
  int isQuad = typeFlags.isQuad();
  switch (typeFlags.getEltType()) {
  case NeonTypeFlags::Int8:
  case NeonTypeFlags::Poly8:
    return cir::VectorType::getFixed(typeFlags.isUnsigned() ? cgf->uInt8Ty
                                                            : cgf->sInt8Ty,
                                     v1Ty ? 1 : (8 << isQuad));
  case NeonTypeFlags::MFloat8:
    return cir::VectorType::getFixed(cgf->uInt8Ty, v1Ty ? 1 : (8 << isQuad));
  case NeonTypeFlags::Int16:
  case NeonTypeFlags::Poly16:
    return cir::VectorType::getFixed(typeFlags.isUnsigned() ? cgf->uInt16Ty
                                                            : cgf->sInt16Ty,
                                     v1Ty ? 1 : (4 << isQuad));
  case NeonTypeFlags::BFloat16:
    if (allowBFloatArgsAndRet)
      return cir::VectorType::getFixed(cgf->getCIRGenModule().bFloat16Ty,
                                       v1Ty ? 1 : (4 << isQuad));
    return cir::VectorType::getFixed(cgf->uInt16Ty, v1Ty ? 1 : (4 << isQuad));
  case NeonTypeFlags::Float16:
    if (hasLegalHalfType)
      return cir::VectorType::getFixed(cgf->getCIRGenModule().fP16Ty,
                                       v1Ty ? 1 : (4 << isQuad));
    return cir::VectorType::getFixed(cgf->uInt16Ty, v1Ty ? 1 : (4 << isQuad));
  case NeonTypeFlags::Int32:
    return cir::VectorType::getFixed(typeFlags.isUnsigned() ? cgf->uInt32Ty
                                                            : cgf->sInt32Ty,
                                     v1Ty ? 1 : (2 << isQuad));
  case NeonTypeFlags::Int64:
  case NeonTypeFlags::Poly64:
    return cir::VectorType::getFixed(typeFlags.isUnsigned() ? cgf->uInt64Ty
                                                            : cgf->sInt64Ty,
                                     v1Ty ? 1 : (1 << isQuad));
  case NeonTypeFlags::Poly128:
    // FIXME: i128 and f128 doesn't get fully support in Clang and llvm.
    // There is a lot of i128 and f128 API missing.
    // so we use v16i8 to represent poly128 and get pattern matched.
    return cir::VectorType::getFixed(cgf->uInt8Ty, 16);
  case NeonTypeFlags::Float32:
    return cir::VectorType::getFixed(cgf->getCIRGenModule().floatTy,
                                     v1Ty ? 1 : (2 << isQuad));
  case NeonTypeFlags::Float64:
    return cir::VectorType::getFixed(cgf->getCIRGenModule().doubleTy,
                                     v1Ty ? 1 : (1 << isQuad));
  }
  llvm_unreachable("Unknown vector element type!");
}
 
static mlir::Value emitNeonSplat(CIRGenBuilderTy &builder, mlir::Location loc,
                                 mlir::Value v, mlir::Value lane,
                                 unsigned int resEltCnt) {
  assert(isa<cir::ConstantOp>(lane.getDefiningOp()) &&
         "lane number is not a constant!");
  int64_t laneCst = getIntValueFromConstOp(lane);
  llvm::SmallVector<int64_t, 4> shuffleMask(resEltCnt, laneCst);
  return builder.createVecShuffle(loc, v, shuffleMask);
}
 
/// Flip the signedness of `vecTy`'s element type, keeping the width and
/// number of lanes the same. Used when a NEON intrinsic takes a shift
/// amount vector that must be signed (e.g. aarch64.neon.urshl takes a
/// signed amount even though the data vector is unsigned).
static cir::VectorType getSignChangedVectorType(CIRGenBuilderTy &builder,
                                                cir::VectorType vecTy) {
  auto elemTy = mlir::cast<cir::IntType>(vecTy.getElementType());
  elemTy = elemTy.isSigned() ? builder.getUIntNTy(elemTy.getWidth())
                             : builder.getSIntNTy(elemTy.getWidth());
  return cir::VectorType::get(elemTy, vecTy.getSize());
}
 
static mlir::Value emitCommonNeonShift(CIRGenBuilderTy &builder,
                                       mlir::Location loc,
                                       cir::VectorType resTy,
                                       mlir::Value shifTgt,
                                       mlir::Value shiftAmt, bool shiftLeft) {
  shiftAmt = emitNeonShiftVector(builder, shiftAmt, resTy, loc, /*neg=*/false);
  return cir::ShiftOp::create(builder, loc, resTy,
                              builder.createBitcast(shifTgt, resTy), shiftAmt,
                              shiftLeft);
}
 
// Right-shift a vector by a constant.
static mlir::Value emitNeonRShiftImm(CIRGenFunction &cgf, mlir::Value shiftVec,
                                     mlir::Value shiftVal,
                                     cir::VectorType vecTy, bool usgn,
                                     mlir::Location loc) {
  CIRGenBuilderTy &builder = cgf.getBuilder();
  int64_t shiftAmt = getIntValueFromConstOp(shiftVal);
  int eltSize =
      cgf.cgm.getDataLayout().getTypeSizeInBits(vecTy.getElementType());
 
  shiftVec = builder.createBitcast(shiftVec, vecTy);
  // lshr/ashr are undefined when the shift amount is equal to the vector
  // element size.
  if (shiftAmt == eltSize) {
    if (usgn) {
      // Right-shifting an unsigned value by its size yields 0.
      return builder.getZero(loc, vecTy);
    }
    // Right-shifting a signed value by its size is equivalent
    // to a shift of size-1.
    --shiftAmt;
    shiftVal = builder.getConstInt(loc, vecTy.getElementType(), shiftAmt);
  }
  return emitCommonNeonShift(builder, loc, vecTy, shiftVec, shiftVal,
                             /*shiftLeft=*/false);
}
 
static cir::VectorType getIntVecFromVecTy(CIRGenBuilderTy &builder,
                                          cir::VectorType vecTy) {
  if (!cir::isAnyFloatingPointType(vecTy.getElementType()))
    return vecTy;
 
  if (mlir::isa<cir::SingleType>(vecTy.getElementType()))
    return cir::VectorType::get(builder.getSInt32Ty(), vecTy.getSize());
  if (mlir::isa<cir::DoubleType>(vecTy.getElementType()))
    return cir::VectorType::get(builder.getSInt64Ty(), vecTy.getSize());
  llvm_unreachable(
      "Unsupported element type in getVecOfIntTypeWithSameEltWidth");
}
 
/// The vld1_xN builtins move their data through an aggregate of N vectors.
/// Return N, or 0 if `builtinID` is not one of them.
static unsigned getNeonMultiVecCount(unsigned builtinID) {
  switch (builtinID) {
  default:
    return 0;
  case NEON::BI__builtin_neon_vld1_x2_v:
  case NEON::BI__builtin_neon_vld1q_x2_v:
    return 2;
  case NEON::BI__builtin_neon_vld1_x3_v:
  case NEON::BI__builtin_neon_vld1q_x3_v:
    return 3;
  case NEON::BI__builtin_neon_vld1_x4_v:
  case NEON::BI__builtin_neon_vld1q_x4_v:
    return 4;
  }
}
 
/// Return the anonymous record type `{vecTy, ..., vecTy}` (`numVecs` members)
/// modelling the struct returned by LLVM's multi-vector NEON load intrinsics.
static cir::RecordType getNeonMultiVecTy(CIRGenBuilderTy &builder,
                                         mlir::Type vecTy, unsigned numVecs) {
  llvm::SmallVector<mlir::Type> members(numVecs, vecTy);
  return builder.getAnonRecordTy(members, /*packed=*/false,
                                 cir::RecordType::getAllDataKinds(members));
}
 
/// Call the multi-vector NEON load intrinsic `llvmIntrinsic` with `args` of
/// `argTypes` and store the resulting `numVecs`-vector aggregate to the
/// destination pointer `dest`. Mirrors the `CreateCall` +
/// `CreateDefaultAlignedStore` pair used for `vld1_xN` in
/// `EmitCommonNeonBuiltinExpr` and for `vld[234]{,_dup,_lane}` in
/// `EmitAArch64BuiltinExpr` (ARM.cpp).
static mlir::Value
emitNeonMultiVecLoadCall(CIRGenFunction &cgf, mlir::Value dest,
                         llvm::SmallVector<mlir::Type> argTypes,
                         llvm::SmallVectorImpl<mlir::Value> &args,
                         llvm::Intrinsic::ID llvmIntrinsic, mlir::Type ty,
                         unsigned numVecs, mlir::Location loc) {
  CIRGenBuilderTy &builder = cgf.getBuilder();
  cir::RecordType recTy = getNeonMultiVecTy(builder, ty, numVecs);
  mlir::Value result =
      emitNeonCall(cgf.getCIRGenModule(), builder, std::move(argTypes), args,
                   getLLVMIntrNameNoPrefix(llvmIntrinsic), recTy, loc);
  Address destAddr(builder.createPtrBitcast(dest, recTy), recTy,
                   clang::CharUnits::fromQuantity(
                       cgf.cgm.getDataLayout().getABITypeAlign(recTy)));
  builder.createStore(loc, result, destAddr);
  return nullptr;
}
 
/// Emit a multi-vector NEON load: call `llvmIntrinsic` on the source pointer
/// `ops[1]` and store the resulting aggregate to the destination pointer
/// `ops[0]`.
static mlir::Value emitNeonMultiVecLoad(CIRGenFunction &cgf,
                                        llvm::ArrayRef<mlir::Value> ops,
                                        unsigned llvmIntrinsic, mlir::Type ty,
                                        unsigned numVecs, mlir::Location loc) {
  llvm::SmallVector<mlir::Value> srcOp = {ops[1]};
  return emitNeonMultiVecLoadCall(
      cgf, ops[0], {cgf.getBuilder().getVoidPtrTy()}, srcOp,
      static_cast<llvm::Intrinsic::ID>(llvmIntrinsic), ty, numVecs, loc);
}
 
/// Emit a multi-vector NEON load-lane: call `llvmIntrinsic` on the `numVecs`
/// vector operands, the lane index, and the source pointer `ops[1]`, then
/// store the resulting aggregate to the destination pointer `ops[0]`.
/// `ops` on entry is `[dest, addr, vec0, ..., vec(numVecs-1), lane]`.
static mlir::Value emitNeonMultiVecLoadLane(CIRGenFunction &cgf,
                                            llvm::ArrayRef<mlir::Value> ops,
                                            llvm::Intrinsic::ID llvmIntrinsic,
                                            mlir::Type ty, unsigned numVecs,
                                            mlir::Location loc) {
  assert(ops.size() == numVecs + 3 && "expected [dest, addr, vecs..., lane]");
  CIRGenBuilderTy &builder = cgf.getBuilder();
  // Reorder to [vec0, ..., vec(numVecs-1), lane, addr] for the intrinsic call.
  llvm::SmallVector<mlir::Value> callOps(ops.begin() + 2, ops.end());
  callOps.push_back(ops[1]);
  // The lane is a Sema-range-checked non-negative constant, so this
  // sign-extending cast is equivalent to classic CodeGen's CreateZExt.
  callOps[numVecs] = builder.createIntCast(callOps[numVecs], cgf.sInt64Ty);
  llvm::SmallVector<mlir::Type> argTypes(numVecs, ty);
  argTypes.push_back(cgf.sInt64Ty);
  argTypes.push_back(builder.getVoidPtrTy());
  return emitNeonMultiVecLoadCall(cgf, ops[0], std::move(argTypes), callOps,
                                  llvmIntrinsic, ty, numVecs, loc);
}
 
static mlir::Value emitCommonNeonBuiltinExpr(
    CIRGenFunction &cgf, unsigned builtinID, unsigned llvmIntrinsic,
    unsigned altLLVMIntrinsic, const char *nameHint, unsigned modifier,
    const CallExpr *expr, llvm::SmallVectorImpl<mlir::Value> &ops) {
  mlir::Location loc = cgf.getLoc(expr->getExprLoc());
  clang::ASTContext &ctx = cgf.getContext();
 
  // Extract the trailing immediate argument that encodes the type discriminator
  // for this overloaded intrinsic.
  // TODO: Move to the parent code that takes care of argument processing.
  const clang::Expr *arg = expr->getArg(expr->getNumArgs() - 1);
  std::optional<llvm::APSInt> neonTypeConst = arg->getIntegerConstantExpr(ctx);
  if (!neonTypeConst)
    return nullptr;
 
  // Determine the type of this overloaded NEON intrinsic.
  NeonTypeFlags neonType(neonTypeConst->getZExtValue());
  const bool isUnsigned = neonType.isUnsigned();
  const bool hasLegalHalfType = cgf.getTarget().hasFastHalfType();
  const bool usgn = neonType.isUnsigned();
  const bool isQuad = neonType.isQuad();
 
  // The value of allowBFloatArgsAndRet is true for AArch64, but it should
  // come from ABI info.
  // TODO(cir): Use ABInfo to extract this information
  const bool allowBFloatArgsAndRet = cgf.getTarget().hasFastHalfType();
  // FIXME
  // getTargetHooks().getABIInfo().allowBFloatArgsAndRet();
 
  cir::VectorType vTy = getNeonType(&cgf, neonType, hasLegalHalfType,
                                    /*v1Ty=*/false, allowBFloatArgsAndRet);
  cir::VectorType ty = vTy;
  if (!ty)
    return nullptr;
 
  switch (builtinID) {
  case NEON::BI__builtin_neon_splat_lane_v:
  case NEON::BI__builtin_neon_splat_laneq_v:
  case NEON::BI__builtin_neon_splatq_lane_v:
  case NEON::BI__builtin_neon_splatq_laneq_v: {
    uint64_t numElements = vTy.getFixedSize();
    if (builtinID == NEON::BI__builtin_neon_splatq_lane_v)
      numElements *= 2;
    if (builtinID == NEON::BI__builtin_neon_splat_laneq_v)
      numElements /= 2;
    ops[0] = cgf.getBuilder().createBitcast(loc, ops[0], vTy);
    return emitNeonSplat(cgf.getBuilder(), loc, ops[0], ops[1], numElements);
  }
  case NEON::BI__builtin_neon_vpadd_v:
  case NEON::BI__builtin_neon_vpaddq_v:
  case NEON::BI__builtin_neon_vabs_v:
  case NEON::BI__builtin_neon_vabsq_v:
    cgf.cgm.errorNYI(expr->getSourceRange(),
                     std::string("unimplemented AArch64 builtin call: ") +
                         ctx.BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  case NEON::BI__builtin_neon_vadd_v:
  case NEON::BI__builtin_neon_vaddq_v: {
    unsigned numBytes = (builtinID == NEON::BI__builtin_neon_vaddq_v) ? 16 : 8;
    cir::VectorType byteTy =
        cir::VectorType::getFixed(cgf.getBuilder().getUInt8Ty(), numBytes);
    ops[0] = cgf.getBuilder().createBitcast(ops[0], byteTy);
    ops[1] = cgf.getBuilder().createBitcast(ops[1], byteTy);
    mlir::Value result = cgf.getBuilder().createXor(loc, ops[0], ops[1]);
    return cgf.getBuilder().createBitcast(result, ty);
  }
  case NEON::BI__builtin_neon_vaddhn_v: {
    // srcTy has double-width elements (e.g. <8 x i16> when VTy is <8 x i8>).
    // Unsigned so createShiftRight emits lshr, not ashr.
    cir::VectorType srcTy =
        cgf.getBuilder().getExtendedOrTruncatedElementVectorType(
            vTy, /*isExtended=*/true, /*isSigned=*/false);
 
    // %sum = add <4 x i32> %lhs, %rhs
    ops[0] = cgf.getBuilder().createBitcast(ops[0], srcTy);
    ops[1] = cgf.getBuilder().createBitcast(ops[1], srcTy);
    mlir::Value result = cgf.getBuilder().createAdd(loc, ops[0], ops[1]);
 
    // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16>
    auto wideEltTy = mlir::cast<cir::IntType>(srcTy.getElementType());
    mlir::Value shiftAmt = cgf.getBuilder().getConstantInt(
        loc, wideEltTy, wideEltTy.getWidth() / 2);
    mlir::Value shiftVec =
        emitNeonShiftVector(cgf.getBuilder(), shiftAmt, srcTy, loc,
                            /*neg=*/false);
    result = cgf.getBuilder().createShiftRight(loc, result, shiftVec);
 
    // %res = trunc <4 x i32> %high to <4 x i16>
    return cgf.getBuilder().createIntCast(result, vTy);
  }
  case NEON::BI__builtin_neon_vsubhn_v: {
    // srcTy has double-width elements (e.g. <8 x i16> when VTy is <8 x i8>).
    // Unsigned so createShiftRight emits lshr, not ashr.
    cir::VectorType srcTy =
        cgf.getBuilder().getExtendedOrTruncatedElementVectorType(
            vTy, /*isExtended=*/true, /*isSigned=*/false);
 
    // %diff = sub <4 x i32> %lhs, %rhs
    ops[0] = cgf.getBuilder().createBitcast(ops[0], srcTy);
    ops[1] = cgf.getBuilder().createBitcast(ops[1], srcTy);
    mlir::Value result = cgf.getBuilder().createSub(loc, ops[0], ops[1]);
 
    // %high = lshr <4 x i32> %diff, <i32 16, i32 16, i32 16, i32 16>
    auto wideEltTy = mlir::cast<cir::IntType>(srcTy.getElementType());
    mlir::Value shiftAmt = cgf.getBuilder().getConstantInt(
        loc, wideEltTy, wideEltTy.getWidth() / 2);
    mlir::Value shiftVec =
        emitNeonShiftVector(cgf.getBuilder(), shiftAmt, srcTy, loc,
                            /*neg=*/false);
    result = cgf.getBuilder().createShiftRight(loc, result, shiftVec);
 
    // %res = trunc <4 x i32> %high to <4 x i16>
    return cgf.getBuilder().createIntCast(result, vTy);
  }
  case NEON::BI__builtin_neon_vcale_v:
  case NEON::BI__builtin_neon_vcaleq_v:
  case NEON::BI__builtin_neon_vcalt_v:
  case NEON::BI__builtin_neon_vcaltq_v:
  case NEON::BI__builtin_neon_vcage_v:
  case NEON::BI__builtin_neon_vcageq_v:
  case NEON::BI__builtin_neon_vcagt_v:
  case NEON::BI__builtin_neon_vcagtq_v:
    cgf.cgm.errorNYI(expr->getSourceRange(),
                     std::string("unimplemented AArch64 builtin call: ") +
                         ctx.BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  case NEON::BI__builtin_neon_vceqz_v:
  case NEON::BI__builtin_neon_vceqzq_v:
    return emitAArch64CompareBuiltinExpr(cgf, cgf.getBuilder(), loc, ops[0],
                                         vTy, cir::CmpOpKind::eq);
  case NEON::BI__builtin_neon_vcgez_v:
  case NEON::BI__builtin_neon_vcgezq_v:
  case NEON::BI__builtin_neon_vclez_v:
  case NEON::BI__builtin_neon_vclezq_v:
  case NEON::BI__builtin_neon_vcgtz_v:
  case NEON::BI__builtin_neon_vcgtzq_v:
  case NEON::BI__builtin_neon_vcltz_v:
  case NEON::BI__builtin_neon_vcltzq_v:
  case NEON::BI__builtin_neon_vclz_v:
  case NEON::BI__builtin_neon_vclzq_v:
    cgf.cgm.errorNYI(expr->getSourceRange(),
                     std::string("unimplemented AArch64 builtin call: ") +
                         ctx.BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  case NEON::BI__builtin_neon_vcvt_f32_v:
  case NEON::BI__builtin_neon_vcvtq_f32_v: {
    ops[0] = cgf.getBuilder().createBitcast(ops[0], ty);
    ty = getNeonType(&cgf, NeonTypeFlags(NeonTypeFlags::Float32, false, isQuad),
                     hasLegalHalfType);
    return cgf.getBuilder().createCast(cir::CastKind::int_to_float, ops[0], ty);
  }
  case NEON::BI__builtin_neon_vcvt_f16_s16:
  case NEON::BI__builtin_neon_vcvt_f16_u16:
  case NEON::BI__builtin_neon_vcvtq_f16_s16:
  case NEON::BI__builtin_neon_vcvtq_f16_u16:
  case NEON::BI__builtin_neon_vcvt_n_f16_s16:
  case NEON::BI__builtin_neon_vcvt_n_f16_u16:
  case NEON::BI__builtin_neon_vcvtq_n_f16_s16:
  case NEON::BI__builtin_neon_vcvtq_n_f16_u16:
    cgf.cgm.errorNYI(expr->getSourceRange(),
                     std::string("unimplemented AArch64 builtin call: ") +
                         ctx.BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  case NEON::BI__builtin_neon_vcvt_n_f32_v:
  case NEON::BI__builtin_neon_vcvt_n_f64_v:
  case NEON::BI__builtin_neon_vcvtq_n_f32_v:
  case NEON::BI__builtin_neon_vcvtq_n_f64_v: {
    // The constant argument to an _n_ intrinsic always is Int32Ty.
    mlir::Type cstIntTy = cgf.sInt32Ty;
    llvm::StringRef llvmIntrName =
        getLLVMIntrNameNoPrefix(static_cast<llvm::Intrinsic::ID>(
            usgn ? llvmIntrinsic : altLLVMIntrinsic));
    return emitNeonCall(cgf.getCIRGenModule(), cgf.getBuilder(),
                        /*argTypes=*/{vTy, cstIntTy}, ops, llvmIntrName,
                        /*funcResTy=*/getFloatNeonType(cgf, neonType), loc);
  }
  case NEON::BI__builtin_neon_vcvt_n_s16_f16:
  case NEON::BI__builtin_neon_vcvt_n_s32_v:
  case NEON::BI__builtin_neon_vcvt_n_u16_f16:
  case NEON::BI__builtin_neon_vcvt_n_u32_v:
  case NEON::BI__builtin_neon_vcvt_n_s64_v:
  case NEON::BI__builtin_neon_vcvt_n_u64_v:
  case NEON::BI__builtin_neon_vcvtq_n_s16_f16:
  case NEON::BI__builtin_neon_vcvtq_n_s32_v:
  case NEON::BI__builtin_neon_vcvtq_n_u16_f16:
  case NEON::BI__builtin_neon_vcvtq_n_u32_v:
  case NEON::BI__builtin_neon_vcvtq_n_s64_v:
  case NEON::BI__builtin_neon_vcvtq_n_u64_v: {
    // The constant argument to an _n_ intrinsic always is Int32Ty.
    mlir::Type cstIntTy = cgf.sInt32Ty;
    llvm::StringRef llvmIntrName = getLLVMIntrNameNoPrefix(
        static_cast<llvm::Intrinsic::ID>(llvmIntrinsic));
    return emitNeonCall(
        cgf.getCIRGenModule(), cgf.getBuilder(),
        /*argTypes=*/{getFloatNeonType(cgf, neonType), cstIntTy}, ops,
        llvmIntrName,
        /*funcResTy=*/vTy, loc);
  }
  case NEON::BI__builtin_neon_vcvt_s32_v:
  case NEON::BI__builtin_neon_vcvt_u32_v:
  case NEON::BI__builtin_neon_vcvt_s64_v:
  case NEON::BI__builtin_neon_vcvt_u64_v:
  case NEON::BI__builtin_neon_vcvt_s16_f16:
  case NEON::BI__builtin_neon_vcvt_u16_f16:
  case NEON::BI__builtin_neon_vcvtq_s32_v:
  case NEON::BI__builtin_neon_vcvtq_u32_v:
  case NEON::BI__builtin_neon_vcvtq_s64_v:
  case NEON::BI__builtin_neon_vcvtq_u64_v:
  case NEON::BI__builtin_neon_vcvtq_s16_f16:
  case NEON::BI__builtin_neon_vcvtq_u16_f16: {
    auto ty = getFloatNeonType(cgf, neonType);
    // Undo the bitcast inserted by intrinsics that expand to this builtin
    // (e.g. vcvt_u32_f32).
    // TODO: While the bitcasts eventually cancel each other out, we should
    // avoid them altogether.
    ops[0] =
        cgf.getBuilder().createCast(loc, cir::CastKind::bitcast, ops[0], ty);
    assert(!cir::MissingFeatures::emitConstrainedFPCall());
    // AArch64: use fptosi.sat/fptoui.sat unless under strict FP.
    llvm::StringRef llvmIntrName = usgn ? "fptoui.sat" : "fptosi.sat";
    return emitNeonCall(cgf.getCIRGenModule(), cgf.getBuilder(),
                        /*argTypes=*/{ty}, ops, llvmIntrName, vTy, loc);
  }
  case NEON::BI__builtin_neon_vcvta_s16_f16:
  case NEON::BI__builtin_neon_vcvta_s32_v:
  case NEON::BI__builtin_neon_vcvta_s64_v:
  case NEON::BI__builtin_neon_vcvta_u16_f16:
  case NEON::BI__builtin_neon_vcvta_u32_v:
  case NEON::BI__builtin_neon_vcvta_u64_v:
  case NEON::BI__builtin_neon_vcvtaq_s16_f16:
  case NEON::BI__builtin_neon_vcvtaq_s32_v:
  case NEON::BI__builtin_neon_vcvtaq_s64_v:
  case NEON::BI__builtin_neon_vcvtaq_u16_f16:
  case NEON::BI__builtin_neon_vcvtaq_u32_v:
  case NEON::BI__builtin_neon_vcvtaq_u64_v:
  case NEON::BI__builtin_neon_vcvtn_s16_f16:
  case NEON::BI__builtin_neon_vcvtn_s32_v:
  case NEON::BI__builtin_neon_vcvtn_s64_v:
  case NEON::BI__builtin_neon_vcvtn_u16_f16:
  case NEON::BI__builtin_neon_vcvtn_u32_v:
  case NEON::BI__builtin_neon_vcvtn_u64_v:
  case NEON::BI__builtin_neon_vcvtnq_s16_f16:
  case NEON::BI__builtin_neon_vcvtnq_s32_v:
  case NEON::BI__builtin_neon_vcvtnq_s64_v:
  case NEON::BI__builtin_neon_vcvtnq_u16_f16:
  case NEON::BI__builtin_neon_vcvtnq_u32_v:
  case NEON::BI__builtin_neon_vcvtnq_u64_v:
  case NEON::BI__builtin_neon_vcvtp_s16_f16:
  case NEON::BI__builtin_neon_vcvtp_s32_v:
  case NEON::BI__builtin_neon_vcvtp_s64_v:
  case NEON::BI__builtin_neon_vcvtp_u16_f16:
  case NEON::BI__builtin_neon_vcvtp_u32_v:
  case NEON::BI__builtin_neon_vcvtp_u64_v:
  case NEON::BI__builtin_neon_vcvtpq_s16_f16:
  case NEON::BI__builtin_neon_vcvtpq_s32_v:
  case NEON::BI__builtin_neon_vcvtpq_s64_v:
  case NEON::BI__builtin_neon_vcvtpq_u16_f16:
  case NEON::BI__builtin_neon_vcvtpq_u32_v:
  case NEON::BI__builtin_neon_vcvtpq_u64_v:
  case NEON::BI__builtin_neon_vcvtm_s16_f16:
  case NEON::BI__builtin_neon_vcvtm_s32_v:
  case NEON::BI__builtin_neon_vcvtm_s64_v:
  case NEON::BI__builtin_neon_vcvtm_u16_f16:
  case NEON::BI__builtin_neon_vcvtm_u32_v:
  case NEON::BI__builtin_neon_vcvtm_u64_v:
  case NEON::BI__builtin_neon_vcvtmq_s16_f16:
  case NEON::BI__builtin_neon_vcvtmq_s32_v:
  case NEON::BI__builtin_neon_vcvtmq_s64_v:
  case NEON::BI__builtin_neon_vcvtmq_u16_f16:
  case NEON::BI__builtin_neon_vcvtmq_u32_v:
  case NEON::BI__builtin_neon_vcvtmq_u64_v:
    cgf.cgm.errorNYI(expr->getSourceRange(),
                     std::string("unimplemented AArch64 builtin call: ") +
                         ctx.BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  case NEON::BI__builtin_neon_vcvtx_f32_v: {
    llvm::StringRef llvmIntrName = getLLVMIntrNameNoPrefix(
        static_cast<llvm::Intrinsic::ID>(llvmIntrinsic));
    auto v2f64 = cir::VectorType::getFixed(cgf.cgm.doubleTy, 2);
    auto v2f32 = cir::VectorType::getFixed(cgf.cgm.floatTy, 2);
    return emitNeonCall(cgf.getCIRGenModule(), cgf.getBuilder(),
                        /*argTypes=*/{v2f64}, ops, llvmIntrName,
                        /*funcResTy=*/v2f32, loc);
  }
  case NEON::BI__builtin_neon_vext_v:
  case NEON::BI__builtin_neon_vextq_v:
    cgf.cgm.errorNYI(expr->getSourceRange(),
                     std::string("unimplemented AArch64 builtin call: ") +
                         ctx.BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  case NEON::BI__builtin_neon_vfma_v:
  case NEON::BI__builtin_neon_vfmaq_v: {
    // NEON intrinsic: vfma(q)(accumulator, multiplicand1, multiplicand2)
    // CIR fma: fma(multiplicand1, multiplicand2, accumulator)
    // Reorder arguments to match fma signature.
    mlir::Value op0 = cgf.getBuilder().createBitcast(ops[0], ty);
    mlir::Value op1 = cgf.getBuilder().createBitcast(ops[1], ty);
    mlir::Value op2 = cgf.getBuilder().createBitcast(ops[2], ty);
    llvm::SmallVector<mlir::Value> fmaOps = {op1, op2, op0};
    return emitNeonCallToOp<cir::FMAOp>(cgf.cgm, cgf.getBuilder(), {ty, ty, ty},
                                        fmaOps, std::nullopt, ty, loc);
  }
  case NEON::BI__builtin_neon_vld1_x2_v:
  case NEON::BI__builtin_neon_vld1q_x2_v:
  case NEON::BI__builtin_neon_vld1_x3_v:
  case NEON::BI__builtin_neon_vld1q_x3_v:
  case NEON::BI__builtin_neon_vld1_x4_v:
  case NEON::BI__builtin_neon_vld1q_x4_v:
    return emitNeonMultiVecLoad(cgf, ops, llvmIntrinsic, ty,
                                getNeonMultiVecCount(builtinID), loc);
  case NEON::BI__builtin_neon_vld1_v:
  case NEON::BI__builtin_neon_vld1q_v:
  case NEON::BI__builtin_neon_vld2_v:
  case NEON::BI__builtin_neon_vld2q_v:
  case NEON::BI__builtin_neon_vld3_v:
  case NEON::BI__builtin_neon_vld3q_v:
  case NEON::BI__builtin_neon_vld4_v:
  case NEON::BI__builtin_neon_vld4q_v:
  case NEON::BI__builtin_neon_vld2_dup_v:
  case NEON::BI__builtin_neon_vld2q_dup_v:
  case NEON::BI__builtin_neon_vld3_dup_v:
  case NEON::BI__builtin_neon_vld3q_dup_v:
  case NEON::BI__builtin_neon_vld4_dup_v:
  case NEON::BI__builtin_neon_vld4q_dup_v:
  case NEON::BI__builtin_neon_vld1_dup_v:
  case NEON::BI__builtin_neon_vld1q_dup_v:
  case NEON::BI__builtin_neon_vld2_lane_v:
  case NEON::BI__builtin_neon_vld2q_lane_v:
  case NEON::BI__builtin_neon_vld3_lane_v:
  case NEON::BI__builtin_neon_vld3q_lane_v:
  case NEON::BI__builtin_neon_vld4_lane_v:
  case NEON::BI__builtin_neon_vld4q_lane_v:
    cgf.cgm.errorNYI(expr->getSourceRange(),
                     std::string("Reached code-path for ARM builtin call ") +
                         ctx.BuiltinInfo.getName(builtinID) +
                         "(ARM builtins are not supported ATM)");
    return mlir::Value{};
  case NEON::BI__builtin_neon_vmovl_v: {
    cir::VectorType dTy =
        cgf.getBuilder().getExtendedOrTruncatedElementVectorType(
            ty, /*isExtended=*/false, !usgn);
    ops[0] = cgf.getBuilder().createBitcast(loc, ops[0], dTy);
    return cgf.getBuilder().createIntCast(ops[0], ty);
  }
  case NEON::BI__builtin_neon_vmovn_v:
  case NEON::BI__builtin_neon_vmull_v:
  case NEON::BI__builtin_neon_vpadal_v:
  case NEON::BI__builtin_neon_vpadalq_v:
    cgf.cgm.errorNYI(expr->getSourceRange(),
                     std::string("Reached code-path for ARM builtin call ") +
                         ctx.BuiltinInfo.getName(builtinID) +
                         "(ARM builtins are not supported ATM)");
    return mlir::Value{};
  case NEON::BI__builtin_neon_vpaddl_v:
  case NEON::BI__builtin_neon_vpaddlq_v: {
    llvm::StringRef llvmIntrName =
        getLLVMIntrNameNoPrefix(static_cast<llvm::Intrinsic::ID>(
            usgn ? llvmIntrinsic : altLLVMIntrinsic));
    return emitNeonCall(cgf.getCIRGenModule(), cgf.getBuilder(),
                        /*argTypes=*/{getNeonPairwiseWidenInputType(vTy, usgn)},
                        ops, llvmIntrName,
                        /*funcResTy=*/vTy, loc);
  }
  case NEON::BI__builtin_neon_vqdmlal_v:
  case NEON::BI__builtin_neon_vqdmlsl_v:
  case NEON::BI__builtin_neon_vqdmulhq_lane_v:
  case NEON::BI__builtin_neon_vqdmulh_lane_v:
  case NEON::BI__builtin_neon_vqrdmulhq_lane_v:
  case NEON::BI__builtin_neon_vqrdmulh_lane_v:
  case NEON::BI__builtin_neon_vqdmulhq_laneq_v:
  case NEON::BI__builtin_neon_vqdmulh_laneq_v:
  case NEON::BI__builtin_neon_vqrdmulhq_laneq_v:
  case NEON::BI__builtin_neon_vqrdmulh_laneq_v:
    cgf.cgm.errorNYI(expr->getSourceRange(),
                     std::string("unimplemented AArch64 builtin call: ") +
                         ctx.BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  case NEON::BI__builtin_neon_vqshl_n_v:
  case NEON::BI__builtin_neon_vqshlq_n_v: {
    llvm::StringRef intrName =
        usgn ? "aarch64.neon.uqshl" : "aarch64.neon.sqshl";
    return emitNeonCall(cgf.cgm, cgf.getBuilder(), {ty, ty}, ops, intrName, ty,
                        loc, /*isConstrainedFPIntrinsic=*/false, /*shift=*/1,
                        /*rightshift=*/false);
  }
  case NEON::BI__builtin_neon_vqshlu_n_v:
  case NEON::BI__builtin_neon_vqshluq_n_v:
    return emitNeonCall(cgf.cgm, cgf.getBuilder(), {ty, ty}, ops,
                        "aarch64.neon.sqshlu", ty, loc,
                        /*isConstrainedFPIntrinsic=*/false, /*shift=*/1,
                        /*rightshift=*/false);
  case NEON::BI__builtin_neon_vrecpe_v:
  case NEON::BI__builtin_neon_vrecpeq_v:
  case NEON::BI__builtin_neon_vrsqrte_v:
  case NEON::BI__builtin_neon_vrsqrteq_v:
    cgf.cgm.errorNYI(expr->getSourceRange(),
                     std::string("unimplemented AArch64 builtin call: ") +
                         ctx.BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  case NEON::BI__builtin_neon_vrndi_v:
  case NEON::BI__builtin_neon_vrndiq_v:
    assert(!cir::MissingFeatures::emitConstrainedFPCall());
    return emitNeonCallToOp<cir::NearbyintOp>(cgf.cgm, cgf.getBuilder(), {ty},
                                              ops, std::nullopt, ty, loc);
  case NEON::BI__builtin_neon_vrshr_n_v:
  case NEON::BI__builtin_neon_vrshrq_n_v: {
    llvm::StringRef intrName =
        usgn ? "aarch64.neon.urshl" : "aarch64.neon.srshl";
    return emitNeonCall(
        cgf.cgm, cgf.getBuilder(),
        {ty, usgn ? getSignChangedVectorType(cgf.getBuilder(), ty) : ty}, ops,
        intrName, ty, loc, /*isConstrainedFPIntrinsic=*/false,
        /*shift=*/1,
        /*rightshift=*/true);
  }
  case NEON::BI__builtin_neon_vsha512hq_u64:
  case NEON::BI__builtin_neon_vsha512h2q_u64:
  case NEON::BI__builtin_neon_vsha512su0q_u64:
  case NEON::BI__builtin_neon_vsha512su1q_u64:
    cgf.cgm.errorNYI(expr->getSourceRange(),
                     std::string("unimplemented AArch64 builtin call: ") +
                         ctx.BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  case NEON::BI__builtin_neon_vshl_n_v:
  case NEON::BI__builtin_neon_vshlq_n_v:
    return emitCommonNeonShift(cgf.getBuilder(), loc, vTy, ops[0], ops[1],
                               /*shiftLeft=*/true);
  case NEON::BI__builtin_neon_vshll_n_v: {
    CIRGenBuilderTy &builder = cgf.getBuilder();
    cir::VectorType narrowVecTy =
        builder.getExtendedOrTruncatedElementVectorType(vTy,
                                                        /*isExtended=*/false,
                                                        /*isSigned=*/!usgn);
    mlir::Value src = builder.createBitcast(ops[0], narrowVecTy);
    mlir::Value extended = builder.createIntCast(src, vTy);
    return emitCommonNeonShift(builder, loc, vTy, extended, ops[1],
                               /*shiftLeft=*/true);
  }
  case NEON::BI__builtin_neon_vshrn_n_v: {
    CIRGenBuilderTy &builder = cgf.getBuilder();
    cir::VectorType wideVecTy =
        builder.getExtendedOrTruncatedElementVectorType(vTy,
                                                        /*isExtended=*/true,
                                                        /*isSigned=*/!usgn);
    mlir::Value src = builder.createBitcast(ops[0], wideVecTy);
    mlir::Value shifted = emitCommonNeonShift(builder, loc, wideVecTy, src,
                                              ops[1], /*shiftLeft=*/false);
    return builder.createIntCast(shifted, vTy);
  }
  case NEON::BI__builtin_neon_vshr_n_v:
  case NEON::BI__builtin_neon_vshrq_n_v:
    return emitNeonRShiftImm(cgf, ops[0], ops[1], vTy, isUnsigned, loc);
  case NEON::BI__builtin_neon_vst1_v:
  case NEON::BI__builtin_neon_vst1q_v:
  case NEON::BI__builtin_neon_vst2_v:
  case NEON::BI__builtin_neon_vst2q_v:
  case NEON::BI__builtin_neon_vst3_v:
  case NEON::BI__builtin_neon_vst3q_v:
  case NEON::BI__builtin_neon_vst4_v:
  case NEON::BI__builtin_neon_vst4q_v:
  case NEON::BI__builtin_neon_vst2_lane_v:
  case NEON::BI__builtin_neon_vst2q_lane_v:
  case NEON::BI__builtin_neon_vst3_lane_v:
  case NEON::BI__builtin_neon_vst3q_lane_v:
  case NEON::BI__builtin_neon_vst4_lane_v:
  case NEON::BI__builtin_neon_vst4q_lane_v:
  case NEON::BI__builtin_neon_vsm3partw1q_u32:
  case NEON::BI__builtin_neon_vsm3partw2q_u32:
  case NEON::BI__builtin_neon_vsm3ss1q_u32:
  case NEON::BI__builtin_neon_vsm4ekeyq_u32:
  case NEON::BI__builtin_neon_vsm4eq_u32:
  case NEON::BI__builtin_neon_vsm3tt1aq_u32:
  case NEON::BI__builtin_neon_vsm3tt1bq_u32:
  case NEON::BI__builtin_neon_vsm3tt2aq_u32:
  case NEON::BI__builtin_neon_vsm3tt2bq_u32:
    cgf.cgm.errorNYI(expr->getSourceRange(),
                     std::string("unimplemented AArch64 builtin call: ") +
                         ctx.BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  case NEON::BI__builtin_neon_vst1_x2_v:
  case NEON::BI__builtin_neon_vst1q_x2_v:
  case NEON::BI__builtin_neon_vst1_x3_v:
  case NEON::BI__builtin_neon_vst1q_x3_v:
  case NEON::BI__builtin_neon_vst1_x4_v:
  case NEON::BI__builtin_neon_vst1q_x4_v: {
    // The builtin call has the pointer first, but the AArch64 st1x2/3/4
    // intrinsics take the vector operands first and the pointer last.
    std::rotate(ops.begin(), ops.begin() + 1, ops.end());
    llvm::SmallVector<mlir::Type> argTypes(ops.size() - 1, ty);
    argTypes.push_back(cgf.getBuilder().getVoidPtrTy());
    llvm::StringRef llvmIntrName = getLLVMIntrNameNoPrefix(
        static_cast<llvm::Intrinsic::ID>(llvmIntrinsic));
    return emitNeonCall(cgf.getCIRGenModule(), cgf.getBuilder(), argTypes, ops,
                        llvmIntrName, cgf.cgm.voidTy, loc);
  }
  case NEON::BI__builtin_neon_vtrn_v:
  case NEON::BI__builtin_neon_vtrnq_v:
  case NEON::BI__builtin_neon_vtst_v:
  case NEON::BI__builtin_neon_vtstq_v:
  case NEON::BI__builtin_neon_vuzp_v:
  case NEON::BI__builtin_neon_vuzpq_v:
  case NEON::BI__builtin_neon_vxarq_u64:
  case NEON::BI__builtin_neon_vzip_v:
  case NEON::BI__builtin_neon_vzipq_v:
  case NEON::BI__builtin_neon_vdot_s32:
  case NEON::BI__builtin_neon_vdot_u32:
  case NEON::BI__builtin_neon_vdotq_s32:
  case NEON::BI__builtin_neon_vdotq_u32:
  case NEON::BI__builtin_neon_vfmlal_low_f16:
  case NEON::BI__builtin_neon_vfmlalq_low_f16:
  case NEON::BI__builtin_neon_vfmlsl_low_f16:
  case NEON::BI__builtin_neon_vfmlslq_low_f16:
  case NEON::BI__builtin_neon_vfmlal_high_f16:
  case NEON::BI__builtin_neon_vfmlalq_high_f16:
  case NEON::BI__builtin_neon_vfmlsl_high_f16:
  case NEON::BI__builtin_neon_vfmlslq_high_f16:
  case NEON::BI__builtin_neon_vmmlaq_s32:
  case NEON::BI__builtin_neon_vmmlaq_u32:
    cgf.cgm.errorNYI(expr->getSourceRange(),
                     std::string("unimplemented AArch64 builtin call: ") +
                         ctx.BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  case NEON::BI__builtin_neon_vmul_v:
  case NEON::BI__builtin_neon_vmulq_v:
    return cgf.getBuilder().emitIntrinsicCallOp(loc, "aarch64.neon.pmul", vTy,
                                                ops);
  case NEON::BI__builtin_neon_vusmmlaq_s32:
  case NEON::BI__builtin_neon_vusdot_s32:
  case NEON::BI__builtin_neon_vusdotq_s32:
  case NEON::BI__builtin_neon_vbfdot_f32:
  case NEON::BI__builtin_neon_vbfdotq_f32:
  case NEON::BI__builtin_neon___a32_vcvt_bf16_f32:
    cgf.cgm.errorNYI(expr->getSourceRange(),
                     std::string("unimplemented AArch64 builtin call: ") +
                         ctx.BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  // The switch stmt is intended to help catch NYI cases and will be removed
  // once the CIR implementation is complete. Avoid adding specialized
  // code in cases - that should only be required for a handful of examples.
  switch (builtinID) {
  default:
    cgf.cgm.errorNYI(expr->getSourceRange(),
                     std::string("unimplemented AArch64 builtin call: ") +
                         cgf.getContext().BuiltinInfo.getName(builtinID));
    break;
  case NEON::BI__builtin_neon_vrnd32x_f32:
  case NEON::BI__builtin_neon_vrnd32xq_f32:
  case NEON::BI__builtin_neon_vrnd32x_f64:
  case NEON::BI__builtin_neon_vrnd32xq_f64:
  case NEON::BI__builtin_neon_vrnd32z_f32:
  case NEON::BI__builtin_neon_vrnd32zq_f32:
  case NEON::BI__builtin_neon_vrnd32z_f64:
  case NEON::BI__builtin_neon_vrnd32zq_f64:
  case NEON::BI__builtin_neon_vrnd64x_f32:
  case NEON::BI__builtin_neon_vrnd64xq_f32:
  case NEON::BI__builtin_neon_vrnd64x_f64:
  case NEON::BI__builtin_neon_vrnd64xq_f64:
  case NEON::BI__builtin_neon_vrnd64z_f32:
  case NEON::BI__builtin_neon_vrnd64zq_f32:
  case NEON::BI__builtin_neon_vrnd64z_f64:
  case NEON::BI__builtin_neon_vrnd64zq_f64: {
    llvm::StringRef llvmIntrName = getLLVMIntrNameNoPrefix(
        static_cast<llvm::Intrinsic::ID>(llvmIntrinsic));
    mlir::Value result =
        emitNeonCall(cgf.cgm, cgf.getBuilder(), /*argTypes=*/{vTy}, ops,
                     llvmIntrName, /*funcResTy=*/vTy, loc);
    mlir::Type resultType = cgf.convertType(expr->getType());
    return cgf.getBuilder().createBitcast(result, resultType);
  }
  case NEON::BI__builtin_neon_vhadd_v:
  case NEON::BI__builtin_neon_vhaddq_v:
  case NEON::BI__builtin_neon_vhsub_v:
  case NEON::BI__builtin_neon_vhsubq_v:
  case NEON::BI__builtin_neon_vqadd_v:
  case NEON::BI__builtin_neon_vqaddq_v:
  case NEON::BI__builtin_neon_vrhadd_v:
  case NEON::BI__builtin_neon_vrhaddq_v:
  case NEON::BI__builtin_neon_vqsub_v:
  case NEON::BI__builtin_neon_vqsubq_v:
  case NEON::BI__builtin_neon_vshl_v:
  case NEON::BI__builtin_neon_vshlq_v:
  case NEON::BI__builtin_neon_vraddhn_v:
  case NEON::BI__builtin_neon_vrsubhn_v:
  case NEON::BI__builtin_neon_vqshl_v:
  case NEON::BI__builtin_neon_vqshlq_v: {
    // Pick the signed/unsigned intrinsic when the builtin has both
    // (UnsignedAlts); otherwise there is a single intrinsic.
    unsigned intrinsic =
        ((modifier & UnsignedAlts) && !usgn) ? altLLVMIntrinsic : llvmIntrinsic;
    llvm::StringRef llvmIntrName =
        getLLVMIntrNameNoPrefix(static_cast<llvm::Intrinsic::ID>(intrinsic));
 
    cir::VectorType argTy =
        deriveNeonBinaryArgType(cgf.getBuilder(), modifier, vTy);
 
    mlir::Value result =
        emitNeonCall(cgf.getCIRGenModule(), cgf.getBuilder(),
                     /*argTypes=*/{argTy, argTy}, ops, llvmIntrName,
                     /*funcResTy=*/vTy, loc);
    mlir::Type resultType = cgf.convertType(expr->getType());
    return cgf.getBuilder().createBitcast(result, resultType);
  }
  }
 
  // NYI
  return nullptr;
}
 
bool CIRGenFunction::getAArch64SVEProcessedOperands(
    unsigned builtinID, const CallExpr *expr, SmallVectorImpl<mlir::Value> &ops,
    SVETypeFlags typeFlags) {
  // Find out if any arguments are required to be integer constant expressions.
  unsigned iceArguments = 0;
  ASTContext::GetBuiltinTypeError error;
  getContext().GetBuiltinType(builtinID, error, &iceArguments);
  assert(error == ASTContext::GE_None && "Should not codegen an error");
 
  for (unsigned i = 0, e = expr->getNumArgs(); i != e; i++) {
    bool isIce = iceArguments & (1 << i);
    mlir::Value arg = emitScalarExpr(expr->getArg(i));
 
    if (isIce) {
      cgm.errorNYI(expr->getSourceRange(),
                   std::string("unimplemented AArch64 builtin call: ") +
                       getContext().BuiltinInfo.getName(builtinID));
    }
 
    // FIXME: Handle types like svint16x2_t, which are currently incorrectly
    // converted to i32. These should be treated as structs and unpacked.
 
    ops.push_back(arg);
  }
  return true;
}
 
// Reinterpret the input predicate so that it can be used to correctly isolate
// the elements of the specified datatype.
mlir::Value CIRGenFunction::emitSVEPredicateCast(mlir::Value pred,
                                                 unsigned minNumElts,
                                                 mlir::Location loc) {
 
  // TODO: Handle "aarch64.svcount" once we get round to supporting SME.
 
  auto retTy = cir::VectorType::getScalable(builder.getUIntNTy(1), minNumElts);
  if (pred.getType() == retTy)
    return pred;
 
  llvm::Intrinsic::ID intID;
  switch (minNumElts) {
  default:
    llvm_unreachable("unsupported element count!");
  case 1:
  case 2:
  case 4:
  case 8:
    intID = Intrinsic::aarch64_sve_convert_from_svbool;
    break;
  case 16:
    intID = Intrinsic::aarch64_sve_convert_to_svbool;
    break;
  }
 
  llvm::StringRef llvmIntrName = getLLVMIntrNameNoPrefix(intID);
  auto call = builder.emitIntrinsicCallOp(loc, llvmIntrName, retTy,
                                          mlir::ValueRange{pred});
  assert(call.getType() == retTy && "Unexpected return type!");
  return call;
}
 
//===----------------------------------------------------------------------===//
//  SVE helpers
//===----------------------------------------------------------------------===//
// Get the minimum number of elements in an SVE vector for the given element
// type. The actual number of elements in the vector would be an integer (power
// of two) multiple of this value.
static unsigned getSVEMinEltCount(clang::SVETypeFlags::EltType sveType) {
  switch (sveType) {
  default:
    llvm_unreachable("Invalid SVETypeFlag!");
 
  case SVETypeFlags::EltTyInt8:
    return 16;
  case SVETypeFlags::EltTyInt16:
    return 8;
  case SVETypeFlags::EltTyInt32:
    return 4;
  case SVETypeFlags::EltTyInt64:
    return 2;
 
  case SVETypeFlags::EltTyMFloat8:
    return 16;
  case SVETypeFlags::EltTyFloat16:
  case SVETypeFlags::EltTyBFloat16:
    return 8;
  case SVETypeFlags::EltTyFloat32:
    return 4;
  case SVETypeFlags::EltTyFloat64:
    return 2;
 
  case SVETypeFlags::EltTyBool8:
    return 16;
  case SVETypeFlags::EltTyBool16:
    return 8;
  case SVETypeFlags::EltTyBool32:
    return 4;
  case SVETypeFlags::EltTyBool64:
    return 2;
  }
}
 
cir::VectorType CIRGenFunction::getSVEType(const SVETypeFlags &typeFlags) {
  switch (typeFlags.getEltType()) {
  default:
    llvm_unreachable("Invalid SVETypeFlag!");
 
  case SVETypeFlags::EltTyInt8:
    return cir::VectorType::getScalable(builder.getUInt8Ty(), 16);
  case SVETypeFlags::EltTyInt16:
    return cir::VectorType::getScalable(builder.getUInt16Ty(), 8);
  case SVETypeFlags::EltTyInt32:
    return cir::VectorType::getScalable(builder.getUInt32Ty(), 4);
  case SVETypeFlags::EltTyInt64:
    return cir::VectorType::getScalable(builder.getUInt64Ty(), 2);
 
  case SVETypeFlags::EltTyMFloat8:
    return cir::VectorType::getScalable(builder.getUInt8Ty(), 16);
  case SVETypeFlags::EltTyFloat16:
    return cir::VectorType::getScalable(builder.getFp16Ty(), 8);
  case SVETypeFlags::EltTyBFloat16:
    return cir::VectorType::getScalable(builder.getBfloat16Ty(), 8);
  case SVETypeFlags::EltTyFloat32:
    return cir::VectorType::getScalable(builder.getSingleTy(), 4);
  case SVETypeFlags::EltTyFloat64:
    return cir::VectorType::getScalable(builder.getDoubleTy(), 2);
 
  case SVETypeFlags::EltTyBool8:
    return cir::VectorType::getScalable(builder.getUIntNTy(1), 16);
  case SVETypeFlags::EltTyBool16:
    return cir::VectorType::getScalable(builder.getUIntNTy(1), 8);
  case SVETypeFlags::EltTyBool32:
    return cir::VectorType::getScalable(builder.getUIntNTy(1), 4);
  case SVETypeFlags::EltTyBool64:
    return cir::VectorType::getScalable(builder.getUIntNTy(1), 2);
  }
}
 
// TODO(cir): Share with OGCG
constexpr unsigned sveBitsPerBlock = 128;
 
static cir::VectorType getSVEVectorForElementType(CIRGenModule &cgm,
                                                  mlir::Type eltTy) {
  unsigned numElts =
      sveBitsPerBlock / cgm.getDataLayout().getTypeSizeInBits(eltTy);
  return cir::VectorType::getScalable(eltTy, numElts);
}
 
//===----------------------------------------------------------------------===//
//  SVE helpers
//===----------------------------------------------------------------------===//
std::optional<mlir::Value>
CIRGenFunction::emitAArch64SVEBuiltinExpr(unsigned builtinID,
                                          const CallExpr *expr) {
  mlir::Type ty = convertType(expr->getType());
 
  if (builtinID >= SVE::BI__builtin_sve_reinterpret_s8_s8 &&
      builtinID <= SVE::BI__builtin_sve_reinterpret_f64_f64_x4) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  assert(!cir::MissingFeatures::aarch64SVEIntrinsics());
 
  auto *builtinIntrInfo =
      findARMVectorIntrinsicInMap(ArrayRef(aarch64SVEIntrinsicMap), builtinID,
                                  aarch64SVEIntrinsicsProvenSorted);
 
  // The operands of the builtin call
  llvm::SmallVector<mlir::Value> ops;
 
  SVETypeFlags typeFlags(builtinIntrInfo->TypeModifier);
  if (!CIRGenFunction::getAArch64SVEProcessedOperands(builtinID, expr, ops,
                                                      typeFlags))
    return mlir::Value{};
 
  if (typeFlags.isLoad() || typeFlags.isStore() || typeFlags.isGatherLoad() ||
      typeFlags.isScatterStore() || typeFlags.isPrefetch() ||
      typeFlags.isGatherPrefetch() || typeFlags.isStructLoad() ||
      typeFlags.isStructStore() || typeFlags.isTupleSet() ||
      typeFlags.isTupleGet() || typeFlags.isTupleCreate() ||
      typeFlags.isUndef())
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
 
  mlir::Location loc = getLoc(expr->getExprLoc());
 
  // Handle built-ins for which there is a corresponding LLVM Intrinsic.
  // -------------------------------------------------------------------
  if (builtinIntrInfo->LLVMIntrinsic != 0) {
    // Emit set FPMR for intrinsics that require it.
    if (typeFlags.setsFPMR())
      cgm.errorNYI(expr->getSourceRange(),
                   std::string("unimplemented AArch64 builtin call: ") +
                       getContext().BuiltinInfo.getName(builtinID));
 
    // Zero-ing predication
    if (typeFlags.getMergeType() == SVETypeFlags::MergeZeroExp) {
      auto null = builder.getNullValue(convertType(expr->getType()),
                                       getLoc(expr->getExprLoc()));
      ops.insert(ops.begin(), null);
    }
 
    if (typeFlags.getMergeType() == SVETypeFlags::MergeAnyExp)
      ops.insert(ops.begin(),
                 builder.getConstant(loc, cir::UndefAttr::get(ty)));
 
    // Some ACLE builtins leave out the argument to specify the predicate
    // pattern, which is expected to be expanded to an SV_ALL pattern.
    if (typeFlags.isAppendSVALL())
      cgm.errorNYI(expr->getSourceRange(),
                   std::string("unimplemented AArch64 builtin call: ") +
                       getContext().BuiltinInfo.getName(builtinID));
    if (typeFlags.isInsertOp1SVALL())
      cgm.errorNYI(expr->getSourceRange(),
                   std::string("unimplemented AArch64 builtin call: ") +
                       getContext().BuiltinInfo.getName(builtinID));
 
    // Predicates must match the main datatype.
    for (mlir::Value &op : ops)
      if (auto predTy = dyn_cast<cir::VectorType>(op.getType()))
        if (auto cirInt = dyn_cast<cir::IntType>(predTy.getElementType()))
          if (cirInt.getWidth() == 1)
            op = emitSVEPredicateCast(
                op, getSVEMinEltCount(typeFlags.getEltType()), loc);
 
    // Splat scalar operand to vector (intrinsics with _n infix)
    if (typeFlags.hasSplatOperand()) {
      unsigned opNo = typeFlags.getSplatOperand();
      ops[opNo] = cir::VecSplatOp::create(
          builder, loc, getSVEVectorForElementType(cgm, ops[opNo].getType()),
          ops[opNo]);
    }
 
    if (typeFlags.isReverseCompare())
      cgm.errorNYI(expr->getSourceRange(),
                   std::string("unimplemented AArch64 builtin call: ") +
                       getContext().BuiltinInfo.getName(builtinID));
    if (typeFlags.isReverseUSDOT())
      cgm.errorNYI(expr->getSourceRange(),
                   std::string("unimplemented AArch64 builtin call: ") +
                       getContext().BuiltinInfo.getName(builtinID));
    if (typeFlags.isReverseMergeAnyBinOp() &&
        typeFlags.getMergeType() == SVETypeFlags::MergeAny)
      cgm.errorNYI(expr->getSourceRange(),
                   std::string("unimplemented AArch64 builtin call: ") +
                       getContext().BuiltinInfo.getName(builtinID));
    if (typeFlags.isReverseMergeAnyAccOp() &&
        typeFlags.getMergeType() == SVETypeFlags::MergeAny)
      cgm.errorNYI(expr->getSourceRange(),
                   std::string("unimplemented AArch64 builtin call: ") +
                       getContext().BuiltinInfo.getName(builtinID));
 
    // Predicated intrinsics with _z suffix.
    if (typeFlags.getMergeType() == SVETypeFlags::MergeZero) {
      cgm.errorNYI(expr->getSourceRange(),
                   std::string("unimplemented AArch64 builtin call: ") +
                       getContext().BuiltinInfo.getName(builtinID));
    }
 
    llvm::StringRef llvmIntrName = getLLVMIntrNameNoPrefix(
        static_cast<llvm::Intrinsic::ID>(builtinIntrInfo->LLVMIntrinsic));
    auto retTy = convertType(expr->getType());
 
    auto call = builder.emitIntrinsicCallOp(loc, llvmIntrName, retTy,
                                            mlir::ValueRange{ops});
    if (call.getType() == retTy)
      return call;
 
    // Predicate results must be converted to svbool_t.
    if (isa<mlir::VectorType>(retTy) &&
        cast<mlir::VectorType>(retTy).isScalable())
      cgm.errorNYI(expr->getSourceRange(),
                   std::string("unimplemented AArch64 builtin call: ") +
                       getContext().BuiltinInfo.getName(builtinID));
    // TODO Handle struct types, e.g. svint8x2_t (update the converter first).
 
    llvm_unreachable("unsupported element count!");
  }
 
  // Handle the remaining built-ins.
  // -------------------------------
  switch (builtinID) {
  default:
    return std::nullopt;
 
  case SVE::BI__builtin_sve_svreinterpret_b:
  case SVE::BI__builtin_sve_svreinterpret_c:
  case SVE::BI__builtin_sve_svpsel_lane_b8:
  case SVE::BI__builtin_sve_svpsel_lane_b16:
  case SVE::BI__builtin_sve_svpsel_lane_b32:
  case SVE::BI__builtin_sve_svpsel_lane_b64:
  case SVE::BI__builtin_sve_svpsel_lane_c8:
  case SVE::BI__builtin_sve_svpsel_lane_c16:
  case SVE::BI__builtin_sve_svpsel_lane_c32:
  case SVE::BI__builtin_sve_svpsel_lane_c64:
  case SVE::BI__builtin_sve_svmov_b_z:
  case SVE::BI__builtin_sve_svnot_b_z:
  case SVE::BI__builtin_sve_svmovlb_u16:
  case SVE::BI__builtin_sve_svmovlb_u32:
  case SVE::BI__builtin_sve_svmovlb_u64:
  case SVE::BI__builtin_sve_svmovlb_s16:
  case SVE::BI__builtin_sve_svmovlb_s32:
  case SVE::BI__builtin_sve_svmovlb_s64:
  case SVE::BI__builtin_sve_svmovlt_u16:
  case SVE::BI__builtin_sve_svmovlt_u32:
  case SVE::BI__builtin_sve_svmovlt_u64:
  case SVE::BI__builtin_sve_svmovlt_s16:
  case SVE::BI__builtin_sve_svmovlt_s32:
  case SVE::BI__builtin_sve_svmovlt_s64:
  case SVE::BI__builtin_sve_svpmullt_u16:
  case SVE::BI__builtin_sve_svpmullt_u64:
  case SVE::BI__builtin_sve_svpmullt_n_u16:
  case SVE::BI__builtin_sve_svpmullt_n_u64:
  case SVE::BI__builtin_sve_svpmullb_u16:
  case SVE::BI__builtin_sve_svpmullb_u64:
  case SVE::BI__builtin_sve_svpmullb_n_u16:
  case SVE::BI__builtin_sve_svpmullb_n_u64:
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
 
  case SVE::BI__builtin_sve_svdup_n_b8:
  case SVE::BI__builtin_sve_svdup_n_b16:
  case SVE::BI__builtin_sve_svdup_n_b32:
  case SVE::BI__builtin_sve_svdup_n_b64: {
    // Cast from cir.bool (input type) to cir.int<u, 1> (element type of the
    // result vector).
    auto dup = builder.createBitcast(ops[0], builder.getUIntNTy(1));
 
    // Splat
    dup = cir::VecSplatOp::create(builder, loc, getSVEType(typeFlags), dup);
 
    // Cast to svbool_t, i.e. <vscale x 16 x i1>. The actual result could be
    // e.g. <vscale x 8 x i1> (for b16), but only svbool_t (i.e. full
    // predicate register) is "storable" (as per SVE ABI).
    return builtinID == SVE::BI__builtin_sve_svdup_n_b8
               ? dup
               : emitSVEPredicateCast(dup, 16, loc);
  }
 
  case SVE::BI__builtin_sve_svdupq_n_b8:
  case SVE::BI__builtin_sve_svdupq_n_b16:
  case SVE::BI__builtin_sve_svdupq_n_b32:
  case SVE::BI__builtin_sve_svdupq_n_b64:
  case SVE::BI__builtin_sve_svdupq_n_u8:
  case SVE::BI__builtin_sve_svdupq_n_s8:
  case SVE::BI__builtin_sve_svdupq_n_u64:
  case SVE::BI__builtin_sve_svdupq_n_f64:
  case SVE::BI__builtin_sve_svdupq_n_s64:
  case SVE::BI__builtin_sve_svdupq_n_u16:
  case SVE::BI__builtin_sve_svdupq_n_f16:
  case SVE::BI__builtin_sve_svdupq_n_bf16:
  case SVE::BI__builtin_sve_svdupq_n_s16:
  case SVE::BI__builtin_sve_svdupq_n_u32:
  case SVE::BI__builtin_sve_svdupq_n_f32:
  case SVE::BI__builtin_sve_svdupq_n_s32:
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
 
  case SVE::BI__builtin_sve_svpfalse_b:
    return builder.getNullValue(convertType(expr->getType()),
                                getLoc(expr->getExprLoc()));
 
  case SVE::BI__builtin_sve_svpfalse_c:
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
 
  case SVE::BI__builtin_sve_svlen_u8:
  case SVE::BI__builtin_sve_svlen_s8:
    return genVscaleTimesFactor(loc, builder, convertType(expr->getType()), 16);
 
  case SVE::BI__builtin_sve_svlen_u16:
  case SVE::BI__builtin_sve_svlen_s16:
  case SVE::BI__builtin_sve_svlen_f16:
  case SVE::BI__builtin_sve_svlen_bf16:
    return genVscaleTimesFactor(loc, builder, convertType(expr->getType()), 8);
 
  case SVE::BI__builtin_sve_svlen_u32:
  case SVE::BI__builtin_sve_svlen_s32:
  case SVE::BI__builtin_sve_svlen_f32:
    return genVscaleTimesFactor(loc, builder, convertType(expr->getType()), 4);
 
  case SVE::BI__builtin_sve_svlen_u64:
  case SVE::BI__builtin_sve_svlen_s64:
  case SVE::BI__builtin_sve_svlen_f64:
    return genVscaleTimesFactor(loc, builder, convertType(expr->getType()), 2);
 
  case SVE::BI__builtin_sve_svtbl2_u8:
  case SVE::BI__builtin_sve_svtbl2_s8:
  case SVE::BI__builtin_sve_svtbl2_u16:
  case SVE::BI__builtin_sve_svtbl2_s16:
  case SVE::BI__builtin_sve_svtbl2_u32:
  case SVE::BI__builtin_sve_svtbl2_s32:
  case SVE::BI__builtin_sve_svtbl2_u64:
  case SVE::BI__builtin_sve_svtbl2_s64:
  case SVE::BI__builtin_sve_svtbl2_f16:
  case SVE::BI__builtin_sve_svtbl2_bf16:
  case SVE::BI__builtin_sve_svtbl2_f32:
  case SVE::BI__builtin_sve_svtbl2_f64:
  case SVE::BI__builtin_sve_svset_neonq_s8:
  case SVE::BI__builtin_sve_svset_neonq_s16:
  case SVE::BI__builtin_sve_svset_neonq_s32:
  case SVE::BI__builtin_sve_svset_neonq_s64:
  case SVE::BI__builtin_sve_svset_neonq_u8:
  case SVE::BI__builtin_sve_svset_neonq_u16:
  case SVE::BI__builtin_sve_svset_neonq_u32:
  case SVE::BI__builtin_sve_svset_neonq_u64:
  case SVE::BI__builtin_sve_svset_neonq_f16:
  case SVE::BI__builtin_sve_svset_neonq_f32:
  case SVE::BI__builtin_sve_svset_neonq_f64:
  case SVE::BI__builtin_sve_svset_neonq_bf16:
  case SVE::BI__builtin_sve_svset_neonq_mf8:
  case SVE::BI__builtin_sve_svget_neonq_s8:
  case SVE::BI__builtin_sve_svget_neonq_s16:
  case SVE::BI__builtin_sve_svget_neonq_s32:
  case SVE::BI__builtin_sve_svget_neonq_s64:
  case SVE::BI__builtin_sve_svget_neonq_u8:
  case SVE::BI__builtin_sve_svget_neonq_u16:
  case SVE::BI__builtin_sve_svget_neonq_u32:
  case SVE::BI__builtin_sve_svget_neonq_u64:
  case SVE::BI__builtin_sve_svget_neonq_f16:
  case SVE::BI__builtin_sve_svget_neonq_f32:
  case SVE::BI__builtin_sve_svget_neonq_f64:
  case SVE::BI__builtin_sve_svget_neonq_bf16:
  case SVE::BI__builtin_sve_svget_neonq_mf8:
  case SVE::BI__builtin_sve_svdup_neonq_s8:
  case SVE::BI__builtin_sve_svdup_neonq_s16:
  case SVE::BI__builtin_sve_svdup_neonq_s32:
  case SVE::BI__builtin_sve_svdup_neonq_s64:
  case SVE::BI__builtin_sve_svdup_neonq_u8:
  case SVE::BI__builtin_sve_svdup_neonq_u16:
  case SVE::BI__builtin_sve_svdup_neonq_u32:
  case SVE::BI__builtin_sve_svdup_neonq_u64:
  case SVE::BI__builtin_sve_svdup_neonq_f16:
  case SVE::BI__builtin_sve_svdup_neonq_f32:
  case SVE::BI__builtin_sve_svdup_neonq_f64:
  case SVE::BI__builtin_sve_svdup_neonq_bf16:
  case SVE::BI__builtin_sve_svdup_neonq_mf8:
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  // Unreachable: All cases in the switch above return.
}
 
std::optional<mlir::Value>
CIRGenFunction::emitAArch64SMEBuiltinExpr(unsigned builtinID,
                                          const CallExpr *expr) {
  assert(!cir::MissingFeatures::aarch64SMEIntrinsics());
 
  cgm.errorNYI(expr->getSourceRange(),
               std::string("unimplemented AArch64 builtin call: ") +
                   getContext().BuiltinInfo.getName(builtinID));
  return mlir::Value{};
}
 
// Some intrinsics are equivalent for codegen.
static const std::pair<unsigned, unsigned> neonEquivalentIntrinsicMap[] = {
    {
        NEON::BI__builtin_neon_vabd_f16,
        NEON::BI__builtin_neon_vabd_v,
    },
    {
        NEON::BI__builtin_neon_vabdq_f16,
        NEON::BI__builtin_neon_vabdq_v,
    },
    {
        NEON::BI__builtin_neon_vabs_f16,
        NEON::BI__builtin_neon_vabs_v,
    },
    {
        NEON::BI__builtin_neon_vabsq_f16,
        NEON::BI__builtin_neon_vabsq_v,
    },
    {
        NEON::BI__builtin_neon_vcage_f16,
        NEON::BI__builtin_neon_vcage_v,
    },
    {
        NEON::BI__builtin_neon_vcageq_f16,
        NEON::BI__builtin_neon_vcageq_v,
    },
    {
        NEON::BI__builtin_neon_vcagt_f16,
        NEON::BI__builtin_neon_vcagt_v,
    },
    {
        NEON::BI__builtin_neon_vcagtq_f16,
        NEON::BI__builtin_neon_vcagtq_v,
    },
    {
        NEON::BI__builtin_neon_vcale_f16,
        NEON::BI__builtin_neon_vcale_v,
    },
    {
        NEON::BI__builtin_neon_vcaleq_f16,
        NEON::BI__builtin_neon_vcaleq_v,
    },
    {
        NEON::BI__builtin_neon_vcalt_f16,
        NEON::BI__builtin_neon_vcalt_v,
    },
    {
        NEON::BI__builtin_neon_vcaltq_f16,
        NEON::BI__builtin_neon_vcaltq_v,
    },
    {
        NEON::BI__builtin_neon_vceqz_f16,
        NEON::BI__builtin_neon_vceqz_v,
    },
    {
        NEON::BI__builtin_neon_vceqzq_f16,
        NEON::BI__builtin_neon_vceqzq_v,
    },
    {
        NEON::BI__builtin_neon_vcgez_f16,
        NEON::BI__builtin_neon_vcgez_v,
    },
    {
        NEON::BI__builtin_neon_vcgezq_f16,
        NEON::BI__builtin_neon_vcgezq_v,
    },
    {
        NEON::BI__builtin_neon_vcgtz_f16,
        NEON::BI__builtin_neon_vcgtz_v,
    },
    {
        NEON::BI__builtin_neon_vcgtzq_f16,
        NEON::BI__builtin_neon_vcgtzq_v,
    },
    {
        NEON::BI__builtin_neon_vclez_f16,
        NEON::BI__builtin_neon_vclez_v,
    },
    {
        NEON::BI__builtin_neon_vclezq_f16,
        NEON::BI__builtin_neon_vclezq_v,
    },
    {
        NEON::BI__builtin_neon_vcltz_f16,
        NEON::BI__builtin_neon_vcltz_v,
    },
    {
        NEON::BI__builtin_neon_vcltzq_f16,
        NEON::BI__builtin_neon_vcltzq_v,
    },
    {
        NEON::BI__builtin_neon_vfma_f16,
        NEON::BI__builtin_neon_vfma_v,
    },
    {
        NEON::BI__builtin_neon_vfma_lane_f16,
        NEON::BI__builtin_neon_vfma_lane_v,
    },
    {
        NEON::BI__builtin_neon_vfma_laneq_f16,
        NEON::BI__builtin_neon_vfma_laneq_v,
    },
    {
        NEON::BI__builtin_neon_vfmaq_f16,
        NEON::BI__builtin_neon_vfmaq_v,
    },
    {
        NEON::BI__builtin_neon_vfmaq_lane_f16,
        NEON::BI__builtin_neon_vfmaq_lane_v,
    },
    {
        NEON::BI__builtin_neon_vfmaq_laneq_f16,
        NEON::BI__builtin_neon_vfmaq_laneq_v,
    },
    {
        NEON::BI__builtin_neon_vmax_f16,
        NEON::BI__builtin_neon_vmax_v,
    },
    {
        NEON::BI__builtin_neon_vmaxnm_f16,
        NEON::BI__builtin_neon_vmaxnm_v,
    },
    {
        NEON::BI__builtin_neon_vmaxnmq_f16,
        NEON::BI__builtin_neon_vmaxnmq_v,
    },
    {
        NEON::BI__builtin_neon_vmaxq_f16,
        NEON::BI__builtin_neon_vmaxq_v,
    },
    {
        NEON::BI__builtin_neon_vmin_f16,
        NEON::BI__builtin_neon_vmin_v,
    },
    {
        NEON::BI__builtin_neon_vminnm_f16,
        NEON::BI__builtin_neon_vminnm_v,
    },
    {
        NEON::BI__builtin_neon_vminnmq_f16,
        NEON::BI__builtin_neon_vminnmq_v,
    },
    {
        NEON::BI__builtin_neon_vminq_f16,
        NEON::BI__builtin_neon_vminq_v,
    },
    {
        NEON::BI__builtin_neon_vmulx_f16,
        NEON::BI__builtin_neon_vmulx_v,
    },
    {
        NEON::BI__builtin_neon_vmulxq_f16,
        NEON::BI__builtin_neon_vmulxq_v,
    },
    {
        NEON::BI__builtin_neon_vpadd_f16,
        NEON::BI__builtin_neon_vpadd_v,
    },
    {
        NEON::BI__builtin_neon_vpaddq_f16,
        NEON::BI__builtin_neon_vpaddq_v,
    },
    {
        NEON::BI__builtin_neon_vpmax_f16,
        NEON::BI__builtin_neon_vpmax_v,
    },
    {
        NEON::BI__builtin_neon_vpmaxnm_f16,
        NEON::BI__builtin_neon_vpmaxnm_v,
    },
    {
        NEON::BI__builtin_neon_vpmaxnmq_f16,
        NEON::BI__builtin_neon_vpmaxnmq_v,
    },
    {
        NEON::BI__builtin_neon_vpmaxq_f16,
        NEON::BI__builtin_neon_vpmaxq_v,
    },
    {
        NEON::BI__builtin_neon_vpmin_f16,
        NEON::BI__builtin_neon_vpmin_v,
    },
    {
        NEON::BI__builtin_neon_vpminnm_f16,
        NEON::BI__builtin_neon_vpminnm_v,
    },
    {
        NEON::BI__builtin_neon_vpminnmq_f16,
        NEON::BI__builtin_neon_vpminnmq_v,
    },
    {
        NEON::BI__builtin_neon_vpminq_f16,
        NEON::BI__builtin_neon_vpminq_v,
    },
    {
        NEON::BI__builtin_neon_vrecpe_f16,
        NEON::BI__builtin_neon_vrecpe_v,
    },
    {
        NEON::BI__builtin_neon_vrecpeq_f16,
        NEON::BI__builtin_neon_vrecpeq_v,
    },
    {
        NEON::BI__builtin_neon_vrecps_f16,
        NEON::BI__builtin_neon_vrecps_v,
    },
    {
        NEON::BI__builtin_neon_vrecpsq_f16,
        NEON::BI__builtin_neon_vrecpsq_v,
    },
    {
        NEON::BI__builtin_neon_vrnd_f16,
        NEON::BI__builtin_neon_vrnd_v,
    },
    {
        NEON::BI__builtin_neon_vrnda_f16,
        NEON::BI__builtin_neon_vrnda_v,
    },
    {
        NEON::BI__builtin_neon_vrndaq_f16,
        NEON::BI__builtin_neon_vrndaq_v,
    },
    {
        NEON::BI__builtin_neon_vrndi_f16,
        NEON::BI__builtin_neon_vrndi_v,
    },
    {
        NEON::BI__builtin_neon_vrndiq_f16,
        NEON::BI__builtin_neon_vrndiq_v,
    },
    {
        NEON::BI__builtin_neon_vrndm_f16,
        NEON::BI__builtin_neon_vrndm_v,
    },
    {
        NEON::BI__builtin_neon_vrndmq_f16,
        NEON::BI__builtin_neon_vrndmq_v,
    },
    {
        NEON::BI__builtin_neon_vrndn_f16,
        NEON::BI__builtin_neon_vrndn_v,
    },
    {
        NEON::BI__builtin_neon_vrndnq_f16,
        NEON::BI__builtin_neon_vrndnq_v,
    },
    {
        NEON::BI__builtin_neon_vrndp_f16,
        NEON::BI__builtin_neon_vrndp_v,
    },
    {
        NEON::BI__builtin_neon_vrndpq_f16,
        NEON::BI__builtin_neon_vrndpq_v,
    },
    {
        NEON::BI__builtin_neon_vrndq_f16,
        NEON::BI__builtin_neon_vrndq_v,
    },
    {
        NEON::BI__builtin_neon_vrndx_f16,
        NEON::BI__builtin_neon_vrndx_v,
    },
    {
        NEON::BI__builtin_neon_vrndxq_f16,
        NEON::BI__builtin_neon_vrndxq_v,
    },
    {
        NEON::BI__builtin_neon_vrsqrte_f16,
        NEON::BI__builtin_neon_vrsqrte_v,
    },
    {
        NEON::BI__builtin_neon_vrsqrteq_f16,
        NEON::BI__builtin_neon_vrsqrteq_v,
    },
    {
        NEON::BI__builtin_neon_vrsqrts_f16,
        NEON::BI__builtin_neon_vrsqrts_v,
    },
    {
        NEON::BI__builtin_neon_vrsqrtsq_f16,
        NEON::BI__builtin_neon_vrsqrtsq_v,
    },
    {
        NEON::BI__builtin_neon_vsqrt_f16,
        NEON::BI__builtin_neon_vsqrt_v,
    },
    {
        NEON::BI__builtin_neon_vsqrtq_f16,
        NEON::BI__builtin_neon_vsqrtq_v,
    },
    // The mangling rules cause us to have one ID for each type for
    // vldap1(q)_lane and vstl1(q)_lane, but codegen is equivalent for all of
    // them. Choose an arbitrary one to be handled as tha canonical variation.
    {NEON::BI__builtin_neon_vldap1_lane_u64,
     NEON::BI__builtin_neon_vldap1_lane_s64},
    {NEON::BI__builtin_neon_vldap1_lane_f64,
     NEON::BI__builtin_neon_vldap1_lane_s64},
    {NEON::BI__builtin_neon_vldap1_lane_p64,
     NEON::BI__builtin_neon_vldap1_lane_s64},
    {NEON::BI__builtin_neon_vldap1q_lane_u64,
     NEON::BI__builtin_neon_vldap1q_lane_s64},
    {NEON::BI__builtin_neon_vldap1q_lane_f64,
     NEON::BI__builtin_neon_vldap1q_lane_s64},
    {NEON::BI__builtin_neon_vldap1q_lane_p64,
     NEON::BI__builtin_neon_vldap1q_lane_s64},
    {NEON::BI__builtin_neon_vstl1_lane_u64,
     NEON::BI__builtin_neon_vstl1_lane_s64},
    {NEON::BI__builtin_neon_vstl1_lane_f64,
     NEON::BI__builtin_neon_vstl1_lane_s64},
    {NEON::BI__builtin_neon_vstl1_lane_p64,
     NEON::BI__builtin_neon_vstl1_lane_s64},
    {NEON::BI__builtin_neon_vstl1q_lane_u64,
     NEON::BI__builtin_neon_vstl1q_lane_s64},
    {NEON::BI__builtin_neon_vstl1q_lane_f64,
     NEON::BI__builtin_neon_vstl1q_lane_s64},
    {NEON::BI__builtin_neon_vstl1q_lane_p64,
     NEON::BI__builtin_neon_vstl1q_lane_s64},
};
 
std::optional<mlir::Value>
CIRGenFunction::emitAArch64BuiltinExpr(unsigned builtinID, const CallExpr *expr,
                                       ReturnValueSlot returnValue,
                                       llvm::Triple::ArchType arch) {
  if (builtinID >= clang::AArch64::FirstSVEBuiltin &&
      builtinID <= clang::AArch64::LastSVEBuiltin)
    return emitAArch64SVEBuiltinExpr(builtinID, expr);
 
  if (builtinID >= clang::AArch64::FirstSMEBuiltin &&
      builtinID <= clang::AArch64::LastSMEBuiltin)
    return emitAArch64SMEBuiltinExpr(builtinID, expr);
 
  if (builtinID == Builtin::BI__builtin_cpu_supports) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  switch (builtinID) {
  default:
    break;
  case clang::AArch64::BI__builtin_arm_nop:
  case clang::AArch64::BI__builtin_arm_yield:
  case clang::AArch64::BI__yield:
  case clang::AArch64::BI__builtin_arm_wfe:
  case clang::AArch64::BI__wfe:
  case clang::AArch64::BI__builtin_arm_wfi:
  case clang::AArch64::BI__wfi:
  case clang::AArch64::BI__builtin_arm_sev:
  case clang::AArch64::BI__sev:
  case clang::AArch64::BI__builtin_arm_sevl:
  case clang::AArch64::BI__sevl:
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  if (builtinID == clang::AArch64::BI__builtin_arm_trap) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  if (builtinID == clang::AArch64::BI__builtin_arm_get_sme_state) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  if (builtinID == clang::AArch64::BI__builtin_arm_rbit) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
  if (builtinID == clang::AArch64::BI__builtin_arm_rbit64) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  if (builtinID == clang::AArch64::BI__builtin_arm_clz ||
      builtinID == clang::AArch64::BI__builtin_arm_clz64) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  if (builtinID == clang::AArch64::BI__builtin_arm_cls) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
  if (builtinID == clang::AArch64::BI__builtin_arm_cls64) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  if (builtinID == clang::AArch64::BI__builtin_arm_rint32zf ||
      builtinID == clang::AArch64::BI__builtin_arm_rint32z) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  if (builtinID == clang::AArch64::BI__builtin_arm_rint64zf ||
      builtinID == clang::AArch64::BI__builtin_arm_rint64z) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  if (builtinID == clang::AArch64::BI__builtin_arm_rint32xf ||
      builtinID == clang::AArch64::BI__builtin_arm_rint32x) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  if (builtinID == clang::AArch64::BI__builtin_arm_rint64xf ||
      builtinID == clang::AArch64::BI__builtin_arm_rint64x) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  if (builtinID == clang::AArch64::BI__builtin_arm_jcvt) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  if (builtinID == clang::AArch64::BI__builtin_arm_ld64b ||
      builtinID == clang::AArch64::BI__builtin_arm_st64b ||
      builtinID == clang::AArch64::BI__builtin_arm_st64bv ||
      builtinID == clang::AArch64::BI__builtin_arm_st64bv0) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  if (builtinID == clang::AArch64::BI__builtin_arm_rndr ||
      builtinID == clang::AArch64::BI__builtin_arm_rndrrs) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  if (builtinID == clang::AArch64::BI__clear_cache) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  if ((builtinID == clang::AArch64::BI__builtin_arm_ldrex ||
       builtinID == clang::AArch64::BI__builtin_arm_ldaex) &&
      getContext().getTypeSize(expr->getType()) == 128) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
  if (builtinID == clang::AArch64::BI__builtin_arm_ldrex ||
      builtinID == clang::AArch64::BI__builtin_arm_ldaex) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  if ((builtinID == clang::AArch64::BI__builtin_arm_strex ||
       builtinID == clang::AArch64::BI__builtin_arm_stlex) &&
      getContext().getTypeSize(expr->getArg(0)->getType()) == 128) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  if (builtinID == clang::AArch64::BI__builtin_arm_strex ||
      builtinID == clang::AArch64::BI__builtin_arm_stlex) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  if (builtinID == clang::AArch64::BI__getReg ||
      builtinID == clang::AArch64::BI__setReg ||
      builtinID == clang::AArch64::BI__getRegFp ||
      builtinID == clang::AArch64::BI__setRegFp) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  if (builtinID == clang::AArch64::BI__break) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  if (builtinID == clang::AArch64::BI__builtin_arm_clrex) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  if (builtinID == clang::AArch64::BI_ReadWriteBarrier) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  // CRC32
  Intrinsic::ID crcIntrinsicID = Intrinsic::not_intrinsic;
  switch (builtinID) {
  case clang::AArch64::BI__builtin_arm_crc32b:
    crcIntrinsicID = Intrinsic::aarch64_crc32b;
    break;
  case clang::AArch64::BI__builtin_arm_crc32cb:
    crcIntrinsicID = Intrinsic::aarch64_crc32cb;
    break;
  case clang::AArch64::BI__builtin_arm_crc32h:
    crcIntrinsicID = Intrinsic::aarch64_crc32h;
    break;
  case clang::AArch64::BI__builtin_arm_crc32ch:
    crcIntrinsicID = Intrinsic::aarch64_crc32ch;
    break;
  case clang::AArch64::BI__builtin_arm_crc32w:
    crcIntrinsicID = Intrinsic::aarch64_crc32w;
    break;
  case clang::AArch64::BI__builtin_arm_crc32cw:
    crcIntrinsicID = Intrinsic::aarch64_crc32cw;
    break;
  case clang::AArch64::BI__builtin_arm_crc32d:
    crcIntrinsicID = Intrinsic::aarch64_crc32x;
    break;
  case clang::AArch64::BI__builtin_arm_crc32cd:
    crcIntrinsicID = Intrinsic::aarch64_crc32cx;
    break;
  }
 
  if (crcIntrinsicID != Intrinsic::not_intrinsic) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  // Memory Operations (MOPS)
  if (builtinID == AArch64::BI__builtin_arm_mops_memset_tag) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  // Memory Tagging Extensions (MTE) Intrinsics
  Intrinsic::ID mteIntrinsicID = Intrinsic::not_intrinsic;
  switch (builtinID) {
  case clang::AArch64::BI__builtin_arm_irg:
    mteIntrinsicID = Intrinsic::aarch64_irg;
    break;
  case clang::AArch64::BI__builtin_arm_addg:
    mteIntrinsicID = Intrinsic::aarch64_addg;
    break;
  case clang::AArch64::BI__builtin_arm_gmi:
    mteIntrinsicID = Intrinsic::aarch64_gmi;
    break;
  case clang::AArch64::BI__builtin_arm_ldg:
    mteIntrinsicID = Intrinsic::aarch64_ldg;
    break;
  case clang::AArch64::BI__builtin_arm_stg:
    mteIntrinsicID = Intrinsic::aarch64_stg;
    break;
  case clang::AArch64::BI__builtin_arm_subp:
    mteIntrinsicID = Intrinsic::aarch64_subp;
    break;
  }
 
  if (mteIntrinsicID != Intrinsic::not_intrinsic) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  if (builtinID == clang::AArch64::BI__builtin_arm_rsr ||
      builtinID == clang::AArch64::BI__builtin_arm_rsr64 ||
      builtinID == clang::AArch64::BI__builtin_arm_rsr128 ||
      builtinID == clang::AArch64::BI__builtin_arm_rsrp ||
      builtinID == clang::AArch64::BI__builtin_arm_wsr ||
      builtinID == clang::AArch64::BI__builtin_arm_wsr64 ||
      builtinID == clang::AArch64::BI__builtin_arm_wsr128 ||
      builtinID == clang::AArch64::BI__builtin_arm_wsrp) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  if (builtinID == clang::AArch64::BI_ReadStatusReg ||
      builtinID == clang::AArch64::BI_WriteStatusReg ||
      builtinID == clang::AArch64::BI__sys) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  if (builtinID == clang::AArch64::BI_AddressOfReturnAddress) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  if (builtinID == clang::AArch64::BI__builtin_sponentry) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  if (builtinID == clang::AArch64::BI__mulh ||
      builtinID == clang::AArch64::BI__umulh) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  if (builtinID == AArch64::BI__writex18byte ||
      builtinID == AArch64::BI__writex18word ||
      builtinID == AArch64::BI__writex18dword ||
      builtinID == AArch64::BI__writex18qword) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  if (builtinID == AArch64::BI__readx18byte ||
      builtinID == AArch64::BI__readx18word ||
      builtinID == AArch64::BI__readx18dword ||
      builtinID == AArch64::BI__readx18qword) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  if (builtinID == AArch64::BI__addx18byte ||
      builtinID == AArch64::BI__addx18word ||
      builtinID == AArch64::BI__addx18dword ||
      builtinID == AArch64::BI__addx18qword ||
      builtinID == AArch64::BI__incx18byte ||
      builtinID == AArch64::BI__incx18word ||
      builtinID == AArch64::BI__incx18dword ||
      builtinID == AArch64::BI__incx18qword) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  if (builtinID == AArch64::BI_CopyDoubleFromInt64 ||
      builtinID == AArch64::BI_CopyFloatFromInt32 ||
      builtinID == AArch64::BI_CopyInt32FromFloat ||
      builtinID == AArch64::BI_CopyInt64FromDouble) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  if (builtinID == AArch64::BI_CountLeadingOnes ||
      builtinID == AArch64::BI_CountLeadingOnes64 ||
      builtinID == AArch64::BI_CountLeadingZeros ||
      builtinID == AArch64::BI_CountLeadingZeros64) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  if (builtinID == AArch64::BI_CountLeadingSigns ||
      builtinID == AArch64::BI_CountLeadingSigns64) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  if (builtinID == AArch64::BI_CountOneBits ||
      builtinID == AArch64::BI_CountOneBits64 ||
      builtinID == AArch64::BI_CountTrailingZeros ||
      builtinID == AArch64::BI_CountTrailingZeros64) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  if (builtinID == AArch64::BI__prefetch ||
      builtinID == AArch64::BI__prefetch2) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  if (builtinID == AArch64::BI__hlt) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  if (builtinID == NEON::BI__builtin_neon_vcvth_bf16_f32) {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  // Handle MSVC intrinsics before argument evaluation to prevent double
  // evaluation.
  assert(!cir::MissingFeatures::msvcBuiltins());
 
  CIRGenFPOptionsRAII fpOptsRAII(*this, expr);
 
  // Some intrinsics are equivalent - if they are use the base intrinsic ID.
  auto it = llvm::find_if(neonEquivalentIntrinsicMap, [builtinID](auto &p) {
    return p.first == builtinID;
  });
  if (it != end(neonEquivalentIntrinsicMap))
    builtinID = it->second;
 
  // Find out if any arguments are required to be integer constant
  // expressions.
  assert(!cir::MissingFeatures::handleBuiltinICEArguments());
  unsigned iceArguments = 0;
  ASTContext::GetBuiltinTypeError error;
  getContext().GetBuiltinType(builtinID, error, &iceArguments);
  assert(error == ASTContext::GE_None && "Should not codegen an error");
  llvm::SmallVector<mlir::Value> ops;
 
  // Captures the pointer and its alignment for builtins that store/load
  // directly through the first argument, i.e. operand 0 (set in the
  // arg-gathering loop below, used later when lowering the store/load
  // itself).
  Address ptrOp0 = Address::invalid();
 
  // Skip extra arguments used to discriminate vector types and that are
  // intended for Sema checking.
  bool hasExtraArg = CodeGenUtils::hasExtraNeonArgument(builtinID);
  unsigned numArgs = expr->getNumArgs() - (hasExtraArg ? 1 : 0);
  for (unsigned i = 0, e = numArgs; i != e; i++) {
    if (i == 0) {
      switch (builtinID) {
      case NEON::BI__builtin_neon_vld1_v:
      case NEON::BI__builtin_neon_vld1q_v:
      case NEON::BI__builtin_neon_vld1_dup_v:
      case NEON::BI__builtin_neon_vld1q_dup_v:
      case NEON::BI__builtin_neon_vld1_lane_v:
      case NEON::BI__builtin_neon_vld1q_lane_v:
      case NEON::BI__builtin_neon_vst1_v:
      case NEON::BI__builtin_neon_vst1q_v:
      case NEON::BI__builtin_neon_vst1_lane_v:
      case NEON::BI__builtin_neon_vst1q_lane_v:
      case NEON::BI__builtin_neon_vstrq_p128:
      case NEON::BI__builtin_neon_vst2_v:
      case NEON::BI__builtin_neon_vst2q_v:
      case NEON::BI__builtin_neon_vst2_lane_v:
      case NEON::BI__builtin_neon_vst2q_lane_v:
      case NEON::BI__builtin_neon_vst3_v:
      case NEON::BI__builtin_neon_vst3q_v:
      case NEON::BI__builtin_neon_vst3_lane_v:
      case NEON::BI__builtin_neon_vst3q_lane_v:
      case NEON::BI__builtin_neon_vst4_v:
      case NEON::BI__builtin_neon_vst4q_v:
      case NEON::BI__builtin_neon_vst4_lane_v:
      case NEON::BI__builtin_neon_vst4q_lane_v:
        // Get the alignment for the argument in addition to the value;
        // we'll use it later.
        ptrOp0 = emitPointerWithAlignment(expr->getArg(0));
        ops.push_back(ptrOp0.getPointer());
        continue;
      case NEON::BI__builtin_neon_vldap1_lane_s64:
      case NEON::BI__builtin_neon_vldap1q_lane_s64:
      case NEON::BI__builtin_neon_vstl1_lane_s64:
      case NEON::BI__builtin_neon_vstl1q_lane_s64:
        cgm.errorNYI(
            expr->getSourceRange(),
            std::string("unimplemented AArch64 builtin argument handling ") +
                getContext().BuiltinInfo.getName(builtinID));
        break;
      }
    }
    ops.push_back(
        emitScalarOrConstFoldImmArg(iceArguments, i, expr->getArg(i)));
  }
 
  const ARMNeonVectorIntrinsicInfo *builtin =
      findARMVectorIntrinsicInMap(ArrayRef(AArch64SISDIntrinsicMap), builtinID,
                                  aarch64SISDIntrinsicsProvenSorted);
  if (builtin)
    return emitCommonNeonSISDBuiltinExpr(*this, *builtin, ops, expr,
                                         iceArguments);
 
  // Not all intrinsics handled by the common case work for AArch64 yet, so only
  // defer to common code if it's been added to our special map.
  assert(!cir::MissingFeatures::aarch64SIMDIntrinsics());
 
  assert(!cir::MissingFeatures::aarch64TblBuiltinExpr());
 
  const Expr *arg = expr->getArg(expr->getNumArgs() - 1);
  NeonTypeFlags type(0);
  // A trailing constant integer is used for discriminating overloaded builtin
  // calls. Use it to determine the type of this overloaded NEON intrinsic.
  if (std::optional<llvm::APSInt> result =
          arg->getIntegerConstantExpr(getContext()))
    type = NeonTypeFlags(result->getZExtValue());
 
  bool usgn = type.isUnsigned();
 
  mlir::Location loc = getLoc(expr->getExprLoc());
 
  // Not all intrinsics handled by the common case work for AArch64 yet, so only
  // defer to common code if it's been added to our special map.
  builtin =
      findARMVectorIntrinsicInMap(ArrayRef(AArch64SIMDIntrinsicMap), builtinID,
                                  aarch64SIMDIntrinsicsProvenSorted);
  if (builtin)
    return emitCommonNeonBuiltinExpr(
        *this, builtin->BuiltinID, builtin->LLVMIntrinsic,
        builtin->AltLLVMIntrinsic, builtin->NameHint, builtin->TypeModifier,
        expr, ops);
 
  // Handle non-overloaded intrinsics first.
  switch (builtinID) {
  default:
    break;
  case NEON::BI__builtin_neon_vabsh_f16: {
    mlir::Type halfTy = builder.getFp16Ty();
    return emitNeonCallToOp<cir::FAbsOp>(cgm, builder, {halfTy}, ops,
                                         std::nullopt, halfTy, loc);
  }
  case NEON::BI__builtin_neon_vaddq_p128: {
    cir::VectorType byteTy =
        cir::VectorType::getFixed(builder.getUInt8Ty(), 16);
    ops[0] = builder.createBitcast(ops[0], byteTy);
    ops[1] = builder.createBitcast(ops[1], byteTy);
    mlir::Value result = builder.createXor(loc, ops[0], ops[1]);
    return builder.createBitcast(result, convertType(expr->getType()));
  }
  case NEON::BI__builtin_neon_vldrq_p128:
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  case NEON::BI__builtin_neon_vstrq_p128: {
    builder.createStore(loc, ops[1], ptrOp0);
    return nullptr;
  }
  case NEON::BI__builtin_neon_vcvts_f32_u32:
  case NEON::BI__builtin_neon_vcvtd_f64_u64:
  case NEON::BI__builtin_neon_vcvts_f32_s32:
  case NEON::BI__builtin_neon_vcvtd_f64_s64: {
    return builder.createCast(loc, cir::CastKind::int_to_float, ops[0],
                              convertType(expr->getType()));
  }
  case NEON::BI__builtin_neon_vcvth_f16_u16:
  case NEON::BI__builtin_neon_vcvth_f16_u32:
  case NEON::BI__builtin_neon_vcvth_f16_u64:
  case NEON::BI__builtin_neon_vcvth_f16_s16:
  case NEON::BI__builtin_neon_vcvth_f16_s32:
  case NEON::BI__builtin_neon_vcvth_f16_s64:
  case NEON::BI__builtin_neon_vcvtah_u16_f16:
  case NEON::BI__builtin_neon_vcvtmh_u16_f16:
  case NEON::BI__builtin_neon_vcvtnh_u16_f16:
  case NEON::BI__builtin_neon_vcvtph_u16_f16:
  case NEON::BI__builtin_neon_vcvth_u16_f16:
  case NEON::BI__builtin_neon_vcvtah_s16_f16:
  case NEON::BI__builtin_neon_vcvtmh_s16_f16:
  case NEON::BI__builtin_neon_vcvtnh_s16_f16:
  case NEON::BI__builtin_neon_vcvtph_s16_f16:
  case NEON::BI__builtin_neon_vcvth_s16_f16:
  case NEON::BI__builtin_neon_vcaleh_f16:
  case NEON::BI__builtin_neon_vcalth_f16:
  case NEON::BI__builtin_neon_vcageh_f16:
  case NEON::BI__builtin_neon_vcagth_f16:
  case NEON::BI__builtin_neon_vcvth_n_s16_f16:
  case NEON::BI__builtin_neon_vcvth_n_u16_f16:
  case NEON::BI__builtin_neon_vcvth_n_f16_s16:
  case NEON::BI__builtin_neon_vcvth_n_f16_u16:
  case NEON::BI__builtin_neon_vpaddd_s64:
  case NEON::BI__builtin_neon_vpaddd_f64:
  case NEON::BI__builtin_neon_vpadds_f32:
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  case NEON::BI__builtin_neon_vceqzd_s64:
  case NEON::BI__builtin_neon_vceqzd_f64:
  case NEON::BI__builtin_neon_vceqzs_f32:
  case NEON::BI__builtin_neon_vceqzh_f16:
    return emitAArch64CompareBuiltinExpr(
        *this, builder, loc, ops[0],
        convertType(expr->getCallReturnType(getContext())), cir::CmpOpKind::eq);
  case NEON::BI__builtin_neon_vcgezd_s64:
  case NEON::BI__builtin_neon_vcgezd_f64:
  case NEON::BI__builtin_neon_vcgezs_f32:
  case NEON::BI__builtin_neon_vcgezh_f16:
  case NEON::BI__builtin_neon_vclezd_s64:
  case NEON::BI__builtin_neon_vclezd_f64:
  case NEON::BI__builtin_neon_vclezs_f32:
  case NEON::BI__builtin_neon_vclezh_f16:
  case NEON::BI__builtin_neon_vcgtzd_s64:
  case NEON::BI__builtin_neon_vcgtzd_f64:
  case NEON::BI__builtin_neon_vcgtzs_f32:
  case NEON::BI__builtin_neon_vcgtzh_f16:
  case NEON::BI__builtin_neon_vcltzd_s64:
  case NEON::BI__builtin_neon_vcltzd_f64:
  case NEON::BI__builtin_neon_vcltzs_f32:
  case NEON::BI__builtin_neon_vcltzh_f16:
  case NEON::BI__builtin_neon_vceqzd_u64: {
    return emitAArch64CompareBuiltinExpr(
        *this, builder, loc, ops[0],
        convertType(expr->getCallReturnType(getContext())), cir::CmpOpKind::eq);
  }
  case NEON::BI__builtin_neon_vceqd_f64:
  case NEON::BI__builtin_neon_vcled_f64:
  case NEON::BI__builtin_neon_vcltd_f64:
  case NEON::BI__builtin_neon_vcged_f64:
  case NEON::BI__builtin_neon_vcgtd_f64:
  case NEON::BI__builtin_neon_vceqs_f32:
  case NEON::BI__builtin_neon_vcles_f32:
  case NEON::BI__builtin_neon_vclts_f32:
  case NEON::BI__builtin_neon_vcges_f32:
  case NEON::BI__builtin_neon_vcgts_f32:
  case NEON::BI__builtin_neon_vceqh_f16:
  case NEON::BI__builtin_neon_vcleh_f16:
  case NEON::BI__builtin_neon_vclth_f16:
  case NEON::BI__builtin_neon_vcgeh_f16:
  case NEON::BI__builtin_neon_vcgth_f16:
  case NEON::BI__builtin_neon_vceqd_s64:
  case NEON::BI__builtin_neon_vceqd_u64:
  case NEON::BI__builtin_neon_vcgtd_s64:
  case NEON::BI__builtin_neon_vcgtd_u64:
  case NEON::BI__builtin_neon_vcltd_s64:
  case NEON::BI__builtin_neon_vcltd_u64:
  case NEON::BI__builtin_neon_vcged_u64:
  case NEON::BI__builtin_neon_vcged_s64:
  case NEON::BI__builtin_neon_vcled_u64:
  case NEON::BI__builtin_neon_vcled_s64:
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  case NEON::BI__builtin_neon_vnegd_s64: {
    return builder.createNeg(loc, ops[0]);
  }
  case NEON::BI__builtin_neon_vnegh_f16: {
    return builder.createFNeg(loc, ops[0]);
  }
  case NEON::BI__builtin_neon_vtstd_s64:
  case NEON::BI__builtin_neon_vtstd_u64:
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  case NEON::BI__builtin_neon_vset_lane_i8:
  case NEON::BI__builtin_neon_vset_lane_i16:
  case NEON::BI__builtin_neon_vset_lane_i32:
  case NEON::BI__builtin_neon_vset_lane_i64:
  case NEON::BI__builtin_neon_vset_lane_bf16:
  case NEON::BI__builtin_neon_vset_lane_f32:
  case NEON::BI__builtin_neon_vsetq_lane_i8:
  case NEON::BI__builtin_neon_vsetq_lane_i16:
  case NEON::BI__builtin_neon_vsetq_lane_i32:
  case NEON::BI__builtin_neon_vsetq_lane_i64:
  case NEON::BI__builtin_neon_vsetq_lane_bf16:
  case NEON::BI__builtin_neon_vsetq_lane_f32:
  case NEON::BI__builtin_neon_vset_lane_f64:
  case NEON::BI__builtin_neon_vset_lane_mf8:
  case NEON::BI__builtin_neon_vsetq_lane_mf8:
  case NEON::BI__builtin_neon_vsetq_lane_f64:
    return cir::VecInsertOp::create(builder, loc, ops[1], ops[0], ops[2]);
  case NEON::BI__builtin_neon_vget_lane_i8:
  case NEON::BI__builtin_neon_vdupb_lane_i8:
  case NEON::BI__builtin_neon_vgetq_lane_i8:
  case NEON::BI__builtin_neon_vdupb_laneq_i8:
  case NEON::BI__builtin_neon_vget_lane_mf8:
  case NEON::BI__builtin_neon_vdupb_lane_mf8:
  case NEON::BI__builtin_neon_vgetq_lane_mf8:
  case NEON::BI__builtin_neon_vdupb_laneq_mf8:
  case NEON::BI__builtin_neon_vget_lane_i16:
  case NEON::BI__builtin_neon_vduph_lane_i16:
  case NEON::BI__builtin_neon_vgetq_lane_i16:
  case NEON::BI__builtin_neon_vduph_laneq_i16:
  case NEON::BI__builtin_neon_vget_lane_i32:
  case NEON::BI__builtin_neon_vdups_lane_i32:
  case NEON::BI__builtin_neon_vdups_lane_f32:
  case NEON::BI__builtin_neon_vgetq_lane_i32:
  case NEON::BI__builtin_neon_vdups_laneq_i32:
  case NEON::BI__builtin_neon_vget_lane_i64:
  case NEON::BI__builtin_neon_vdupd_lane_i64:
  case NEON::BI__builtin_neon_vdupd_lane_f64:
  case NEON::BI__builtin_neon_vgetq_lane_i64:
  case NEON::BI__builtin_neon_vdupd_laneq_i64:
  case NEON::BI__builtin_neon_vget_lane_f32:
  case NEON::BI__builtin_neon_vget_lane_f64:
  case NEON::BI__builtin_neon_vgetq_lane_f32:
  case NEON::BI__builtin_neon_vdups_laneq_f32:
  case NEON::BI__builtin_neon_vgetq_lane_f64:
  case NEON::BI__builtin_neon_vdupd_laneq_f64:
    return cir::VecExtractOp::create(builder, loc, ops[0],
                                     emitScalarExpr(expr->getArg(1)));
  case NEON::BI__builtin_neon_vaddh_f16:
    return builder.createFAdd(loc, ops[0], ops[1]);
  case NEON::BI__builtin_neon_vsubh_f16:
    return builder.createFSub(loc, ops[0], ops[1]);
  case NEON::BI__builtin_neon_vmulh_f16:
    return builder.createFMul(loc, ops[0], ops[1]);
  case NEON::BI__builtin_neon_vdivh_f16:
    return builder.createFDiv(loc, ops[0], ops[1]);
  case NEON::BI__builtin_neon_vfmah_f16: {
    // NEON intrinsic puts accumulator first, unlike fma.
    std::rotate(ops.begin(), ops.begin() + 1, ops.end());
    mlir::Type ty = convertType(expr->getType());
    return emitNeonCallToOp<cir::FMAOp>(cgm, builder, {ty, ty, ty}, ops,
                                        std::nullopt, ty, loc);
  }
  case NEON::BI__builtin_neon_vfmsh_f16: {
    // NEON intrinsic puts accumulator first, unlike fma.
    std::rotate(ops.begin(), ops.begin() + 1, ops.end());
    ops[0] = builder.createFNeg(loc, ops[0]);
    mlir::Type ty = convertType(expr->getType());
    return emitNeonCallToOp<cir::FMAOp>(cgm, builder, {ty, ty, ty}, ops,
                                        std::nullopt, ty, loc);
  }
  case NEON::BI__builtin_neon_vaddd_s64:
  case NEON::BI__builtin_neon_vaddd_u64:
    return builder.createAdd(loc, ops[0], ops[1]);
  case NEON::BI__builtin_neon_vsubd_s64:
  case NEON::BI__builtin_neon_vsubd_u64:
    return builder.createSub(loc, ops[0], ops[1]);
  case NEON::BI__builtin_neon_vqdmlalh_s16:
  case NEON::BI__builtin_neon_vqdmlslh_s16:
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  case NEON::BI__builtin_neon_vqshlud_n_s64: {
    cir::IntType int64Type = builder.getSInt64Ty();
    ops[1] = builder.getSInt64(getZExtIntValueFromConstOp(ops[1]), loc);
    return emitNeonCall(cgm, builder, {int64Type, int64Type}, ops,
                        "aarch64.neon.sqshlu", convertType(expr->getType()),
                        loc);
  }
  case NEON::BI__builtin_neon_vqshld_n_u64:
  case NEON::BI__builtin_neon_vqshld_n_s64: {
    cir::IntType int64Type = builtinID == NEON::BI__builtin_neon_vqshld_n_u64
                                 ? builder.getUInt64Ty()
                                 : builder.getSInt64Ty();
    llvm::StringRef intrinsicName =
        builtinID == NEON::BI__builtin_neon_vqshld_n_u64 ? "aarch64.neon.uqshl"
                                                         : "aarch64.neon.sqshl";
    ops[1] = builder.getSInt64(getZExtIntValueFromConstOp(ops[1]), loc);
    return emitNeonCall(cgm, builder, {int64Type, int64Type}, ops,
                        intrinsicName, convertType(expr->getType()), loc);
  }
  case NEON::BI__builtin_neon_vrshrd_n_u64:
  case NEON::BI__builtin_neon_vrshrd_n_s64: {
    llvm::StringRef intrName = builtinID == NEON::BI__builtin_neon_vrshrd_n_s64
                                   ? "aarch64.neon.srshl"
                                   : "aarch64.neon.urshl";
    cir::IntType int64Ty = builtinID == NEON::BI__builtin_neon_vqshld_n_u64
                               ? builder.getUInt64Ty()
                               : builder.getSInt64Ty();
    int64_t sv = -cast<cir::IntAttr>(
                      cast<cir::ConstantOp>(ops[1].getDefiningOp()).getValue())
                      .getSInt();
    ops[1] = builder.getSInt64(sv, loc);
    return emitNeonCall(cgm, builder, {int64Ty, builder.getSInt64Ty()}, ops,
                        intrName, int64Ty, loc);
  }
  case NEON::BI__builtin_neon_vrsrad_n_u64:
  case NEON::BI__builtin_neon_vrsrad_n_s64: {
    cir::IntType int64Type = builtinID == NEON::BI__builtin_neon_vrsrad_n_u64
                                 ? builder.getUInt64Ty()
                                 : builder.getSInt64Ty();
    ops[2] = builder.createNeg(loc, ops[2]);
    const StringRef intrName = builtinID == NEON::BI__builtin_neon_vrsrad_n_u64
                                   ? "aarch64.neon.urshl"
                                   : "aarch64.neon.srshl";
 
    llvm::SmallVector<mlir::Value, 2> args = {
        ops[1], builder.createIntCast(ops[2], builder.getSInt64Ty())};
    ops[1] = builder.emitIntrinsicCallOp(loc, intrName, int64Type, args);
    return builder.createAdd(loc, ops[0],
                             builder.createBitcast(ops[1], int64Type));
  }
  case NEON::BI__builtin_neon_vshld_n_s64:
  case NEON::BI__builtin_neon_vshld_n_u64: {
    auto loc = getLoc(expr->getExprLoc());
    std::optional<llvm::APSInt> amt =
        expr->getArg(1)->getIntegerConstantExpr(getContext());
    assert(amt && "Expected argument to be a constant");
    return builder.createShiftLeft(loc, ops[0], amt->getZExtValue());
  }
  case NEON::BI__builtin_neon_vshrd_n_s64: {
    std::optional<llvm::APSInt> amt =
        expr->getArg(1)->getIntegerConstantExpr(getContext());
    assert(amt && "Expected argument to be a constant");
    return builder.createShiftRight(
        loc, ops[0], std::min(static_cast<uint64_t>(63), amt->getZExtValue()));
  }
  case NEON::BI__builtin_neon_vshrd_n_u64: {
    std::optional<llvm::APSInt> amt =
        expr->getArg(1)->getIntegerConstantExpr(getContext());
    assert(amt && "Expected argument to be a constant");
    uint64_t shiftAmt = amt->getZExtValue();
    // Right-shifting an unsigned value by its size yields 0.
    if (shiftAmt == 64)
      return builder.getConstInt(loc, builder.getUInt64Ty(), 0);
    return builder.createShiftRight(loc, ops[0], shiftAmt);
  }
  case NEON::BI__builtin_neon_vsrad_n_s64: {
    std::optional<llvm::APSInt> amt =
        expr->getArg(2)->getIntegerConstantExpr(getContext());
    assert(amt && "Expected argument to be a constant");
    uint64_t shiftAmt =
        std::min(static_cast<uint64_t>(63), amt->getZExtValue());
    mlir::Value shifted =
        builder.createShiftRight(loc, ops[1], static_cast<unsigned>(shiftAmt));
    return builder.createAdd(loc, ops[0], shifted);
  }
  case NEON::BI__builtin_neon_vsrad_n_u64: {
    std::optional<llvm::APSInt> amt =
        expr->getArg(2)->getIntegerConstantExpr(getContext());
    assert(amt && "Expected argument to be a constant");
    uint64_t shiftAmt = amt->getZExtValue();
    // Right-shifting an unsigned value by its size yields 0, so a + 0 = a.
    if (shiftAmt == 64)
      return ops[0];
    mlir::Value shifted =
        builder.createShiftRight(loc, ops[1], static_cast<unsigned>(shiftAmt));
    return builder.createAdd(loc, ops[0], shifted);
  }
  case NEON::BI__builtin_neon_vqdmlalh_lane_s16:
  case NEON::BI__builtin_neon_vqdmlalh_laneq_s16:
  case NEON::BI__builtin_neon_vqdmlslh_lane_s16:
  case NEON::BI__builtin_neon_vqdmlslh_laneq_s16:
  case NEON::BI__builtin_neon_vqdmlals_s32:
  case NEON::BI__builtin_neon_vqdmlsls_s32:
  case NEON::BI__builtin_neon_vqdmlals_lane_s32:
  case NEON::BI__builtin_neon_vqdmlals_laneq_s32:
  case NEON::BI__builtin_neon_vqdmlsls_lane_s32:
  case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: {
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
  case NEON::BI__builtin_neon_vget_lane_bf16:
  case NEON::BI__builtin_neon_vduph_lane_bf16:
  case NEON::BI__builtin_neon_vduph_lane_f16:
  case NEON::BI__builtin_neon_vgetq_lane_bf16:
  case NEON::BI__builtin_neon_vduph_laneq_bf16:
  case NEON::BI__builtin_neon_vduph_laneq_f16: {
    return cir::VecExtractOp::create(builder, loc, ops[0], ops[1]);
  }
  case NEON::BI__builtin_neon_vcvt_bf16_f32:
  case NEON::BI__builtin_neon_vcvtq_low_bf16_f32:
  case NEON::BI__builtin_neon_vcvtq_high_bf16_f32:
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  case NEON::BI__builtin_neon_vcvt_f16_f32: {
    auto v4f32 = cir::VectorType::getFixed(cgm.floatTy, 4);
    auto v4f16 = cir::VectorType::getFixed(cgm.fP16Ty, 4);
    return builder.createFloatingCast(builder.createBitcast(ops[0], v4f32),
                                      v4f16);
  }
  case NEON::BI__builtin_neon_vcvt_f32_f16: {
    auto v4f32 = cir::VectorType::getFixed(cgm.floatTy, 4);
    auto v4f16 = cir::VectorType::getFixed(cgm.fP16Ty, 4);
    return builder.createFloatingCast(builder.createBitcast(ops[0], v4f16),
                                      v4f32);
  }
  case clang::AArch64::BI_InterlockedAdd:
  case clang::AArch64::BI_InterlockedAdd_acq:
  case clang::AArch64::BI_InterlockedAdd_rel:
  case clang::AArch64::BI_InterlockedAdd_nf:
  case clang::AArch64::BI_InterlockedAdd64:
  case clang::AArch64::BI_InterlockedAdd64_acq:
  case clang::AArch64::BI_InterlockedAdd64_rel:
  case clang::AArch64::BI_InterlockedAdd64_nf:
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  cir::VectorType ty = getNeonType(this, type);
  if (!ty)
    return nullptr;
 
  llvm::StringRef intrName;
 
  switch (builtinID) {
  default:
    return std::nullopt;
  case NEON::BI__builtin_neon_vbsl_v:
  case NEON::BI__builtin_neon_vbslq_v: {
 
    cir::VectorType bitTy = getIntVecFromVecTy(builder, ty);
    ops[0] = builder.createBitcast(ops[0], bitTy);
    ops[1] = builder.createBitcast(ops[1], bitTy);
    ops[2] = builder.createBitcast(ops[2], bitTy);
 
    ops[1] = builder.createAnd(loc, ops[0], ops[1]);
    ops[2] = builder.createAnd(loc, builder.createNot(ops[0]), ops[2]);
    ops[0] = builder.createOr(loc, ops[1], ops[2]);
    return builder.createBitcast(ops[0], ty);
  }
  case NEON::BI__builtin_neon_vfma_lane_v:
  case NEON::BI__builtin_neon_vfmaq_lane_v: {
    mlir::Value addend = builder.createBitcast(ops[0], ty);
    mlir::Value multiplicand = builder.createBitcast(ops[1], ty);
    // For vfmaq_lane, the lane source operand is the non-quad vector, so it has
    // half as many lanes as the quad result vector. For vfma_lane, it has the
    // same shape as the result vector.
    cir::VectorType sourceTy = cir::VectorType::getFixed(
        ty.getElementType(), builtinID == NEON::BI__builtin_neon_vfmaq_lane_v
                                 ? ty.getFixedSize() / 2
                                 : ty.getFixedSize());
    mlir::Value laneSource = builder.createBitcast(ops[2], sourceTy);
    laneSource =
        emitNeonSplat(builder, loc, laneSource, ops[3], ty.getFixedSize());
 
    llvm::SmallVector<mlir::Value> fmaOps = {multiplicand, laneSource, addend};
    return emitNeonCallToOp<cir::FMAOp>(cgm, builder, {ty, ty, ty}, fmaOps,
                                        std::nullopt, ty, loc);
  }
  case NEON::BI__builtin_neon_vfma_laneq_v: {
    // v1f64 fma should be mapped to Neon scalar f64 fma.
    if (ty.getElementType() == cgm.doubleTy) {
      mlir::Value addend = builder.createBitcast(ops[0], cgm.doubleTy);
      mlir::Value multiplicand = builder.createBitcast(ops[1], cgm.doubleTy);
      // The laneq source operand is float64x2_t, so the source vector has two
      // double lanes.
      cir::VectorType sourceTy = cir::VectorType::getFixed(cgm.doubleTy, 2);
      mlir::Value laneSource = builder.createBitcast(ops[2], sourceTy);
      laneSource = builder.createExtractElement(
          loc, laneSource,
          static_cast<uint64_t>(getIntValueFromConstOp(ops[3])));
 
      llvm::SmallVector<mlir::Value> fmaOps = {multiplicand, laneSource,
                                               addend};
      return builder.createBitcast(
          emitNeonCallToOp<cir::FMAOp>(
              cgm, builder, {cgm.doubleTy, cgm.doubleTy, cgm.doubleTy}, fmaOps,
              std::nullopt, cgm.doubleTy, loc),
          ty);
    }
 
    mlir::Value addend = builder.createBitcast(ops[0], ty);
    mlir::Value multiplicand = builder.createBitcast(ops[1], ty);
    // The laneq source operand is the quad vector, so it has twice as many
    // lanes as the non-quad result vector.
    cir::VectorType sourceTy =
        cir::VectorType::getFixed(ty.getElementType(), ty.getFixedSize() * 2);
    mlir::Value laneSource = builder.createBitcast(ops[2], sourceTy);
    laneSource =
        emitNeonSplat(builder, loc, laneSource, ops[3], ty.getFixedSize());
 
    llvm::SmallVector<mlir::Value> fmaOps = {laneSource, multiplicand, addend};
    return emitNeonCallToOp<cir::FMAOp>(cgm, builder, {ty, ty, ty}, fmaOps,
                                        std::nullopt, ty, loc);
  }
  case NEON::BI__builtin_neon_vfmaq_laneq_v: {
    mlir::Value addend = builder.createBitcast(ops[0], ty);
    mlir::Value multiplicand = builder.createBitcast(ops[1], ty);
    mlir::Value laneSource = builder.createBitcast(ops[2], ty);
    laneSource =
        emitNeonSplat(builder, loc, laneSource, ops[3], ty.getFixedSize());
 
    llvm::SmallVector<mlir::Value> fmaOps = {laneSource, multiplicand, addend};
    return emitNeonCallToOp<cir::FMAOp>(cgm, builder, {ty, ty, ty}, fmaOps,
                                        std::nullopt, ty, loc);
  }
  case NEON::BI__builtin_neon_vfmah_lane_f16:
  case NEON::BI__builtin_neon_vfmas_lane_f32:
  case NEON::BI__builtin_neon_vfmah_laneq_f16:
  case NEON::BI__builtin_neon_vfmas_laneq_f32: {
    // Scalar lane/laneq forms use one selected element from the lane source.
    mlir::Value laneSource = builder.createExtractElement(
        loc, ops[2], static_cast<uint64_t>(getIntValueFromConstOp(ops[3])));
 
    llvm::SmallVector<mlir::Value> fmaOps = {ops[1], laneSource, ops[0]};
    mlir::Type ty = convertType(expr->getType());
    return emitNeonCallToOp<cir::FMAOp>(cgm, builder, {ty, ty, ty}, fmaOps,
                                        std::nullopt, ty, loc);
  }
  case NEON::BI__builtin_neon_vfmad_lane_f64:
  case NEON::BI__builtin_neon_vfmad_laneq_f64: {
    // The lane source operand is float64x1_t for lane forms and float64x2_t
    // for laneq forms.
    mlir::Value laneSource = builder.createExtractElement(
        loc, ops[2], static_cast<uint64_t>(getIntValueFromConstOp(ops[3])));
 
    llvm::SmallVector<mlir::Value> fmaOps = {ops[1], laneSource, ops[0]};
    return emitNeonCallToOp<cir::FMAOp>(
        cgm, builder, {cgm.doubleTy, cgm.doubleTy, cgm.doubleTy}, fmaOps,
        std::nullopt, cgm.doubleTy, loc);
  }
  case NEON::BI__builtin_neon_vmull_v: {
    intrName = usgn ? "aarch64.neon.umull" : "aarch64.neon.smull";
    if (type.isPoly())
      intrName = "aarch64.neon.pmull";
    cir::VectorType argTy = builder.getExtendedOrTruncatedElementVectorType(
        ty, /*isExtended*/ false, !usgn);
    return emitNeonCall(cgm, builder, {argTy, argTy}, ops, intrName, ty, loc);
  }
  case NEON::BI__builtin_neon_vmax_v:
  case NEON::BI__builtin_neon_vmaxq_v:
    intrName = usgn ? "umax" : "smax";
    if (cir::isFPOrVectorOfFPType(ty))
      intrName = "aarch64.neon.fmax";
    return emitNeonCall(cgm, builder, {ty, ty}, ops, intrName, ty, loc);
  case NEON::BI__builtin_neon_vmaxh_f16: {
    auto halfTy = builder.getFp16Ty();
    return builder.emitIntrinsicCallOp(loc, "aarch64.neon.fmax", halfTy, ops);
  }
  case NEON::BI__builtin_neon_vmin_v:
  case NEON::BI__builtin_neon_vminq_v:
    intrName = usgn ? "umin" : "smin";
    if (cir::isFPOrVectorOfFPType(ty))
      intrName = "aarch64.neon.fmin";
    return emitNeonCall(cgm, builder, {ty, ty}, ops, intrName, ty, loc);
  case NEON::BI__builtin_neon_vminh_f16: {
    auto halfTy = builder.getFp16Ty();
    return builder.emitIntrinsicCallOp(loc, "aarch64.neon.fmin", halfTy, ops);
  }
  case NEON::BI__builtin_neon_vabd_v:
  case NEON::BI__builtin_neon_vabdq_v:
    intrName = usgn ? "aarch64.neon.uabd" : "aarch64.neon.sabd";
    if (cir::isFPOrVectorOfFPType(ty))
      intrName = "aarch64.neon.fabd";
    return emitNeonCall(cgm, builder, {ty, ty}, ops, intrName, ty, loc);
  case NEON::BI__builtin_neon_vpadal_v:
  case NEON::BI__builtin_neon_vpadalq_v: {
    intrName = usgn ? "aarch64.neon.uaddlp" : "aarch64.neon.saddlp";
    llvm::SmallVector<mlir::Value> inputs{ops[1]};
    mlir::Value pairwiseSum =
        emitNeonCall(cgm, builder, {getNeonPairwiseWidenInputType(ty, usgn)},
                     inputs, intrName, ty, loc);
    mlir::Value accumValue = builder.createBitcast(loc, ops[0], ty);
    return cir::AddOp::create(builder, loc, ty, pairwiseSum, accumValue);
  }
  case NEON::BI__builtin_neon_vpmin_v:
  case NEON::BI__builtin_neon_vpminq_v:
    intrName = usgn ? "aarch64.neon.uminp" : "aarch64.neon.sminp";
    if (cir::isFPOrVectorOfFPType(ty))
      intrName = "aarch64.neon.fminp";
    return emitNeonCall(cgm, builder, {ty, ty}, ops, intrName, ty, loc);
  case NEON::BI__builtin_neon_vpmax_v:
  case NEON::BI__builtin_neon_vpmaxq_v:
    intrName = usgn ? "aarch64.neon.umaxp" : "aarch64.neon.smaxp";
    if (cir::isFPOrVectorOfFPType(ty))
      intrName = "aarch64.neon.fmaxp";
    return emitNeonCall(cgm, builder, {ty, ty}, ops, intrName, ty, loc);
  case NEON::BI__builtin_neon_vminnm_v:
  case NEON::BI__builtin_neon_vminnmq_v:
    intrName = "aarch64.neon.fminnm";
    return emitNeonCall(cgm, builder, {ty, ty}, ops, intrName, ty, loc);
  case NEON::BI__builtin_neon_vminnmh_f16: {
    auto halfTy = builder.getFp16Ty();
    return builder.emitIntrinsicCallOp(loc, "aarch64.neon.fminnm", halfTy, ops);
  }
  case NEON::BI__builtin_neon_vmaxnm_v:
  case NEON::BI__builtin_neon_vmaxnmq_v:
    intrName = "aarch64.neon.fmaxnm";
    return emitNeonCall(cgm, builder, {ty, ty}, ops, intrName, ty, loc);
  case NEON::BI__builtin_neon_vmaxnmh_f16: {
    auto halfTy = builder.getFp16Ty();
    return builder.emitIntrinsicCallOp(loc, "aarch64.neon.fmaxnm", halfTy, ops);
  }
  case NEON::BI__builtin_neon_vrecpss_f32:
  case NEON::BI__builtin_neon_vrecpsd_f64:
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  case NEON::BI__builtin_neon_vrecpsh_f16: {
    auto halfTy = builder.getFp16Ty();
    return builder.emitIntrinsicCallOp(loc, "aarch64.neon.frecps", halfTy, ops);
  }
  case NEON::BI__builtin_neon_vqshrun_n_v: {
    cir::VectorType argTy = builder.getExtendedOrTruncatedElementVectorType(
        ty, /*isExtended=*/true, /*isSigned=*/true);
    return emitNeonCall(cgm, builder, {argTy, sInt32Ty}, ops,
                        "aarch64.neon.sqshrun", ty, loc);
  }
  case NEON::BI__builtin_neon_vqrshrun_n_v: {
    cir::VectorType argTy = builder.getExtendedOrTruncatedElementVectorType(
        ty, /*isExtended=*/true, /*isSigned=*/true);
    return emitNeonCall(cgm, builder, {argTy, sInt32Ty}, ops,
                        "aarch64.neon.sqrshrun", ty, loc);
  }
  case NEON::BI__builtin_neon_vqshrn_n_v: {
    cir::VectorType argTy = builder.getExtendedOrTruncatedElementVectorType(
        ty, /*isExtended=*/true, /*isSigned=*/!usgn);
    llvm::StringRef intrName =
        usgn ? "aarch64.neon.uqshrn" : "aarch64.neon.sqshrn";
    return emitNeonCall(cgm, builder, {argTy, sInt32Ty}, ops, intrName, ty,
                        loc);
  }
  case NEON::BI__builtin_neon_vrshrn_n_v:
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  case NEON::BI__builtin_neon_vqrshrn_n_v: {
    cir::VectorType argTy = builder.getExtendedOrTruncatedElementVectorType(
        ty, /*isExtended=*/true, /*isSigned=*/!usgn);
    llvm::StringRef intrName =
        usgn ? "aarch64.neon.uqrshrn" : "aarch64.neon.sqrshrn";
    return emitNeonCall(cgm, builder, {argTy, sInt32Ty}, ops, intrName, ty,
                        loc);
  }
  case NEON::BI__builtin_neon_vrndah_f16: {
    assert(!cir::MissingFeatures::emitConstrainedFPCall());
    mlir::Type halfTy = builder.getFp16Ty();
    return emitNeonCallToOp<cir::RoundOp>(cgm, builder, {halfTy}, ops,
                                          std::nullopt, halfTy, loc);
  }
  case NEON::BI__builtin_neon_vrnda_v:
  case NEON::BI__builtin_neon_vrndaq_v:
    assert(!cir::MissingFeatures::emitConstrainedFPCall());
    return emitNeonCallToOp<cir::RoundOp>(cgm, builder, {ty}, ops, std::nullopt,
                                          ty, loc);
  case NEON::BI__builtin_neon_vrndih_f16: {
    assert(!cir::MissingFeatures::emitConstrainedFPCall());
    mlir::Type halfTy = builder.getFp16Ty();
    return emitNeonCallToOp<cir::NearbyintOp>(cgm, builder, {halfTy}, ops,
                                              std::nullopt, halfTy, loc);
  }
  case NEON::BI__builtin_neon_vrndmh_f16: {
    assert(!cir::MissingFeatures::emitConstrainedFPCall());
    mlir::Type halfTy = builder.getFp16Ty();
    return emitNeonCallToOp<cir::FloorOp>(cgm, builder, {halfTy}, ops,
                                          std::nullopt, halfTy, loc);
  }
  case NEON::BI__builtin_neon_vrndm_v:
  case NEON::BI__builtin_neon_vrndmq_v:
    assert(!cir::MissingFeatures::emitConstrainedFPCall());
    return emitNeonCallToOp<cir::FloorOp>(cgm, builder, {ty}, ops, std::nullopt,
                                          ty, loc);
  case NEON::BI__builtin_neon_vrndnh_f16: {
    assert(!cir::MissingFeatures::emitConstrainedFPCall());
    mlir::Type halfTy = builder.getFp16Ty();
    return emitNeonCallToOp<cir::RoundEvenOp>(cgm, builder, {halfTy}, ops,
                                              std::nullopt, halfTy, loc);
  }
  case NEON::BI__builtin_neon_vrndn_v:
  case NEON::BI__builtin_neon_vrndnq_v:
    assert(!cir::MissingFeatures::emitConstrainedFPCall());
    return emitNeonCallToOp<cir::RoundEvenOp>(cgm, builder, {ty}, ops,
                                              std::nullopt, ty, loc);
  case NEON::BI__builtin_neon_vrndns_f32: {
    assert(!cir::MissingFeatures::emitConstrainedFPCall());
    mlir::Type floatTy = builder.getSingleTy();
    return emitNeonCallToOp<cir::RoundEvenOp>(cgm, builder, {floatTy}, ops,
                                              std::nullopt, floatTy, loc);
  }
  case NEON::BI__builtin_neon_vrndph_f16: {
    assert(!cir::MissingFeatures::emitConstrainedFPCall());
    mlir::Type halfTy = builder.getFp16Ty();
    return emitNeonCallToOp<cir::CeilOp>(cgm, builder, {halfTy}, ops,
                                         std::nullopt, halfTy, loc);
  }
  case NEON::BI__builtin_neon_vrndp_v:
  case NEON::BI__builtin_neon_vrndpq_v:
    assert(!cir::MissingFeatures::emitConstrainedFPCall());
    return emitNeonCallToOp<cir::CeilOp>(cgm, builder, {ty}, ops, std::nullopt,
                                         ty, loc);
  case NEON::BI__builtin_neon_vrndxh_f16: {
    assert(!cir::MissingFeatures::emitConstrainedFPCall());
    mlir::Type halfTy = builder.getFp16Ty();
    return emitNeonCallToOp<cir::RintOp>(cgm, builder, {halfTy}, ops,
                                         std::nullopt, halfTy, loc);
  }
  case NEON::BI__builtin_neon_vrndx_v:
  case NEON::BI__builtin_neon_vrndxq_v:
    assert(!cir::MissingFeatures::emitConstrainedFPCall());
    return emitNeonCallToOp<cir::RintOp>(cgm, builder, {ty}, ops, std::nullopt,
                                         ty, loc);
  case NEON::BI__builtin_neon_vrndh_f16: {
    assert(!cir::MissingFeatures::emitConstrainedFPCall());
    mlir::Type halfTy = builder.getFp16Ty();
    return emitNeonCallToOp<cir::TruncOp>(cgm, builder, {halfTy}, ops,
                                          std::nullopt, halfTy, loc);
  }
  case NEON::BI__builtin_neon_vrnd_v:
  case NEON::BI__builtin_neon_vrndq_v:
    assert(!cir::MissingFeatures::emitConstrainedFPCall());
    return emitNeonCallToOp<cir::TruncOp>(cgm, builder, {ty}, ops, std::nullopt,
                                          ty, loc);
  case NEON::BI__builtin_neon_vcvt_f64_v:
  case NEON::BI__builtin_neon_vcvtq_f64_v:
    ops[0] = builder.createBitcast(ops[0], ty);
    ty = getNeonType(
        this, NeonTypeFlags(NeonTypeFlags::Float64, false, type.isQuad()));
    return builder.createCast(loc, cir::CastKind::int_to_float, ops[0], ty);
  case NEON::BI__builtin_neon_vcvt_f64_f32: {
    auto v2f32 = cir::VectorType::getFixed(cgm.floatTy, 2);
    auto v2f64 = cir::VectorType::getFixed(cgm.doubleTy, 2);
    return builder.createFloatingCast(builder.createBitcast(ops[0], v2f32),
                                      v2f64);
  }
  case NEON::BI__builtin_neon_vcvt_f32_f64: {
    auto v2f32 = cir::VectorType::getFixed(cgm.floatTy, 2);
    auto v2f64 = cir::VectorType::getFixed(cgm.doubleTy, 2);
    return builder.createFloatingCast(builder.createBitcast(ops[0], v2f64),
                                      v2f32);
  }
  case NEON::BI__builtin_neon_vcvt_s32_v:
  case NEON::BI__builtin_neon_vcvt_u32_v:
  case NEON::BI__builtin_neon_vcvt_s64_v:
  case NEON::BI__builtin_neon_vcvt_u64_v:
  case NEON::BI__builtin_neon_vcvt_s16_f16:
  case NEON::BI__builtin_neon_vcvt_u16_f16:
  case NEON::BI__builtin_neon_vcvtq_s32_v:
  case NEON::BI__builtin_neon_vcvtq_u32_v:
  case NEON::BI__builtin_neon_vcvtq_s64_v:
  case NEON::BI__builtin_neon_vcvtq_u64_v:
  case NEON::BI__builtin_neon_vcvtq_s16_f16:
  case NEON::BI__builtin_neon_vcvtq_u16_f16:
 
  case NEON::BI__builtin_neon_vcvta_s16_f16:
  case NEON::BI__builtin_neon_vcvta_u16_f16:
  case NEON::BI__builtin_neon_vcvtaq_s16_f16:
  case NEON::BI__builtin_neon_vcvtaq_u16_f16:
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  case NEON::BI__builtin_neon_vcvta_s32_v:
  case NEON::BI__builtin_neon_vcvtaq_s32_v:
  case NEON::BI__builtin_neon_vcvta_u32_v:
  case NEON::BI__builtin_neon_vcvtaq_u32_v:
  case NEON::BI__builtin_neon_vcvta_s64_v:
  case NEON::BI__builtin_neon_vcvtaq_s64_v:
  case NEON::BI__builtin_neon_vcvta_u64_v:
  case NEON::BI__builtin_neon_vcvtaq_u64_v: {
    auto argTy = getFloatNeonType(*this, type);
    llvm::StringRef intrName =
        usgn ? "aarch64.neon.fcvtau" : "aarch64.neon.fcvtas";
    return emitNeonCall(cgm, builder, {argTy}, ops, intrName, ty, loc);
  }
 
  case NEON::BI__builtin_neon_vcvtm_s16_f16:
  case NEON::BI__builtin_neon_vcvtmq_s16_f16:
  case NEON::BI__builtin_neon_vcvtm_u16_f16:
  case NEON::BI__builtin_neon_vcvtmq_u16_f16:
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  case NEON::BI__builtin_neon_vcvtm_s32_v:
  case NEON::BI__builtin_neon_vcvtmq_s32_v:
  case NEON::BI__builtin_neon_vcvtm_u32_v:
  case NEON::BI__builtin_neon_vcvtmq_u32_v:
  case NEON::BI__builtin_neon_vcvtm_s64_v:
  case NEON::BI__builtin_neon_vcvtmq_s64_v:
  case NEON::BI__builtin_neon_vcvtm_u64_v:
  case NEON::BI__builtin_neon_vcvtmq_u64_v: {
    auto argTy = getFloatNeonType(*this, type);
    llvm::StringRef intrName =
        usgn ? "aarch64.neon.fcvtmu" : "aarch64.neon.fcvtms";
    return emitNeonCall(cgm, builder, {argTy}, ops, intrName, ty, loc);
  }
 
  case NEON::BI__builtin_neon_vcvtn_s16_f16:
  case NEON::BI__builtin_neon_vcvtnq_s16_f16:
  case NEON::BI__builtin_neon_vcvtn_u16_f16:
  case NEON::BI__builtin_neon_vcvtnq_u16_f16:
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
 
  case NEON::BI__builtin_neon_vcvtn_s32_v:
  case NEON::BI__builtin_neon_vcvtnq_s32_v:
  case NEON::BI__builtin_neon_vcvtn_u32_v:
  case NEON::BI__builtin_neon_vcvtnq_u32_v:
  case NEON::BI__builtin_neon_vcvtn_s64_v:
  case NEON::BI__builtin_neon_vcvtnq_s64_v:
  case NEON::BI__builtin_neon_vcvtn_u64_v:
  case NEON::BI__builtin_neon_vcvtnq_u64_v: {
    auto argTy = getFloatNeonType(*this, type);
    llvm::StringRef intrName =
        usgn ? "aarch64.neon.fcvtnu" : "aarch64.neon.fcvtns";
    return emitNeonCall(cgm, builder, {argTy}, ops, intrName, ty, loc);
  }
 
  case NEON::BI__builtin_neon_vcvtp_s16_f16:
  case NEON::BI__builtin_neon_vcvtpq_s16_f16:
  case NEON::BI__builtin_neon_vcvtp_u16_f16:
  case NEON::BI__builtin_neon_vcvtpq_u16_f16:
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  case NEON::BI__builtin_neon_vcvtp_s32_v:
  case NEON::BI__builtin_neon_vcvtpq_s32_v:
  case NEON::BI__builtin_neon_vcvtp_u32_v:
  case NEON::BI__builtin_neon_vcvtpq_u32_v:
  case NEON::BI__builtin_neon_vcvtp_s64_v:
  case NEON::BI__builtin_neon_vcvtpq_s64_v:
  case NEON::BI__builtin_neon_vcvtp_u64_v:
  case NEON::BI__builtin_neon_vcvtpq_u64_v: {
    auto argTy = getFloatNeonType(*this, type);
    llvm::StringRef intrName =
        usgn ? "aarch64.neon.fcvtpu" : "aarch64.neon.fcvtps";
    return emitNeonCall(cgm, builder, {argTy}, ops, intrName, ty, loc);
  }
 
  case NEON::BI__builtin_neon_vmulx_v:
  case NEON::BI__builtin_neon_vmulxq_v:
  case NEON::BI__builtin_neon_vmulxh_lane_f16:
  case NEON::BI__builtin_neon_vmulxh_laneq_f16:
  case NEON::BI__builtin_neon_vmul_lane_v:
  case NEON::BI__builtin_neon_vmul_laneq_v:
  case NEON::BI__builtin_neon_vpmaxnm_v:
  case NEON::BI__builtin_neon_vpmaxnmq_v:
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  case NEON::BI__builtin_neon_vpminnm_v:
  case NEON::BI__builtin_neon_vpminnmq_v:
    intrName = "aarch64.neon.fminnmp";
    return emitNeonCall(cgm, builder, {ty, ty}, ops, intrName, ty, loc);
  case NEON::BI__builtin_neon_vsqrth_f16: {
    mlir::Type halfTy = builder.getFp16Ty();
    return emitNeonCallToOp<cir::SqrtOp>(cgm, builder, {halfTy}, ops,
                                         std::nullopt, halfTy, loc);
  }
  case NEON::BI__builtin_neon_vsqrt_v:
  case NEON::BI__builtin_neon_vsqrtq_v:
    return emitNeonCallToOp<cir::SqrtOp>(cgm, builder, {ty}, ops, std::nullopt,
                                         ty, loc);
  case NEON::BI__builtin_neon_vrbit_v:
  case NEON::BI__builtin_neon_vrbitq_v:
  case NEON::BI__builtin_neon_vmaxv_f16:
  case NEON::BI__builtin_neon_vmaxvq_f16:
  case NEON::BI__builtin_neon_vminv_f16:
  case NEON::BI__builtin_neon_vminvq_f16:
  case NEON::BI__builtin_neon_vmaxnmv_f16:
  case NEON::BI__builtin_neon_vmaxnmvq_f16:
  case NEON::BI__builtin_neon_vminnmv_f16:
  case NEON::BI__builtin_neon_vminnmvq_f16:
  case NEON::BI__builtin_neon_vmul_n_f64:
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  case NEON::BI__builtin_neon_vaddlv_u8:
  case NEON::BI__builtin_neon_vaddlvq_u8:
  case NEON::BI__builtin_neon_vaddlv_u16:
  case NEON::BI__builtin_neon_vaddlvq_u16:
  case NEON::BI__builtin_neon_vaddlv_s8:
  case NEON::BI__builtin_neon_vaddlvq_s8:
  case NEON::BI__builtin_neon_vaddlv_s16:
  case NEON::BI__builtin_neon_vaddlvq_s16: {
    mlir::Type argTy = convertType(expr->getArg(0)->getType());
    mlir::Type userRetTy = convertType(expr->getType());
    auto eltTy = mlir::cast<cir::IntType>(
        mlir::cast<cir::VectorType>(argTy).getElementType());
    bool isUnsigned = !eltTy.isSigned();
    // These builtins only use 8 and 16-bit element vectors; the intrinsic
    // always produces i32. The C result is i32 for 16-bit elements, but i16
    // for 8-bit elements, so we emit at i32 and narrow only in that case.
    bool needsTrunc = eltTy.getWidth() == 8;
    intrName = isUnsigned ? "aarch64.neon.uaddlv" : "aarch64.neon.saddlv";
    mlir::Type intrRetTy = userRetTy;
    if (needsTrunc)
      intrRetTy = isUnsigned ? builder.getUInt32Ty() : builder.getSInt32Ty();
    mlir::Value result =
        emitNeonCall(cgm, builder, {argTy}, ops, intrName, intrRetTy, loc);
    if (needsTrunc)
      result = builder.createIntCast(result, userRetTy);
    return result;
  }
  case NEON::BI__builtin_neon_vsri_n_v:
  case NEON::BI__builtin_neon_vsriq_n_v: {
    llvm::SmallVector<mlir::Value, 4> vsriArgs = {
        ops[0], ops[1], builder.createIntCast(ops[2], builder.getUInt32Ty())};
    return emitNeonCall(cgm, builder, {ty, ty, builder.getUInt32Ty()}, vsriArgs,
                        "aarch64.neon.vsri", ty, loc);
  }
  case NEON::BI__builtin_neon_vsli_n_v:
  case NEON::BI__builtin_neon_vsliq_n_v: {
 
    intrName = "aarch64.neon.vsli";
 
    llvm::SmallVector<mlir::Type> argTypes = {ty, ty, ops[2].getType()};
 
    return emitNeonCall(cgm, builder, argTypes, ops, intrName, ty, loc,
                        /*isConstrainedFPIntrinsic=*/false,
                        /*shift=*/0, /*rightshift=*/false);
  }
  case NEON::BI__builtin_neon_vsra_n_v:
  case NEON::BI__builtin_neon_vsraq_n_v: {
    ops[0] = builder.createBitcast(ops[0], ty);
    ops[1] = emitNeonRShiftImm(*this, ops[1], ops[2], ty, usgn, loc);
    return builder.createAdd(loc, ops[0], ops[1]);
  }
  case NEON::BI__builtin_neon_vrsra_n_v:
  case NEON::BI__builtin_neon_vrsraq_n_v: {
    intrName = usgn ? "aarch64.neon.urshl" : "aarch64.neon.srshl";
    // The llvm intrinsic is expecting negative shift amount for right shift.
    // Thus we have to make shift amount vec type to be signed.
    cir::VectorType shiftAmtVecTy =
        usgn ? getSignChangedVectorType(builder, ty) : ty;
    llvm::SmallVector<mlir::Value, 2> tmpOps = {ops[1], ops[2]};
    mlir::Value tmp = emitNeonCall(cgm, builder, {ty, shiftAmtVecTy}, tmpOps,
                                   intrName, ty, loc,
                                   /*isConstrainedFPIntrinsic=*/false,
                                   /*shift=*/1, /*rightshift=*/true);
    ops[0] = builder.createBitcast(ops[0], ty);
    return builder.createAdd(loc, ops[0], tmp);
  }
  case NEON::BI__builtin_neon_vld1_v:
  case NEON::BI__builtin_neon_vld1q_v:
    return builder.createLoad(loc, ptrOp0.withElementType(builder, ty));
  case NEON::BI__builtin_neon_vst1_v:
  case NEON::BI__builtin_neon_vst1q_v: {
    ops[1] = builder.createBitcast(ops[1], ty);
    builder.createStore(loc, ops[1], ptrOp0.withElementType(builder, ty));
    return nullptr;
  }
  case NEON::BI__builtin_neon_vld1_lane_v:
  case NEON::BI__builtin_neon_vld1q_lane_v: {
    ops[1] = builder.createBitcast(ops[1], ty);
    mlir::Value scalar = builder.createLoad(
        loc, ptrOp0.withElementType(builder, ty.getElementType()));
    return cir::VecInsertOp::create(builder, loc, ops[1], scalar, ops[2]);
  }
  case NEON::BI__builtin_neon_vld1_dup_v:
  case NEON::BI__builtin_neon_vld1q_dup_v: {
    mlir::Value scalar = builder.createLoad(
        loc, ptrOp0.withElementType(builder, ty.getElementType()));
    return cir::VecSplatOp::create(builder, loc, ty, scalar);
  }
  case NEON::BI__builtin_neon_vldap1_lane_s64:
  case NEON::BI__builtin_neon_vldap1q_lane_s64:
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  case NEON::BI__builtin_neon_vst1_lane_v:
  case NEON::BI__builtin_neon_vst1q_lane_v: {
    ops[1] = builder.createBitcast(ops[1], ty);
    mlir::Value scalar = builder.createExtractElement(
        loc, ops[1], static_cast<uint64_t>(getIntValueFromConstOp(ops[2])));
    builder.createStore(loc, scalar,
                        ptrOp0.withElementType(builder, scalar.getType()));
    return nullptr;
  }
  case NEON::BI__builtin_neon_vstl1_lane_s64:
  case NEON::BI__builtin_neon_vstl1q_lane_s64:
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  case NEON::BI__builtin_neon_vld2_v:
  case NEON::BI__builtin_neon_vld2q_v:
    return emitNeonMultiVecLoad(*this, ops, llvm::Intrinsic::aarch64_neon_ld2,
                                ty, 2, loc);
  case NEON::BI__builtin_neon_vld3_v:
  case NEON::BI__builtin_neon_vld3q_v:
    return emitNeonMultiVecLoad(*this, ops, llvm::Intrinsic::aarch64_neon_ld3,
                                ty, 3, loc);
  case NEON::BI__builtin_neon_vld4_v:
  case NEON::BI__builtin_neon_vld4q_v:
    return emitNeonMultiVecLoad(*this, ops, llvm::Intrinsic::aarch64_neon_ld4,
                                ty, 4, loc);
  case NEON::BI__builtin_neon_vld2_dup_v:
  case NEON::BI__builtin_neon_vld2q_dup_v:
    return emitNeonMultiVecLoad(*this, ops, llvm::Intrinsic::aarch64_neon_ld2r,
                                ty, 2, loc);
  case NEON::BI__builtin_neon_vld3_dup_v:
  case NEON::BI__builtin_neon_vld3q_dup_v:
    return emitNeonMultiVecLoad(*this, ops, llvm::Intrinsic::aarch64_neon_ld3r,
                                ty, 3, loc);
  case NEON::BI__builtin_neon_vld4_dup_v:
  case NEON::BI__builtin_neon_vld4q_dup_v:
    return emitNeonMultiVecLoad(*this, ops, llvm::Intrinsic::aarch64_neon_ld4r,
                                ty, 4, loc);
  case NEON::BI__builtin_neon_vld2_lane_v:
  case NEON::BI__builtin_neon_vld2q_lane_v:
    return emitNeonMultiVecLoadLane(
        *this, ops, llvm::Intrinsic::aarch64_neon_ld2lane, ty, 2, loc);
  case NEON::BI__builtin_neon_vld3_lane_v:
  case NEON::BI__builtin_neon_vld3q_lane_v:
    return emitNeonMultiVecLoadLane(
        *this, ops, llvm::Intrinsic::aarch64_neon_ld3lane, ty, 3, loc);
  case NEON::BI__builtin_neon_vld4_lane_v:
  case NEON::BI__builtin_neon_vld4q_lane_v:
    return emitNeonMultiVecLoadLane(
        *this, ops, llvm::Intrinsic::aarch64_neon_ld4lane, ty, 4, loc);
  case NEON::BI__builtin_neon_vst2_v:
  case NEON::BI__builtin_neon_vst2q_v: {
    // The builtin call has the pointer first, but the AArch64 st2 intrinsic
    // takes the vector operands first and the pointer last.
    std::rotate(ops.begin(), ops.begin() + 1, ops.end());
    llvm::SmallVector<mlir::Type> argTypes = {ty, ty, builder.getVoidPtrTy()};
    return emitNeonCall(cgm, builder, argTypes, ops, "aarch64.neon.st2",
                        cgm.voidTy, loc);
  }
  case NEON::BI__builtin_neon_vst2_lane_v:
  case NEON::BI__builtin_neon_vst2q_lane_v: {
    std::rotate(ops.begin(), ops.begin() + 1, ops.end());
    ops[2] = builder.createIntCast(ops[2], sInt64Ty);
    llvm::SmallVector<mlir::Type> argTypes = {ty, ty, sInt64Ty,
                                              builder.getVoidPtrTy()};
    return emitNeonCall(cgm, builder, argTypes, ops, "aarch64.neon.st2lane",
                        cgm.voidTy, loc);
  }
  case NEON::BI__builtin_neon_vst3_v:
  case NEON::BI__builtin_neon_vst3q_v: {
    std::rotate(ops.begin(), ops.begin() + 1, ops.end());
    llvm::SmallVector<mlir::Type> argTypes = {ty, ty, ty,
                                              builder.getVoidPtrTy()};
    return emitNeonCall(cgm, builder, argTypes, ops, "aarch64.neon.st3",
                        cgm.voidTy, loc);
  }
  case NEON::BI__builtin_neon_vst3_lane_v:
  case NEON::BI__builtin_neon_vst3q_lane_v: {
    std::rotate(ops.begin(), ops.begin() + 1, ops.end());
    ops[3] = builder.createIntCast(ops[3], sInt64Ty);
    llvm::SmallVector<mlir::Type> argTypes = {ty, ty, ty, sInt64Ty,
                                              builder.getVoidPtrTy()};
    return emitNeonCall(cgm, builder, argTypes, ops, "aarch64.neon.st3lane",
                        cgm.voidTy, loc);
  }
  case NEON::BI__builtin_neon_vst4_v:
  case NEON::BI__builtin_neon_vst4q_v: {
    std::rotate(ops.begin(), ops.begin() + 1, ops.end());
    llvm::SmallVector<mlir::Type> argTypes = {ty, ty, ty, ty,
                                              builder.getVoidPtrTy()};
    return emitNeonCall(cgm, builder, argTypes, ops, "aarch64.neon.st4",
                        cgm.voidTy, loc);
  }
  case NEON::BI__builtin_neon_vst4_lane_v:
  case NEON::BI__builtin_neon_vst4q_lane_v: {
    std::rotate(ops.begin(), ops.begin() + 1, ops.end());
    ops[4] = builder.createIntCast(ops[4], sInt64Ty);
    llvm::SmallVector<mlir::Type> argTypes = {
        ty, ty, ty, ty, sInt64Ty, builder.getVoidPtrTy()};
    return emitNeonCall(cgm, builder, argTypes, ops, "aarch64.neon.st4lane",
                        cgm.voidTy, loc);
  }
  case NEON::BI__builtin_neon_vtrn_v:
  case NEON::BI__builtin_neon_vtrnq_v: {
    ops[1] = builder.createBitcast(ops[1], ty);
    ops[2] = builder.createBitcast(ops[2], ty);
    // Adding a bitcast here as Ops[0] might be a void pointer.
    mlir::Value baseAddr =
        builder.createBitcast(ops[0], builder.getPointerTo(ty));
    mlir::Value sv;
 
    for (unsigned vi = 0; vi != 2; ++vi) {
      llvm::SmallVector<int64_t, 16> indices;
      for (unsigned i = 0, e = ty.getFixedSize(); i != e; i += 2) {
        indices.push_back(i + vi);
        indices.push_back(i + e + vi);
      }
      cir::ConstantOp idx = builder.getConstInt(loc, builder.getSInt32Ty(), vi);
      mlir::Value addr = builder.createPtrStride(loc, baseAddr, idx);
      sv = builder.createVecShuffle(loc, ops[1], ops[2], indices);
      (void)builder.CIRBaseBuilderTy::createStore(loc, sv, addr);
    }
    return sv;
  }
  case NEON::BI__builtin_neon_vuzp_v:
  case NEON::BI__builtin_neon_vuzpq_v: {
    ops[1] = builder.createBitcast(ops[1], ty);
    ops[2] = builder.createBitcast(ops[2], ty);
    // Adding a bitcast here as Ops[0] might be a void pointer.
    mlir::Value baseAddr =
        builder.createBitcast(ops[0], builder.getPointerTo(ty));
    mlir::Value sv;
    for (unsigned vi = 0; vi != 2; ++vi) {
      llvm::SmallVector<int64_t, 16> indices;
      for (unsigned i = 0, e = ty.getFixedSize(); i != e; ++i) {
        indices.push_back(2 * i + vi);
      }
      cir::ConstantOp idx = builder.getConstInt(loc, builder.getSInt32Ty(), vi);
      mlir::Value addr = builder.createPtrStride(loc, baseAddr, idx);
      sv = builder.createVecShuffle(loc, ops[1], ops[2], indices);
      (void)builder.CIRBaseBuilderTy::createStore(loc, sv, addr);
    }
    return sv;
  }
  case NEON::BI__builtin_neon_vzip_v:
  case NEON::BI__builtin_neon_vzipq_v: {
    ops[1] = builder.createBitcast(ops[1], ty);
    ops[2] = builder.createBitcast(ops[2], ty);
    // Adding a bitcast here as Ops[0] might be a void pointer.
    mlir::Value baseAddr =
        builder.createBitcast(ops[0], builder.getPointerTo(ty));
    mlir::Value sv;
    for (unsigned vi = 0; vi != 2; ++vi) {
      llvm::SmallVector<int64_t, 16> indices;
      for (unsigned i = 0, e = ty.getFixedSize(); i != e; i += 2) {
        indices.push_back((i + vi * e) >> 1);
        indices.push_back(((i + vi * e) >> 1) + e);
      }
      cir::ConstantOp idx = builder.getConstInt(loc, builder.getSInt32Ty(), vi);
      mlir::Value addr = builder.createPtrStride(loc, baseAddr, idx);
      sv = builder.createVecShuffle(loc, ops[1], ops[2], indices);
      (void)builder.CIRBaseBuilderTy::createStore(loc, sv, addr);
    }
    return sv;
  }
  case NEON::BI__builtin_neon_vqtbl1q_v:
  case NEON::BI__builtin_neon_vqtbl2q_v:
  case NEON::BI__builtin_neon_vqtbl3q_v:
  case NEON::BI__builtin_neon_vqtbl4q_v:
  case NEON::BI__builtin_neon_vqtbx1q_v:
  case NEON::BI__builtin_neon_vqtbx2q_v:
  case NEON::BI__builtin_neon_vqtbx3q_v:
  case NEON::BI__builtin_neon_vqtbx4q_v:
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  case NEON::BI__builtin_neon_vsqadd_v:
  case NEON::BI__builtin_neon_vsqaddq_v:
    return emitNeonCall(cgm, builder, {ty, ty}, ops, "aarch64.neon.usqadd", ty,
                        loc);
  case NEON::BI__builtin_neon_vuqadd_v:
  case NEON::BI__builtin_neon_vuqaddq_v:
    return emitNeonCall(cgm, builder, {ty, ty}, ops, "aarch64.neon.suqadd", ty,
                        loc);
  case NEON::BI__builtin_neon_vluti2_laneq_mf8:
  case NEON::BI__builtin_neon_vluti2_laneq_bf16:
  case NEON::BI__builtin_neon_vluti2_laneq_f16:
  case NEON::BI__builtin_neon_vluti2_laneq_p16:
  case NEON::BI__builtin_neon_vluti2_laneq_p8:
  case NEON::BI__builtin_neon_vluti2_laneq_s16:
  case NEON::BI__builtin_neon_vluti2_laneq_s8:
  case NEON::BI__builtin_neon_vluti2_laneq_u16:
  case NEON::BI__builtin_neon_vluti2_laneq_u8:
  case NEON::BI__builtin_neon_vluti2q_laneq_mf8:
  case NEON::BI__builtin_neon_vluti2q_laneq_bf16:
  case NEON::BI__builtin_neon_vluti2q_laneq_f16:
  case NEON::BI__builtin_neon_vluti2q_laneq_p16:
  case NEON::BI__builtin_neon_vluti2q_laneq_p8:
  case NEON::BI__builtin_neon_vluti2q_laneq_s16:
  case NEON::BI__builtin_neon_vluti2q_laneq_s8:
  case NEON::BI__builtin_neon_vluti2q_laneq_u16:
  case NEON::BI__builtin_neon_vluti2q_laneq_u8:
  case NEON::BI__builtin_neon_vluti2_lane_mf8:
  case NEON::BI__builtin_neon_vluti2_lane_bf16:
  case NEON::BI__builtin_neon_vluti2_lane_f16:
  case NEON::BI__builtin_neon_vluti2_lane_p16:
  case NEON::BI__builtin_neon_vluti2_lane_p8:
  case NEON::BI__builtin_neon_vluti2_lane_s16:
  case NEON::BI__builtin_neon_vluti2_lane_s8:
  case NEON::BI__builtin_neon_vluti2_lane_u16:
  case NEON::BI__builtin_neon_vluti2_lane_u8:
  case NEON::BI__builtin_neon_vluti2q_lane_mf8:
  case NEON::BI__builtin_neon_vluti2q_lane_bf16:
  case NEON::BI__builtin_neon_vluti2q_lane_f16:
  case NEON::BI__builtin_neon_vluti2q_lane_p16:
  case NEON::BI__builtin_neon_vluti2q_lane_p8:
  case NEON::BI__builtin_neon_vluti2q_lane_s16:
  case NEON::BI__builtin_neon_vluti2q_lane_s8:
  case NEON::BI__builtin_neon_vluti2q_lane_u16:
  case NEON::BI__builtin_neon_vluti2q_lane_u8:
  case NEON::BI__builtin_neon_vluti4q_lane_mf8:
  case NEON::BI__builtin_neon_vluti4q_lane_p8:
  case NEON::BI__builtin_neon_vluti4q_lane_s8:
  case NEON::BI__builtin_neon_vluti4q_lane_u8:
  case NEON::BI__builtin_neon_vluti4q_laneq_mf8:
  case NEON::BI__builtin_neon_vluti4q_laneq_p8:
  case NEON::BI__builtin_neon_vluti4q_laneq_s8:
  case NEON::BI__builtin_neon_vluti4q_laneq_u8:
  case NEON::BI__builtin_neon_vluti4q_lane_bf16_x2:
  case NEON::BI__builtin_neon_vluti4q_lane_f16_x2:
  case NEON::BI__builtin_neon_vluti4q_lane_p16_x2:
  case NEON::BI__builtin_neon_vluti4q_lane_s16_x2:
  case NEON::BI__builtin_neon_vluti4q_lane_u16_x2:
  case NEON::BI__builtin_neon_vluti4q_laneq_bf16_x2:
  case NEON::BI__builtin_neon_vluti4q_laneq_f16_x2:
  case NEON::BI__builtin_neon_vluti4q_laneq_p16_x2:
  case NEON::BI__builtin_neon_vluti4q_laneq_s16_x2:
  case NEON::BI__builtin_neon_vluti4q_laneq_u16_x2:
  case NEON::BI__builtin_neon_vmmlaq_f16_mf8_fpm:
  case NEON::BI__builtin_neon_vmmlaq_f32_mf8_fpm:
  case NEON::BI__builtin_neon_vcvt1_low_bf16_mf8_fpm:
  case NEON::BI__builtin_neon_vcvt1_bf16_mf8_fpm:
  case NEON::BI__builtin_neon_vcvt1_high_bf16_mf8_fpm:
  case NEON::BI__builtin_neon_vcvt2_low_bf16_mf8_fpm:
  case NEON::BI__builtin_neon_vcvt2_bf16_mf8_fpm:
  case NEON::BI__builtin_neon_vcvt2_high_bf16_mf8_fpm:
  case NEON::BI__builtin_neon_vcvt1_low_f16_mf8_fpm:
  case NEON::BI__builtin_neon_vcvt1_f16_mf8_fpm:
  case NEON::BI__builtin_neon_vcvt1_high_f16_mf8_fpm:
  case NEON::BI__builtin_neon_vcvt2_low_f16_mf8_fpm:
  case NEON::BI__builtin_neon_vcvt2_f16_mf8_fpm:
  case NEON::BI__builtin_neon_vcvt2_high_f16_mf8_fpm:
  case NEON::BI__builtin_neon_vcvt_mf8_f32_fpm:
  case NEON::BI__builtin_neon_vcvt_mf8_f16_fpm:
  case NEON::BI__builtin_neon_vcvtq_mf8_f16_fpm:
  case NEON::BI__builtin_neon_vcvt_high_mf8_f32_fpm:
  case NEON::BI__builtin_neon_vdot_f16_mf8_fpm:
  case NEON::BI__builtin_neon_vdotq_f16_mf8_fpm:
  case NEON::BI__builtin_neon_vdot_lane_f16_mf8_fpm:
  case NEON::BI__builtin_neon_vdotq_lane_f16_mf8_fpm:
  case NEON::BI__builtin_neon_vdot_laneq_f16_mf8_fpm:
  case NEON::BI__builtin_neon_vdotq_laneq_f16_mf8_fpm:
  case NEON::BI__builtin_neon_vdot_f32_mf8_fpm:
  case NEON::BI__builtin_neon_vdotq_f32_mf8_fpm:
  case NEON::BI__builtin_neon_vdot_lane_f32_mf8_fpm:
  case NEON::BI__builtin_neon_vdotq_lane_f32_mf8_fpm:
  case NEON::BI__builtin_neon_vdot_laneq_f32_mf8_fpm:
  case NEON::BI__builtin_neon_vdotq_laneq_f32_mf8_fpm:
  case NEON::BI__builtin_neon_vmlalbq_f16_mf8_fpm:
  case NEON::BI__builtin_neon_vmlaltq_f16_mf8_fpm:
  case NEON::BI__builtin_neon_vmlallbbq_f32_mf8_fpm:
  case NEON::BI__builtin_neon_vmlallbtq_f32_mf8_fpm:
  case NEON::BI__builtin_neon_vmlalltbq_f32_mf8_fpm:
  case NEON::BI__builtin_neon_vmlallttq_f32_mf8_fpm:
  case NEON::BI__builtin_neon_vmlalbq_lane_f16_mf8_fpm:
  case NEON::BI__builtin_neon_vmlalbq_laneq_f16_mf8_fpm:
  case NEON::BI__builtin_neon_vmlaltq_lane_f16_mf8_fpm:
  case NEON::BI__builtin_neon_vmlaltq_laneq_f16_mf8_fpm:
  case NEON::BI__builtin_neon_vmlallbbq_lane_f32_mf8_fpm:
  case NEON::BI__builtin_neon_vmlallbbq_laneq_f32_mf8_fpm:
  case NEON::BI__builtin_neon_vmlallbtq_lane_f32_mf8_fpm:
  case NEON::BI__builtin_neon_vmlallbtq_laneq_f32_mf8_fpm:
  case NEON::BI__builtin_neon_vmlalltbq_lane_f32_mf8_fpm:
  case NEON::BI__builtin_neon_vmlalltbq_laneq_f32_mf8_fpm:
  case NEON::BI__builtin_neon_vmlallttq_lane_f32_mf8_fpm:
  case NEON::BI__builtin_neon_vmlallttq_laneq_f32_mf8_fpm:
  case NEON::BI__builtin_neon_vamin_f16:
  case NEON::BI__builtin_neon_vaminq_f16:
  case NEON::BI__builtin_neon_vamin_f32:
  case NEON::BI__builtin_neon_vaminq_f32:
  case NEON::BI__builtin_neon_vaminq_f64:
  case NEON::BI__builtin_neon_vamax_f16:
  case NEON::BI__builtin_neon_vamaxq_f16:
  case NEON::BI__builtin_neon_vamax_f32:
  case NEON::BI__builtin_neon_vamaxq_f32:
  case NEON::BI__builtin_neon_vamaxq_f64:
  case NEON::BI__builtin_neon_vscale_f16:
  case NEON::BI__builtin_neon_vscaleq_f16:
  case NEON::BI__builtin_neon_vscale_f32:
  case NEON::BI__builtin_neon_vscaleq_f32:
  case NEON::BI__builtin_neon_vscaleq_f64:
    cgm.errorNYI(expr->getSourceRange(),
                 std::string("unimplemented AArch64 builtin call: ") +
                     getContext().BuiltinInfo.getName(builtinID));
    return mlir::Value{};
  }
 
  // Unreachable: All cases in the switch above return.
}