CIRGenBuiltin.cpp

Go to the documentation of this file.

//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This contains code to emit Builtin calls as CIR or a function call to be
// later resolved.
//
//===----------------------------------------------------------------------===//
 
#include "CIRGenCall.h"
#include "CIRGenFunction.h"
#include "CIRGenModule.h"
#include "CIRGenValue.h"
#include "mlir/IR/BuiltinAttributes.h"
#include "mlir/IR/Value.h"
#include "mlir/Support/LLVM.h"
#include "clang/AST/DeclBase.h"
#include "clang/AST/Expr.h"
#include "clang/AST/GlobalDecl.h"
#include "clang/Basic/Builtins.h"
#include "clang/Basic/DiagnosticFrontend.h"
#include "clang/Basic/OperatorKinds.h"
#include "clang/CIR/Dialect/IR/CIRTypes.h"
#include "clang/CIR/MissingFeatures.h"
#include "clang/CodeGenUtils/CodeGenUtils.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/Support/ErrorHandling.h"
 
using namespace clang;
using namespace clang::CIRGen;
using namespace llvm;
 
static bool shouldEmitBuiltinAsIR(unsigned builtinID,
                                  const Builtin::Context &bi,
                                  const CIRGenFunction &cgf) {
  if (!cgf.cgm.getLangOpts().MathErrno &&
      cgf.curFPFeatures.getExceptionMode() ==
          LangOptions::FPExceptionModeKind::FPE_Ignore &&
      !cgf.cgm.getTargetCIRGenInfo().supportsLibCall()) {
    switch (builtinID) {
    default:
      return false;
    case Builtin::BIlogbf:
    case Builtin::BI__builtin_logbf:
    case Builtin::BIlogb:
    case Builtin::BI__builtin_logb:
    case Builtin::BIscalbnf:
    case Builtin::BI__builtin_scalbnf:
    case Builtin::BIscalbn:
    case Builtin::BI__builtin_scalbn:
      return true;
    }
  }
  return false;
}
 
static RValue emitLibraryCall(CIRGenFunction &cgf, const FunctionDecl *fd,
                              const CallExpr *e, mlir::Operation *calleeValue) {
  CIRGenCallee callee = CIRGenCallee::forDirect(calleeValue, GlobalDecl(fd));
  return cgf.emitCall(e->getCallee()->getType(), callee, e, ReturnValueSlot());
}
 
template <typename Op, typename... Args>
static mlir::Value createBuiltinBitOp(CIRGenFunction &cgf, const CallExpr *e,
                                      mlir::Value arg, Args... args) {
  CIRGenBuilderTy &builder = cgf.getBuilder();
  mlir::Location loc = cgf.getLoc(e->getSourceRange());
  auto op = Op::create(builder, loc, arg, args...);
  mlir::Value result = op.getResult();
  mlir::Type resultTy = cgf.convertType(e->getType());
  if (resultTy != result.getType())
    result = builder.createIntCast(result, resultTy);
  return result;
}
 
template <typename Op, typename... Args>
static RValue emitBuiltinBitOp(CIRGenFunction &cgf, const CallExpr *e,
                               Args... args) {
  mlir::Value arg = cgf.emitScalarExpr(e->getArg(0));
  return RValue::get(createBuiltinBitOp<Op>(cgf, e, arg, args...));
}
 
/// Emit a clz/ctz bit op with optional fallback for __builtin_c[lt]zg.
/// When a fallback is present, the result is the fallback value if the input is
/// zero, otherwise the bit count.
template <typename Op>
static RValue emitBuiltinBitOpWithFallback(CIRGenFunction &cgf,
                                           const CallExpr *e) {
  bool hasFallback = e->getNumArgs() > 1;
  bool poisonZero = hasFallback || cgf.getTarget().isCLZForZeroUndef();
 
  if (!hasFallback) {
    assert(!cir::MissingFeatures::builtinCheckKind());
    return emitBuiltinBitOp<Op>(cgf, e, poisonZero);
  }
 
  assert(!cir::MissingFeatures::builtinBitCountExpr());
  mlir::Value arg = cgf.emitScalarExpr(e->getArg(0));
  mlir::Value result = createBuiltinBitOp<Op>(cgf, e, arg, poisonZero);
 
  CIRGenBuilderTy &builder = cgf.getBuilder();
  mlir::Location loc = cgf.getLoc(e->getSourceRange());
  mlir::Value zero = builder.getNullValue(arg.getType(), loc);
  mlir::Value isZero =
      builder.createCompare(loc, cir::CmpOpKind::eq, arg, zero);
  mlir::Value fallbackValue = cgf.emitScalarExpr(e->getArg(1));
  return RValue::get(builder.createSelect(loc, isZero, fallbackValue, result));
}
 
static bool isStdcBitOpWidthSupported(cir::IntType intTy) {
  switch (intTy.getWidth()) {
  case 8:
  case 16:
  case 32:
  case 64:
  case 128:
    return true;
  default:
    return false;
  }
}
 
static RValue errorStdcBitOpWidthNYI(CIRGenFunction &cgf, const CallExpr *e) {
  cgf.cgm.errorNYI(e->getSourceRange(),
                   "stdc bit builtin with unsupported argument integer width");
  return cgf.getUndefRValue(e->getType());
}
 
template <typename Op, typename... Args>
static RValue emitStdcFirstBit(CIRGenFunction &cgf, const CallExpr *e,
                               bool invertArg, Args... args) {
  CIRGenBuilderTy &builder = cgf.getBuilder();
  mlir::Location loc = cgf.getLoc(e->getSourceRange());
  mlir::Value arg = cgf.emitScalarExpr(e->getArg(0));
  auto argTy = mlir::cast<cir::IntType>(arg.getType());
  if (!isStdcBitOpWidthSupported(argTy))
    return errorStdcBitOpWidthNYI(cgf, e);
 
  mlir::Value actualArg = invertArg ? builder.createNot(loc, arg) : arg;
  mlir::Value count = Op::create(builder, loc, actualArg, args...).getResult();
 
  mlir::Value zero = builder.getNullValue(argTy, loc);
  mlir::Value one = builder.getConstInt(loc, argTy, 1);
  mlir::Value countPlusOne = builder.createAdd(loc, count, one);
  mlir::Value isZero =
      builder.createCompare(loc, cir::CmpOpKind::eq, actualArg, zero);
  mlir::Value result = builder.createSelect(loc, isZero, zero, countPlusOne);
 
  mlir::Type resultTy = cgf.convertType(e->getType());
  if (result.getType() != resultTy)
    result = builder.createIntCast(result, resultTy);
 
  return RValue::get(result);
}
 
static RValue emitStdcBitCeil(CIRGenFunction &cgf, const CallExpr *e) {
  CIRGenBuilderTy &builder = cgf.getBuilder();
  mlir::Location loc = cgf.getLoc(e->getSourceRange());
  mlir::Value arg = cgf.emitScalarExpr(e->getArg(0));
  auto argTy = mlir::cast<cir::IntType>(arg.getType());
  if (!isStdcBitOpWidthSupported(argTy))
    return errorStdcBitOpWidthNYI(cgf, e);
 
  mlir::Value one = builder.getConstInt(loc, argTy, 1);
  mlir::Value argMinusOne = builder.createSub(loc, arg, one);
  mlir::Value lz =
      cir::BitClzOp::create(builder, loc, argMinusOne, /*poisonZero=*/false)
          .getResult();
  mlir::Value widthMinusOne =
      builder.getConstInt(loc, argTy, argTy.getWidth() - 1);
  mlir::Value shiftAmt = builder.createSub(loc, widthMinusOne, lz);
  mlir::Value isLeOne =
      builder.createCompare(loc, cir::CmpOpKind::le, arg, one);
  mlir::Value two = builder.createShiftLeft(loc, one, one);
  mlir::Value ceil = builder.createShiftLeft(loc, two, shiftAmt);
  mlir::Value result = builder.createSelect(loc, isLeOne, one, ceil);
  return RValue::get(result);
}
 
// stdc_{leading,trailing}_{zeros,ones} and stdc_count_ones: counts bits using
// clz, ctz, or popcount. InvertArg flips the input to count the opposite bit
// value.
template <typename Op, typename... Args>
static RValue emitStdcCountOp(CIRGenFunction &cgf, const CallExpr *e,
                              bool invertArg, Args... args) {
  CIRGenBuilderTy &builder = cgf.getBuilder();
  mlir::Location loc = cgf.getLoc(e->getSourceRange());
  mlir::Value arg = cgf.emitScalarExpr(e->getArg(0));
  if (!isStdcBitOpWidthSupported(mlir::cast<cir::IntType>(arg.getType())))
    return errorStdcBitOpWidthNYI(cgf, e);
  mlir::Value actualArg = invertArg ? builder.createNot(loc, arg) : arg;
  mlir::Value result = Op::create(builder, loc, actualArg, args...).getResult();
 
  mlir::Type resultTy = cgf.convertType(e->getType());
  if (result.getType() != resultTy)
    result = builder.createIntCast(result, resultTy);
  return RValue::get(result);
}
 
// stdc_count_zeros (BitWidth - popcount) and stdc_bit_width (BitWidth - clz).
// The subtract is performed in the argument type and cast once at the end.
template <typename Op, typename... Args>
static RValue emitStdcBitWidthMinus(CIRGenFunction &cgf, const CallExpr *e,
                                    Args... args) {
  CIRGenBuilderTy &builder = cgf.getBuilder();
  mlir::Location loc = cgf.getLoc(e->getSourceRange());
  mlir::Value arg = cgf.emitScalarExpr(e->getArg(0));
  auto argTy = mlir::cast<cir::IntType>(arg.getType());
  if (!isStdcBitOpWidthSupported(argTy))
    return errorStdcBitOpWidthNYI(cgf, e);
 
  mlir::Value cnt = Op::create(builder, loc, arg, args...).getResult();
  mlir::Value width = builder.getConstInt(loc, argTy, argTy.getWidth());
  mlir::Value result = builder.createSub(loc, width, cnt);
 
  mlir::Type resultTy = cgf.convertType(e->getType());
  if (result.getType() != resultTy)
    result = builder.createIntCast(result, resultTy);
 
  return RValue::get(result);
}
 
static RValue emitStdcHasSingleBit(CIRGenFunction &cgf, const CallExpr *e) {
  CIRGenBuilderTy &builder = cgf.getBuilder();
  mlir::Location loc = cgf.getLoc(e->getSourceRange());
  mlir::Value arg = cgf.emitScalarExpr(e->getArg(0));
  if (!isStdcBitOpWidthSupported(mlir::cast<cir::IntType>(arg.getType())))
    return errorStdcBitOpWidthNYI(cgf, e);
 
  mlir::Value popCount =
      cir::BitPopcountOp::create(builder, loc, arg).getResult();
  mlir::Value one = builder.getConstInt(loc, popCount.getType(), 1);
  mlir::Value result =
      builder.createCompare(loc, cir::CmpOpKind::eq, popCount, one);
  return RValue::get(result);
}
 
static RValue emitStdcBitFloor(CIRGenFunction &cgf, const CallExpr *e) {
  CIRGenBuilderTy &builder = cgf.getBuilder();
  mlir::Location loc = cgf.getLoc(e->getSourceRange());
  mlir::Value arg = cgf.emitScalarExpr(e->getArg(0));
  auto argTy = mlir::cast<cir::IntType>(arg.getType());
  if (!isStdcBitOpWidthSupported(argTy))
    return errorStdcBitOpWidthNYI(cgf, e);
 
  mlir::Value widthMinusOne =
      builder.getConstInt(loc, argTy, argTy.getWidth() - 1);
  mlir::Value one = builder.getConstInt(loc, argTy, 1);
  mlir::Value lz =
      cir::BitClzOp::create(builder, loc, arg, /*poisonZero=*/true).getResult();
  mlir::Value shiftAmt = builder.createSub(loc, widthMinusOne, lz);
  mlir::Value zero = builder.getNullValue(argTy, loc);
  mlir::Value isZero =
      builder.createCompare(loc, cir::CmpOpKind::eq, arg, zero);
  mlir::Value floor = builder.createShiftLeft(loc, one, shiftAmt);
  mlir::Value result = builder.createSelect(loc, isZero, zero, floor);
  return RValue::get(result);
}
 
/// Emit the conversions required to turn the given value into an
/// integer of the given size.
static mlir::Value emitToInt(CIRGenFunction &cgf, mlir::Value v, QualType t,
                             cir::IntType intType) {
  v = cgf.emitToMemory(v, t);
 
  if (mlir::isa<cir::PointerType>(v.getType()))
    return cgf.getBuilder().createPtrToInt(v, intType);
 
  assert(v.getType() == intType);
  return v;
}
 
static mlir::Value emitFromInt(CIRGenFunction &cgf, mlir::Value v, QualType t,
                               mlir::Type resultType) {
  v = cgf.emitFromMemory(v, t);
 
  if (mlir::isa<cir::PointerType>(resultType))
    return cgf.getBuilder().createIntToPtr(v, resultType);
 
  assert(v.getType() == resultType);
  return v;
}
 
static mlir::Value emitSignBit(mlir::Location loc, CIRGenFunction &cgf,
                               mlir::Value val) {
  assert(!::cir::MissingFeatures::isPPC_FP128Ty());
  cir::SignBitOp returnValue = cgf.getBuilder().createSignBit(loc, val);
  return returnValue->getResult(0);
}
 
static Address checkAtomicAlignment(CIRGenFunction &cgf, const CallExpr *e) {
  ASTContext &astContext = cgf.getContext();
  Address ptr = cgf.emitPointerWithAlignment(e->getArg(0));
  unsigned bytes =
      mlir::isa<cir::PointerType>(ptr.getElementType())
          ? astContext.getTypeSizeInChars(astContext.VoidPtrTy).getQuantity()
          : cgf.cgm.getDataLayout().getTypeSizeInBits(ptr.getElementType()) /
                cgf.cgm.getASTContext().getCharWidth();
 
  unsigned align = ptr.getAlignment().getQuantity();
  if (align % bytes != 0) {
    DiagnosticsEngine &diags = cgf.cgm.getDiags();
    diags.Report(e->getBeginLoc(), diag::warn_sync_op_misaligned);
    // Force address to be at least naturally-aligned.
    return ptr.withAlignment(CharUnits::fromQuantity(bytes));
  }
  return ptr;
}
 
/// Utility to insert an atomic instruction based on Intrinsic::ID
/// and the expression node.
mlir::Value CIRGenFunction::makeBinaryAtomicValue(cir::AtomicFetchKind kind,
                                                  const CallExpr *expr,
                                                  mlir::Type *originalArgType,
                                                  mlir::Value *emittedArgValue,
                                                  cir::MemOrder ordering) {
  CIRGenFunction &cgf = *this;
 
  QualType type = expr->getType();
  QualType ptrType = expr->getArg(0)->getType();
 
  assert(ptrType->isPointerType());
  assert(
      cgf.getContext().hasSameUnqualifiedType(type, ptrType->getPointeeType()));
  assert(cgf.getContext().hasSameUnqualifiedType(type,
                                                 expr->getArg(1)->getType()));
 
  Address destAddr = checkAtomicAlignment(cgf, expr);
  CIRGenBuilderTy &builder = cgf.getBuilder();
 
  mlir::Value val = cgf.emitScalarExpr(expr->getArg(1));
  mlir::Type valueType = val.getType();
  mlir::Value destValue = destAddr.emitRawPointer();
 
  if (ptrType->getPointeeType()->isPointerType()) {
    // Pointer to pointer
    // `cir.atomic.fetch` expects a pointer to an integer type, so we cast
    // ptr<ptr<T>> to ptr<intPtrSize>
    cir::IntType ptrSizeInt =
        builder.getSIntNTy(cgf.getContext().getTypeSize(ptrType));
    destValue =
        builder.createBitcast(destValue, builder.getPointerTo(ptrSizeInt));
    val = emitToInt(cgf, val, type, ptrSizeInt);
  } else {
    // Pointer to integer type
    cir::IntType intType =
        ptrType->getPointeeType()->isUnsignedIntegerType()
            ? builder.getUIntNTy(cgf.getContext().getTypeSize(type))
            : builder.getSIntNTy(cgf.getContext().getTypeSize(type));
    val = emitToInt(cgf, val, type, intType);
  }
 
  // This output argument is needed for post atomic fetch operations
  // that calculate the result of the operation as return value of
  // <binop>_and_fetch builtins. The `AtomicFetch` operation only updates the
  // memory location and returns the old value.
  if (emittedArgValue) {
    *emittedArgValue = val;
    assert(originalArgType != nullptr &&
           "originalArgType must be provided when emittedArgValue is set");
    *originalArgType = valueType;
  }
 
  auto rmwi = cir::AtomicFetchOp::create(
      builder, cgf.getLoc(expr->getSourceRange()), destValue, val, kind,
      ordering, cir::SyncScopeKind::System, false, /* is volatile */
      true);                                       /* fetch first */
  return rmwi->getResult(0);
}
 
static RValue emitBinaryAtomic(CIRGenFunction &cgf,
                               cir::AtomicFetchKind atomicOpkind,
                               const CallExpr *e) {
  return RValue::get(cgf.makeBinaryAtomicValue(atomicOpkind, e));
}
 
template <typename BinOp>
static RValue emitBinaryAtomicPost(CIRGenFunction &cgf,
                                   cir::AtomicFetchKind atomicOpkind,
                                   const CallExpr *e, bool invert = false) {
  mlir::Value emittedArgValue;
  mlir::Type originalArgType;
  clang::QualType typ = e->getType();
  mlir::Value result = cgf.makeBinaryAtomicValue(
      atomicOpkind, e, &originalArgType, &emittedArgValue);
  clang::CIRGen::CIRGenBuilderTy &builder = cgf.getBuilder();
  result = BinOp::create(builder, result.getLoc(), result, emittedArgValue);
 
  if (invert)
    result = builder.createNot(result);
 
  result = emitFromInt(cgf, result, typ, originalArgType);
  return RValue::get(result);
}
 
mlir::Value CIRGenFunction::emitAtomicCmpXchg(const CallExpr *e,
                                              bool returnBool,
                                              cir::MemOrder successOrder,
                                              cir::MemOrder failureOrder,
                                              cir::SyncScopeKind scope) {
  Address destAddr = checkAtomicAlignment(*this, e);
  CIRGenBuilderTy &builder = getBuilder();
  mlir::Value destValue = destAddr.emitRawPointer();
  mlir::Value expected = emitScalarExpr(e->getArg(1));
  mlir::Value desired = emitScalarExpr(e->getArg(2));
 
  auto cmpxchg = cir::AtomicCmpXchgOp::create(
      builder, getLoc(e->getSourceRange()), destValue, expected, desired,
      successOrder, failureOrder, scope,
      /*alignment=*/nullptr, /*weak=*/false, /*is_volatile=*/false);
 
  if (returnBool)
    return cmpxchg.getSuccess();
  return cmpxchg.getOld();
}
 
/// Emit a `cir.atomic.xchg` for __sync_swap_N and __sync_lock_test_and_set_N.
static RValue emitAtomicXchg(CIRGenFunction &cgf, const CallExpr *e,
                             cir::MemOrder ordering) {
  Address destAddr = checkAtomicAlignment(cgf, e);
  CIRGenBuilderTy &builder = cgf.getBuilder();
  mlir::Value destValue = destAddr.emitRawPointer();
  mlir::Value val = cgf.emitScalarExpr(e->getArg(1));
 
  auto xchg = cir::AtomicXchgOp::create(
      builder, cgf.getLoc(e->getSourceRange()), destValue, val, ordering,
      cir::SyncScopeKind::System, /*is_volatile=*/false);
  return RValue::get(xchg.getResult());
}
 
/// Emit a release store of 0 for __sync_lock_release_N.
static void emitAtomicLockRelease(CIRGenFunction &cgf, const CallExpr *e) {
  CIRGenBuilderTy &builder = cgf.getBuilder();
  Address destAddr = checkAtomicAlignment(cgf, e);
  mlir::Location loc = cgf.getLoc(e->getSourceRange());
  mlir::Type elemTy = destAddr.getElementType();
  mlir::Value zero = builder.getConstant(loc, builder.getZeroInitAttr(elemTy));
  auto orderAttr =
      cir::MemOrderAttr::get(&cgf.getMLIRContext(), cir::MemOrder::Release);
  auto scopeAttr = cir::SyncScopeKindAttr::get(&cgf.getMLIRContext(),
                                               cir::SyncScopeKind::System);
  builder.createStore(loc, zero, destAddr, /*isVolatile=*/false,
                      /*isNontemporal=*/false,
                      /*align=*/mlir::IntegerAttr{}, scopeAttr, orderAttr);
}
 
static void emitAtomicFenceOp(CIRGenFunction &cgf, const CallExpr *expr,
                              cir::SyncScopeKind syncScope) {
  CIRGenBuilderTy &builder = cgf.getBuilder();
  mlir::Location loc = cgf.getLoc(expr->getSourceRange());
 
  auto emitAtomicOpCallBackFn = [&](cir::MemOrder memOrder) {
    cir::AtomicFenceOp::create(
        builder, loc, memOrder,
        cir::SyncScopeKindAttr::get(&cgf.getMLIRContext(), syncScope));
  };
 
  cgf.emitAtomicExprWithMemOrder(expr->getArg(0), /*isStore*/ false,
                                 /*isLoad*/ false, /*isFence*/ true,
                                 emitAtomicOpCallBackFn);
}
 
// Emit a runtime call to bool __atomic_is_lock_free(size_t size, void *ptr).
// For the __c11 builtin the pointer is null, since an _Atomic object is always
// suitably aligned.
static RValue emitAtomicIsLockFree(CIRGenFunction &cgf, const CallExpr *e,
                                   unsigned builtinID) {
  CIRGenBuilderTy &builder = cgf.getBuilder();
  mlir::Location loc = cgf.getLoc(e->getExprLoc());
 
  mlir::Type sizeTy = cgf.convertType(cgf.getContext().getSizeType());
  mlir::Value size = cgf.emitScalarExpr(e->getArg(0));
  mlir::Value ptr;
  if (builtinID == Builtin::BI__atomic_is_lock_free)
    ptr = builder.createBitcast(cgf.emitScalarExpr(e->getArg(1)),
                                builder.getVoidPtrTy());
  else
    ptr = builder.getNullPtr(builder.getVoidPtrTy(), loc);
 
  cir::FuncOp func = cgf.cgm.createRuntimeFunction(
      cir::FuncType::get({sizeTy, builder.getVoidPtrTy()}, builder.getBoolTy()),
      "__atomic_is_lock_free");
  return RValue::get(
      builder.createCallOp(loc, func, mlir::ValueRange{size, ptr}).getResult());
}
 
namespace {
struct WidthAndSignedness {
  unsigned width;
  bool isSigned;
};
} // namespace
 
static WidthAndSignedness
getIntegerWidthAndSignedness(const clang::ASTContext &astContext,
                             const clang::QualType type) {
  assert(type->isIntegerType() && "Given type is not an integer.");
  unsigned width = type->isBooleanType()  ? 1
                   : type->isBitIntType() ? astContext.getIntWidth(type)
                                          : astContext.getTypeInfo(type).Width;
  bool isSigned = type->isSignedIntegerType();
  return {width, isSigned};
}
 
/// Create a checked overflow arithmetic op and return its result and overflow
/// flag.
template <typename OpTy>
static std::pair<mlir::Value, mlir::Value>
emitOverflowOp(CIRGenBuilderTy &builder, mlir::Location loc,
               mlir::Type resultTy, mlir::Value lhs, mlir::Value rhs) {
  auto op = OpTy::create(builder, loc, resultTy, lhs, rhs);
  return {op.getResult(), op.getOverflow()};
}
 
// Given one or more integer types, this function produces an integer type that
// encompasses them: any value in one of the given types could be expressed in
// the encompassing type.
static struct WidthAndSignedness
EncompassingIntegerType(ArrayRef<struct WidthAndSignedness> types) {
  assert(types.size() > 0 && "Empty list of types.");
 
  // If any of the given types is signed, we must return a signed type.
  bool isSigned = llvm::any_of(types, [](const auto &t) { return t.isSigned; });
 
  // The encompassing type must have a width greater than or equal to the width
  // of the specified types.  Additionally, if the encompassing type is signed,
  // its width must be strictly greater than the width of any unsigned types
  // given.
  unsigned width = 0;
  for (const auto &type : types)
    width = std::max(width, type.width + (isSigned && !type.isSigned));
 
  return {width, isSigned};
}
 
RValue CIRGenFunction::emitRotate(const CallExpr *e, bool isRotateLeft) {
  mlir::Value input = emitScalarExpr(e->getArg(0));
  mlir::Value amount = emitScalarExpr(e->getArg(1));
 
  if (amount.getType() != input.getType())
    amount = builder.createIntCast(amount, input.getType());
 
  auto r = cir::RotateOp::create(builder, getLoc(e->getSourceRange()), input,
                                 amount, isRotateLeft);
  return RValue::get(r);
}
 
template <class Operation>
static RValue emitUnaryMaybeConstrainedFPBuiltin(CIRGenFunction &cgf,
                                                 const CallExpr &e) {
  mlir::Value arg = cgf.emitScalarExpr(e.getArg(0));
 
  CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, &e);
 
  auto call = Operation::create(cgf.getBuilder(), arg.getLoc(), arg.getType(),
                                arg, cgf.getBuilder().getConstrainedFPAttr());
  return RValue::get(call->getResult(0));
}
 
template <class Operation>
static RValue emitUnaryFPBuiltin(CIRGenFunction &cgf, const CallExpr &e) {
  mlir::Value arg = cgf.emitScalarExpr(e.getArg(0));
  auto call = Operation::create(cgf.getBuilder(), arg.getLoc(), arg);
  return RValue::get(call->getResult(0));
}
 
template <typename Op>
static RValue emitUnaryMaybeConstrainedFPToIntBuiltin(CIRGenFunction &cgf,
                                                      const CallExpr &e) {
  mlir::Type resultType = cgf.convertType(e.getType());
  mlir::Value src = cgf.emitScalarExpr(e.getArg(0));
 
  CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, &e);
 
  auto call = Op::create(cgf.getBuilder(), src.getLoc(), resultType, src,
                         cgf.getBuilder().getConstrainedFPAttr());
  return RValue::get(call->getResult(0));
}
 
template <typename Op>
static RValue emitBinaryFPBuiltin(CIRGenFunction &cgf, const CallExpr &e) {
  mlir::Value arg0 = cgf.emitScalarExpr(e.getArg(0));
  mlir::Value arg1 = cgf.emitScalarExpr(e.getArg(1));
 
  mlir::Location loc = cgf.getLoc(e.getExprLoc());
  mlir::Type ty = cgf.convertType(e.getType());
  auto call = Op::create(cgf.getBuilder(), loc, ty, arg0, arg1);
 
  return RValue::get(call->getResult(0));
}
 
template <typename Op>
static RValue emitTernarySameTypeBuiltin(CIRGenFunction &cgf,
                                         const CallExpr &e) {
  mlir::Value arg0 = cgf.emitScalarExpr(e.getArg(0));
  mlir::Value arg1 = cgf.emitScalarExpr(e.getArg(1));
  mlir::Value arg2 = cgf.emitScalarExpr(e.getArg(2));
 
  mlir::Location loc = cgf.getLoc(e.getExprLoc());
  mlir::Type ty = cgf.convertType(e.getType());
  auto call = Op::create(cgf.getBuilder(), loc, ty, arg0, arg1, arg2);
 
  return RValue::get(call->getResult(0));
}
 
template <typename Op>
static RValue emitTernaryMaybeConstrainedFPBuiltin(CIRGenFunction &cgf,
                                                   const CallExpr &e) {
  CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, &e);
 
  mlir::Value arg0 = cgf.emitScalarExpr(e.getArg(0));
  mlir::Value arg1 = cgf.emitScalarExpr(e.getArg(1));
  mlir::Value arg2 = cgf.emitScalarExpr(e.getArg(2));
 
  mlir::Location loc = cgf.getLoc(e.getExprLoc());
  mlir::Type ty = cgf.convertType(e.getType());
 
  auto call = Op::create(cgf.getBuilder(), loc, ty, arg0, arg1, arg2,
                         cgf.getBuilder().getConstrainedFPAttr());
  return RValue::get(call->getResult(0));
}
 
template <typename Op>
static mlir::Value emitBinaryMaybeConstrainedFPBuiltin(CIRGenFunction &cgf,
                                                       const CallExpr &e) {
  mlir::Value arg0 = cgf.emitScalarExpr(e.getArg(0));
  mlir::Value arg1 = cgf.emitScalarExpr(e.getArg(1));
 
  CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, &e);
 
  mlir::Location loc = cgf.getLoc(e.getExprLoc());
  mlir::Type ty = cgf.convertType(e.getType());
 
  auto call = Op::create(cgf.getBuilder(), loc, ty, arg0, arg1,
                         cgf.getBuilder().getConstrainedFPAttr());
  return call->getResult(0);
}
 
static RValue errorBuiltinNYI(CIRGenFunction &cgf, const CallExpr *e,
                              unsigned builtinID) {
 
  if (cgf.getContext().BuiltinInfo.isLibFunction(builtinID)) {
    cgf.cgm.errorNYI(
        e->getSourceRange(),
        std::string("unimplemented library function builtin call: ") +
            cgf.getContext().BuiltinInfo.getName(builtinID));
  } else {
    cgf.cgm.errorNYI(e->getSourceRange(),
                     std::string("unimplemented builtin call: ") +
                         cgf.getContext().BuiltinInfo.getName(builtinID));
  }
 
  return cgf.getUndefRValue(e->getType());
}
 
static RValue emitBuiltinAlloca(CIRGenFunction &cgf, const CallExpr *e,
                                unsigned builtinID) {
  assert(builtinID == Builtin::BI__builtin_alloca ||
         builtinID == Builtin::BI__builtin_alloca_uninitialized ||
         builtinID == Builtin::BIalloca || builtinID == Builtin::BI_alloca);
 
  // Get alloca size input
  mlir::Value size = cgf.emitScalarExpr(e->getArg(0));
 
  // The alignment of the alloca should correspond to __BIGGEST_ALIGNMENT__.
  const TargetInfo &ti = cgf.getContext().getTargetInfo();
  const CharUnits suitableAlignmentInBytes =
      cgf.getContext().toCharUnitsFromBits(ti.getSuitableAlign());
 
  // Emit the alloca op with type `u8 *` to match the semantics of
  // `llvm.alloca`. We later bitcast the type to `void *` to match the
  // semantics of C/C++
  // FIXME(cir): It may make sense to allow AllocaOp of type `u8` to return a
  // pointer of type `void *`. This will require a change to the allocaOp
  // verifier.
  CIRGenBuilderTy &builder = cgf.getBuilder();
  mlir::Value allocaAddr = builder.createAlloca(
      cgf.getLoc(e->getSourceRange()), builder.getUInt8PtrTy(),
      builder.getUInt8Ty(), "bi_alloca", suitableAlignmentInBytes, size);
 
  // Initialize the allocated buffer if required.
  if (builtinID != Builtin::BI__builtin_alloca_uninitialized) {
    // Initialize the alloca with the given size and alignment according to
    // the lang opts. Only the trivial non-initialization is supported for
    // now.
 
    switch (cgf.getLangOpts().getTrivialAutoVarInit()) {
    case LangOptions::TrivialAutoVarInitKind::Uninitialized:
      // Nothing to initialize.
      break;
    case LangOptions::TrivialAutoVarInitKind::Zero:
    case LangOptions::TrivialAutoVarInitKind::Pattern:
      cgf.cgm.errorNYI("trivial auto var init");
      break;
    }
  }
 
  // An alloca will always return a pointer to the alloca (stack) address
  // space. This address space need not be the same as the AST / Language
  // default (e.g. in C / C++ auto vars are in the generic address space). At
  // the AST level this is handled within CreateTempAlloca et al., but for the
  // builtin / dynamic alloca we have to handle it here.
 
  if (!cir::isMatchingAddressSpace(
          cgf.getCIRAllocaAddressSpace(),
          e->getType()->getPointeeType().getAddressSpace())) {
    cgf.cgm.errorNYI(e->getSourceRange(),
                     "Address Space Cast for builtin alloca");
  }
 
  // Bitcast the alloca to the expected type.
  return RValue::get(builder.createBitcast(
      allocaAddr, builder.getVoidPtrTy(cgf.getCIRAllocaAddressSpace())));
}
 
static bool shouldCIREmitFPMathIntrinsic(CIRGenFunction &cgf, const CallExpr *e,
                                         unsigned builtinID) {
  std::optional<bool> errnoOverriden;
  // ErrnoOverriden is true if math-errno is overriden via the
  // '#pragma float_control(precise, on)'. This pragma disables fast-math,
  // which implies math-errno.
  if (e->hasStoredFPFeatures()) {
    FPOptionsOverride op = e->getFPFeatures();
    if (op.hasMathErrnoOverride())
      errnoOverriden = op.getMathErrnoOverride();
  }
  // True if 'attribute__((optnone))' is used. This attribute overrides
  // fast-math which implies math-errno.
  bool optNone =
      cgf.curFuncDecl && cgf.curFuncDecl->hasAttr<OptimizeNoneAttr>();
  bool isOptimizationEnabled = cgf.cgm.getCodeGenOpts().OptimizationLevel != 0;
  bool generateFPMathIntrinsics =
      cgf.getContext().BuiltinInfo.shouldGenerateFPMathIntrinsic(
          builtinID, cgf.cgm.getTriple(), errnoOverriden,
          cgf.getLangOpts().MathErrno, optNone, isOptimizationEnabled);
  return generateFPMathIntrinsics;
}
 
static RValue tryEmitFPMathIntrinsic(CIRGenFunction &cgf, const CallExpr *e,
                                     unsigned builtinID) {
  assert(!cir::MissingFeatures::fastMathFlags());
  switch (builtinID) {
  case Builtin::BIacos:
  case Builtin::BIacosf:
  case Builtin::BIacosl:
  case Builtin::BI__builtin_acos:
  case Builtin::BI__builtin_acosf:
  case Builtin::BI__builtin_acosf16:
  case Builtin::BI__builtin_acosl:
  case Builtin::BI__builtin_acosf128:
  case Builtin::BI__builtin_elementwise_acos:
    return emitUnaryMaybeConstrainedFPBuiltin<cir::ACosOp>(cgf, *e);
  case Builtin::BIasin:
  case Builtin::BIasinf:
  case Builtin::BIasinl:
  case Builtin::BI__builtin_asin:
  case Builtin::BI__builtin_asinf:
  case Builtin::BI__builtin_asinf16:
  case Builtin::BI__builtin_asinl:
  case Builtin::BI__builtin_asinf128:
  case Builtin::BI__builtin_elementwise_asin:
    return emitUnaryMaybeConstrainedFPBuiltin<cir::ASinOp>(cgf, *e);
  case Builtin::BIatan:
  case Builtin::BIatanf:
  case Builtin::BIatanl:
  case Builtin::BI__builtin_atan:
  case Builtin::BI__builtin_atanf:
  case Builtin::BI__builtin_atanf16:
  case Builtin::BI__builtin_atanl:
  case Builtin::BI__builtin_atanf128:
  case Builtin::BI__builtin_elementwise_atan:
    return emitUnaryMaybeConstrainedFPBuiltin<cir::ATanOp>(cgf, *e);
  case Builtin::BIatan2:
  case Builtin::BIatan2f:
  case Builtin::BIatan2l:
  case Builtin::BI__builtin_atan2:
  case Builtin::BI__builtin_atan2f:
  case Builtin::BI__builtin_atan2f16:
  case Builtin::BI__builtin_atan2l:
  case Builtin::BI__builtin_atan2f128:
  case Builtin::BI__builtin_elementwise_atan2:
    return RValue::get(
        emitBinaryMaybeConstrainedFPBuiltin<cir::ATan2Op>(cgf, *e));
  case Builtin::BIceil:
  case Builtin::BIceilf:
  case Builtin::BIceill:
  case Builtin::BI__builtin_ceil:
  case Builtin::BI__builtin_ceilf:
  case Builtin::BI__builtin_ceilf16:
  case Builtin::BI__builtin_ceill:
  case Builtin::BI__builtin_ceilf128:
  case Builtin::BI__builtin_elementwise_ceil:
    return emitUnaryMaybeConstrainedFPBuiltin<cir::CeilOp>(cgf, *e);
  case Builtin::BIcopysign:
  case Builtin::BIcopysignf:
  case Builtin::BIcopysignl:
  case Builtin::BI__builtin_copysign:
  case Builtin::BI__builtin_copysignf:
  case Builtin::BI__builtin_copysignf16:
  case Builtin::BI__builtin_copysignl:
  case Builtin::BI__builtin_copysignf128:
    return emitBinaryFPBuiltin<cir::CopysignOp>(cgf, *e);
  case Builtin::BIcos:
  case Builtin::BIcosf:
  case Builtin::BIcosl:
  case Builtin::BI__builtin_cos:
  case Builtin::BI__builtin_cosf:
  case Builtin::BI__builtin_cosf16:
  case Builtin::BI__builtin_cosl:
  case Builtin::BI__builtin_cosf128:
  case Builtin::BI__builtin_elementwise_cos:
    return emitUnaryMaybeConstrainedFPBuiltin<cir::CosOp>(cgf, *e);
  case Builtin::BIcosh:
  case Builtin::BIcoshf:
  case Builtin::BIcoshl:
  case Builtin::BI__builtin_cosh:
  case Builtin::BI__builtin_coshf:
  case Builtin::BI__builtin_coshf16:
  case Builtin::BI__builtin_coshl:
  case Builtin::BI__builtin_coshf128:
  case Builtin::BI__builtin_elementwise_cosh:
    return emitUnaryMaybeConstrainedFPBuiltin<cir::CoshOp>(cgf, *e);
  case Builtin::BIexp:
  case Builtin::BIexpf:
  case Builtin::BIexpl:
  case Builtin::BI__builtin_exp:
  case Builtin::BI__builtin_expf:
  case Builtin::BI__builtin_expf16:
  case Builtin::BI__builtin_expl:
  case Builtin::BI__builtin_expf128:
  case Builtin::BI__builtin_elementwise_exp:
    return emitUnaryMaybeConstrainedFPBuiltin<cir::ExpOp>(cgf, *e);
  case Builtin::BIexp2:
  case Builtin::BIexp2f:
  case Builtin::BIexp2l:
  case Builtin::BI__builtin_exp2:
  case Builtin::BI__builtin_exp2f:
  case Builtin::BI__builtin_exp2f16:
  case Builtin::BI__builtin_exp2l:
  case Builtin::BI__builtin_exp2f128:
  case Builtin::BI__builtin_elementwise_exp2:
    return emitUnaryMaybeConstrainedFPBuiltin<cir::Exp2Op>(cgf, *e);
  case Builtin::BI__builtin_exp10:
  case Builtin::BI__builtin_exp10f:
  case Builtin::BI__builtin_exp10f16:
  case Builtin::BI__builtin_exp10l:
  case Builtin::BI__builtin_exp10f128:
  case Builtin::BI__builtin_elementwise_exp10:
    return emitUnaryMaybeConstrainedFPBuiltin<cir::Exp10Op>(cgf, *e);
  case Builtin::BIfabs:
  case Builtin::BIfabsf:
  case Builtin::BIfabsl:
  case Builtin::BI__builtin_fabs:
  case Builtin::BI__builtin_fabsf:
  case Builtin::BI__builtin_fabsf16:
  case Builtin::BI__builtin_fabsl:
  case Builtin::BI__builtin_fabsf128:
    return emitUnaryMaybeConstrainedFPBuiltin<cir::FAbsOp>(cgf, *e);
  case Builtin::BIfloor:
  case Builtin::BIfloorf:
  case Builtin::BIfloorl:
  case Builtin::BI__builtin_floor:
  case Builtin::BI__builtin_floorf:
  case Builtin::BI__builtin_floorf16:
  case Builtin::BI__builtin_floorl:
  case Builtin::BI__builtin_floorf128:
  case Builtin::BI__builtin_elementwise_floor:
    return emitUnaryMaybeConstrainedFPBuiltin<cir::FloorOp>(cgf, *e);
  case Builtin::BIfma:
  case Builtin::BIfmaf:
  case Builtin::BIfmal:
  case Builtin::BI__builtin_fma:
  case Builtin::BI__builtin_fmaf:
  case Builtin::BI__builtin_fmaf16:
  case Builtin::BI__builtin_fmal:
  case Builtin::BI__builtin_fmaf128:
  case Builtin::BI__builtin_elementwise_fma:
    return emitTernaryMaybeConstrainedFPBuiltin<cir::FMAOp>(cgf, *e);
  case Builtin::BIfmax:
  case Builtin::BIfmaxf:
  case Builtin::BIfmaxl:
  case Builtin::BI__builtin_fmax:
  case Builtin::BI__builtin_fmaxf:
  case Builtin::BI__builtin_fmaxf16:
  case Builtin::BI__builtin_fmaxl:
  case Builtin::BI__builtin_fmaxf128:
    return RValue::get(
        emitBinaryMaybeConstrainedFPBuiltin<cir::FMaxNumOp>(cgf, *e));
  case Builtin::BIfmin:
  case Builtin::BIfminf:
  case Builtin::BIfminl:
  case Builtin::BI__builtin_fmin:
  case Builtin::BI__builtin_fminf:
  case Builtin::BI__builtin_fminf16:
  case Builtin::BI__builtin_fminl:
  case Builtin::BI__builtin_fminf128:
    return RValue::get(
        emitBinaryMaybeConstrainedFPBuiltin<cir::FMinNumOp>(cgf, *e));
  case Builtin::BIfmaximum_num:
  case Builtin::BIfmaximum_numf:
  case Builtin::BIfmaximum_numl:
  case Builtin::BI__builtin_fmaximum_num:
  case Builtin::BI__builtin_fmaximum_numf:
  case Builtin::BI__builtin_fmaximum_numf16:
  case Builtin::BI__builtin_fmaximum_numl:
  case Builtin::BI__builtin_fmaximum_numf128:
  case Builtin::BIfminimum_num:
  case Builtin::BIfminimum_numf:
  case Builtin::BIfminimum_numl:
  case Builtin::BI__builtin_fminimum_num:
  case Builtin::BI__builtin_fminimum_numf:
  case Builtin::BI__builtin_fminimum_numf16:
  case Builtin::BI__builtin_fminimum_numl:
  case Builtin::BI__builtin_fminimum_numf128:
    return errorBuiltinNYI(cgf, e, builtinID);
  case Builtin::BIfmod:
  case Builtin::BIfmodf:
  case Builtin::BIfmodl:
  case Builtin::BI__builtin_fmod:
  case Builtin::BI__builtin_fmodf:
  case Builtin::BI__builtin_fmodf16:
  case Builtin::BI__builtin_fmodl:
  case Builtin::BI__builtin_fmodf128:
  case Builtin::BI__builtin_elementwise_fmod:
    return RValue::get(
        emitBinaryMaybeConstrainedFPBuiltin<cir::FModOp>(cgf, *e));
  case Builtin::BIlog:
  case Builtin::BIlogf:
  case Builtin::BIlogl:
  case Builtin::BI__builtin_log:
  case Builtin::BI__builtin_logf:
  case Builtin::BI__builtin_logf16:
  case Builtin::BI__builtin_logl:
  case Builtin::BI__builtin_logf128:
  case Builtin::BI__builtin_elementwise_log:
    return emitUnaryMaybeConstrainedFPBuiltin<cir::LogOp>(cgf, *e);
  case Builtin::BIlog10:
  case Builtin::BIlog10f:
  case Builtin::BIlog10l:
  case Builtin::BI__builtin_log10:
  case Builtin::BI__builtin_log10f:
  case Builtin::BI__builtin_log10f16:
  case Builtin::BI__builtin_log10l:
  case Builtin::BI__builtin_log10f128:
  case Builtin::BI__builtin_elementwise_log10:
    return emitUnaryMaybeConstrainedFPBuiltin<cir::Log10Op>(cgf, *e);
  case Builtin::BIlog2:
  case Builtin::BIlog2f:
  case Builtin::BIlog2l:
  case Builtin::BI__builtin_log2:
  case Builtin::BI__builtin_log2f:
  case Builtin::BI__builtin_log2f16:
  case Builtin::BI__builtin_log2l:
  case Builtin::BI__builtin_log2f128:
  case Builtin::BI__builtin_elementwise_log2:
    return emitUnaryMaybeConstrainedFPBuiltin<cir::Log2Op>(cgf, *e);
  case Builtin::BInearbyint:
  case Builtin::BInearbyintf:
  case Builtin::BInearbyintl:
  case Builtin::BI__builtin_nearbyint:
  case Builtin::BI__builtin_nearbyintf:
  case Builtin::BI__builtin_nearbyintl:
  case Builtin::BI__builtin_nearbyintf128:
  case Builtin::BI__builtin_elementwise_nearbyint:
    return emitUnaryMaybeConstrainedFPBuiltin<cir::NearbyintOp>(cgf, *e);
  case Builtin::BIpow:
  case Builtin::BIpowf:
  case Builtin::BIpowl:
  case Builtin::BI__builtin_pow:
  case Builtin::BI__builtin_powf:
  case Builtin::BI__builtin_powf16:
  case Builtin::BI__builtin_powl:
  case Builtin::BI__builtin_powf128:
    return RValue::get(
        emitBinaryMaybeConstrainedFPBuiltin<cir::PowOp>(cgf, *e));
  case Builtin::BI__builtin_elementwise_pow:
    return errorBuiltinNYI(cgf, e, builtinID);
  case Builtin::BIrint:
  case Builtin::BIrintf:
  case Builtin::BIrintl:
  case Builtin::BI__builtin_rint:
  case Builtin::BI__builtin_rintf:
  case Builtin::BI__builtin_rintf16:
  case Builtin::BI__builtin_rintl:
  case Builtin::BI__builtin_rintf128:
  case Builtin::BI__builtin_elementwise_rint:
    return emitUnaryMaybeConstrainedFPBuiltin<cir::RintOp>(cgf, *e);
  case Builtin::BIround:
  case Builtin::BIroundf:
  case Builtin::BIroundl:
  case Builtin::BI__builtin_round:
  case Builtin::BI__builtin_roundf:
  case Builtin::BI__builtin_roundf16:
  case Builtin::BI__builtin_roundl:
  case Builtin::BI__builtin_roundf128:
  case Builtin::BI__builtin_elementwise_round:
    return emitUnaryMaybeConstrainedFPBuiltin<cir::RoundOp>(cgf, *e);
  case Builtin::BIroundeven:
  case Builtin::BIroundevenf:
  case Builtin::BIroundevenl:
  case Builtin::BI__builtin_roundeven:
  case Builtin::BI__builtin_roundevenf:
  case Builtin::BI__builtin_roundevenf16:
  case Builtin::BI__builtin_roundevenl:
  case Builtin::BI__builtin_roundevenf128:
  case Builtin::BI__builtin_elementwise_roundeven:
    return emitUnaryMaybeConstrainedFPBuiltin<cir::RoundEvenOp>(cgf, *e);
  case Builtin::BIsin:
  case Builtin::BIsinf:
  case Builtin::BIsinl:
  case Builtin::BI__builtin_sin:
  case Builtin::BI__builtin_sinf:
  case Builtin::BI__builtin_sinf16:
  case Builtin::BI__builtin_sinl:
  case Builtin::BI__builtin_sinf128:
  case Builtin::BI__builtin_elementwise_sin:
    return emitUnaryMaybeConstrainedFPBuiltin<cir::SinOp>(cgf, *e);
  case Builtin::BIsinh:
  case Builtin::BIsinhf:
  case Builtin::BIsinhl:
  case Builtin::BI__builtin_sinh:
  case Builtin::BI__builtin_sinhf:
  case Builtin::BI__builtin_sinhf16:
  case Builtin::BI__builtin_sinhl:
  case Builtin::BI__builtin_sinhf128:
  case Builtin::BI__builtin_elementwise_sinh:
    return emitUnaryMaybeConstrainedFPBuiltin<cir::SinhOp>(cgf, *e);
  case Builtin::BI__builtin_sincospi:
  case Builtin::BI__builtin_sincospif:
  case Builtin::BI__builtin_sincospil:
  case Builtin::BIsincos:
  case Builtin::BIsincosf:
  case Builtin::BIsincosl:
  case Builtin::BI__builtin_sincos:
  case Builtin::BI__builtin_sincosf:
  case Builtin::BI__builtin_sincosf16:
  case Builtin::BI__builtin_sincosl:
  case Builtin::BI__builtin_sincosf128:
    return errorBuiltinNYI(cgf, e, builtinID);
  case Builtin::BIsqrt:
  case Builtin::BIsqrtf:
  case Builtin::BIsqrtl:
  case Builtin::BI__builtin_sqrt:
  case Builtin::BI__builtin_sqrtf:
  case Builtin::BI__builtin_sqrtf16:
  case Builtin::BI__builtin_sqrtl:
  case Builtin::BI__builtin_sqrtf128:
  case Builtin::BI__builtin_elementwise_sqrt:
    return emitUnaryMaybeConstrainedFPBuiltin<cir::SqrtOp>(cgf, *e);
  case Builtin::BItan:
  case Builtin::BItanf:
  case Builtin::BItanl:
  case Builtin::BI__builtin_tan:
  case Builtin::BI__builtin_tanf:
  case Builtin::BI__builtin_tanf16:
  case Builtin::BI__builtin_tanl:
  case Builtin::BI__builtin_tanf128:
  case Builtin::BI__builtin_elementwise_tan:
    return emitUnaryMaybeConstrainedFPBuiltin<cir::TanOp>(cgf, *e);
  case Builtin::BItanh:
  case Builtin::BItanhf:
  case Builtin::BItanhl:
  case Builtin::BI__builtin_tanh:
  case Builtin::BI__builtin_tanhf:
  case Builtin::BI__builtin_tanhf16:
  case Builtin::BI__builtin_tanhl:
  case Builtin::BI__builtin_tanhf128:
  case Builtin::BI__builtin_elementwise_tanh:
    return emitUnaryMaybeConstrainedFPBuiltin<cir::TanhOp>(cgf, *e);
  case Builtin::BItrunc:
  case Builtin::BItruncf:
  case Builtin::BItruncl:
  case Builtin::BI__builtin_trunc:
  case Builtin::BI__builtin_truncf:
  case Builtin::BI__builtin_truncf16:
  case Builtin::BI__builtin_truncl:
  case Builtin::BI__builtin_truncf128:
  case Builtin::BI__builtin_elementwise_trunc:
    return emitUnaryMaybeConstrainedFPBuiltin<cir::TruncOp>(cgf, *e);
  case Builtin::BIlround:
  case Builtin::BIlroundf:
  case Builtin::BIlroundl:
  case Builtin::BI__builtin_lround:
  case Builtin::BI__builtin_lroundf:
  case Builtin::BI__builtin_lroundl:
  case Builtin::BI__builtin_lroundf128:
    return emitUnaryMaybeConstrainedFPToIntBuiltin<cir::LroundOp>(cgf, *e);
  case Builtin::BIllround:
  case Builtin::BIllroundf:
  case Builtin::BIllroundl:
  case Builtin::BI__builtin_llround:
  case Builtin::BI__builtin_llroundf:
  case Builtin::BI__builtin_llroundl:
  case Builtin::BI__builtin_llroundf128:
    return emitUnaryMaybeConstrainedFPToIntBuiltin<cir::LlroundOp>(cgf, *e);
  case Builtin::BIlrint:
  case Builtin::BIlrintf:
  case Builtin::BIlrintl:
  case Builtin::BI__builtin_lrint:
  case Builtin::BI__builtin_lrintf:
  case Builtin::BI__builtin_lrintl:
  case Builtin::BI__builtin_lrintf128:
    return emitUnaryMaybeConstrainedFPToIntBuiltin<cir::LrintOp>(cgf, *e);
  case Builtin::BIllrint:
  case Builtin::BIllrintf:
  case Builtin::BIllrintl:
  case Builtin::BI__builtin_llrint:
  case Builtin::BI__builtin_llrintf:
  case Builtin::BI__builtin_llrintl:
  case Builtin::BI__builtin_llrintf128:
    return emitUnaryMaybeConstrainedFPToIntBuiltin<cir::LlrintOp>(cgf, *e);
  case Builtin::BI__builtin_ldexp:
  case Builtin::BI__builtin_ldexpf:
  case Builtin::BI__builtin_ldexpl:
  case Builtin::BI__builtin_ldexpf16:
  case Builtin::BI__builtin_ldexpf128:
  case Builtin::BI__builtin_elementwise_ldexp:
    return errorBuiltinNYI(cgf, e, builtinID);
  default:
    break;
  }
 
  return RValue::getIgnored();
}
 
// FIXME: Remove cgf parameter when all descriptor kinds are implemented
static mlir::Type
decodeFixedType(CIRGenFunction &cgf,
                ArrayRef<llvm::Intrinsic::IITDescriptor> &infos,
                mlir::MLIRContext *context) {
  using namespace llvm::Intrinsic;
 
  IITDescriptor descriptor = infos.front();
  infos = infos.slice(1);
 
  switch (descriptor.Kind) {
  case IITDescriptor::Void:
    return cir::VoidType::get(context);
  case IITDescriptor::Half:
    return cir::FP16Type::get(context);
  case IITDescriptor::BFloat:
    return cir::BF16Type::get(context);
  case IITDescriptor::Float:
    return cir::SingleType::get(context);
  case IITDescriptor::Double:
    return cir::DoubleType::get(context);
  case IITDescriptor::Quad:
    return cir::FP128Type::get(context);
  // If the intrinsic expects unsigned integers, the signedness is corrected in
  // correctIntegerSignedness()
  case IITDescriptor::Integer:
    return cir::IntType::get(context, descriptor.IntegerWidth,
                             /*isSigned=*/true);
  case IITDescriptor::Vector: {
    mlir::Type elementType = decodeFixedType(cgf, infos, context);
    unsigned numElements = descriptor.VectorWidth.getFixedValue();
    return cir::VectorType::get(elementType, numElements);
  }
  case IITDescriptor::Pointer: {
    mlir::Builder builder(context);
    auto addrSpace = cir::TargetAddressSpaceAttr::get(
        context, descriptor.PointerAddressSpace);
    return cir::PointerType::get(cir::VoidType::get(context), addrSpace);
  }
  default:
    cgf.cgm.errorNYI("Unimplemented intrinsic type descriptor");
    return cir::VoidType::get(context);
  }
}
 
/// Helper function to correct integer signedness for intrinsic arguments and
/// return type. IIT always returns signed integers, but the actual intrinsic
/// may expect unsigned integers based on the AST FunctionDecl parameter types.
static mlir::Type correctIntegerSignedness(mlir::Type iitType, QualType astType,
                                           mlir::MLIRContext *context) {
  auto intTy = dyn_cast<cir::IntType>(iitType);
  if (!intTy)
    return iitType;
 
  if (astType->isUnsignedIntegerType())
    return cir::IntType::get(context, intTy.getWidth(), /*isSigned=*/false);
 
  return iitType;
}
 
/// Helper function to correct the return type for intrinsic calls. This is
/// needed because the AST FunctionDecl may have a different return type than
/// the intrinsic's IIT descriptor. For example, builtins may need their
/// signedness corrected, or a builtin may return a bool while the intrinsic
/// returns an i1.
static mlir::Type correctReturnType(mlir::Type iitType,
                                    const FunctionDecl *funcDecl,
                                    mlir::MLIRContext *context) {
  if (!funcDecl)
    return iitType;
  QualType astType = funcDecl->getReturnType();
 
  // Relabel the return type to cir.bool if the builtin returns a bool and
  // the intrinsic returns an i1.
  auto intTy = mlir::dyn_cast<cir::IntType>(iitType);
  if (intTy && intTy.getWidth() == 1 && astType->isBooleanType())
    return cir::BoolType::get(context);
 
  return correctIntegerSignedness(iitType, astType, context);
}
 
static mlir::Value getCorrectedPtr(mlir::Value argValue, mlir::Type expectedTy,
                                   CIRGenBuilderTy &builder) {
  auto ptrType = mlir::cast<cir::PointerType>(argValue.getType());
 
  auto expectedPtrType = mlir::cast<cir::PointerType>(expectedTy);
  assert(ptrType != expectedPtrType && "types should not match");
 
  if (ptrType.getAddrSpace() != expectedPtrType.getAddrSpace()) {
    assert(!cir::MissingFeatures::addressSpace() &&
           "address space handling not yet implemented");
    auto newPtrType = cir::PointerType::get(ptrType.getPointee(),
                                            expectedPtrType.getAddrSpace());
    return builder.createAddrSpaceCast(argValue, newPtrType);
  }
 
  return builder.createBitcast(argValue, expectedTy);
}
 
static cir::FuncType getIntrinsicType(CIRGenFunction &cgf,
                                      mlir::MLIRContext *context,
                                      llvm::Intrinsic::ID id) {
  using namespace llvm::Intrinsic;
 
  SmallVector<IITDescriptor, 8> table;
  auto [tableRef, _, isVarArg] = getIntrinsicInfoTableEntries(id, table);
 
  mlir::Type resultTy = decodeFixedType(cgf, tableRef, context);
 
  SmallVector<mlir::Type, 8> argTypes;
  while (!tableRef.empty())
    argTypes.push_back(decodeFixedType(cgf, tableRef, context));
 
  // CIR convention: no explicit void return type
  if (isa<cir::VoidType>(resultTy))
    return cir::FuncType::get(context, argTypes, /*optionalReturnType=*/nullptr,
                              isVarArg);
 
  return cir::FuncType::get(context, argTypes, resultTy, isVarArg);
}
 
void CIRGenFunction::checkTargetFeatures(const CallExpr *e,
                                         const FunctionDecl *targetDecl) {
  const FunctionDecl *fd = dyn_cast_or_null<FunctionDecl>(curCodeDecl);
  CodeGenUtils::checkTargetFeatures(getContext(), cgm.getDiags(), getLangOpts(),
                                    e, fd, targetDecl);
}
 
void CIRGenFunction::checkTargetFeatures(SourceLocation loc,
                                         const FunctionDecl *targetDecl) {
  const FunctionDecl *fd = dyn_cast_or_null<FunctionDecl>(curCodeDecl);
  CodeGenUtils::checkTargetFeatures(getContext(), cgm.getDiags(), getLangOpts(),
                                    loc, fd, targetDecl);
}
 
RValue CIRGenFunction::emitBuiltinExpr(const GlobalDecl &gd, unsigned builtinID,
                                       const CallExpr *e,
                                       ReturnValueSlot returnValue) {
  mlir::Location loc = getLoc(e->getSourceRange());
 
  // See if we can constant fold this builtin.  If so, don't emit it at all.
  // TODO: Extend this handling to all builtin calls that we can constant-fold.
  // Do not constant-fold immediate (target-specific) builtins; their ASTs can
  // trigger the constant evaluator in cases it cannot safely handle.
  // Skip EvaluateAsRValue for those.
  Expr::EvalResult result;
  if (e->isPRValue() && !getContext().BuiltinInfo.isImmediate(builtinID) &&
      e->EvaluateAsRValue(result, cgm.getASTContext()) &&
      !result.hasSideEffects()) {
    if (result.Val.isInt()) {
      QualType type = e->getType();
      if (type->isBooleanType())
        return RValue::get(
            builder.getBool(result.Val.getInt().getBoolValue(), loc));
      return RValue::get(builder.getConstInt(loc, result.Val.getInt()));
    }
    if (result.Val.isFloat()) {
      // Note: we are using result type of CallExpr to determine the type of
      // the constant. Classic codegen uses the result value to determine the
      // type. We feel it should be Ok to use expression type because it is
      // hard to imagine a builtin function evaluates to a value that
      // over/underflows its own defined type.
      mlir::Type type = convertType(e->getType());
      return RValue::get(builder.getConstFP(loc, type, result.Val.getFloat()));
    }
  }
 
  const FunctionDecl *fd = gd.getDecl()->getAsFunction();
 
  assert(!cir::MissingFeatures::builtinCallF128());
 
  // If the builtin has been declared explicitly with an assembler label,
  // disable the specialized emitting below. Ideally we should communicate the
  // rename in IR, or at least avoid generating the intrinsic calls that are
  // likely to get lowered to the renamed library functions.
  unsigned builtinIDIfNoAsmLabel = fd->hasAttr<AsmLabelAttr>() ? 0 : builtinID;
 
  bool generateFPMathIntrinsics =
      shouldCIREmitFPMathIntrinsic(*this, e, builtinID);
 
  if (generateFPMathIntrinsics) {
    // Try to match the builtinID with a floating point math builtin.
    RValue rv = tryEmitFPMathIntrinsic(*this, e, builtinIDIfNoAsmLabel);
 
    // Return the result directly if a math intrinsic was generated.
    if (!rv.isIgnored()) {
      return rv;
    }
  }
 
  assert(!cir::MissingFeatures::builtinCall());
 
  switch (builtinIDIfNoAsmLabel) {
  default:
    break;
 
  // C stdarg builtins.
  case Builtin::BI__builtin_stdarg_start:
  case Builtin::BI__builtin_va_start:
  case Builtin::BI__builtin_c23_va_start:
  case Builtin::BI__va_start: {
    mlir::Value vaList = builtinID == Builtin::BI__va_start
                             ? emitScalarExpr(e->getArg(0))
                             : emitVAListRef(e->getArg(0)).getPointer();
    emitVAStart(vaList);
    return {};
  }
 
  case Builtin::BI__builtin_va_end:
    emitVAEnd(emitVAListRef(e->getArg(0)).getPointer());
    return {};
  case Builtin::BI__builtin_va_copy: {
    mlir::Value dstPtr = emitVAListRef(e->getArg(0)).getPointer();
    mlir::Value srcPtr = emitVAListRef(e->getArg(1)).getPointer();
    cir::VACopyOp::create(builder, dstPtr.getLoc(), dstPtr, srcPtr);
    return {};
  }
 
  case Builtin::BIabs:
  case Builtin::BIlabs:
  case Builtin::BIllabs:
  case Builtin::BI__builtin_abs:
  case Builtin::BI__builtin_labs:
  case Builtin::BI__builtin_llabs: {
    bool sanitizeOverflow = sanOpts.has(SanitizerKind::SignedIntegerOverflow);
    mlir::Value arg = emitScalarExpr(e->getArg(0));
    mlir::Value result;
    switch (getLangOpts().getSignedOverflowBehavior()) {
    case LangOptions::SOB_Defined:
      result = cir::AbsOp::create(builder, loc, arg.getType(), arg,
                                  /*minIsPoison=*/false);
      break;
    case LangOptions::SOB_Undefined:
      if (!sanitizeOverflow) {
        result = cir::AbsOp::create(builder, loc, arg.getType(), arg,
                                    /*minIsPoison=*/true);
        break;
      }
      [[fallthrough]];
    case LangOptions::SOB_Trapping:
      cgm.errorNYI(e->getSourceRange(), "abs with overflow handling");
      return RValue::get(nullptr);
    }
    return RValue::get(result);
  }
 
  case Builtin::BI__assume:
  case Builtin::BI__builtin_assume: {
    if (e->getArg(0)->HasSideEffects(getContext()))
      return RValue::get(nullptr);
 
    mlir::Value argValue = emitCheckedArgForAssume(e->getArg(0));
    cir::AssumeOp::create(builder, loc, argValue, cir::AssumeBundleKind::None,
                          mlir::ValueRange{});
    return RValue::get(nullptr);
  }
 
  case Builtin::BI__builtin_assume_separate_storage: {
    mlir::Value value0 = emitScalarExpr(e->getArg(0));
    mlir::Value value1 = emitScalarExpr(e->getArg(1));
    mlir::Value cond = builder.getBool(true, loc);
    cir::AssumeOp::create(builder, loc, cond,
                          cir::AssumeBundleKind::SeparateStorage,
                          mlir::ValueRange{value0, value1});
    return RValue::get(nullptr);
  }
 
  case Builtin::BI__arithmetic_fence: {
    assert(!cir::MissingFeatures::fastMathFlags());
    QualType argType = e->getArg(0)->getType();
    FPOptions fpFeatures = e->getFPFeaturesInEffect(getLangOpts());
    if (fpFeatures.getAllowFPReassociate() &&
        getContext().getTargetInfo().checkArithmeticFenceSupported()) {
      cgm.errorNYI(e->getSourceRange(), "__arithmetic_fence with reassocc");
    }
    if (argType->isComplexType())
      return RValue::getComplex(emitComplexExpr(e->getArg(0)));
    return RValue::get(emitScalarExpr(e->getArg(0)));
  }
 
  case Builtin::BI__builtin_assume_dereferenceable: {
    mlir::Value ptrValue = emitScalarExpr(e->getArg(0));
    mlir::Value sizeValue = emitScalarExpr(e->getArg(1));
    // The `dereferenceable` operand bundle expects a pointer-sized unsigned
    // integer; widen/narrow as needed.
    mlir::Type uintPtrTy = convertType(getContext().getUIntPtrType());
    if (sizeValue.getType() != uintPtrTy)
      sizeValue = builder.createIntCast(sizeValue, uintPtrTy);
    mlir::Value cond = builder.getBool(true, loc);
    cir::AssumeOp::create(builder, loc, cond,
                          cir::AssumeBundleKind::Dereferenceable,
                          mlir::ValueRange{ptrValue, sizeValue});
    return RValue::get(nullptr);
  }
 
  case Builtin::BI__builtin_assume_aligned: {
    const Expr *ptrExpr = e->getArg(0);
    mlir::Value ptrValue = emitScalarExpr(ptrExpr);
    mlir::Value offsetValue =
        (e->getNumArgs() > 2) ? emitScalarExpr(e->getArg(2)) : nullptr;
 
    std::optional<llvm::APSInt> alignment =
        e->getArg(1)->getIntegerConstantExpr(getContext());
    assert(alignment.has_value() &&
           "the second argument to __builtin_assume_aligned must be an "
           "integral constant expression");
 
    mlir::Value result =
        emitAlignmentAssumption(ptrValue, ptrExpr, ptrExpr->getExprLoc(),
                                alignment->getSExtValue(), offsetValue);
    return RValue::get(result);
  }
 
  case Builtin::BI__builtin_complex: {
    mlir::Value real = emitScalarExpr(e->getArg(0));
    mlir::Value imag = emitScalarExpr(e->getArg(1));
    mlir::Value complex = builder.createComplexCreate(loc, real, imag);
    return RValue::getComplex(complex);
  }
 
  case Builtin::BI__builtin_creal:
  case Builtin::BI__builtin_crealf:
  case Builtin::BI__builtin_creall:
  case Builtin::BIcreal:
  case Builtin::BIcrealf:
  case Builtin::BIcreall: {
    mlir::Value complex = emitComplexExpr(e->getArg(0));
    mlir::Value real = builder.createComplexReal(loc, complex);
    return RValue::get(real);
  }
 
  case Builtin::BI__builtin_cimag:
  case Builtin::BI__builtin_cimagf:
  case Builtin::BI__builtin_cimagl:
  case Builtin::BIcimag:
  case Builtin::BIcimagf:
  case Builtin::BIcimagl: {
    mlir::Value complex = emitComplexExpr(e->getArg(0));
    mlir::Value imag = builder.createComplexImag(loc, complex);
    return RValue::get(imag);
  }
 
  case Builtin::BI__builtin_conj:
  case Builtin::BI__builtin_conjf:
  case Builtin::BI__builtin_conjl:
  case Builtin::BIconj:
  case Builtin::BIconjf:
  case Builtin::BIconjl: {
    mlir::Value complex = emitComplexExpr(e->getArg(0));
    mlir::Value conj = builder.createComplexConj(loc, complex);
    return RValue::getComplex(conj);
  }
 
  case Builtin::BI__builtin_clrsb:
  case Builtin::BI__builtin_clrsbl:
  case Builtin::BI__builtin_clrsbll:
    return emitBuiltinBitOp<cir::BitClrsbOp>(*this, e);
 
  case Builtin::BI__builtin_ctzs:
  case Builtin::BI__builtin_ctz:
  case Builtin::BI__builtin_ctzl:
  case Builtin::BI__builtin_ctzll:
    assert(!cir::MissingFeatures::builtinCheckKind());
    return emitBuiltinBitOp<cir::BitCtzOp>(*this, e,
                                           getTarget().isCLZForZeroUndef());
  case Builtin::BI__builtin_ctzg:
    return emitBuiltinBitOpWithFallback<cir::BitCtzOp>(*this, e);
  case Builtin::BIstdc_trailing_zeros_uc:
  case Builtin::BIstdc_trailing_zeros_us:
  case Builtin::BIstdc_trailing_zeros_ui:
  case Builtin::BIstdc_trailing_zeros_ul:
  case Builtin::BIstdc_trailing_zeros_ull:
  case Builtin::BI__builtin_stdc_trailing_zeros:
    return emitStdcCountOp<cir::BitCtzOp>(*this, e, /*invertArg=*/false,
                                          /*poisonZero=*/false);
 
  case Builtin::BIstdc_leading_zeros_uc:
  case Builtin::BIstdc_leading_zeros_us:
  case Builtin::BIstdc_leading_zeros_ui:
  case Builtin::BIstdc_leading_zeros_ul:
  case Builtin::BIstdc_leading_zeros_ull:
  case Builtin::BI__builtin_stdc_leading_zeros:
    return emitStdcCountOp<cir::BitClzOp>(*this, e, /*invertArg=*/false,
                                          /*poisonZero=*/false);
 
  case Builtin::BIstdc_trailing_ones_uc:
  case Builtin::BIstdc_trailing_ones_us:
  case Builtin::BIstdc_trailing_ones_ui:
  case Builtin::BIstdc_trailing_ones_ul:
  case Builtin::BIstdc_trailing_ones_ull:
  case Builtin::BI__builtin_stdc_trailing_ones:
    return emitStdcCountOp<cir::BitCtzOp>(*this, e, /*invertArg=*/true,
                                          /*poisonZero=*/false);
 
  case Builtin::BIstdc_leading_ones_uc:
  case Builtin::BIstdc_leading_ones_us:
  case Builtin::BIstdc_leading_ones_ui:
  case Builtin::BIstdc_leading_ones_ul:
  case Builtin::BIstdc_leading_ones_ull:
  case Builtin::BI__builtin_stdc_leading_ones:
    return emitStdcCountOp<cir::BitClzOp>(*this, e, /*invertArg=*/true,
                                          /*poisonZero=*/false);
 
  case Builtin::BIstdc_bit_width_uc:
  case Builtin::BIstdc_bit_width_us:
  case Builtin::BIstdc_bit_width_ui:
  case Builtin::BIstdc_bit_width_ul:
  case Builtin::BIstdc_bit_width_ull:
  case Builtin::BI__builtin_stdc_bit_width:
    return emitStdcBitWidthMinus<cir::BitClzOp>(*this, e,
                                                /*poisonZero=*/false);
 
  case Builtin::BIstdc_count_zeros_uc:
  case Builtin::BIstdc_count_zeros_us:
  case Builtin::BIstdc_count_zeros_ui:
  case Builtin::BIstdc_count_zeros_ul:
  case Builtin::BIstdc_count_zeros_ull:
  case Builtin::BI__builtin_stdc_count_zeros:
    return emitStdcBitWidthMinus<cir::BitPopcountOp>(*this, e);
 
  case Builtin::BIstdc_count_ones_uc:
  case Builtin::BIstdc_count_ones_us:
  case Builtin::BIstdc_count_ones_ui:
  case Builtin::BIstdc_count_ones_ul:
  case Builtin::BIstdc_count_ones_ull:
  case Builtin::BI__builtin_stdc_count_ones:
    return emitStdcCountOp<cir::BitPopcountOp>(*this, e,
                                               /*invertArg=*/false);
 
  case Builtin::BIstdc_has_single_bit_uc:
  case Builtin::BIstdc_has_single_bit_us:
  case Builtin::BIstdc_has_single_bit_ui:
  case Builtin::BIstdc_has_single_bit_ul:
  case Builtin::BIstdc_has_single_bit_ull:
  case Builtin::BI__builtin_stdc_has_single_bit:
    return emitStdcHasSingleBit(*this, e);
 
  case Builtin::BIstdc_first_leading_zero_uc:
  case Builtin::BIstdc_first_leading_zero_us:
  case Builtin::BIstdc_first_leading_zero_ui:
  case Builtin::BIstdc_first_leading_zero_ul:
  case Builtin::BIstdc_first_leading_zero_ull:
  case Builtin::BI__builtin_stdc_first_leading_zero:
    return emitStdcFirstBit<cir::BitClzOp>(*this, e, /*invertArg=*/true,
                                           /*poisonZero=*/false);
 
  case Builtin::BIstdc_first_leading_one_uc:
  case Builtin::BIstdc_first_leading_one_us:
  case Builtin::BIstdc_first_leading_one_ui:
  case Builtin::BIstdc_first_leading_one_ul:
  case Builtin::BIstdc_first_leading_one_ull:
  case Builtin::BI__builtin_stdc_first_leading_one:
    return emitStdcFirstBit<cir::BitClzOp>(*this, e, /*invertArg=*/false,
                                           /*poisonZero=*/false);
 
  case Builtin::BIstdc_first_trailing_zero_uc:
  case Builtin::BIstdc_first_trailing_zero_us:
  case Builtin::BIstdc_first_trailing_zero_ui:
  case Builtin::BIstdc_first_trailing_zero_ul:
  case Builtin::BIstdc_first_trailing_zero_ull:
  case Builtin::BI__builtin_stdc_first_trailing_zero:
    return emitStdcFirstBit<cir::BitCtzOp>(*this, e, /*invertArg=*/true,
                                           /*poisonZero=*/false);
 
  case Builtin::BIstdc_first_trailing_one_uc:
  case Builtin::BIstdc_first_trailing_one_us:
  case Builtin::BIstdc_first_trailing_one_ui:
  case Builtin::BIstdc_first_trailing_one_ul:
  case Builtin::BIstdc_first_trailing_one_ull:
  case Builtin::BI__builtin_stdc_first_trailing_one:
    return emitStdcFirstBit<cir::BitCtzOp>(*this, e, /*invertArg=*/false,
                                           /*poisonZero=*/false);
 
  case Builtin::BIstdc_bit_ceil_uc:
  case Builtin::BIstdc_bit_ceil_us:
  case Builtin::BIstdc_bit_ceil_ui:
  case Builtin::BIstdc_bit_ceil_ul:
  case Builtin::BIstdc_bit_ceil_ull:
  case Builtin::BI__builtin_stdc_bit_ceil:
    return emitStdcBitCeil(*this, e);
 
  case Builtin::BIstdc_bit_floor_uc:
  case Builtin::BIstdc_bit_floor_us:
  case Builtin::BIstdc_bit_floor_ui:
  case Builtin::BIstdc_bit_floor_ul:
  case Builtin::BIstdc_bit_floor_ull:
  case Builtin::BI__builtin_stdc_bit_floor:
    return emitStdcBitFloor(*this, e);
 
  case Builtin::BI__builtin_clzs:
  case Builtin::BI__builtin_clz:
  case Builtin::BI__builtin_clzl:
  case Builtin::BI__builtin_clzll:
    assert(!cir::MissingFeatures::builtinCheckKind());
    return emitBuiltinBitOp<cir::BitClzOp>(*this, e,
                                           getTarget().isCLZForZeroUndef());
  case Builtin::BI__builtin_clzg:
    return emitBuiltinBitOpWithFallback<cir::BitClzOp>(*this, e);
 
  case Builtin::BI__builtin_elementwise_ctzg:
    cgm.errorNYI(e->getSourceRange(), "__builtin_elementwise_ctzg");
    return RValue::get(nullptr);
  case Builtin::BI__builtin_elementwise_clzg:
    cgm.errorNYI(e->getSourceRange(), "__builtin_elementwise_clzg");
    return RValue::get(nullptr);
 
  case Builtin::BI__builtin_ffs:
  case Builtin::BI__builtin_ffsl:
  case Builtin::BI__builtin_ffsll:
    return emitBuiltinBitOp<cir::BitFfsOp>(*this, e);
 
  case Builtin::BI__builtin_parity:
  case Builtin::BI__builtin_parityl:
  case Builtin::BI__builtin_parityll:
    return emitBuiltinBitOp<cir::BitParityOp>(*this, e);
 
  case Builtin::BI__lzcnt16:
  case Builtin::BI__lzcnt:
  case Builtin::BI__lzcnt64:
    return emitBuiltinBitOp<cir::BitClzOp>(*this, e);
 
  case Builtin::BI__popcnt16:
  case Builtin::BI__popcnt:
  case Builtin::BI__popcnt64:
  case Builtin::BI__builtin_popcount:
  case Builtin::BI__builtin_popcountl:
  case Builtin::BI__builtin_popcountll:
  case Builtin::BI__builtin_popcountg:
    return emitBuiltinBitOp<cir::BitPopcountOp>(*this, e);
 
  // Always return the argument of __builtin_unpredictable.  LLVM does not
  // have an intrinsic corresponding to this builtin.  Metadata for this
  // builtin should be added directly to instructions such as branches or
  // switches that use it.
  case Builtin::BI__builtin_unpredictable: {
    return RValue::get(emitScalarExpr(e->getArg(0)));
  }
 
  case Builtin::BI__builtin_expect:
  case Builtin::BI__builtin_expect_with_probability: {
    mlir::Value argValue = emitScalarExpr(e->getArg(0));
    if (cgm.getCodeGenOpts().OptimizationLevel == 0)
      return RValue::get(argValue);
 
    mlir::Value expectedValue = emitScalarExpr(e->getArg(1));
 
    mlir::FloatAttr probAttr;
    if (builtinIDIfNoAsmLabel == Builtin::BI__builtin_expect_with_probability) {
      llvm::APFloat probability(0.0);
      const Expr *probArg = e->getArg(2);
      [[maybe_unused]] bool evalSucceeded =
          probArg->EvaluateAsFloat(probability, cgm.getASTContext());
      assert(evalSucceeded &&
             "probability should be able to evaluate as float");
      bool loseInfo = false; // ignored
      probability.convert(llvm::APFloat::IEEEdouble(),
                          llvm::RoundingMode::Dynamic, &loseInfo);
      probAttr = mlir::FloatAttr::get(mlir::Float64Type::get(&getMLIRContext()),
                                      probability);
    }
 
    auto result = cir::ExpectOp::create(builder, loc, argValue.getType(),
                                        argValue, expectedValue, probAttr);
    return RValue::get(result);
  }
 
  case Builtin::BI__builtin_bswapg: {
    mlir::Value arg = emitScalarExpr(e->getArg(0));
    // CIR models bool as cir.bool rather than an integer, so peel it off
    // before the cast below.  Like classic codegen's i1 case, it byte-swaps
    // to itself.
    if (mlir::isa<cir::BoolType>(arg.getType()))
      return RValue::get(arg);
    auto argTy = mlir::cast<cir::IntType>(arg.getType());
    // A single bit or a single byte byte-swaps to itself.
    if (argTy.getWidth() == 1 || argTy.getWidth() == 8)
      return RValue::get(arg);
    assert(argTy.getWidth() % 16 == 0 &&
           "__builtin_bswapg requires a single byte or a multiple of 16 bits");
    // cir.byte_swap requires an unsigned operand.  Reinterpret a signed
    // argument as unsigned of the same width; createBuiltinBitOp casts the
    // swapped result back to the builtin's (possibly signed) return type.
    if (argTy.isSigned())
      arg = builder.createIntCast(arg, builder.getUIntNTy(argTy.getWidth()));
    return RValue::get(createBuiltinBitOp<cir::ByteSwapOp>(*this, e, arg));
  }
 
  case Builtin::BI__builtin_bswap16:
  case Builtin::BI__builtin_bswap32:
  case Builtin::BI__builtin_bswap64:
  case Builtin::BI_byteswap_ushort:
  case Builtin::BI_byteswap_ulong:
  case Builtin::BI_byteswap_uint64: {
    mlir::Value arg = emitScalarExpr(e->getArg(0));
    return RValue::get(cir::ByteSwapOp::create(builder, loc, arg));
  }
 
  case Builtin::BI__builtin_bitreverse8:
  case Builtin::BI__builtin_bitreverse16:
  case Builtin::BI__builtin_bitreverse32:
  case Builtin::BI__builtin_bitreverse64: {
    mlir::Value arg = emitScalarExpr(e->getArg(0));
    return RValue::get(cir::BitReverseOp::create(builder, loc, arg));
  }
 
  case Builtin::BI__builtin_rotateleft8:
  case Builtin::BI__builtin_rotateleft16:
  case Builtin::BI__builtin_rotateleft32:
  case Builtin::BI__builtin_rotateleft64:
  case Builtin::BI__builtin_stdc_rotate_left:
  case Builtin::BIstdc_rotate_left_uc:
  case Builtin::BIstdc_rotate_left_us:
  case Builtin::BIstdc_rotate_left_ui:
  case Builtin::BIstdc_rotate_left_ul:
  case Builtin::BIstdc_rotate_left_ull:
    return emitRotate(e, /*isRotateLeft=*/true);
 
  case Builtin::BI__builtin_rotateright8:
  case Builtin::BI__builtin_rotateright16:
  case Builtin::BI__builtin_rotateright32:
  case Builtin::BI__builtin_rotateright64:
  case Builtin::BI__builtin_stdc_rotate_right:
  case Builtin::BIstdc_rotate_right_uc:
  case Builtin::BIstdc_rotate_right_us:
  case Builtin::BIstdc_rotate_right_ui:
  case Builtin::BIstdc_rotate_right_ul:
  case Builtin::BIstdc_rotate_right_ull:
    return emitRotate(e, /*isRotateLeft=*/false);
 
  // stdc_memreverse8u8 is a no-op (single byte, nothing to swap).
  case Builtin::BIstdc_memreverse8u8:
    return RValue::get(emitScalarExpr(e->getArg(0)));
  case Builtin::BIstdc_memreverse8u16:
  case Builtin::BIstdc_memreverse8u32:
  case Builtin::BIstdc_memreverse8u64: {
    mlir::Value arg = emitScalarExpr(e->getArg(0));
    return RValue::get(cir::ByteSwapOp::create(builder, loc, arg));
  }
  case Builtin::BIstdc_memreverse8:
  case Builtin::BI__builtin_stdc_memreverse8: {
    Expr::EvalResult result;
    if (e->getArg(0)->EvaluateAsInt(result, getContext())) {
      uint64_t size = result.Val.getInt().getZExtValue();
      if (size <= 1) {
        emitIgnoredExpr(e->getArg(1));
        return RValue::get(nullptr);
      }
      if (size == 2 || size == 4 || size == 8) {
        mlir::Location loc = getLoc(e->getSourceRange());
        Address ptrAddr = emitPointerWithAlignment(e->getArg(1));
        mlir::Type intTy = builder.getUIntNTy(size * 8);
        Address addr = builder.createElementBitCast(loc, ptrAddr, intTy);
        mlir::Value val = builder.createLoad(loc, addr);
        mlir::Value swapped = cir::ByteSwapOp::create(builder, loc, val);
        builder.createStore(loc, swapped, addr);
        return RValue::get(nullptr);
      }
    }
    // General case: fall back to the library function stdc_memreverse8.
    break;
  }
 
  case Builtin::BI__builtin_coro_id:
    return RValue::get(emitCoroIDBuiltinCall(e).getResult());
  case Builtin::BI__builtin_coro_alloc: {
    cir::CoroAllocOp coroAlloc = emitCoroAllocBuiltinCall(e);
    return coroAlloc ? RValue::get(coroAlloc.getResult())
                     : getUndefRValue(e->getType());
  }
  case Builtin::BI__builtin_coro_begin: {
    cir::CoroBeginOp coroBeg = emitCoroBeginBuiltinCall(e);
    return coroBeg ? RValue::get(coroBeg.getResult())
                   : getUndefRValue(e->getType());
  }
  case Builtin::BI__builtin_coro_end:
    return RValue::get(emitCoroEndBuiltinCall(e).getResult());
  case Builtin::BI__builtin_coro_promise:
    return RValue::get(emitCoroPromiseBuiltinCall(e).getResult());
  case Builtin::BI__builtin_coro_resume: {
    emitCoroResumeBuiltinCall(e);
    return RValue::get(nullptr);
  }
  case Builtin::BI__builtin_coro_noop:
    cgm.errorNYI(e->getSourceRange(), "BI__builtin_coro_noop NYI");
    return getUndefRValue(e->getType());
  case Builtin::BI__builtin_coro_destroy: {
    emitCoroDestroyBuiltinCall(e);
    return RValue::get(nullptr);
  }
  case Builtin::BI__builtin_coro_done:
    return RValue::get(emitCoroDoneBuiltinCall(e).getResult());
  case Builtin::BI__builtin_coro_suspend:
    cgm.errorNYI(e->getSourceRange(), "BI__builtin_coro_suspend NYI");
    return getUndefRValue(e->getType());
  case Builtin::BI__builtin_coro_align:
    cgm.errorNYI(e->getSourceRange(), "BI__builtin_coro_align NYI");
    return getUndefRValue(e->getType());
 
  case Builtin::BI__builtin_coro_frame: {
    return emitCoroutineFrame();
  }
  case Builtin::BI__builtin_coro_free: {
    cir::CoroFreeOp coroFree = emitCoroFreeBuiltin(e);
    return coroFree ? RValue::get(coroFree.getResult())
                    : getUndefRValue(e->getType());
  }
 
  case Builtin::BI__builtin_coro_size: {
    return RValue::get(emitCoroSizeBuiltinCall(e).getResult());
  }
 
  case Builtin::BI__builtin_constant_p: {
    mlir::Type resultType = convertType(e->getType());
 
    const Expr *arg = e->getArg(0);
    QualType argType = arg->getType();
    // FIXME: The allowance for Obj-C pointers and block pointers is historical
    // and likely a mistake.
    if (!argType->isIntegralOrEnumerationType() && !argType->isFloatingType() &&
        !argType->isObjCObjectPointerType() && !argType->isBlockPointerType()) {
      // Per the GCC documentation, only numeric constants are recognized after
      // inlining.
      return RValue::get(
          builder.getConstInt(getLoc(e->getSourceRange()),
                              mlir::cast<cir::IntType>(resultType), 0));
    }
 
    if (arg->HasSideEffects(getContext())) {
      // The argument is unevaluated, so be conservative if it might have
      // side-effects.
      return RValue::get(
          builder.getConstInt(getLoc(e->getSourceRange()),
                              mlir::cast<cir::IntType>(resultType), 0));
    }
 
    mlir::Value argValue = emitScalarExpr(arg);
    if (argType->isObjCObjectPointerType()) {
      cgm.errorNYI(e->getSourceRange(),
                   "__builtin_constant_p: Obj-C object pointer");
      return {};
    }
    argValue = builder.createBitcast(argValue, convertType(argType));
 
    mlir::Value result = cir::IsConstantOp::create(
        builder, getLoc(e->getSourceRange()), argValue);
    // IsConstantOp returns a bool, but __builtin_constant_p returns an int.
    result = builder.createBoolToInt(result, resultType);
    return RValue::get(result);
  }
  case Builtin::BI__builtin_dynamic_object_size:
  case Builtin::BI__builtin_object_size: {
    unsigned type =
        e->getArg(1)->EvaluateKnownConstInt(getContext()).getZExtValue();
    auto resType = mlir::cast<cir::IntType>(convertType(e->getType()));
 
    // We pass this builtin onto the optimizer so that it can figure out the
    // object size in more complex cases.
    bool isDynamic = builtinID == Builtin::BI__builtin_dynamic_object_size;
    return RValue::get(emitBuiltinObjectSize(e->getArg(0), type, resType,
                                             /*EmittedE=*/nullptr, isDynamic));
  }
 
  case Builtin::BI__builtin_prefetch: {
    auto evaluateOperandAsInt = [&](const Expr *arg) {
      Expr::EvalResult res;
      [[maybe_unused]] bool evalSucceed =
          arg->EvaluateAsInt(res, cgm.getASTContext());
      assert(evalSucceed && "expression should be able to evaluate as int");
      return res.Val.getInt().getZExtValue();
    };
 
    bool isWrite = false;
    if (e->getNumArgs() > 1)
      isWrite = evaluateOperandAsInt(e->getArg(1));
 
    int locality = 3;
    if (e->getNumArgs() > 2)
      locality = evaluateOperandAsInt(e->getArg(2));
 
    mlir::Value address = emitScalarExpr(e->getArg(0));
    cir::PrefetchOp::create(builder, loc, address, locality, isWrite);
    return RValue::get(nullptr);
  }
  case Builtin::BI__builtin_readcyclecounter:
  case Builtin::BI__builtin_readsteadycounter:
    return errorBuiltinNYI(*this, e, builtinID);
  case Builtin::BI__builtin___clear_cache: {
    mlir::Value begin =
        builder.createPtrBitcast(emitScalarExpr(e->getArg(0)), cgm.voidTy);
    mlir::Value end =
        builder.createPtrBitcast(emitScalarExpr(e->getArg(1)), cgm.voidTy);
    cir::ClearCacheOp::create(builder, getLoc(e->getSourceRange()), begin, end);
    return RValue::get(nullptr);
  }
  case Builtin::BI__builtin_trap:
    emitTrap(loc, /*createNewBlock=*/true);
    return RValue::getIgnored();
  case Builtin::BI__builtin_verbose_trap:
    assert(!cir::MissingFeatures::generateDebugInfo());
    emitTrap(loc, /*createNewBlock=*/true);
    return RValue::getIgnored();
  case Builtin::BI__debugbreak:
    return errorBuiltinNYI(*this, e, builtinID);
  case Builtin::BI__builtin_unreachable:
    emitUnreachable(e->getExprLoc(), /*createNewBlock=*/true);
    return RValue::getIgnored();
  case Builtin::BI__builtin_powi:
  case Builtin::BI__builtin_powif:
  case Builtin::BI__builtin_powil: {
    mlir::Value src0 = emitScalarExpr(e->getArg(0));
    mlir::Value src1 = emitScalarExpr(e->getArg(1));
    return RValue::get(builder.emitIntrinsicCallOp(
        getLoc(e->getExprLoc()), "powi", src0.getType(),
        mlir::ValueRange{src0, src1}));
  }
  case Builtin::BI__builtin_frexpl:
  case Builtin::BI__builtin_frexp:
  case Builtin::BI__builtin_frexpf:
  case Builtin::BI__builtin_frexpf128:
  case Builtin::BI__builtin_frexpf16: {
    mlir::Value val = emitScalarExpr(e->getArg(0));
    mlir::Value ptr = emitScalarExpr(e->getArg(1));
    mlir::Type fpTy = val.getType();
    QualType intQualTy = e->getArg(1)->getType()->getPointeeType();
    mlir::Type intTy = convertType(intQualTy);
    mlir::Location callLoc = getLoc(e->getExprLoc());
    auto frexpOp = cir::FrexpOp::create(builder, callLoc, fpTy, intTy, val);
    LValue lv = makeNaturalAlignAddrLValue(ptr, intQualTy);
    emitStoreOfScalar(frexpOp.getExp(), lv, /*isInit=*/false);
    return RValue::get(frexpOp.getResult());
  }
  case Builtin::BImodf:
  case Builtin::BImodff:
  case Builtin::BImodfl:
  case Builtin::BI__builtin_modf:
  case Builtin::BI__builtin_modff:
  case Builtin::BI__builtin_modfl: {
    mlir::Value val = emitScalarExpr(e->getArg(0));
    mlir::Value ptr = emitScalarExpr(e->getArg(1));
    mlir::Type fpTy = val.getType();
    mlir::Location callLoc = getLoc(e->getExprLoc());
    auto modfOp = cir::ModfOp::create(builder, callLoc, fpTy, fpTy, val);
    QualType destPtrTy = e->getArg(1)->getType()->getPointeeType();
    LValue lv = makeNaturalAlignAddrLValue(ptr, destPtrTy);
    emitStoreOfScalar(modfOp.getIntegral(), lv, /*isInit=*/false);
    return RValue::get(modfOp.getFractional());
  }
  case Builtin::BI__builtin_isgreater:
  case Builtin::BI__builtin_isgreaterequal:
  case Builtin::BI__builtin_isless:
  case Builtin::BI__builtin_islessequal:
  case Builtin::BI__builtin_islessgreater:
  case Builtin::BI__builtin_isunordered: {
    CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(*this, e);
    mlir::Value lhs = emitScalarExpr(e->getArg(0));
    mlir::Value rhs = emitScalarExpr(e->getArg(1));
    mlir::Location loc = getLoc(e->getBeginLoc());
    mlir::Type intTy = convertType(e->getType());
 
    mlir::Value cmpResult;
    switch (builtinID) {
    case Builtin::BI__builtin_isgreater:
      cmpResult = builder.createCompare(loc, cir::CmpOpKind::gt, lhs, rhs);
      break;
    case Builtin::BI__builtin_isgreaterequal:
      cmpResult = builder.createCompare(loc, cir::CmpOpKind::ge, lhs, rhs);
      break;
    case Builtin::BI__builtin_isless:
      cmpResult = builder.createCompare(loc, cir::CmpOpKind::lt, lhs, rhs);
      break;
    case Builtin::BI__builtin_islessequal:
      cmpResult = builder.createCompare(loc, cir::CmpOpKind::le, lhs, rhs);
      break;
    case Builtin::BI__builtin_islessgreater:
      cmpResult = builder.createCompare(loc, cir::CmpOpKind::one, lhs, rhs);
      break;
    case Builtin::BI__builtin_isunordered:
      cmpResult = builder.createCompare(loc, cir::CmpOpKind::uno, lhs, rhs);
      break;
    default:
      llvm_unreachable("Unknown ordered comparison");
    }
    return RValue::get(builder.createBoolToInt(cmpResult, intTy));
  }
  // From https://clang.llvm.org/docs/LanguageExtensions.html#builtin-isfpclass
  //
  //  The `__builtin_isfpclass()` builtin is a generalization of functions
  //  isnan, isinf, isfinite and some others defined by the C standard. It tests
  //  if the floating-point value, specified by the first argument, falls into
  //  any of data classes, specified by the second argument.
  case Builtin::BI__builtin_isnan: {
    CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(*this, e);
    mlir::Value v = emitScalarExpr(e->getArg(0));
    mlir::Location loc = getLoc(e->getBeginLoc());
    return RValue::get(builder.createBoolToInt(
        builder.createIsFPClass(loc, v, cir::FPClassTest::Nan),
        convertType(e->getType())));
  }
 
  case Builtin::BI__builtin_issignaling: {
    CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(*this, e);
    mlir::Value v = emitScalarExpr(e->getArg(0));
    mlir::Location loc = getLoc(e->getBeginLoc());
    return RValue::get(builder.createBoolToInt(
        builder.createIsFPClass(loc, v, cir::FPClassTest::SignalingNaN),
        convertType(e->getType())));
  }
 
  case Builtin::BI__builtin_isinf: {
    CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(*this, e);
    mlir::Value v = emitScalarExpr(e->getArg(0));
    mlir::Location loc = getLoc(e->getBeginLoc());
    return RValue::get(builder.createBoolToInt(
        builder.createIsFPClass(loc, v, cir::FPClassTest::Infinity),
        convertType(e->getType())));
  }
  case Builtin::BIfinite:
  case Builtin::BI__finite:
  case Builtin::BIfinitef:
  case Builtin::BI__finitef:
  case Builtin::BIfinitel:
  case Builtin::BI__finitel:
  case Builtin::BI__builtin_isfinite: {
    mlir::Value v = emitScalarExpr(e->getArg(0));
    mlir::Location loc = getLoc(e->getBeginLoc());
    return RValue::get(builder.createBoolToInt(
        builder.createIsFPClass(loc, v, cir::FPClassTest::Finite),
        convertType(e->getType())));
  }
 
  case Builtin::BI__builtin_isnormal: {
    mlir::Value v = emitScalarExpr(e->getArg(0));
    mlir::Location loc = getLoc(e->getBeginLoc());
    return RValue::get(builder.createBoolToInt(
        builder.createIsFPClass(loc, v, cir::FPClassTest::Normal),
        convertType(e->getType())));
  }
 
  case Builtin::BI__builtin_issubnormal: {
    mlir::Value v = emitScalarExpr(e->getArg(0));
    mlir::Location loc = getLoc(e->getBeginLoc());
    return RValue::get(builder.createBoolToInt(
        builder.createIsFPClass(loc, v, cir::FPClassTest::Subnormal),
        convertType(e->getType())));
  }
 
  case Builtin::BI__builtin_iszero: {
    mlir::Value v = emitScalarExpr(e->getArg(0));
    mlir::Location loc = getLoc(e->getBeginLoc());
    return RValue::get(builder.createBoolToInt(
        builder.createIsFPClass(loc, v, cir::FPClassTest::Zero),
        convertType(e->getType())));
  }
  case Builtin::BI__builtin_isfpclass: {
    Expr::EvalResult result;
    if (!e->getArg(1)->EvaluateAsInt(result, cgm.getASTContext()))
      break;
 
    CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(*this, e);
    mlir::Value v = emitScalarExpr(e->getArg(0));
    uint64_t test = result.Val.getInt().getLimitedValue();
    mlir::Location loc = getLoc(e->getBeginLoc());
    return RValue::get(builder.createBoolToInt(
        builder.createIsFPClass(loc, v, cir::FPClassTest(test)),
        convertType(e->getType())));
  }
  case Builtin::BI__builtin_nondeterministic_value: {
    mlir::Type ty = convertType(e->getArg(0)->getType());
    mlir::Value result =
        cir::ConstantOp::create(builder, loc, ty, cir::PoisonAttr::get(ty));
    result = cir::FreezeOp::create(builder, loc, result);
    return RValue::get(result);
  }
  case Builtin::BI__builtin_elementwise_abs: {
    mlir::Type cirTy = convertType(e->getArg(0)->getType());
    bool isIntTy = cir::isIntOrVectorOfIntType(cirTy);
    if (!isIntTy)
      return emitUnaryFPBuiltin<cir::FAbsOp>(*this, *e);
    mlir::Value arg = emitScalarExpr(e->getArg(0));
    mlir::Value result = cir::AbsOp::create(builder, getLoc(e->getExprLoc()),
                                            arg.getType(), arg, false);
    return RValue::get(result);
  }
  case Builtin::BI__builtin_elementwise_bitreverse:
  case Builtin::BI__builtin_elementwise_popcount:
  case Builtin::BI__builtin_elementwise_canonicalize:
  case Builtin::BI__builtin_elementwise_copysign:
    return errorBuiltinNYI(*this, e, builtinID);
  case Builtin::BI__builtin_elementwise_fshl:
    return emitTernarySameTypeBuiltin<cir::FshlOp>(*this, *e);
  case Builtin::BI__builtin_elementwise_fshr:
    return emitTernarySameTypeBuiltin<cir::FshrOp>(*this, *e);
  case Builtin::BI__builtin_elementwise_clmul:
  case Builtin::BI__builtin_elementwise_pext:
  case Builtin::BI__builtin_elementwise_pdep:
    return errorBuiltinNYI(*this, e, builtinID);
  case Builtin::BI__builtin_elementwise_add_sat:
  case Builtin::BI__builtin_elementwise_sub_sat: {
    // cir.add/cir.sub do not model i1 arithmetic, so a bool-element
    // saturating add/sub is not representable through the saturated op.
    // Bail before emitScalarExpr: an ext-vector-of-bool operand would
    // otherwise hit the NYI bool-vector load, which returns a null value
    // and would crash op0.getType().
    QualType argTy = e->getArg(0)->getType();
    if (argTy->isBooleanType() || argTy->isExtVectorBoolType()) {
      cgm.errorNYI(e->getSourceRange(),
                   "saturating add/sub on a boolean operand");
      return RValue::get(nullptr);
    }
    mlir::Location loc = getLoc(e->getExprLoc());
    mlir::Value op0 = emitScalarExpr(e->getArg(0));
    mlir::Value op1 = emitScalarExpr(e->getArg(1));
    assert(cir::isIntOrVectorOfIntType(op0.getType()) &&
           "elementwise saturating add/sub requires integer operands");
    mlir::Value val =
        builtinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_add_sat
            ? builder.createAdd(loc, op0, op1, cir::OverflowBehavior::Saturated)
            : builder.createSub(loc, op0, op1,
                                cir::OverflowBehavior::Saturated);
    return RValue::get(val);
  }
  case Builtin::BI__builtin_elementwise_max: {
    if (cir::isIntOrVectorOfIntType(convertType(e->getArg(0)->getType()))) {
      mlir::Location loc = getLoc(e->getExprLoc());
      mlir::Value op0 = emitScalarExpr(e->getArg(0));
      mlir::Value op1 = emitScalarExpr(e->getArg(1));
      return RValue::get(builder.createMax(loc, op0, op1));
    }
    return RValue::get(
        emitBinaryMaybeConstrainedFPBuiltin<cir::FMaxNumOp>(*this, *e));
  }
  case Builtin::BI__builtin_elementwise_min: {
    if (cir::isIntOrVectorOfIntType(convertType(e->getArg(0)->getType()))) {
      mlir::Location loc = getLoc(e->getExprLoc());
      mlir::Value op0 = emitScalarExpr(e->getArg(0));
      mlir::Value op1 = emitScalarExpr(e->getArg(1));
      return RValue::get(builder.createMin(loc, op0, op1));
    }
    return RValue::get(
        emitBinaryMaybeConstrainedFPBuiltin<cir::FMinNumOp>(*this, *e));
  }
  case Builtin::BI__builtin_elementwise_maxnum:
    return RValue::get(
        emitBinaryMaybeConstrainedFPBuiltin<cir::FMaxNumOp>(*this, *e));
  case Builtin::BI__builtin_elementwise_minnum:
    return RValue::get(
        emitBinaryMaybeConstrainedFPBuiltin<cir::FMinNumOp>(*this, *e));
  case Builtin::BI__builtin_elementwise_maximum:
    return RValue::get(
        emitBinaryMaybeConstrainedFPBuiltin<cir::FMaximumOp>(*this, *e));
  case Builtin::BI__builtin_elementwise_minimum:
    return RValue::get(
        emitBinaryMaybeConstrainedFPBuiltin<cir::FMinimumOp>(*this, *e));
 
  case Builtin::BI__builtin_elementwise_maximumnum:
  case Builtin::BI__builtin_elementwise_minimumnum:
    return errorBuiltinNYI(*this, e, builtinID);
  case Builtin::BI__builtin_reduce_max:
  case Builtin::BI__builtin_reduce_min: {
    auto getIntrinsicName = [this, builtinIDIfNoAsmLabel](QualType type) {
      if (const auto *vecTy = type->getAs<VectorType>())
        type = vecTy->getElementType();
      else if (type->isSizelessVectorType())
        type = type->getSizelessVectorEltType(getContext());
 
      if (builtinIDIfNoAsmLabel == Builtin::BI__builtin_reduce_max) {
        if (type->isSignedIntegerType())
          return "vector.reduce.smax";
        if (type->isUnsignedIntegerType())
          return "vector.reduce.umax";
        assert(type->isFloatingType() && "must have a float here");
        return "vector.reduce.fmax";
      }
 
      if (type->isSignedIntegerType())
        return "vector.reduce.smin";
      if (type->isUnsignedIntegerType())
        return "vector.reduce.umin";
      assert(type->isFloatingType() && "must have a float here");
      return "vector.reduce.fmin";
    };
    return emitBuiltinWithOneOverloadedType<1>(
        e, getIntrinsicName(e->getArg(0)->getType()),
        cast<cir::VectorType>(convertType(e->getArg(0)->getType()))
            .getElementType());
  }
  case Builtin::BI__builtin_reduce_add:
    return emitBuiltinWithOneOverloadedType<1>(
        e, "vector.reduce.add",
        cast<cir::VectorType>(convertType(e->getArg(0)->getType()))
            .getElementType());
  case Builtin::BI__builtin_reduce_mul:
    return emitBuiltinWithOneOverloadedType<1>(
        e, "vector.reduce.mul",
        cast<cir::VectorType>(convertType(e->getArg(0)->getType()))
            .getElementType());
  case Builtin::BI__builtin_reduce_xor:
    return emitBuiltinWithOneOverloadedType<1>(
        e, "vector.reduce.xor",
        cast<cir::VectorType>(convertType(e->getArg(0)->getType()))
            .getElementType());
  case Builtin::BI__builtin_reduce_or:
    return emitBuiltinWithOneOverloadedType<1>(
        e, "vector.reduce.or",
        cast<cir::VectorType>(convertType(e->getArg(0)->getType()))
            .getElementType());
  case Builtin::BI__builtin_reduce_and:
    return emitBuiltinWithOneOverloadedType<1>(
        e, "vector.reduce.and",
        cast<cir::VectorType>(convertType(e->getArg(0)->getType()))
            .getElementType());
  case Builtin::BI__builtin_reduce_assoc_fadd:
    return errorBuiltinNYI(*this, e, builtinID);
  case Builtin::BI__builtin_reduce_in_order_fadd: {
    assert(e->getNumArgs() == 2 &&
           "__builtin_reduce_in_order_fadd requires a start value");
    mlir::Value vector = emitScalarExpr(e->getArg(0));
    auto vectorTy = cast<cir::VectorType>(vector.getType());
    mlir::Type scalarTy = vectorTy.getElementType();
    mlir::Location loc = getLoc(e->getExprLoc());
    mlir::Value startValue = emitScalarExpr(e->getArg(1));
    if (startValue.getType() != scalarTy)
      startValue =
          builder.createCast(getLoc(e->getArg(1)->getExprLoc()),
                             cir::CastKind::floating, startValue, scalarTy);
    SmallVector<mlir::Value, 2> args = {startValue, vector};
    mlir::Value result =
        builder.emitIntrinsicCallOp(loc, "vector.reduce.fadd", scalarTy, args);
    return RValue::get(result);
  }
  case Builtin::BI__builtin_reduce_maximum:
  case Builtin::BI__builtin_reduce_minimum:
  case Builtin::BI__builtin_matrix_transpose:
  case Builtin::BI__builtin_matrix_column_major_load:
  case Builtin::BI__builtin_matrix_column_major_store:
  case Builtin::BI__builtin_masked_load:
  case Builtin::BI__builtin_masked_expand_load:
  case Builtin::BI__builtin_masked_gather:
  case Builtin::BI__builtin_masked_store:
  case Builtin::BI__builtin_masked_compress_store:
  case Builtin::BI__builtin_masked_scatter:
    return errorBuiltinNYI(*this, e, builtinID);
  case Builtin::BI__builtin_isinf_sign: {
    CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(*this, e);
    mlir::Location loc = getLoc(e->getBeginLoc());
    mlir::Value arg = emitScalarExpr(e->getArg(0));
    mlir::Value isInf =
        builder.createIsFPClass(loc, arg, cir::FPClassTest::Infinity);
    mlir::Value isNeg = emitSignBit(loc, *this, arg);
    mlir::Type intTy = convertType(e->getType());
    cir::ConstantOp zero = builder.getNullValue(intTy, loc);
    cir::ConstantOp one = builder.getConstant(loc, cir::IntAttr::get(intTy, 1));
    cir::ConstantOp negativeOne =
        builder.getConstant(loc, cir::IntAttr::get(intTy, -1));
    mlir::Value signResult = builder.createSelect(loc, isNeg, negativeOne, one);
    mlir::Value result = builder.createSelect(loc, isInf, signResult, zero);
    return RValue::get(result);
  }
  case Builtin::BI__builtin_flt_rounds: {
    mlir::Location loc = getLoc(e->getExprLoc());
    mlir::Type resultType = convertType(e->getType());
    mlir::Value result =
        builder.emitIntrinsicCallOp(loc, "get.rounding", resultType);
    if (result.getType() != resultType)
      result =
          builder.createCast(loc, cir::CastKind::integral, result, resultType);
    return RValue::get(result);
  }
  case Builtin::BI__builtin_set_flt_rounds: {
    mlir::Location loc = getLoc(e->getExprLoc());
    mlir::Value v = emitScalarExpr(e->getArg(0));
    builder.emitIntrinsicCallOp(loc, "set.rounding", builder.getVoidTy(),
                                mlir::ValueRange{v});
    return RValue::get(nullptr);
  }
  case Builtin::BI__builtin_fpclassify: {
    CIRGenFunction::CIRGenFPOptionsRAII fPOptsRAII(*this, e);
    mlir::Location loc = getLoc(e->getBeginLoc());
    mlir::Value value = emitScalarExpr(e->getArg(5));
    mlir::Type resultTy = convertType(e->getType());
    // if isZero then
    //     result = FP_ZERO
    // elseif isNan then
    //     result = FP_NAN
    // elseif isInfinity then
    //     result = FP_INFINITE
    // elseif isNormal then
    //     result = FP_NORMAL
    // else
    //     result = FP_SUBNORMAL
    auto isZero =
        cir::IsFPClassOp::create(builder, loc, value, cir::FPClassTest::Zero);
    mlir::Value result =
        cir::TernaryOp::create(
            builder, loc, isZero,
            /*thenBuilder=*/
            [&](mlir::OpBuilder &opBuilder, mlir::Location location) {
              mlir::Value zeroLiteral = emitScalarExpr(e->getArg(4));
              cir::YieldOp::create(opBuilder, location, zeroLiteral);
            },
            /*elseBuilder=*/
            [&](mlir::OpBuilder &opBuilder, mlir::Location location) {
              auto isNan = cir::IsFPClassOp::create(opBuilder, location, value,
                                                    cir::FPClassTest::Nan);
              mlir::Value nanResult =
                  cir::TernaryOp::create(
                      opBuilder, location, isNan,
                      /*thenBuilder=*/
                      [&](mlir::OpBuilder &opBuilder, mlir::Location location) {
                        mlir::Value nanLiteral = emitScalarExpr(e->getArg(0));
                        cir::YieldOp::create(opBuilder, location, nanLiteral);
                      },
                      /*elseBuilder=*/
                      [&](mlir::OpBuilder &opBuilder, mlir::Location location) {
                        auto isInfinity = cir::IsFPClassOp::create(
                            opBuilder, location, value,
                            cir::FPClassTest::Infinity);
                        mlir::Value infResult =
                            cir::TernaryOp::create(
                                opBuilder, location, isInfinity,
                                /*thenBuilder=*/
                                [&](mlir::OpBuilder &opBuilder,
                                    mlir::Location location) {
                                  mlir::Value infinityLiteral =
                                      emitScalarExpr(e->getArg(1));
                                  cir::YieldOp::create(opBuilder, location,
                                                       infinityLiteral);
                                },
                                /*elseBuilder=*/
                                [&](mlir::OpBuilder &opBuilder,
                                    mlir::Location location) {
                                  auto isNormal = cir::IsFPClassOp::create(
                                      opBuilder, location, value,
                                      cir::FPClassTest::Normal);
                                  mlir::Value fpNormal =
                                      emitScalarExpr(e->getArg(2));
                                  mlir::Value fpSubnormal =
                                      emitScalarExpr(e->getArg(3));
                                  mlir::Value returnValue =
                                      cir::SelectOp::create(
                                          opBuilder, location, resultTy,
                                          isNormal, fpNormal, fpSubnormal);
                                  cir::YieldOp::create(opBuilder, location,
                                                       returnValue);
                                })
                                .getResult();
                        cir::YieldOp::create(opBuilder, location, infResult);
                      })
                      .getResult();
              cir::YieldOp::create(opBuilder, location, nanResult);
            })
            .getResult();
    return RValue::get(result);
  }
  case Builtin::BIalloca:
  case Builtin::BI_alloca:
  case Builtin::BI__builtin_alloca_uninitialized:
  case Builtin::BI__builtin_alloca:
    return emitBuiltinAlloca(*this, e, builtinID);
  case Builtin::BI__builtin_alloca_with_align_uninitialized:
  case Builtin::BI__builtin_alloca_with_align:
  case Builtin::BI__builtin_infer_alloc_token:
    return errorBuiltinNYI(*this, e, builtinID);
  case Builtin::BIbzero:
  case Builtin::BI__builtin_bzero: {
    mlir::Location loc = getLoc(e->getSourceRange());
    Address destPtr = emitPointerWithAlignment(e->getArg(0));
    Address destPtrCast = destPtr.withElementType(builder, cgm.voidTy);
    mlir::Value size = emitScalarExpr(e->getArg(1));
    mlir::Value zero = builder.getNullValue(builder.getUInt8Ty(), loc);
    assert(!cir::MissingFeatures::sanitizers());
    builder.createMemSet(loc, destPtrCast, zero, size);
    assert(!cir::MissingFeatures::generateDebugInfo());
    return RValue::getIgnored();
  }
  case Builtin::BIbcopy:
  case Builtin::BI__builtin_bcopy: {
    Address src = emitPointerWithAlignment(e->getArg(0));
    Address dest = emitPointerWithAlignment(e->getArg(1));
    mlir::Value sizeVal = emitScalarExpr(e->getArg(2));
    emitNonNullArgCheck(RValue::get(src.getPointer()), e->getArg(0)->getType(),
                        e->getArg(0)->getExprLoc(), fd, 0);
    emitNonNullArgCheck(RValue::get(dest.getPointer()), e->getArg(1)->getType(),
                        e->getArg(1)->getExprLoc(), fd, 0);
    builder.createMemMove(getLoc(e->getSourceRange()), dest.getPointer(),
                          src.getPointer(), sizeVal);
    return RValue::get(nullptr);
  }
  case Builtin::BI__builtin_char_memchr:
  case Builtin::BI__builtin_memchr: {
    Address srcPtr = emitPointerWithAlignment(e->getArg(0));
    mlir::Value src =
        builder.createBitcast(srcPtr.getPointer(), builder.getVoidPtrTy());
    mlir::Value pattern = emitScalarExpr(e->getArg(1));
    mlir::Value len = emitScalarExpr(e->getArg(2));
    mlir::Value res = cir::MemChrOp::create(builder, getLoc(e->getExprLoc()),
                                            src, pattern, len);
    return RValue::get(res);
  }
  case Builtin::BImemcpy:
  case Builtin::BI__builtin_memcpy:
  case Builtin::BImempcpy:
  case Builtin::BI__builtin_mempcpy:
  case Builtin::BI__builtin_memcpy_inline:
  case Builtin::BI__builtin___memcpy_chk:
  case Builtin::BI__builtin_objc_memmove_collectable:
  case Builtin::BI__builtin___memmove_chk:
  case Builtin::BI__builtin_trivially_relocate:
  case Builtin::BImemmove:
  case Builtin::BI__builtin_memmove:
  case Builtin::BImemset:
  case Builtin::BI__builtin_memset:
  case Builtin::BI__builtin_memset_inline:
  case Builtin::BI__builtin___memset_chk:
  case Builtin::BI__builtin_wmemchr:
  case Builtin::BI__builtin_wmemcmp:
    break; // Handled as library calls below.
  case Builtin::BI__builtin_dwarf_cfa:
    return errorBuiltinNYI(*this, e, builtinID);
  case Builtin::BI__builtin_return_address: {
    llvm::APSInt level = e->getArg(0)->EvaluateKnownConstInt(getContext());
    return RValue::get(cir::ReturnAddrOp::create(
        builder, getLoc(e->getExprLoc()),
        builder.getConstAPInt(loc, builder.getUInt32Ty(), level)));
  }
  case Builtin::BI_ReturnAddress: {
    return RValue::get(cir::ReturnAddrOp::create(
        builder, getLoc(e->getExprLoc()),
        builder.getConstInt(loc, builder.getUInt32Ty(), 0)));
  }
  case Builtin::BI__builtin_frame_address: {
    llvm::APSInt level = e->getArg(0)->EvaluateKnownConstInt(getContext());
    mlir::Location loc = getLoc(e->getExprLoc());
    mlir::Value addr = cir::FrameAddrOp::create(
        builder, loc, allocaInt8PtrTy,
        builder.getConstAPInt(loc, builder.getUInt32Ty(), level));
    return RValue::get(
        builder.createCast(loc, cir::CastKind::bitcast, addr, voidPtrTy));
  }
  case Builtin::BI__builtin_extract_return_addr:
  case Builtin::BI__builtin_frob_return_addr:
  case Builtin::BI__builtin_dwarf_sp_column:
  case Builtin::BI__builtin_init_dwarf_reg_size_table:
  case Builtin::BI__builtin_eh_return:
  case Builtin::BI__builtin_unwind_init:
  case Builtin::BI__builtin_extend_pointer:
    return errorBuiltinNYI(*this, e, builtinID);
  case Builtin::BI__builtin_setjmp: {
    Address buf = emitPointerWithAlignment(e->getArg(0));
    mlir::Location loc = getLoc(e->getExprLoc());
 
    cir::PointerType voidPtrTy = builder.getVoidPtrTy();
    cir::PointerType ppTy = builder.getPointerTo(voidPtrTy);
    Address castBuf = buf.withElementType(builder, voidPtrTy);
 
    assert(!cir::MissingFeatures::emitCheckedInBoundsGEP());
    if (getTarget().getTriple().isSystemZ()) {
      cgm.errorNYI(e->getExprLoc(), "setjmp on SystemZ");
      return {};
    }
 
    mlir::Value frameAddress =
        cir::FrameAddrOp::create(builder, loc, voidPtrTy,
                                 mlir::ValueRange{builder.getUInt32(0, loc)})
            .getResult();
 
    builder.createStore(loc, frameAddress, castBuf);
 
    mlir::Value stacksave =
        cir::StackSaveOp::create(builder, loc, voidPtrTy).getResult();
    cir::PtrStrideOp stackSaveSlot = cir::PtrStrideOp::create(
        builder, loc, ppTy, castBuf.getPointer(), builder.getSInt32(2, loc));
    llvm::TypeSize voidPtrTySize =
        cgm.getDataLayout().getTypeAllocSize(voidPtrTy);
    CharUnits slotAlign = castBuf.getAlignment().alignmentAtOffset(
        CharUnits().fromQuantity(2 * voidPtrTySize));
    Address slotAddr = Address(stackSaveSlot, voidPtrTy, slotAlign);
    builder.createStore(loc, stacksave, slotAddr);
    auto op = cir::EhSetjmpOp::create(builder, loc, castBuf.getPointer());
    return RValue::get(op);
  }
  case Builtin::BI__builtin_longjmp: {
    mlir::Value buf = emitScalarExpr(e->getArg(0));
    mlir::Location loc = getLoc(e->getExprLoc());
 
    cir::EhLongjmpOp::create(builder, loc, buf);
    cir::UnreachableOp::create(builder, loc);
    return RValue::get(nullptr);
  }
  case Builtin::BI__builtin_launder: {
    const Expr *arg = e->getArg(0);
    QualType argTy = arg->getType()->getPointeeType();
    mlir::Value ptr = emitScalarExpr(arg);
 
    if (cgm.getCodeGenOpts().StrictVTablePointers &&
        argTy.requiresBuiltinLaunder(cgm.getASTContext())) {
      mlir::Location loc = getLoc(e->getExprLoc());
      ptr = cir::LaunderOp::create(builder, loc, ptr).getResult();
    }
    return RValue::get(ptr);
  }
  case Builtin::BI__builtin_clear_padding: {
    Address addr = emitPointerWithAlignment(e->getArg(0));
    mlir::Location loc = getLoc(e->getExprLoc());
    QualType pointeeTy = e->getArg(0)->getType()->getPointeeType();
 
    llvm::ArrayRef<ASTContext::BitInterval> padding =
        cgm.getASTContext().getPaddingIntervals(pointeeTy);
 
    llvm::SmallVector<mlir::Attribute> paddingLocs;
    for (const auto &interval : padding)
      paddingLocs.push_back(cir::OffsetPairAttr::get(
          &getMLIRContext(), interval.First, interval.Last));
 
    cir::ClearPaddingOp::create(
        builder, loc, addr.getPointer(),
        builder.getI64IntegerAttr(addr.getAlignment().getQuantity()),
        mlir::ArrayAttr::get(&getMLIRContext(), paddingLocs));
    return RValue::get(nullptr);
  }
  case Builtin::BI__sync_fetch_and_add:
  case Builtin::BI__sync_fetch_and_sub:
  case Builtin::BI__sync_fetch_and_or:
  case Builtin::BI__sync_fetch_and_and:
  case Builtin::BI__sync_fetch_and_xor:
  case Builtin::BI__sync_fetch_and_nand:
  case Builtin::BI__sync_add_and_fetch:
  case Builtin::BI__sync_sub_and_fetch:
  case Builtin::BI__sync_and_and_fetch:
  case Builtin::BI__sync_or_and_fetch:
  case Builtin::BI__sync_xor_and_fetch:
  case Builtin::BI__sync_nand_and_fetch:
  case Builtin::BI__sync_val_compare_and_swap:
  case Builtin::BI__sync_bool_compare_and_swap:
  case Builtin::BI__sync_lock_test_and_set:
  case Builtin::BI__sync_lock_release:
  case Builtin::BI__sync_swap:
    llvm_unreachable("Shouldn't make it through sema");
  case Builtin::BI__sync_fetch_and_add_1:
  case Builtin::BI__sync_fetch_and_add_2:
  case Builtin::BI__sync_fetch_and_add_4:
  case Builtin::BI__sync_fetch_and_add_8:
  case Builtin::BI__sync_fetch_and_add_16:
    return emitBinaryAtomic(*this, cir::AtomicFetchKind::Add, e);
  case Builtin::BI__sync_fetch_and_sub_1:
  case Builtin::BI__sync_fetch_and_sub_2:
  case Builtin::BI__sync_fetch_and_sub_4:
  case Builtin::BI__sync_fetch_and_sub_8:
  case Builtin::BI__sync_fetch_and_sub_16:
    return emitBinaryAtomic(*this, cir::AtomicFetchKind::Sub, e);
  case Builtin::BI__sync_fetch_and_or_1:
  case Builtin::BI__sync_fetch_and_or_2:
  case Builtin::BI__sync_fetch_and_or_4:
  case Builtin::BI__sync_fetch_and_or_8:
  case Builtin::BI__sync_fetch_and_or_16:
    return emitBinaryAtomic(*this, cir::AtomicFetchKind::Or, e);
  case Builtin::BI__sync_fetch_and_and_1:
  case Builtin::BI__sync_fetch_and_and_2:
  case Builtin::BI__sync_fetch_and_and_4:
  case Builtin::BI__sync_fetch_and_and_8:
  case Builtin::BI__sync_fetch_and_and_16:
    return emitBinaryAtomic(*this, cir::AtomicFetchKind::And, e);
  case Builtin::BI__sync_fetch_and_xor_1:
  case Builtin::BI__sync_fetch_and_xor_2:
  case Builtin::BI__sync_fetch_and_xor_4:
  case Builtin::BI__sync_fetch_and_xor_8:
  case Builtin::BI__sync_fetch_and_xor_16:
    return emitBinaryAtomic(*this, cir::AtomicFetchKind::Xor, e);
  case Builtin::BI__sync_fetch_and_nand_1:
  case Builtin::BI__sync_fetch_and_nand_2:
  case Builtin::BI__sync_fetch_and_nand_4:
  case Builtin::BI__sync_fetch_and_nand_8:
  case Builtin::BI__sync_fetch_and_nand_16:
    return emitBinaryAtomic(*this, cir::AtomicFetchKind::Nand, e);
  case Builtin::BI__sync_fetch_and_min:
  case Builtin::BI__sync_fetch_and_max:
  case Builtin::BI__sync_fetch_and_umin:
  case Builtin::BI__sync_fetch_and_umax:
    return errorBuiltinNYI(*this, e, builtinID);
    return getUndefRValue(e->getType());
  case Builtin::BI__sync_add_and_fetch_1:
  case Builtin::BI__sync_add_and_fetch_2:
  case Builtin::BI__sync_add_and_fetch_4:
  case Builtin::BI__sync_add_and_fetch_8:
  case Builtin::BI__sync_add_and_fetch_16:
    return emitBinaryAtomicPost<cir::AddOp>(*this, cir::AtomicFetchKind::Add,
                                            e);
  case Builtin::BI__sync_sub_and_fetch_1:
  case Builtin::BI__sync_sub_and_fetch_2:
  case Builtin::BI__sync_sub_and_fetch_4:
  case Builtin::BI__sync_sub_and_fetch_8:
  case Builtin::BI__sync_sub_and_fetch_16:
    return emitBinaryAtomicPost<cir::SubOp>(*this, cir::AtomicFetchKind::Sub,
                                            e);
  case Builtin::BI__sync_and_and_fetch_1:
  case Builtin::BI__sync_and_and_fetch_2:
  case Builtin::BI__sync_and_and_fetch_4:
  case Builtin::BI__sync_and_and_fetch_8:
  case Builtin::BI__sync_and_and_fetch_16:
    return emitBinaryAtomicPost<cir::AndOp>(*this, cir::AtomicFetchKind::And,
                                            e);
  case Builtin::BI__sync_or_and_fetch_1:
  case Builtin::BI__sync_or_and_fetch_2:
  case Builtin::BI__sync_or_and_fetch_4:
  case Builtin::BI__sync_or_and_fetch_8:
  case Builtin::BI__sync_or_and_fetch_16:
    return emitBinaryAtomicPost<cir::OrOp>(*this, cir::AtomicFetchKind::Or, e);
  case Builtin::BI__sync_xor_and_fetch_1:
  case Builtin::BI__sync_xor_and_fetch_2:
  case Builtin::BI__sync_xor_and_fetch_4:
  case Builtin::BI__sync_xor_and_fetch_8:
  case Builtin::BI__sync_xor_and_fetch_16:
    return emitBinaryAtomicPost<cir::XorOp>(*this, cir::AtomicFetchKind::Xor,
                                            e);
  case Builtin::BI__sync_nand_and_fetch_1:
  case Builtin::BI__sync_nand_and_fetch_2:
  case Builtin::BI__sync_nand_and_fetch_4:
  case Builtin::BI__sync_nand_and_fetch_8:
  case Builtin::BI__sync_nand_and_fetch_16:
    return emitBinaryAtomicPost<cir::AndOp>(*this, cir::AtomicFetchKind::Nand,
                                            e, /*invert=*/true);
  case Builtin::BI__sync_val_compare_and_swap_1:
  case Builtin::BI__sync_val_compare_and_swap_2:
  case Builtin::BI__sync_val_compare_and_swap_4:
  case Builtin::BI__sync_val_compare_and_swap_8:
    return RValue::get(emitAtomicCmpXchg(e, /*returnBool=*/false));
  case Builtin::BI__sync_bool_compare_and_swap_1:
  case Builtin::BI__sync_bool_compare_and_swap_2:
  case Builtin::BI__sync_bool_compare_and_swap_4:
  case Builtin::BI__sync_bool_compare_and_swap_8:
    return RValue::get(emitAtomicCmpXchg(e, /*returnBool=*/true));
  case Builtin::BI__sync_swap_1:
  case Builtin::BI__sync_swap_2:
  case Builtin::BI__sync_swap_4:
  case Builtin::BI__sync_swap_8:
    return emitAtomicXchg(*this, e, cir::MemOrder::SequentiallyConsistent);
  case Builtin::BI__sync_lock_test_and_set_1:
  case Builtin::BI__sync_lock_test_and_set_2:
  case Builtin::BI__sync_lock_test_and_set_4:
  case Builtin::BI__sync_lock_test_and_set_8:
    return emitAtomicXchg(*this, e, cir::MemOrder::SequentiallyConsistent);
  case Builtin::BI__sync_lock_release_1:
  case Builtin::BI__sync_lock_release_2:
  case Builtin::BI__sync_lock_release_4:
  case Builtin::BI__sync_lock_release_8:
    emitAtomicLockRelease(*this, e);
    return RValue::get(nullptr);
  case Builtin::BI__sync_val_compare_and_swap_16:
  case Builtin::BI__sync_bool_compare_and_swap_16:
  case Builtin::BI__sync_swap_16:
  case Builtin::BI__sync_lock_test_and_set_16:
  case Builtin::BI__sync_lock_release_16:
    return errorBuiltinNYI(*this, e, builtinID);
  case Builtin::BI__sync_synchronize: {
    // We assume this is supposed to correspond to a C++0x-style
    // sequentially-consistent fence (i.e. this is only usable for
    // synchronization, not device I/O or anything like that). This intrinsic
    // is really badly designed in the sense that in theory, there isn't
    // any way to safely use it... but in practice, it mostly works
    // to use it with non-atomic loads and stores to get acquire/release
    // semantics.
    cir::AtomicFenceOp::create(
        builder, getLoc(e->getSourceRange()),
        cir::MemOrder::SequentiallyConsistent,
        cir::SyncScopeKindAttr::get(&getMLIRContext(),
                                    cir::SyncScopeKind::System));
    return RValue::get(nullptr);
  }
  case Builtin::BI__builtin_nontemporal_load: {
    Address addr = emitPointerWithAlignment(e->getArg(0));
    LValue lv = makeAddrLValue(addr, e->getType(),
                               LValueBaseInfo(AlignmentSource::Type));
    lv.setNontemporal(true);
    mlir::Value val = emitLoadOfScalar(lv, e->getExprLoc());
    return RValue::get(val);
  }
  case Builtin::BI__builtin_nontemporal_store: {
    mlir::Value val = emitScalarExpr(e->getArg(0));
    Address addr = emitPointerWithAlignment(e->getArg(1));
    val = emitToMemory(val, e->getArg(0)->getType());
    LValue lv = makeAddrLValue(addr, e->getArg(0)->getType(),
                               LValueBaseInfo(AlignmentSource::Type));
    lv.setNontemporal(true);
    emitStoreOfScalar(val, lv, /*isInit=*/false);
    return RValue::get(nullptr);
  }
  case Builtin::BI__c11_atomic_is_lock_free:
  case Builtin::BI__atomic_is_lock_free:
    return emitAtomicIsLockFree(*this, e, builtinID);
  case Builtin::BI__atomic_test_and_set:
  case Builtin::BI__atomic_clear:
    return errorBuiltinNYI(*this, e, builtinID);
  case Builtin::BI__atomic_thread_fence:
  case Builtin::BI__c11_atomic_thread_fence: {
    emitAtomicFenceOp(*this, e, cir::SyncScopeKind::System);
    return RValue::get(nullptr);
  }
  case Builtin::BI__atomic_signal_fence:
  case Builtin::BI__c11_atomic_signal_fence: {
    emitAtomicFenceOp(*this, e, cir::SyncScopeKind::SingleThread);
    return RValue::get(nullptr);
  }
  case Builtin::BI__scoped_atomic_thread_fence:
    return errorBuiltinNYI(*this, e, builtinID);
  case Builtin::BI__builtin_signbit:
  case Builtin::BI__builtin_signbitf:
  case Builtin::BI__builtin_signbitl: {
    CIRGenFunction::CIRGenFPOptionsRAII fPOptsRAII(*this, e);
    mlir::Location loc = getLoc(e->getBeginLoc());
    mlir::Value value = emitScalarExpr(e->getArg(0));
    mlir::Operation *signBitOp = cir::SignBitOp::create(builder, loc, value);
    mlir::Value result = builder.createBoolToInt(signBitOp->getResult(0),
                                                 convertType(e->getType()));
    return RValue::get(result);
  }
  case Builtin::BI__warn_memset_zero_len:
  case Builtin::BI__annotation:
  case Builtin::BI__builtin_annotation:
    return errorBuiltinNYI(*this, e, builtinID);
 
  case Builtin::BI__builtin_addcb:
  case Builtin::BI__builtin_addcs:
  case Builtin::BI__builtin_addc:
  case Builtin::BI__builtin_addcl:
  case Builtin::BI__builtin_addcll:
  case Builtin::BI__builtin_subcb:
  case Builtin::BI__builtin_subcs:
  case Builtin::BI__builtin_subc:
  case Builtin::BI__builtin_subcl:
  case Builtin::BI__builtin_subcll: {
    // Multiprecision add/sub-with-carry.  Lower as two chained checked
    // add/sub overflow ops, matching classic CodeGen:
    //   sum1, carry1 = x +/- y
    //   result, carry2 = sum1 +/- carryin
    //   *carryout = carry1 | carry2
    // All operands and the result share the builtin's integer type, so no
    // encompassing-type widening is needed.
    mlir::Value x = emitScalarExpr(e->getArg(0));
    mlir::Value y = emitScalarExpr(e->getArg(1));
    mlir::Value carryin = emitScalarExpr(e->getArg(2));
    Address carryOutPtr = emitPointerWithAlignment(e->getArg(3));
 
    mlir::Location loc = getLoc(e->getSourceRange());
    mlir::Type resultTy = convertType(e->getType());
 
    static constexpr unsigned addcBuiltins[] = {
        Builtin::BI__builtin_addcb, Builtin::BI__builtin_addcs,
        Builtin::BI__builtin_addc, Builtin::BI__builtin_addcl,
        Builtin::BI__builtin_addcll};
    bool isAdd = llvm::is_contained(addcBuiltins, builtinID);
 
    mlir::Value sum1, carry1, sum2, carry2;
    if (isAdd) {
      std::tie(sum1, carry1) =
          emitOverflowOp<cir::AddOverflowOp>(builder, loc, resultTy, x, y);
      std::tie(sum2, carry2) = emitOverflowOp<cir::AddOverflowOp>(
          builder, loc, resultTy, sum1, carryin);
    } else {
      std::tie(sum1, carry1) =
          emitOverflowOp<cir::SubOverflowOp>(builder, loc, resultTy, x, y);
      std::tie(sum2, carry2) = emitOverflowOp<cir::SubOverflowOp>(
          builder, loc, resultTy, sum1, carryin);
    }
 
    // Combine the two carry bits, then widen to the result integer type.
    mlir::Value carryOut = builder.createBoolToInt(
        builder.createOr(loc, carry1, carry2), resultTy);
    builder.createStore(loc, carryOut, carryOutPtr);
    return RValue::get(sum2);
  }
 
  case Builtin::BI__builtin_add_overflow:
  case Builtin::BI__builtin_sub_overflow:
  case Builtin::BI__builtin_mul_overflow: {
    const clang::Expr *leftArg = e->getArg(0);
    const clang::Expr *rightArg = e->getArg(1);
    const clang::Expr *resultArg = e->getArg(2);
 
    clang::QualType resultQTy =
        resultArg->getType()->castAs<clang::PointerType>()->getPointeeType();
 
    WidthAndSignedness leftInfo =
        getIntegerWidthAndSignedness(cgm.getASTContext(), leftArg->getType());
    WidthAndSignedness rightInfo =
        getIntegerWidthAndSignedness(cgm.getASTContext(), rightArg->getType());
    WidthAndSignedness resultInfo =
        getIntegerWidthAndSignedness(cgm.getASTContext(), resultQTy);
 
    // Note we compute the encompassing type with the consideration to the
    // result type, so later in LLVM lowering we don't get redundant integral
    // extension casts.
    WidthAndSignedness encompassingInfo =
        EncompassingIntegerType({leftInfo, rightInfo, resultInfo});
 
    auto encompassingCIRTy = cir::IntType::get(
        &getMLIRContext(), encompassingInfo.width, encompassingInfo.isSigned);
    mlir::Type resultCIRTy = cgm.convertType(resultQTy);
 
    mlir::Value x = emitScalarExpr(leftArg);
    mlir::Value y = emitScalarExpr(rightArg);
    Address resultPtr = emitPointerWithAlignment(resultArg);
 
    // Extend each operand to the encompassing type, if necessary.
    if (x.getType() != encompassingCIRTy) {
      x = builder.createCast(mlir::isa<cir::BoolType>(x.getType())
                                 ? cir::CastKind::bool_to_int
                                 : cir::CastKind::integral,
                             x, encompassingCIRTy);
    }
 
    if (y.getType() != encompassingCIRTy) {
      y = builder.createCast(mlir::isa<cir::BoolType>(y.getType())
                                 ? cir::CastKind::bool_to_int
                                 : cir::CastKind::integral,
                             y, encompassingCIRTy);
    }
 
    // Perform the operation on the extended values.
    mlir::Location loc = getLoc(e->getSourceRange());
    mlir::Value result, overflow;
    switch (builtinID) {
    default:
      llvm_unreachable("Unknown overflow builtin id.");
    case Builtin::BI__builtin_add_overflow:
      std::tie(result, overflow) =
          emitOverflowOp<cir::AddOverflowOp>(builder, loc, resultCIRTy, x, y);
      break;
    case Builtin::BI__builtin_sub_overflow:
      std::tie(result, overflow) =
          emitOverflowOp<cir::SubOverflowOp>(builder, loc, resultCIRTy, x, y);
      break;
    case Builtin::BI__builtin_mul_overflow:
      std::tie(result, overflow) =
          emitOverflowOp<cir::MulOverflowOp>(builder, loc, resultCIRTy, x, y);
      break;
    }
 
    // Here is a slight difference from the original clang CodeGen:
    //   - In the original clang CodeGen, the checked arithmetic result is
    //     first computed as a value of the encompassing type, and then it is
    //     truncated to the actual result type with a second overflow checking.
    //   - In CIRGen, the checked arithmetic operation directly produce the
    //     checked arithmetic result in its expected type, which may be a
    //     `cir.bool`.
    //
    // So we don't need a truncation and a second overflow checking here.
 
    // Finally, store the result using the pointer.
    bool isVolatile =
        resultArg->getType()->getPointeeType().isVolatileQualified();
    builder.createStore(loc, result, resultPtr, isVolatile);
 
    return RValue::get(overflow);
  }
 
  case Builtin::BI__builtin_uadd_overflow:
  case Builtin::BI__builtin_uaddl_overflow:
  case Builtin::BI__builtin_uaddll_overflow:
  case Builtin::BI__builtin_usub_overflow:
  case Builtin::BI__builtin_usubl_overflow:
  case Builtin::BI__builtin_usubll_overflow:
  case Builtin::BI__builtin_umul_overflow:
  case Builtin::BI__builtin_umull_overflow:
  case Builtin::BI__builtin_umulll_overflow:
  case Builtin::BI__builtin_sadd_overflow:
  case Builtin::BI__builtin_saddl_overflow:
  case Builtin::BI__builtin_saddll_overflow:
  case Builtin::BI__builtin_ssub_overflow:
  case Builtin::BI__builtin_ssubl_overflow:
  case Builtin::BI__builtin_ssubll_overflow:
  case Builtin::BI__builtin_smul_overflow:
  case Builtin::BI__builtin_smull_overflow:
  case Builtin::BI__builtin_smulll_overflow: {
    // Scalarize our inputs.
    mlir::Value x = emitScalarExpr(e->getArg(0));
    mlir::Value y = emitScalarExpr(e->getArg(1));
 
    const clang::Expr *resultArg = e->getArg(2);
    Address resultPtr = emitPointerWithAlignment(resultArg);
 
    clang::QualType resultQTy =
        resultArg->getType()->castAs<clang::PointerType>()->getPointeeType();
    auto resultCIRTy = mlir::cast<cir::IntType>(cgm.convertType(resultQTy));
 
    // Create the appropriate overflow-checked arithmetic operation.
    mlir::Location loc = getLoc(e->getSourceRange());
    mlir::Value result, overflow;
    switch (builtinID) {
    default:
      llvm_unreachable("Unknown overflow builtin id.");
    case Builtin::BI__builtin_uadd_overflow:
    case Builtin::BI__builtin_uaddl_overflow:
    case Builtin::BI__builtin_uaddll_overflow:
    case Builtin::BI__builtin_sadd_overflow:
    case Builtin::BI__builtin_saddl_overflow:
    case Builtin::BI__builtin_saddll_overflow:
      std::tie(result, overflow) =
          emitOverflowOp<cir::AddOverflowOp>(builder, loc, resultCIRTy, x, y);
      break;
    case Builtin::BI__builtin_usub_overflow:
    case Builtin::BI__builtin_usubl_overflow:
    case Builtin::BI__builtin_usubll_overflow:
    case Builtin::BI__builtin_ssub_overflow:
    case Builtin::BI__builtin_ssubl_overflow:
    case Builtin::BI__builtin_ssubll_overflow:
      std::tie(result, overflow) =
          emitOverflowOp<cir::SubOverflowOp>(builder, loc, resultCIRTy, x, y);
      break;
    case Builtin::BI__builtin_umul_overflow:
    case Builtin::BI__builtin_umull_overflow:
    case Builtin::BI__builtin_umulll_overflow:
    case Builtin::BI__builtin_smul_overflow:
    case Builtin::BI__builtin_smull_overflow:
    case Builtin::BI__builtin_smulll_overflow:
      std::tie(result, overflow) =
          emitOverflowOp<cir::MulOverflowOp>(builder, loc, resultCIRTy, x, y);
      break;
    }
 
    bool isVolatile =
        resultArg->getType()->getPointeeType().isVolatileQualified();
    builder.createStore(loc, emitToMemory(result, resultQTy), resultPtr,
                        isVolatile);
 
    return RValue::get(overflow);
  }
 
  case Builtin::BIaddressof:
  case Builtin::BI__addressof:
  case Builtin::BI__builtin_addressof:
    return RValue::get(emitLValue(e->getArg(0)).getPointer());
  case Builtin::BI__builtin_function_start:
    return errorBuiltinNYI(*this, e, builtinID);
  case Builtin::BI__builtin_operator_new:
    return emitNewOrDeleteBuiltinCall(
        e->getCallee()->getType()->castAs<FunctionProtoType>(), e, OO_New);
  case Builtin::BI__builtin_operator_delete:
    emitNewOrDeleteBuiltinCall(
        e->getCallee()->getType()->castAs<FunctionProtoType>(), e, OO_Delete);
    return RValue::get(nullptr);
  case Builtin::BI__builtin_is_aligned:
  case Builtin::BI__builtin_align_up:
  case Builtin::BI__builtin_align_down:
  case Builtin::BI__noop:
  case Builtin::BI__builtin_call_with_static_chain:
  case Builtin::BI_InterlockedExchange8:
  case Builtin::BI_InterlockedExchange16:
  case Builtin::BI_InterlockedExchange:
  case Builtin::BI_InterlockedExchangePointer:
  case Builtin::BI_InterlockedCompareExchangePointer:
  case Builtin::BI_InterlockedCompareExchangePointer_nf:
  case Builtin::BI_InterlockedCompareExchange8:
  case Builtin::BI_InterlockedCompareExchange16:
  case Builtin::BI_InterlockedCompareExchange:
  case Builtin::BI_InterlockedCompareExchange64:
  case Builtin::BI_InterlockedIncrement16:
  case Builtin::BI_InterlockedIncrement:
  case Builtin::BI_InterlockedDecrement16:
  case Builtin::BI_InterlockedDecrement:
  case Builtin::BI_InterlockedAnd8:
  case Builtin::BI_InterlockedAnd16:
  case Builtin::BI_InterlockedAnd:
  case Builtin::BI_InterlockedExchangeAdd8:
  case Builtin::BI_InterlockedExchangeAdd16:
  case Builtin::BI_InterlockedExchangeAdd:
  case Builtin::BI_InterlockedExchangeSub8:
  case Builtin::BI_InterlockedExchangeSub16:
  case Builtin::BI_InterlockedExchangeSub:
  case Builtin::BI_InterlockedOr8:
  case Builtin::BI_InterlockedOr16:
  case Builtin::BI_InterlockedOr:
  case Builtin::BI_InterlockedXor8:
  case Builtin::BI_InterlockedXor16:
  case Builtin::BI_InterlockedXor:
  case Builtin::BI_bittest64:
  case Builtin::BI_bittest:
  case Builtin::BI_bittestandcomplement64:
  case Builtin::BI_bittestandcomplement:
  case Builtin::BI_bittestandreset64:
  case Builtin::BI_bittestandreset:
  case Builtin::BI_bittestandset64:
  case Builtin::BI_bittestandset:
  case Builtin::BI_interlockedbittestandreset:
  case Builtin::BI_interlockedbittestandreset64:
  case Builtin::BI_interlockedbittestandreset64_acq:
  case Builtin::BI_interlockedbittestandreset64_rel:
  case Builtin::BI_interlockedbittestandreset64_nf:
  case Builtin::BI_interlockedbittestandset64:
  case Builtin::BI_interlockedbittestandset64_acq:
  case Builtin::BI_interlockedbittestandset64_rel:
  case Builtin::BI_interlockedbittestandset64_nf:
  case Builtin::BI_interlockedbittestandset:
  case Builtin::BI_interlockedbittestandset_acq:
  case Builtin::BI_interlockedbittestandset_rel:
  case Builtin::BI_interlockedbittestandset_nf:
  case Builtin::BI_interlockedbittestandreset_acq:
  case Builtin::BI_interlockedbittestandreset_rel:
  case Builtin::BI_interlockedbittestandreset_nf:
  case Builtin::BI__iso_volatile_load8:
  case Builtin::BI__iso_volatile_load16:
  case Builtin::BI__iso_volatile_load32:
  case Builtin::BI__iso_volatile_load64:
  case Builtin::BI__iso_volatile_store8:
  case Builtin::BI__iso_volatile_store16:
  case Builtin::BI__iso_volatile_store32:
  case Builtin::BI__iso_volatile_store64:
  case Builtin::BI__builtin_ptrauth_sign_constant:
  case Builtin::BI__builtin_ptrauth_auth:
  case Builtin::BI__builtin_ptrauth_auth_and_resign:
  case Builtin::BI__builtin_ptrauth_blend_discriminator:
  case Builtin::BI__builtin_ptrauth_sign_generic_data:
  case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
  case Builtin::BI__builtin_ptrauth_strip:
  case Builtin::BI__builtin_get_vtable_pointer:
  case Builtin::BI__exception_code:
  case Builtin::BI_exception_code:
  case Builtin::BI__exception_info:
  case Builtin::BI_exception_info:
  case Builtin::BI__abnormal_termination:
  case Builtin::BI_abnormal_termination:
    return errorBuiltinNYI(*this, e, builtinID);
  case Builtin::BI_setjmpex:
  case Builtin::BI_setjmp:
    if (getTarget().getTriple().isOSMSVCRT()) {
      cgm.errorNYI(e->getSourceRange(), "setjmp/setjmpex on MSVCRT");
      return getUndefRValue(e->getType());
    }
    // Else break and this will be handled as a library call.
    break;
  case Builtin::BImove:
  case Builtin::BImove_if_noexcept:
  case Builtin::BIforward:
  case Builtin::BIforward_like:
  case Builtin::BIas_const:
    return RValue::get(emitLValue(e->getArg(0)).getPointer());
  case Builtin::BI__GetExceptionInfo:
  case Builtin::BI__fastfail:
  case Builtin::BIread_pipe:
  case Builtin::BIwrite_pipe:
  case Builtin::BIreserve_read_pipe:
  case Builtin::BIreserve_write_pipe:
  case Builtin::BIwork_group_reserve_read_pipe:
  case Builtin::BIwork_group_reserve_write_pipe:
  case Builtin::BIsub_group_reserve_read_pipe:
  case Builtin::BIsub_group_reserve_write_pipe:
  case Builtin::BIcommit_read_pipe:
  case Builtin::BIcommit_write_pipe:
  case Builtin::BIwork_group_commit_read_pipe:
  case Builtin::BIwork_group_commit_write_pipe:
  case Builtin::BIsub_group_commit_read_pipe:
  case Builtin::BIsub_group_commit_write_pipe:
  case Builtin::BIget_pipe_num_packets:
  case Builtin::BIget_pipe_max_packets:
  case Builtin::BIto_global:
  case Builtin::BIto_local:
  case Builtin::BIto_private:
  case Builtin::BIenqueue_kernel:
  case Builtin::BIget_kernel_work_group_size:
  case Builtin::BIget_kernel_preferred_work_group_size_multiple:
  case Builtin::BIget_kernel_max_sub_group_size_for_ndrange:
  case Builtin::BIget_kernel_sub_group_count_for_ndrange:
  case Builtin::BI__builtin_store_half:
  case Builtin::BI__builtin_store_halff:
  case Builtin::BI__builtin_load_half:
  case Builtin::BI__builtin_load_halff:
    return errorBuiltinNYI(*this, e, builtinID);
  case Builtin::BI__builtin_printf:
  case Builtin::BIprintf:
    if (getTarget().getTriple().isNVPTX() ||
        getTarget().getTriple().isAMDGCN() ||
        (getTarget().getTriple().isSPIRV() &&
         getTarget().getTriple().getVendor() == llvm::Triple::AMD)) {
      if (getTarget().getTriple().isNVPTX())
        return RValue::get(emitNVPTXDevicePrintfCallExpr(e));
      if ((getTarget().getTriple().isAMDGCN() ||
           getTarget().getTriple().isSPIRV()) &&
          getLangOpts().HIP)
        return errorBuiltinNYI(*this, e, builtinID);
    }
    break;
  case Builtin::BI__builtin_canonicalize:
  case Builtin::BI__builtin_canonicalizef:
  case Builtin::BI__builtin_canonicalizef16:
  case Builtin::BI__builtin_canonicalizel:
  case Builtin::BI__builtin_thread_pointer:
  case Builtin::BI__builtin_os_log_format:
  case Builtin::BI__xray_customevent:
  case Builtin::BI__xray_typedevent:
  case Builtin::BI__builtin_ms_va_start:
  case Builtin::BI__builtin_ms_va_end:
  case Builtin::BI__builtin_ms_va_copy:
  case Builtin::BI__builtin_get_device_side_mangled_name:
    return errorBuiltinNYI(*this, e, builtinID);
  }
 
  // If this is an alias for a lib function (e.g. __builtin_sin), emit
  // the call using the normal call path, but using the unmangled
  // version of the function name.
  if (!shouldEmitBuiltinAsIR(builtinID, getContext().BuiltinInfo, *this) &&
      getContext().BuiltinInfo.isLibFunction(builtinID))
    return emitLibraryCall(*this, fd, e,
                           cgm.getBuiltinLibFunction(fd, builtinID));
 
  // If this is a predefined lib function (e.g. malloc), emit the call
  // using exactly the normal call path.
  if (getContext().BuiltinInfo.isPredefinedLibFunction(builtinID))
    return emitLibraryCall(*this, fd, e,
                           emitScalarExpr(e->getCallee()).getDefiningOp());
 
  // Check that a call to a target specific builtin has the correct target
  // features.
  // This is down here to avoid non-target specific builtins, however, if
  // generic builtins start to require generic target features then we
  // can move this up to the beginning of the function.
  checkTargetFeatures(e, fd);
 
  // See if we have a target specific intrinsic.
  std::string name = getContext().BuiltinInfo.getName(builtinID);
  Intrinsic::ID intrinsicID = Intrinsic::not_intrinsic;
  StringRef prefix =
      llvm::Triple::getArchTypePrefix(getTarget().getTriple().getArch());
  if (!prefix.empty()) {
    intrinsicID = Intrinsic::getIntrinsicForClangBuiltin(prefix, name);
    // NOTE we don't need to perform a compatibility flag check here since the
    // intrinsics are declared in Builtins*.def via LANGBUILTIN which filter the
    // MS builtins via ALL_MS_LANGUAGES and are filtered earlier.
    if (intrinsicID == Intrinsic::not_intrinsic)
      intrinsicID = Intrinsic::getIntrinsicForMSBuiltin(prefix, name);
  }
 
  if (intrinsicID != Intrinsic::not_intrinsic) {
    unsigned iceArguments = 0;
    ASTContext::GetBuiltinTypeError error;
    getContext().GetBuiltinType(builtinID, error, &iceArguments);
    assert(error == ASTContext::GE_None && "Should not codegen an error");
 
    StringRef name = Intrinsic::getName(intrinsicID);
    // cir::LLVMIntrinsicCallOp expects intrinsic name to not have prefix
    // "llvm." For example, `llvm.nvvm.barrier0` should be passed as
    // `nvvm.barrier0`.
    assert(name.starts_with("llvm.") && "expected llvm. prefix");
    name = name.drop_front(/*strlen("llvm.")=*/5);
 
    cir::FuncType intrinsicType =
        getIntrinsicType(*this, &getMLIRContext(), intrinsicID);
 
    SmallVector<mlir::Value> args;
    const FunctionDecl *fd = e->getDirectCallee();
    for (unsigned i = 0; i < e->getNumArgs(); i++) {
      mlir::Value argValue =
          emitScalarOrConstFoldImmArg(iceArguments, i, e->getArg(i));
      // If the intrinsic arg type is different from the builtin arg type
      // we need to do a bit cast.
      mlir::Type argType = argValue.getType();
      mlir::Type expectedTy = intrinsicType.getInput(i);
 
      // Correct integer signedness based on AST parameter type
      mlir::Type correctedExpectedTy = expectedTy;
      if (fd && i < fd->getNumParams()) {
        correctedExpectedTy = correctIntegerSignedness(
            expectedTy, fd->getParamDecl(i)->getType(), &getMLIRContext());
      }
 
      if (mlir::isa<cir::PointerType>(expectedTy)) {
        bool argIsPointer = mlir::isa<cir::PointerType>(argType);
        bool argIsVectorOfPointer = false;
        if (auto vecTy = dyn_cast<mlir::VectorType>(argType))
          argIsVectorOfPointer =
              mlir::isa<cir::PointerType>(vecTy.getElementType());
 
        if (!argIsPointer && !argIsVectorOfPointer) {
          cgm.errorNYI(
              e->getSourceRange(),
              "intrinsic expects a pointer type (NYI for non-pointer)");
          return getUndefRValue(e->getType());
        }
 
        // Pointer handling (address-space cast / bitcast fallback).
        if (argType != expectedTy)
          argValue = getCorrectedPtr(argValue, expectedTy, builder);
      } else {
        // Non-pointer expected type: if needed, bitcast to the corrected
        // expected type to match signedness/representation.
        if (argType != correctedExpectedTy)
          argValue = builder.createBitcast(argValue, correctedExpectedTy);
      }
 
      args.push_back(argValue);
    }
 
    // Correct the builtin type based on the AST function declaration's return
    // type, if available.
    mlir::Type correctedReturnType =
        correctReturnType(intrinsicType.getReturnType(), fd, &getMLIRContext());
 
    cir::LLVMIntrinsicCallOp intrinsicCall = cir::LLVMIntrinsicCallOp::create(
        builder, getLoc(e->getExprLoc()), builder.getStringAttr(name),
        correctedReturnType, args);
 
    mlir::Value intrinsicRes = intrinsicCall.getResult();
 
    if (isa<cir::VoidType>(correctedReturnType))
      return RValue::get(nullptr);
 
    return RValue::get(intrinsicRes);
  }
 
  // Some target-specific builtins can have aggregate return values, e.g.
  // __builtin_arm_mve_vld2q_u32. So if the result is an aggregate, force
  // returnValue to be non-null, so that the target-specific emission code can
  // always just emit into it.
  cir::TypeEvaluationKind evalKind = getEvaluationKind(e->getType());
  if (evalKind == cir::TEK_Aggregate && returnValue.isNull()) {
    cgm.errorNYI(e->getSourceRange(), "aggregate return value from builtin");
    return getUndefRValue(e->getType());
  }
 
  // Now see if we can emit a target-specific builtin.
  // FIXME: This is a temporary mechanism (double-optional semantics) that will
  // go away once everything is implemented:
  //   1. return `mlir::Value{}` for cases where we have issued the diagnostic.
  //   2. return `std::nullopt` in cases where we didn't issue a diagnostic
  //      but also didn't handle the builtin.
  if (std::optional<mlir::Value> rst =
          emitTargetBuiltinExpr(builtinID, e, returnValue)) {
    mlir::Value v = rst.value();
    // CIR dialect operations may have no results, no values will be returned
    // even if it executes successfully.
    if (!v)
      return RValue::get(nullptr);
 
    switch (evalKind) {
    case cir::TEK_Scalar:
      if (mlir::isa<cir::VoidType>(v.getType()))
        return RValue::get(nullptr);
      return RValue::get(v);
    case cir::TEK_Aggregate:
      cgm.errorNYI(e->getSourceRange(), "aggregate return value from builtin");
      return getUndefRValue(e->getType());
    case cir::TEK_Complex:
      llvm_unreachable("No current target builtin returns complex");
    }
    llvm_unreachable("Bad evaluation kind in EmitBuiltinExpr");
  }
 
  cgm.errorNYI(e->getSourceRange(),
               std::string("unimplemented builtin call: ") +
                   getContext().BuiltinInfo.getName(builtinID));
  return getUndefRValue(e->getType());
}
 
static std::optional<mlir::Value>
emitTargetArchBuiltinExpr(CIRGenFunction *cgf, unsigned builtinID,
                          const CallExpr *e, ReturnValueSlot &returnValue,
                          llvm::Triple::ArchType arch) {
  // When compiling in HipStdPar mode we have to be conservative in rejecting
  // target specific features in the FE, and defer the possible error to the
  // AcceleratorCodeSelection pass, wherein iff an unsupported target builtin is
  // referenced by an accelerator executable function, we emit an error.
  // Returning nullptr here leads to the builtin being handled in
  // EmitStdParUnsupportedBuiltin.
  if (cgf->getLangOpts().HIPStdPar && cgf->getLangOpts().CUDAIsDevice &&
      arch != cgf->getTarget().getTriple().getArch())
    return std::nullopt;
 
  switch (arch) {
  case llvm::Triple::arm:
  case llvm::Triple::armeb:
  case llvm::Triple::thumb:
  case llvm::Triple::thumbeb:
    // These are actually NYI, but that will be reported by emitBuiltinExpr.
    // At this point, we don't even know that the builtin is target-specific.
    return std::nullopt;
  case llvm::Triple::aarch64:
  case llvm::Triple::aarch64_32:
  case llvm::Triple::aarch64_be:
    return cgf->emitAArch64BuiltinExpr(builtinID, e, returnValue, arch);
  case llvm::Triple::bpfeb:
  case llvm::Triple::bpfel:
    // These are actually NYI, but that will be reported by emitBuiltinExpr.
    // At this point, we don't even know that the builtin is target-specific.
    return std::nullopt;
 
  case llvm::Triple::x86:
  case llvm::Triple::x86_64:
    return cgf->emitX86BuiltinExpr(builtinID, e);
 
  case llvm::Triple::ppc:
  case llvm::Triple::ppcle:
  case llvm::Triple::ppc64:
  case llvm::Triple::ppc64le:
  case llvm::Triple::r600:
    // These are actually NYI, but that will be reported by emitBuiltinExpr.
    // At this point, we don't even know that the builtin is target-specific.
    return std::nullopt;
  case llvm::Triple::amdgpu:
    return cgf->emitAMDGPUBuiltinExpr(builtinID, e);
  case llvm::Triple::systemz:
    return std::nullopt;
  case llvm::Triple::nvptx:
  case llvm::Triple::nvptx64:
    return cgf->emitNVPTXBuiltinExpr(builtinID, e);
  case llvm::Triple::wasm32:
  case llvm::Triple::wasm64:
  case llvm::Triple::hexagon:
    // These are actually NYI, but that will be reported by emitBuiltinExpr.
    // At this point, we don't even know that the builtin is target-specific.
    return std::nullopt;
  case llvm::Triple::riscv32:
  case llvm::Triple::riscv64:
    return cgf->emitRISCVBuiltinExpr(builtinID, e);
  default:
    return std::nullopt;
  }
}
 
std::optional<mlir::Value>
CIRGenFunction::emitTargetBuiltinExpr(unsigned builtinID, const CallExpr *e,
                                      ReturnValueSlot &returnValue) {
  if (getContext().BuiltinInfo.isAuxBuiltinID(builtinID)) {
    assert(getContext().getAuxTargetInfo() && "Missing aux target info");
    return emitTargetArchBuiltinExpr(
        this, getContext().BuiltinInfo.getAuxBuiltinID(builtinID), e,
        returnValue, getContext().getAuxTargetInfo()->getTriple().getArch());
  }
 
  return emitTargetArchBuiltinExpr(this, builtinID, e, returnValue,
                                   getTarget().getTriple().getArch());
}
 
mlir::Value CIRGenFunction::emitScalarOrConstFoldImmArg(
    const unsigned iceArguments, const unsigned idx, const Expr *argExpr) {
  mlir::Value arg = {};
  if ((iceArguments & (1 << idx)) == 0) {
    arg = emitScalarExpr(argExpr);
  } else {
    // If this is required to be a constant, constant fold it so that we
    // know that the generated intrinsic gets a ConstantInt.
    const std::optional<llvm::APSInt> result =
        argExpr->getIntegerConstantExpr(getContext());
    assert(result && "Expected argument to be a constant");
    arg = builder.getConstInt(getLoc(argExpr->getSourceRange()), *result);
  }
  return arg;
}
 
/// Given a builtin id for a function like "__builtin_fabsf", return a Function*
/// for "fabsf".
cir::FuncOp CIRGenModule::getBuiltinLibFunction(const FunctionDecl *fd,
                                                unsigned builtinID) {
  assert(astContext.BuiltinInfo.isLibFunction(builtinID));
 
  // Get the name, skip over the __builtin_ prefix (if necessary). We may have
  // to build this up so provide a small stack buffer to handle the vast
  // majority of names.
  llvm::SmallString<64> name;
 
  assert(!cir::MissingFeatures::asmLabelAttr());
  name = astContext.BuiltinInfo.getName(builtinID).substr(10);
 
  GlobalDecl d(fd);
  mlir::Type type = convertType(fd->getType());
  return getOrCreateCIRFunction(name, type, d, /*forVTable=*/false);
}
 
mlir::Value CIRGenFunction::emitCheckedArgForAssume(const Expr *e) {
  mlir::Value argValue = evaluateExprAsBool(e);
  if (!sanOpts.has(SanitizerKind::Builtin))
    return argValue;
 
  assert(!cir::MissingFeatures::sanitizers());
  cgm.errorNYI(e->getSourceRange(),
               "emitCheckedArgForAssume: sanitizers are NYI");
  return {};
}
 
void CIRGenFunction::emitVAStart(mlir::Value vaList) {
  // LLVM codegen casts to *i8, no real gain on doing this for CIRGen this
  // early, defer to LLVM lowering.
  cir::VAStartOp::create(builder, vaList.getLoc(), vaList);
}
 
void CIRGenFunction::emitVAEnd(mlir::Value vaList) {
  cir::VAEndOp::create(builder, vaList.getLoc(), vaList);
}
 
// FIXME(cir): This completely abstracts away the ABI with a generic CIR Op. By
// default this lowers to llvm.va_arg which is incomplete and not ABI-compliant
// on most targets so cir.va_arg will need some ABI handling in LoweringPrepare
mlir::Value CIRGenFunction::emitVAArg(VAArgExpr *ve) {
  assert(!cir::MissingFeatures::msabi());
  assert(!cir::MissingFeatures::vlas());
  mlir::Location loc = cgm.getLoc(ve->getExprLoc());
  mlir::Type type = convertType(ve->getType());
  mlir::Value vaList = emitVAListRef(ve->getSubExpr()).getPointer();
  return cir::VAArgOp::create(builder, loc, type, vaList);
}
 
mlir::Value CIRGenFunction::emitBuiltinObjectSize(const Expr *e, unsigned type,
                                                  cir::IntType resType,
                                                  mlir::Value emittedE,
                                                  bool isDynamic) {
  // If this is a pass_object_size parameter, load the implicit size arg.
  //
  // BOS type compatibility: a pass_object_size annotation with one type can
  // satisfy a __builtin_object_size query with a different type when the
  // annotated type is a safe approximation.  Type 0 (max, whole object) is
  // an overestimate for type 1 (max, closest surrounding subobject), and
  // type 3 (min, closest surrounding subobject) is an underestimate for
  // type 2 (min, whole object).
  enum BOSType {
    MaxWholeObject = 0,
    MaxSubobject = 1,
    MinWholeObject = 2,
    MinSubobject = 3,
  };
  if (auto *dre = dyn_cast<DeclRefExpr>(e->IgnoreParenImpCasts())) {
    auto *param = dyn_cast<ParmVarDecl>(dre->getDecl());
    auto *objSizeAttr = dre->getDecl()->getAttr<PassObjectSizeAttr>();
    if (param && objSizeAttr) {
      auto from = objSizeAttr->getType();
      bool compatible = from == static_cast<int>(type) ||
                        (from == MaxWholeObject && type == MaxSubobject) ||
                        (from == MinSubobject && type == MinWholeObject);
      if (compatible) {
        const ImplicitParamDecl *sizeDecl = sizeArguments.lookup(param);
        assert(sizeDecl && "expected pass_object_size implicit param");
 
        DeclMapTy::iterator declIter = localDeclMap.find(sizeDecl);
        assert(declIter != localDeclMap.end());
        Address addr = declIter->second;
 
        return emitLoadOfScalar(addr, /*volatile=*/false,
                                getContext().getSizeType(), e->getBeginLoc(),
                                LValueBaseInfo(AlignmentSource::Decl));
      }
    }
  }
 
  // LLVM can't handle type=3 appropriately, and __builtin_object_size shouldn't
  // evaluate e for side-effects. In either case, just like original LLVM
  // lowering, we shouldn't lower to `cir.objsize` but to a constant instead.
  if (type == 3 || (!emittedE && e->HasSideEffects(getContext())))
    return builder.getConstInt(getLoc(e->getSourceRange()), resType,
                               (type & 2) ? 0 : -1);
 
  mlir::Value ptr = emittedE ? emittedE : emitScalarExpr(e);
  assert(mlir::isa<cir::PointerType>(ptr.getType()) &&
         "Non-pointer passed to __builtin_object_size?");
 
  assert(!cir::MissingFeatures::countedBySize());
 
  // Extract the min/max mode from type. CIR only supports type 0
  // (max, whole object) and type 2 (min, whole object), not type 1 or 3
  // (closest subobject variants).
  const bool min = ((type & 2) != 0);
  // For GCC compatibility, __builtin_object_size treats NULL as unknown size.
  auto op =
      cir::ObjSizeOp::create(builder, getLoc(e->getSourceRange()), resType, ptr,
                             min, /*nullUnknown=*/true, isDynamic);
  return op.getResult();
}
 
mlir::Value CIRGenFunction::evaluateOrEmitBuiltinObjectSize(
    const Expr *e, unsigned type, cir::IntType resType, mlir::Value emittedE,
    bool isDynamic) {
  if (std::optional<uint64_t> objectSize =
          e->tryEvaluateObjectSize(getContext(), type))
    return builder.getConstInt(getLoc(e->getSourceRange()), resType,
                               *objectSize);
  return emitBuiltinObjectSize(e, type, resType, emittedE, isDynamic);
}