CIRGenExprScalar.cpp

Go to the documentation of this file.

//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// Emit Expr nodes with scalar CIR types as CIR code.
//
//===----------------------------------------------------------------------===//
 
#include "CIRGenConstantEmitter.h"
#include "CIRGenFixedPointBuilder.h"
#include "CIRGenFunction.h"
#include "CIRGenValue.h"
 
#include "clang/AST/Expr.h"
#include "clang/AST/StmtVisitor.h"
#include "clang/CIR/Dialect/IR/CIRTypes.h"
#include "clang/CIR/MissingFeatures.h"
#include "clang/CodeGenUtils/ExprUtils.h"
 
#include "mlir/Dialect/Ptr/IR/MemorySpaceInterfaces.h"
#include "mlir/IR/Location.h"
#include "mlir/IR/Value.h"
 
#include <cassert>
#include <numeric>
#include <utility>
 
using namespace clang;
using namespace clang::CIRGen;
 
namespace {
 
struct BinOpInfo {
  mlir::Value lhs;
  mlir::Value rhs;
  SourceRange loc;
  QualType fullType;             // Type of operands and result
  QualType compType;             // Type used for computations. Element type
                                 // for vectors, otherwise same as FullType.
  BinaryOperator::Opcode opcode; // Opcode of BinOp to perform
  FPOptions fpFeatures;
  const Expr *e; // Entire expr, for error unsupported.  May not be binop.
 
  /// Check if the binop computes a division or a remainder.
  bool isDivRemOp() const {
    return opcode == BO_Div || opcode == BO_Rem || opcode == BO_DivAssign ||
           opcode == BO_RemAssign;
  }
 
  /// Check if the binop can result in integer overflow.
  bool mayHaveIntegerOverflow() const {
    // Without constant input, we can't rule out overflow.
    auto lhsci = lhs.getDefiningOp<cir::ConstantOp>();
    auto rhsci = rhs.getDefiningOp<cir::ConstantOp>();
    if (!lhsci || !rhsci)
      return true;
 
    assert(!cir::MissingFeatures::mayHaveIntegerOverflow());
    // TODO(cir): For now we just assume that we might overflow
    return true;
  }
 
  /// Check if at least one operand is a fixed point type. In such cases,
  /// this operation did not follow usual arithmetic conversion and both
  /// operands might not be of the same type.
  bool isFixedPointOp() const {
    // We cannot simply check the result type since comparison operations
    // return an int.
    if (const auto *binOp = llvm::dyn_cast<BinaryOperator>(e)) {
      QualType lhstype = binOp->getLHS()->getType();
      QualType rhstype = binOp->getRHS()->getType();
      return lhstype->isFixedPointType() || rhstype->isFixedPointType();
    }
    if (const auto *unop = llvm::dyn_cast<UnaryOperator>(e))
      return unop->getSubExpr()->getType()->isFixedPointType();
    return false;
  }
};
 
class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
  CIRGenFunction &cgf;
  CIRGenBuilderTy &builder;
  // Unlike classic codegen we set this to false or use std::exchange to read
  // the value instead of calling TestAndClearIgnoreResultAssign to make it
  // explicit when the value is used
  bool ignoreResultAssign;
 
public:
  ScalarExprEmitter(CIRGenFunction &cgf, CIRGenBuilderTy &builder,
                    bool ignoreResultAssign = false)
      : cgf(cgf), builder(builder), ignoreResultAssign(ignoreResultAssign) {}
 
  //===--------------------------------------------------------------------===//
  //                               Utilities
  //===--------------------------------------------------------------------===//
  mlir::Type convertType(QualType ty) { return cgf.convertType(ty); }
 
  mlir::Value emitComplexToScalarConversion(mlir::Location loc,
                                            mlir::Value value, CastKind kind,
                                            QualType destTy);
 
  mlir::Value emitNullValue(QualType ty, mlir::Location loc) {
    return cgf.cgm.emitNullConstant(ty, loc);
  }
 
  mlir::Value emitPromotedValue(mlir::Value result, QualType promotionType) {
    return builder.createFloatingCast(result, cgf.convertType(promotionType));
  }
 
  mlir::Value emitUnPromotedValue(mlir::Value result, QualType exprType) {
    return builder.createFloatingCast(result, cgf.convertType(exprType));
  }
 
  mlir::Value emitPromoted(const Expr *e, QualType promotionType);
 
  mlir::Value maybePromoteBoolResult(mlir::Value value,
                                     mlir::Type dstTy) const {
    if (mlir::isa<cir::IntType>(dstTy))
      return builder.createBoolToInt(value, dstTy);
    if (mlir::isa<cir::BoolType>(dstTy))
      return value;
    llvm_unreachable("Can only promote integer or boolean types");
  }
 
  //===--------------------------------------------------------------------===//
  //                            Visitor Methods
  //===--------------------------------------------------------------------===//
 
  mlir::Value Visit(Expr *e) {
    return StmtVisitor<ScalarExprEmitter, mlir::Value>::Visit(e);
  }
 
  mlir::Value VisitStmt(Stmt *s) {
    llvm_unreachable("Statement passed to ScalarExprEmitter");
  }
 
  mlir::Value VisitExpr(Expr *e) {
    cgf.getCIRGenModule().errorNYI(
        e->getSourceRange(), "scalar expression kind: ", e->getStmtClassName());
    return {};
  }
 
  mlir::Value VisitConstantExpr(ConstantExpr *e) {
    // A constant expression of type 'void' generates no code and produces no
    // value.
    if (e->getType()->isVoidType())
      return {};
 
    if (mlir::Attribute result = ConstantEmitter(cgf).tryEmitConstantExpr(e)) {
      if (e->isGLValue()) {
        cgf.cgm.errorNYI(e->getSourceRange(),
                         "ScalarExprEmitter: constant expr GL Value");
        return {};
      }
 
      return builder.getConstant(cgf.getLoc(e->getSourceRange()),
                                 mlir::cast<mlir::TypedAttr>(result));
    }
 
    cgf.cgm.errorNYI(e->getSourceRange(), "ScalarExprEmitter: constant expr");
    return {};
  }
 
  mlir::Value VisitPackIndexingExpr(PackIndexingExpr *e) {
    return Visit(e->getSelectedExpr());
  }
 
  mlir::Value VisitParenExpr(ParenExpr *pe) { return Visit(pe->getSubExpr()); }
 
  mlir::Value VisitGenericSelectionExpr(GenericSelectionExpr *ge) {
    return Visit(ge->getResultExpr());
  }
 
  /// Emits the address of the l-value, then loads and returns the result.
  mlir::Value emitLoadOfLValue(const Expr *e) {
    LValue lv = cgf.emitLValue(e);
    // FIXME: add some akin to EmitLValueAlignmentAssumption(E, V);
    return cgf.emitLoadOfLValue(lv, e->getExprLoc()).getValue();
  }
 
  mlir::Value VisitCoawaitExpr(CoawaitExpr *s) {
    return cgf.emitCoawaitExpr(*s).getValue();
  }
 
  mlir::Value VisitCoyieldExpr(CoyieldExpr *e) {
    return cgf.emitCoyieldExpr(*e).getValue();
  }
 
  mlir::Value VisitUnaryCoawait(const UnaryOperator *e) {
    cgf.cgm.errorNYI(e->getSourceRange(), "ScalarExprEmitter: unary coawait");
    return {};
  }
 
  mlir::Value emitLoadOfLValue(LValue lv, SourceLocation loc) {
    return cgf.emitLoadOfLValue(lv, loc).getValue();
  }
 
  // l-values
  mlir::Value VisitDeclRefExpr(DeclRefExpr *e) {
    if (CIRGenFunction::ConstantEmission constant = cgf.tryEmitAsConstant(e))
      return cgf.emitScalarConstant(constant, e);
 
    return emitLoadOfLValue(e);
  }
 
  mlir::Value VisitAddrLabelExpr(const AddrLabelExpr *e) {
    auto func = cast<cir::FuncOp>(cgf.curFn);
    cir::BlockAddrInfoAttr blockInfoAttr = cir::BlockAddrInfoAttr::get(
        &cgf.getMLIRContext(), func.getSymName(), e->getLabel()->getName());
    // GotoSolver collects this cir.block_address op after FlattenCFG to keep
    // the label and wire it as an indirect-branch successor.
    return cir::BlockAddressOp::create(builder, cgf.getLoc(e->getSourceRange()),
                                       cgf.convertType(e->getType()),
                                       blockInfoAttr);
  }
 
  mlir::Value VisitIntegerLiteral(const IntegerLiteral *e) {
    mlir::Type type = cgf.convertType(e->getType());
    return cir::ConstantOp::create(builder, cgf.getLoc(e->getExprLoc()),
                                   cir::IntAttr::get(type, e->getValue()));
  }
 
  mlir::Value VisitFixedPointLiteral(const FixedPointLiteral *e) {
    mlir::Type type = cgf.convertType(e->getType());
    return cir::ConstantOp::create(builder, cgf.getLoc(e->getExprLoc()),
                                   cir::IntAttr::get(type, e->getValue()));
  }
 
  mlir::Value VisitFloatingLiteral(const FloatingLiteral *e) {
    mlir::Type type = cgf.convertType(e->getType());
    assert(mlir::isa<cir::FPTypeInterface>(type) &&
           "expect floating-point type");
    return cir::ConstantOp::create(builder, cgf.getLoc(e->getExprLoc()),
                                   cir::FPAttr::get(type, e->getValue()));
  }
 
  mlir::Value VisitCharacterLiteral(const CharacterLiteral *e) {
    mlir::Type ty = cgf.convertType(e->getType());
    // Character literals are always stored in an unsigned (even for signed
    // char), so allow implicit truncation here.
    auto intTy = mlir::cast<cir::IntTypeInterface>(ty);
    llvm::APInt apValue(intTy.getWidth(), e->getValue(),
                        /*isSigned=*/false, /*implicitTrunc=*/true);
    return cir::ConstantOp::create(builder, cgf.getLoc(e->getExprLoc()),
                                   cir::IntAttr::get(ty, apValue));
  }
 
  mlir::Value VisitCXXBoolLiteralExpr(const CXXBoolLiteralExpr *e) {
    return builder.getBool(e->getValue(), cgf.getLoc(e->getExprLoc()));
  }
 
  mlir::Value VisitCXXScalarValueInitExpr(const CXXScalarValueInitExpr *e) {
    if (e->getType()->isVoidType())
      return {};
 
    return emitNullValue(e->getType(), cgf.getLoc(e->getSourceRange()));
  }
 
  mlir::Value VisitGNUNullExpr(const GNUNullExpr *e) {
    return emitNullValue(e->getType(), cgf.getLoc(e->getSourceRange()));
  }
 
  mlir::Value VisitOffsetOfExpr(OffsetOfExpr *e);
 
  mlir::Value VisitSizeOfPackExpr(SizeOfPackExpr *e) {
    return builder.getConstInt(cgf.getLoc(e->getExprLoc()),
                               convertType(e->getType()), e->getPackLength());
  }
  mlir::Value VisitPseudoObjectExpr(PseudoObjectExpr *e) {
    return cgf.emitPseudoObjectRValue(e).getValue();
  }
  mlir::Value VisitSYCLUniqueStableNameExpr(SYCLUniqueStableNameExpr *e);
  mlir::Value VisitEmbedExpr(EmbedExpr *e) {
    assert(e->getDataElementCount() == 1);
    auto it = e->begin();
    llvm::APInt value = (*it)->getValue();
    return builder.getConstInt(cgf.getLoc(e->getExprLoc()), value,
                               e->getType()->isUnsignedIntegerType());
  }
  mlir::Value VisitOpaqueValueExpr(OpaqueValueExpr *e) {
    if (e->isGLValue())
      return emitLoadOfLValue(cgf.getOrCreateOpaqueLValueMapping(e),
                              e->getExprLoc());
 
    // Otherwise, assume the mapping is the scalar directly.
    return cgf.getOrCreateOpaqueRValueMapping(e).getValue();
  }
 
  mlir::Value VisitObjCSelectorExpr(ObjCSelectorExpr *e) {
    cgf.cgm.errorNYI(e->getSourceRange(), "ScalarExprEmitter: objc selector");
    return {};
  }
  mlir::Value VisitObjCProtocolExpr(ObjCProtocolExpr *e) {
    cgf.cgm.errorNYI(e->getSourceRange(), "ScalarExprEmitter: objc protocol");
    return {};
  }
  mlir::Value VisitObjCIVarRefExpr(ObjCIvarRefExpr *e) {
    cgf.cgm.errorNYI(e->getSourceRange(), "ScalarExprEmitter: objc ivar ref");
    return {};
  }
  mlir::Value VisitObjCMessageExpr(ObjCMessageExpr *e) {
    cgf.cgm.errorNYI(e->getSourceRange(), "ScalarExprEmitter: objc message");
    return {};
  }
  mlir::Value VisitObjCIsaExpr(ObjCIsaExpr *e) {
    cgf.cgm.errorNYI(e->getSourceRange(), "ScalarExprEmitter: objc isa");
    return {};
  }
  mlir::Value VisitObjCAvailabilityCheckExpr(ObjCAvailabilityCheckExpr *e) {
    cgf.cgm.errorNYI(e->getSourceRange(),
                     "ScalarExprEmitter: objc availability check");
    return {};
  }
 
  mlir::Value VisitMatrixSubscriptExpr(MatrixSubscriptExpr *e) {
    mlir::Value rowIdx = cgf.emitScalarExpr(e->getRowIdx());
    mlir::Value columnIdx = cgf.emitScalarExpr(e->getColumnIdx());
    mlir::Value matrix = Visit(e->getBase());
    mlir::Location loc = cgf.getLoc(e->getSourceRange());
    assert(!cir::MissingFeatures::emitMatrixIndexAssumption());
    return builder.createMatrixExtract(loc, matrix, rowIdx, columnIdx);
  }
 
  mlir::Value VisitMatrixSingleSubscriptExpr(MatrixSingleSubscriptExpr *e) {
    mlir::Value rowIdx = cgf.emitScalarExpr(e->getRowIdx());
    assert(!cir::MissingFeatures::emitMatrixIndexAssumption());
    mlir::Value matrix = Visit(e->getBase());
    mlir::Location loc = cgf.getLoc(e->getSourceRange());
    return builder.createMatrixExtractRow(loc, matrix, rowIdx);
  }
 
  mlir::Value VisitCastExpr(CastExpr *e);
  mlir::Value VisitCallExpr(const CallExpr *e);
 
  mlir::Value VisitStmtExpr(StmtExpr *e) {
    CIRGenFunction::StmtExprEvaluation eval(cgf);
    if (e->getType()->isVoidType()) {
      (void)cgf.emitCompoundStmt(*e->getSubStmt());
      return {};
    }
 
    Address retAlloca =
        cgf.createMemTemp(e->getType(), cgf.getLoc(e->getSourceRange()));
    (void)cgf.emitCompoundStmt(*e->getSubStmt(), &retAlloca);
 
    return cgf.emitLoadOfScalar(cgf.makeAddrLValue(retAlloca, e->getType()),
                                e->getExprLoc());
  }
 
  mlir::Value VisitArraySubscriptExpr(ArraySubscriptExpr *e) {
    ignoreResultAssign = false;
 
    if (e->getBase()->getType()->isVectorType()) {
      assert(!cir::MissingFeatures::scalableVectors());
 
      const mlir::Location loc = cgf.getLoc(e->getSourceRange());
      const mlir::Value vecValue = Visit(e->getBase());
      const mlir::Value indexValue = Visit(e->getIdx());
      return cir::VecExtractOp::create(cgf.getBuilder(), loc, vecValue,
                                       indexValue);
    }
    // Just load the lvalue formed by the subscript expression.
    return emitLoadOfLValue(e);
  }
 
  mlir::Value VisitShuffleVectorExpr(ShuffleVectorExpr *e) {
    if (e->getNumSubExprs() == 2) {
      // The undocumented form of __builtin_shufflevector.
      mlir::Value inputVec = Visit(e->getExpr(0));
      mlir::Value indexVec = Visit(e->getExpr(1));
      return cir::VecShuffleDynamicOp::create(cgf.getBuilder(),
                                              cgf.getLoc(e->getSourceRange()),
                                              inputVec, indexVec);
    }
 
    mlir::Value vec1 = Visit(e->getExpr(0));
    mlir::Value vec2 = Visit(e->getExpr(1));
 
    // The documented form of __builtin_shufflevector, where the indices are
    // a variable number of integer constants. The constants will be stored
    // in an ArrayAttr.
    SmallVector<mlir::Attribute, 8> indices;
    for (unsigned i = 2; i < e->getNumSubExprs(); ++i) {
      indices.push_back(
          cir::IntAttr::get(cgf.getBuilder().getSInt64Ty(),
                            e->getExpr(i)
                                ->EvaluateKnownConstInt(cgf.getContext())
                                .getSExtValue()));
    }
 
    return cir::VecShuffleOp::create(cgf.getBuilder(),
                                     cgf.getLoc(e->getSourceRange()),
                                     cgf.convertType(e->getType()), vec1, vec2,
                                     cgf.getBuilder().getArrayAttr(indices));
  }
 
  mlir::Value VisitConvertVectorExpr(ConvertVectorExpr *e) {
    // __builtin_convertvector is an element-wise cast, and is implemented as a
    // regular cast. The back end handles casts of vectors correctly.
    return emitScalarConversion(Visit(e->getSrcExpr()),
                                e->getSrcExpr()->getType(), e->getType(),
                                e->getSourceRange().getBegin());
  }
 
  mlir::Value VisitExtVectorElementExpr(Expr *e) { return emitLoadOfLValue(e); }
 
  mlir::Value VisitMatrixElementExpr(Expr *e) { return emitLoadOfLValue(e); }
 
  mlir::Value VisitMemberExpr(MemberExpr *e);
 
  mlir::Value VisitCompoundLiteralExpr(CompoundLiteralExpr *e) {
    return emitLoadOfLValue(e);
  }
 
  mlir::Value VisitInitListExpr(InitListExpr *e);
  mlir::Value emitVectorInitWithSubvectors(mlir::Location loc,
                                           cir::VectorType vectorType,
                                           ArrayRef<mlir::Value> inits);
 
  mlir::Value VisitArrayInitIndexExpr(ArrayInitIndexExpr *e) {
    assert(cgf.getArrayInitIndex() &&
           "ArrayInitIndexExpr not inside an ArrayInitLoopExpr?");
    return cgf.getArrayInitIndex();
  }
 
  mlir::Value VisitImplicitValueInitExpr(const ImplicitValueInitExpr *e) {
    return emitNullValue(e->getType(), cgf.getLoc(e->getSourceRange()));
  }
 
  mlir::Value VisitExplicitCastExpr(ExplicitCastExpr *e) {
    return VisitCastExpr(e);
  }
 
  mlir::Value VisitCXXNullPtrLiteralExpr(CXXNullPtrLiteralExpr *e) {
    return cgf.cgm.emitNullConstant(e->getType(),
                                    cgf.getLoc(e->getSourceRange()));
  }
 
  /// Perform a pointer to boolean conversion.
  mlir::Value emitPointerToBoolConversion(mlir::Value v, QualType qt) {
    // TODO(cir): comparing the ptr to null is done when lowering CIR to LLVM.
    // We might want to have a separate pass for these types of conversions.
    return cgf.getBuilder().createPtrToBoolCast(v);
  }
 
  mlir::Value emitFloatToBoolConversion(mlir::Value src, mlir::Location loc) {
    cir::BoolType boolTy = builder.getBoolTy();
    return cir::CastOp::create(builder, loc, boolTy,
                               cir::CastKind::float_to_bool, src);
  }
 
  mlir::Value emitIntToBoolConversion(mlir::Value srcVal, mlir::Location loc) {
    // The enumerator of an enum with an integral underlying type is lowered to
    // that type, which can be bool.  In that case the operand is already a
    // !cir.bool, so return it -- int_to_bool requires a !cir.int source.
    if (mlir::isa<cir::BoolType>(srcVal.getType()))
      return srcVal;
 
    // Because of the type rules of C, we often end up computing a
    // logical value, then zero extending it to int, then wanting it
    // as a logical value again.
    // TODO: optimize this common case here or leave it for later
    // CIR passes?
    cir::BoolType boolTy = builder.getBoolTy();
    return cir::CastOp::create(builder, loc, boolTy, cir::CastKind::int_to_bool,
                               srcVal);
  }
 
  /// Convert the specified expression value to a boolean (!cir.bool) truth
  /// value. This is equivalent to "Val != 0".
  mlir::Value emitConversionToBool(mlir::Value src, QualType srcType,
                                   mlir::Location loc) {
    assert(srcType.isCanonical() && "EmitScalarConversion strips typedefs");
 
    if (srcType->isRealFloatingType())
      return emitFloatToBoolConversion(src, loc);
 
    if (llvm::isa<MemberPointerType>(srcType)) {
      cgf.getCIRGenModule().errorNYI(loc, "member pointer to bool conversion");
      return builder.getFalse(loc);
    }
 
    if (srcType->isIntegerType())
      return emitIntToBoolConversion(src, loc);
 
    assert(::mlir::isa<cir::PointerType>(src.getType()));
    return emitPointerToBoolConversion(src, srcType);
  }
 
  // Emit a conversion from the specified type to the specified destination
  // type, both of which are CIR scalar types.
  struct ScalarConversionOpts {
    bool treatBooleanAsSigned;
    bool emitImplicitIntegerTruncationChecks;
    bool emitImplicitIntegerSignChangeChecks;
 
    ScalarConversionOpts()
        : treatBooleanAsSigned(false),
          emitImplicitIntegerTruncationChecks(false),
          emitImplicitIntegerSignChangeChecks(false) {}
 
    ScalarConversionOpts(clang::SanitizerSet sanOpts)
        : treatBooleanAsSigned(false),
          emitImplicitIntegerTruncationChecks(
              sanOpts.hasOneOf(SanitizerKind::ImplicitIntegerTruncation)),
          emitImplicitIntegerSignChangeChecks(
              sanOpts.has(SanitizerKind::ImplicitIntegerSignChange)) {}
  };
 
  // Conversion from bool, integral, or floating-point to integral or
  // floating-point. Conversions involving other types are handled elsewhere.
  // Conversion to bool is handled elsewhere because that's a comparison against
  // zero, not a simple cast. This handles both individual scalars and vectors.
  mlir::Value emitScalarCast(mlir::Value src, QualType srcType,
                             QualType dstType, mlir::Type srcTy,
                             mlir::Type dstTy, ScalarConversionOpts opts) {
    assert(!srcType->isMatrixType() && !dstType->isMatrixType() &&
           "Internal error: matrix types not handled by this function.");
    assert(!(mlir::isa<mlir::IntegerType>(srcTy) ||
             mlir::isa<mlir::IntegerType>(dstTy)) &&
           "Obsolete code. Don't use mlir::IntegerType with CIR.");
 
    mlir::Type fullDstTy = dstTy;
    if (mlir::isa<cir::VectorType>(srcTy) &&
        mlir::isa<cir::VectorType>(dstTy)) {
      // Use the element types of the vectors to figure out the CastKind.
      srcTy = mlir::dyn_cast<cir::VectorType>(srcTy).getElementType();
      dstTy = mlir::dyn_cast<cir::VectorType>(dstTy).getElementType();
    }
 
    std::optional<cir::CastKind> castKind;
 
    // Start with a null fenv attr. If this is a floating point cast, we will
    // get the attribute from the builder.
    cir::FenvAttr fenvAttr;
 
    if (mlir::isa<cir::BoolType>(srcTy)) {
      if (opts.treatBooleanAsSigned)
        cgf.getCIRGenModule().errorNYI("signed bool");
      if (cgf.getBuilder().isInt(dstTy)) {
        castKind = cir::CastKind::bool_to_int;
      } else if (mlir::isa<cir::FPTypeInterface>(dstTy)) {
        fenvAttr = cgf.getBuilder().getConstrainedFPAttr();
        castKind = cir::CastKind::bool_to_float;
      } else {
        llvm_unreachable("Internal error: Cast to unexpected type");
      }
    } else if (cgf.getBuilder().isInt(srcTy)) {
      if (cgf.getBuilder().isInt(dstTy)) {
        castKind = cir::CastKind::integral;
      } else if (mlir::isa<cir::FPTypeInterface>(dstTy)) {
        fenvAttr = cgf.getBuilder().getConstrainedFPAttr();
        castKind = cir::CastKind::int_to_float;
      } else if (mlir::isa<cir::BoolType>(dstTy)) {
        castKind = cir::CastKind::int_to_bool;
      } else {
        llvm_unreachable("Internal error: Cast to unexpected type");
      }
    } else if (mlir::isa<cir::FPTypeInterface>(srcTy)) {
      fenvAttr = cgf.getBuilder().getConstrainedFPAttr();
      if (cgf.getBuilder().isInt(dstTy)) {
        // If we can't recognize overflow as undefined behavior, assume that
        // overflow saturates. This protects against normal optimizations if we
        // are compiling with non-standard FP semantics.
        if (!cgf.cgm.getCodeGenOpts().StrictFloatCastOverflow)
          cgf.getCIRGenModule().errorNYI("strict float cast overflow");
        castKind = cir::CastKind::float_to_int;
      } else if (mlir::isa<cir::FPTypeInterface>(dstTy)) {
        // TODO: split this to createFPExt/createFPTrunc
        return builder.createFloatingCast(src, fullDstTy);
      } else if (mlir::isa<cir::BoolType>(dstTy)) {
        castKind = cir::CastKind::float_to_bool;
      } else {
        llvm_unreachable("Internal error: Cast to unexpected type");
      }
    } else {
      llvm_unreachable("Internal error: Cast from unexpected type");
    }
 
    assert(castKind.has_value() && "Internal error: CastKind not set.");
    return builder.createOrFold<cir::CastOp>(src.getLoc(), fullDstTy, *castKind,
                                             src, fenvAttr);
  }
 
  mlir::Value
  VisitSubstNonTypeTemplateParmExpr(SubstNonTypeTemplateParmExpr *e) {
    return Visit(e->getReplacement());
  }
 
  mlir::Value VisitVAArgExpr(VAArgExpr *ve) {
    QualType ty = ve->getType();
 
    if (ty->isVariablyModifiedType()) {
      cgf.cgm.errorNYI(ve->getSourceRange(),
                       "variably modified types in varargs");
    }
 
    return cgf.emitVAArg(ve);
  }
 
  mlir::Value VisitCXXRewrittenBinaryOperator(CXXRewrittenBinaryOperator *e) {
    return Visit(e->getSemanticForm());
  }
 
  mlir::Value VisitUnaryExprOrTypeTraitExpr(const UnaryExprOrTypeTraitExpr *e);
  mlir::Value
  VisitAbstractConditionalOperator(const AbstractConditionalOperator *e);
 
  // Unary Operators.
  mlir::Value VisitUnaryPrePostIncDec(const UnaryOperator *e) {
    LValue lv = cgf.emitLValue(e->getSubExpr());
    return emitScalarPrePostIncDec(e, lv);
  }
  mlir::Value VisitUnaryPostDec(const UnaryOperator *e) {
    return VisitUnaryPrePostIncDec(e);
  }
  mlir::Value VisitUnaryPostInc(const UnaryOperator *e) {
    return VisitUnaryPrePostIncDec(e);
  }
  mlir::Value VisitUnaryPreDec(const UnaryOperator *e) {
    return VisitUnaryPrePostIncDec(e);
  }
  mlir::Value VisitUnaryPreInc(const UnaryOperator *e) {
    return VisitUnaryPrePostIncDec(e);
  }
 
  mlir::Value emitFixedPointIncDec(const UnaryOperator *e, mlir::Value value,
                                   QualType type);
  mlir::Value emitScalarPrePostIncDec(const UnaryOperator *e, LValue lv) {
    assert(!cir::MissingFeatures::openMPLastprivateConditional());
 
    QualType type = e->getSubExpr()->getType();
 
    mlir::Value value;
    mlir::Value input;
 
    if (const AtomicType *atomicTy = type->getAs<AtomicType>()) {
      QualType valType = atomicTy->getValueType();
      mlir::Location loc = cgf.getLoc(e->getSourceRange());
      bool isInc = e->isIncrementOp();
      bool isPre = e->isPrefix();
 
      // Bools are always set-to-true, as decrement isn't legal on bools.
      if (valType->isBooleanType()) {
        assert(isInc);
        // Atomic operations require an integer type; reinterpret the bool
        // pointer as a pointer to its underlying storage integer type.
        cir::IntType intTy = builder.getBoolMemoryTy();
        mlir::Value one = builder.getConstInt(loc, intTy, 1);
        Address intAddr = lv.getAddress().withElementType(builder, intTy);
 
        if (isPre) {
          // Pre-increment: atomically store true, return true.
          cir::StoreOp store = builder.createStore(loc, one, intAddr);
          store.setMemOrder(cir::MemOrder::SequentiallyConsistent);
          if (lv.isVolatileQualified())
            store.setIsVolatile(true);
          return builder.getTrue(loc);
        }
        // Post-increment: atomically exchange with true, return old value.
        auto xchg = cir::AtomicXchgOp::create(
            builder, loc, intAddr.getPointer(), one,
            cir::MemOrder::SequentiallyConsistent, cir::SyncScopeKind::System,
            lv.isVolatileQualified());
        return builder.createCast(cir::CastKind::int_to_bool, xchg.getResult(),
                                  builder.getBoolTy());
      }
 
      // Special case for atomic increment / decrement on integers, emit
      // atomicrmw instructions.  We skip this if we want to be doing overflow
      // checking, and fall into the slow path with the atomic cmpxchg loop.
      if (valType->isIntegerType() &&
          !(valType->isUnsignedIntegerType() &&
            cgf.sanOpts.has(SanitizerKind::UnsignedIntegerOverflow)) &&
          cgf.getLangOpts().getSignedOverflowBehavior() !=
              LangOptions::SOB_Trapping) {
        mlir::Type intTy = cgf.convertType(valType);
        mlir::Value one = builder.getConstInt(loc, intTy, 1);
        cir::AtomicFetchKind kind =
            isInc ? cir::AtomicFetchKind::Add : cir::AtomicFetchKind::Sub;
        auto rmw = cir::AtomicFetchOp::create(
            builder, loc, lv.getPointer(), one, kind,
            cir::MemOrder::SequentiallyConsistent, cir::SyncScopeKind::System,
            lv.isVolatileQualified(), /*fetch_first=*/true);
        mlir::Value oldVal = rmw->getResult(0);
        // Prefix returns new value; postfix returns old value.
        return isPre ? emitIntIncOrDec(e, oldVal) : oldVal;
      }
 
      // Special case for atomic increment/decrement on floats.
      // Bail out non-power-of-2-sized floating point types (e.g., x86_fp80).
      if (valType->isFloatingType()) {
        mlir::Type fpTy = cgf.convertType(valType);
        auto fpType = mlir::cast<cir::FPTypeInterface>(fpTy);
        // Bail on non-power-of-2 types (e.g., x86_fp80 is 80 bits).
        if (llvm::has_single_bit(fpType.getWidth())) {
          CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, e);
          mlir::Value amt = builder.getConstFP(
              loc, fpTy, llvm::APFloat(fpType.getFloatSemantics(), 1));
          cir::AtomicFetchKind kind =
              isInc ? cir::AtomicFetchKind::Add : cir::AtomicFetchKind::Sub;
          auto rmw = cir::AtomicFetchOp::create(
              builder, loc, lv.getPointer(), amt, kind,
              cir::MemOrder::SequentiallyConsistent, cir::SyncScopeKind::System,
              lv.isVolatileQualified(), /*fetch_first=*/true);
          mlir::Value oldVal = rmw->getResult(0);
          if (isPre)
            return isInc ? builder.createFAdd(loc, oldVal, amt)
                         : builder.createFSub(loc, oldVal, amt);
          return oldVal;
        } else {
          cgf.cgm.errorNYI(e->getSourceRange(),
                           "Atomic inc/dec of non-power-of-2 float");
        }
      }
 
      // Otherwise we need a loop on compare-exchange.
      cgf.cgm.errorNYI(e->getSourceRange(), "Atomic inc/dec");
      value = cgf.emitLoadOfLValue(lv, e->getExprLoc()).getValue();
      input = value;
    } else {
      value = cgf.emitLoadOfLValue(lv, e->getExprLoc()).getValue();
      input = value;
    }
 
    // NOTE: When possible, more frequent cases are handled first.
 
    // Special case of integer increment that we have to check first: bool++.
    // Due to promotion rules, we get:
    //   bool++ -> bool = bool + 1
    //          -> bool = (int)bool + 1
    //          -> bool = ((int)bool + 1 != 0)
    // An interesting aspect of this is that increment is always true.
    // Decrement does not have this property.
    if (e->isIncrementOp() && type->isBooleanType()) {
      value = builder.getTrue(cgf.getLoc(e->getExprLoc()));
    } else if (type->isIntegerType()) {
      QualType promotedType;
      [[maybe_unused]] bool canPerformLossyDemotionCheck = false;
      if (cgf.getContext().isPromotableIntegerType(type)) {
        promotedType = cgf.getContext().getPromotedIntegerType(type);
        assert(promotedType != type && "Shouldn't promote to the same type.");
        canPerformLossyDemotionCheck = true;
        canPerformLossyDemotionCheck &=
            cgf.getContext().getCanonicalType(type) !=
            cgf.getContext().getCanonicalType(promotedType);
        canPerformLossyDemotionCheck &=
            type->isIntegerType() && promotedType->isIntegerType();
 
        // TODO(cir): Currently, we store bitwidths in CIR types only for
        // integers. This might also be required for other types.
 
        assert(
            (!canPerformLossyDemotionCheck ||
             type->isSignedIntegerOrEnumerationType() ||
             promotedType->isSignedIntegerOrEnumerationType() ||
             mlir::cast<cir::IntType>(cgf.convertType(type)).getWidth() ==
                 mlir::cast<cir::IntType>(cgf.convertType(type)).getWidth()) &&
            "The following check expects that if we do promotion to different "
            "underlying canonical type, at least one of the types (either "
            "base or promoted) will be signed, or the bitwidths will match.");
      }
 
      assert(!cir::MissingFeatures::sanitizers());
      if (e->canOverflow() && type->isSignedIntegerOrEnumerationType()) {
        value = emitIncDecConsiderOverflowBehavior(e, value);
      } else {
        // NOTE(CIR): clang calls CreateAdd but folds this to a unary op
        value = emitIntIncOrDec(e, input, /*nsw=*/false);
      }
    } else if (const PointerType *ptr = type->getAs<PointerType>()) {
      QualType type = ptr->getPointeeType();
      if (const VariableArrayType *vla =
              cgf.getContext().getAsVariableArrayType(type)) {
        mlir::Location loc = cgf.getLoc(e->getSourceRange());
        mlir::Value numElts = cgf.getVLASize(vla).numElts;
        if (!e->isIncrementOp())
          numElts = cgf.getBuilder().createNeg(loc, numElts, /*nsw=*/true);
        assert(!cir::MissingFeatures::sanitizers());
        value = cgf.getBuilder().createPtrStride(loc, value, numElts);
      } else {
        // For everything else, we can just do a simple increment.
        mlir::Location loc = cgf.getLoc(e->getSourceRange());
        int amount = e->isIncrementOp() ? 1 : -1;
        mlir::Value amt = builder.getSInt32(amount, loc);
        assert(!cir::MissingFeatures::sanitizers());
        value = builder.createPtrStride(loc, value, amt);
      }
    } else if (type->isVectorType()) {
      if (type->hasIntegerRepresentation()) {
        value = emitIntIncOrDec(e, input, /*nsw=*/false);
      } else {
        CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, e);
        value = emitFloatIncOrDec(e, input);
      }
    } else if (type->isRealFloatingType()) {
      CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, e);
      value = emitFloatIncOrDec(e, input);
    } else if (type->isFixedPointType()) {
      value = emitFixedPointIncDec(e, value, type);
    } else {
      assert(type->castAs<ObjCObjectPointerType>());
      cgf.cgm.errorNYI(e->getSourceRange(), "Unary inc/dec ObjectiveC pointer");
      return {};
    }
 
    CIRGenFunction::SourceLocRAIIObject sourceloc{cgf, e->getSourceRange()};
 
    // Store the updated result through the lvalue
    if (lv.isBitField())
      value = cgf.emitStoreThroughBitfieldLValue(RValue::get(value), lv);
    else
      cgf.emitStoreThroughLValue(RValue::get(value), lv);
 
    // If this is a postinc, return the value read from memory, otherwise use
    // the updated value.
    return e->isPrefix() ? value : input;
  }
 
  mlir::Value emitIncDecConsiderOverflowBehavior(const UnaryOperator *e,
                                                 mlir::Value inVal) {
    switch (cgf.getLangOpts().getSignedOverflowBehavior()) {
    case LangOptions::SOB_Defined:
      return emitIntIncOrDec(e, inVal, /*nsw=*/false);
    case LangOptions::SOB_Undefined:
      assert(!cir::MissingFeatures::sanitizers());
      return emitIntIncOrDec(e, inVal, /*nsw=*/true);
    case LangOptions::SOB_Trapping:
      if (!e->canOverflow())
        return emitIntIncOrDec(e, inVal, /*nsw=*/true);
      cgf.cgm.errorNYI(e->getSourceRange(), "inc/def overflow SOB_Trapping");
      return {};
    }
    llvm_unreachable("Unexpected signed overflow behavior kind");
  }
 
  mlir::Value VisitUnaryAddrOf(const UnaryOperator *e) {
    if (llvm::isa<MemberPointerType>(e->getType()))
      return cgf.cgm.emitMemberPointerConstant(e);
 
    return cgf.emitLValue(e->getSubExpr()).getPointer();
  }
 
  mlir::Value VisitUnaryDeref(const UnaryOperator *e) {
    if (e->getType()->isVoidType())
      return Visit(e->getSubExpr()); // the actual value should be unused
    return emitLoadOfLValue(e);
  }
 
  mlir::Value VisitUnaryPlus(const UnaryOperator *e) {
    QualType promotionType = getPromotionType(e->getSubExpr()->getType());
    mlir::Value result = VisitUnaryPlus(e, promotionType);
    if (result && !promotionType.isNull())
      return emitUnPromotedValue(result, e->getType());
    return result;
  }
 
  mlir::Value VisitUnaryPlus(const UnaryOperator *e, QualType promotionType) {
    ignoreResultAssign = false;
    if (!promotionType.isNull())
      return cgf.emitPromotedScalarExpr(e->getSubExpr(), promotionType);
    return Visit(e->getSubExpr());
  }
 
  mlir::Value VisitUnaryMinus(const UnaryOperator *e) {
    QualType promotionType = getPromotionType(e->getSubExpr()->getType());
    mlir::Value result = VisitUnaryMinus(e, promotionType);
    if (result && !promotionType.isNull())
      return emitUnPromotedValue(result, e->getType());
    return result;
  }
 
  mlir::Value VisitUnaryMinus(const UnaryOperator *e, QualType promotionType) {
    ignoreResultAssign = false;
    mlir::Value operand;
    if (!promotionType.isNull())
      operand = cgf.emitPromotedScalarExpr(e->getSubExpr(), promotionType);
    else
      operand = Visit(e->getSubExpr());
 
    mlir::Location loc = cgf.getLoc(e->getSourceRange().getBegin());
 
    if (cir::isFPOrVectorOfFPType(operand.getType()))
      return builder.createOrFold<cir::FNegOp>(loc, operand);
 
    // TODO(cir): We might have to change this to support overflow trapping.
    //            Classic codegen routes unary minus through emitSub to ensure
    //            that the overflow behavior is handled correctly.
    bool nsw = e->getType()->isSignedIntegerType() &&
               cgf.getLangOpts().getSignedOverflowBehavior() !=
                   LangOptions::SOB_Defined;
 
    return builder.createOrFold<cir::MinusOp>(loc, operand, nsw);
  }
 
  mlir::Value emitIntIncOrDec(const UnaryOperator *e, mlir::Value input,
                              bool nsw = false) {
    mlir::Location loc = cgf.getLoc(e->getSourceRange().getBegin());
    return e->isIncrementOp()
               ? builder.createOrFold<cir::IncOp>(loc, input, nsw)
               : builder.createOrFold<cir::DecOp>(loc, input, nsw);
  }
 
  mlir::Value emitFloatIncOrDec(const UnaryOperator *e, mlir::Value input) {
    assert(cir::isFPOrVectorOfFPType(input.getType()) &&
           "Expect floating-point operand");
    mlir::Location loc = cgf.getLoc(e->getSourceRange().getBegin());
 
    if (auto vecType = mlir::dyn_cast<cir::VectorType>(input.getType())) {
      mlir::Type fpScalarType = vecType.getElementType();
      auto fpInterface = mlir::cast<cir::FPTypeInterface>(fpScalarType);
      auto amount = llvm::APFloat::getOne(fpInterface.getFloatSemantics(),
                                          /*Negative=*/e->isDecrementOp());
      mlir::Value amtValue = builder.getConstFP(loc, fpScalarType, amount);
      amtValue = cir::VecSplatOp::create(builder, loc, vecType, amtValue);
      return builder.createFAdd(loc, input, amtValue);
    }
 
    QualType type = e->getSubExpr()->getType();
    // Another special case: half FP increment should be done via float.
    if (type->isHalfType() && !cgf.getContext().getLangOpts().NativeHalfType)
      input = builder.createFloatingCast(input, builder.getSingleTy());
 
    auto fpInterface = mlir::cast<cir::FPTypeInterface>(input.getType());
    auto amount = llvm::APFloat::getOne(fpInterface.getFloatSemantics(),
                                        /*Negative=*/e->isDecrementOp());
    mlir::Value amtValue = builder.getConstFP(loc, input.getType(), amount);
    mlir::Value output = builder.createFAdd(loc, input, amtValue);
 
    if (type->isHalfType() && !cgf.getContext().getLangOpts().NativeHalfType)
      output = builder.createFloatingCast(output, builder.getFp16Ty());
 
    return output;
  }
 
  mlir::Value VisitUnaryNot(const UnaryOperator *e) {
    ignoreResultAssign = false;
    mlir::Value op = Visit(e->getSubExpr());
    return builder.createOrFold<cir::NotOp>(
        cgf.getLoc(e->getSourceRange().getBegin()), op);
  }
 
  mlir::Value VisitUnaryLNot(const UnaryOperator *e);
 
  mlir::Value VisitUnaryReal(const UnaryOperator *e);
  mlir::Value VisitUnaryImag(const UnaryOperator *e);
  mlir::Value VisitRealImag(const UnaryOperator *e,
                            QualType promotionType = QualType());
 
  mlir::Value VisitUnaryExtension(const UnaryOperator *e) {
    return Visit(e->getSubExpr());
  }
 
  // C++
  mlir::Value VisitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *e) {
    return emitLoadOfLValue(e);
  }
  mlir::Value VisitSourceLocExpr(SourceLocExpr *e) {
    ASTContext &ctx = cgf.getContext();
    APValue evaluated =
        e->EvaluateInContext(ctx, cgf.curSourceLocExprScope.getDefaultExpr());
    mlir::Attribute attribute = ConstantEmitter(cgf).emitAbstract(
        e->getLocation(), evaluated, e->getType());
    mlir::TypedAttr typedAttr = mlir::cast<mlir::TypedAttr>(attribute);
    return cir::ConstantOp::create(builder, cgf.getLoc(e->getExprLoc()),
                                   typedAttr);
  }
  mlir::Value VisitCXXDefaultArgExpr(CXXDefaultArgExpr *dae) {
    CIRGenFunction::CXXDefaultArgExprScope scope(cgf, dae);
    return Visit(dae->getExpr());
  }
  mlir::Value VisitCXXDefaultInitExpr(CXXDefaultInitExpr *die) {
    CIRGenFunction::CXXDefaultInitExprScope scope(cgf, die);
    return Visit(die->getExpr());
  }
 
  mlir::Value VisitCXXThisExpr(CXXThisExpr *te) { return cgf.loadCXXThis(); }
 
  mlir::Value VisitExprWithCleanups(ExprWithCleanups *e);
  mlir::Value VisitCXXNewExpr(const CXXNewExpr *e) {
    return cgf.emitCXXNewExpr(e);
  }
  mlir::Value VisitCXXDeleteExpr(const CXXDeleteExpr *e) {
    cgf.emitCXXDeleteExpr(e);
    return {};
  }
  mlir::Value VisitTypeTraitExpr(const TypeTraitExpr *e) {
    // We diverge slightly from classic codegen here because CIR has stricter
    // typing. In LLVM IR, constant folding covers up some potential type
    // mismatches such as bool-to-int conversions that would fail the verifier
    // in CIR. To make things work, we need to be sure we only emit a bool value
    // if the expression type is bool.
    mlir::Location loc = cgf.getLoc(e->getExprLoc());
    if (e->isStoredAsBoolean()) {
      if (e->getType()->isBooleanType())
        return builder.getBool(e->getBoolValue(), loc);
      assert(e->getType()->isIntegerType() &&
             "Expected int type for TypeTraitExpr");
      return builder.getConstInt(loc, cgf.convertType(e->getType()),
                                 (uint64_t)e->getBoolValue());
    }
    assert(e->getType()->isIntegerType() && "not a scalar type trait");
    return builder.getConstInt(loc, e->getAPValue().getInt());
  }
  mlir::Value
  VisitConceptSpecializationExpr(const ConceptSpecializationExpr *e) {
    return builder.getBool(e->isSatisfied(), cgf.getLoc(e->getExprLoc()));
  }
  mlir::Value VisitRequiresExpr(const RequiresExpr *e) {
    return builder.getBool(e->isSatisfied(), cgf.getLoc(e->getExprLoc()));
  }
  mlir::Value VisitArrayTypeTraitExpr(const ArrayTypeTraitExpr *e) {
    mlir::Type type = cgf.convertType(e->getType());
    mlir::Location loc = cgf.getLoc(e->getExprLoc());
    return builder.getConstInt(loc, type, e->getValue());
  }
  mlir::Value VisitExpressionTraitExpr(const ExpressionTraitExpr *e) {
    return builder.getBool(e->getValue(), cgf.getLoc(e->getExprLoc()));
  }
  mlir::Value VisitCXXPseudoDestructorExpr(const CXXPseudoDestructorExpr *e) {
    cgf.cgm.errorNYI(e->getSourceRange(),
                     "ScalarExprEmitter: cxx pseudo destructor");
    return {};
  }
  mlir::Value VisitCXXThrowExpr(const CXXThrowExpr *e) {
    cgf.emitCXXThrowExpr(e);
    return {};
  }
 
  mlir::Value VisitCXXNoexceptExpr(CXXNoexceptExpr *e) {
    return builder.getBool(e->getValue(), cgf.getLoc(e->getExprLoc()));
  }
 
  mlir::Value emitFixedPointConversion(mlir::Value src, QualType srcTy,
                                       QualType dstTy, mlir::Location loc);
 
  /// Emit a conversion from the specified type to the specified destination
  /// type, both of which are CIR scalar types.
  /// TODO: do we need ScalarConversionOpts here? Should be done in another
  /// pass.
  mlir::Value
  emitScalarConversion(mlir::Value src, QualType srcType, QualType dstType,
                       SourceLocation loc,
                       ScalarConversionOpts opts = ScalarConversionOpts()) {
    // All conversions involving fixed point types should be handled by the
    // emitFixedPoint family functions. This is done to prevent bloating up
    // this function more, and although fixed point numbers are represented by
    // integers, we do not want to follow any logic that assumes they should be
    // treated as integers.
    if (srcType->isFixedPointType()) {
      if (dstType->isBooleanType()) {
        cir::BoolType boolTy = builder.getBoolTy();
        return cir::CastOp::create(builder, cgf.getLoc(loc), boolTy,
                                   cir::CastKind::int_to_bool, src);
      }
 
      if (dstType->isFixedPointType() || dstType->isIntegerType() ||
          dstType->isRealFloatingType())
        return emitFixedPointConversion(src, srcType, dstType, cgf.getLoc(loc));
 
      llvm_unreachable("Unhandled scalar conversion from a fixed point type to "
                       "another type.");
    } else if (dstType->isFixedPointType()) {
      if (srcType->isIntegerType() || srcType->isRealFloatingType()) {
        // This also includes converting booleans and enums to fixed point
        // types.
        return emitFixedPointConversion(src, srcType, dstType, cgf.getLoc(loc));
      }
 
      llvm_unreachable("Unhandled scalar conversion to a fixed point type from "
                       "another type.");
    }
 
    srcType = srcType.getCanonicalType();
    dstType = dstType.getCanonicalType();
    if (srcType == dstType) {
      if (opts.emitImplicitIntegerSignChangeChecks)
        cgf.getCIRGenModule().errorNYI(loc,
                                       "implicit integer sign change checks");
      return src;
    }
 
    if (dstType->isVoidType())
      return {};
 
    mlir::Type mlirSrcType = src.getType();
 
    // Handle conversions to bool first, they are special: comparisons against
    // 0.
    if (dstType->isBooleanType())
      return emitConversionToBool(src, srcType, cgf.getLoc(loc));
 
    mlir::Type mlirDstType = cgf.convertType(dstType);
 
    if (srcType->isHalfType() &&
        !cgf.getContext().getLangOpts().NativeHalfType) {
      // Cast to FP using the intrinsic if the half type itself isn't supported.
      if (!mlir::isa<cir::FPTypeInterface>(mlirDstType)) {
        // Cast to other types through float, using FPExt, depending on whether
        // the half type itself is supported (as opposed to operations on half,
        // available with NativeHalfType).
        src = builder.createCast(cgf.getLoc(loc), cir::CastKind::floating, src,
                                 cgf.floatTy);
        srcType = cgf.getContext().FloatTy;
        mlirSrcType = cgf.floatTy;
      }
    }
 
    // TODO(cir): LLVM codegen ignore conversions like int -> uint,
    // is there anything to be done for CIR here?
    if (mlirSrcType == mlirDstType) {
      if (opts.emitImplicitIntegerSignChangeChecks)
        cgf.getCIRGenModule().errorNYI(loc,
                                       "implicit integer sign change checks");
      return src;
    }
 
    // Handle pointer conversions next: pointers can only be converted to/from
    // other pointers and integers. Check for pointer types in terms of LLVM, as
    // some native types (like Obj-C id) may map to a pointer type.
    if (auto dstPT = dyn_cast<cir::PointerType>(mlirDstType)) {
      cgf.getCIRGenModule().errorNYI(loc, "pointer casts");
      return builder.getNullPtr(dstPT, src.getLoc());
    }
 
    if (isa<cir::PointerType>(mlirSrcType)) {
      // Must be an ptr to int cast.
      assert(isa<cir::IntType>(mlirDstType) && "not ptr->int?");
      return builder.createPtrToInt(src, mlirDstType);
    }
 
    // A scalar can be splatted to an extended vector of the same element type
    if (dstType->isExtVectorType() && !srcType->isVectorType()) {
      // Sema should add casts to make sure that the source expression's type
      // is the same as the vector's element type (sans qualifiers)
      assert(dstType->castAs<ExtVectorType>()->getElementType().getTypePtr() ==
                 srcType.getTypePtr() &&
             "Splatted expr doesn't match with vector element type?");
 
      cgf.getCIRGenModule().errorNYI(loc, "vector splatting");
      return {};
    }
 
    if (srcType->isMatrixType() && dstType->isMatrixType()) {
      cgf.getCIRGenModule().errorNYI(loc,
                                     "matrix type to matrix type conversion");
      return {};
    }
    assert(!srcType->isMatrixType() && !dstType->isMatrixType() &&
           "Internal error: conversion between matrix type and scalar type");
 
    // Finally, we have the arithmetic types or vectors of arithmetic types.
    mlir::Value res = nullptr;
    mlir::Type resTy = mlirDstType;
 
    res = emitScalarCast(src, srcType, dstType, mlirSrcType, mlirDstType, opts);
 
    if (mlirDstType != resTy) {
      res = builder.createCast(cgf.getLoc(loc), cir::CastKind::floating, res,
                               resTy);
    }
 
    if (opts.emitImplicitIntegerTruncationChecks)
      cgf.getCIRGenModule().errorNYI(loc, "implicit integer truncation checks");
 
    if (opts.emitImplicitIntegerSignChangeChecks)
      cgf.getCIRGenModule().errorNYI(loc,
                                     "implicit integer sign change checks");
 
    return res;
  }
 
  BinOpInfo emitBinOps(const BinaryOperator *e,
                       QualType promotionType = QualType()) {
    ignoreResultAssign = false;
    BinOpInfo result;
    result.lhs = cgf.emitPromotedScalarExpr(e->getLHS(), promotionType);
    result.rhs = cgf.emitPromotedScalarExpr(e->getRHS(), promotionType);
    if (!promotionType.isNull())
      result.fullType = promotionType;
    else
      result.fullType = e->getType();
    result.compType = result.fullType;
    if (const auto *vecType = result.fullType->getAs<VectorType>())
      result.compType = vecType->getElementType();
    result.opcode = e->getOpcode();
    result.loc = e->getSourceRange();
    result.fpFeatures = e->getFPFeaturesInEffect(cgf.getLangOpts());
    result.e = e;
    return result;
  }
 
  mlir::Value emitMul(const BinOpInfo &ops);
  mlir::Value emitDiv(const BinOpInfo &ops);
  mlir::Value emitRem(const BinOpInfo &ops);
  mlir::Value emitAdd(const BinOpInfo &ops);
  mlir::Value emitSub(const BinOpInfo &ops);
  mlir::Value emitShl(const BinOpInfo &ops);
  mlir::Value emitShr(const BinOpInfo &ops);
  mlir::Value emitAnd(const BinOpInfo &ops);
  mlir::Value emitXor(const BinOpInfo &ops);
  mlir::Value emitOr(const BinOpInfo &ops);
 
  mlir::Value emitFixedPointBinOp(const BinOpInfo &ops);
 
  LValue emitCompoundAssignLValue(
      const CompoundAssignOperator *e,
      mlir::Value (ScalarExprEmitter::*f)(const BinOpInfo &),
      mlir::Value &result);
  mlir::Value
  emitCompoundAssign(const CompoundAssignOperator *e,
                     mlir::Value (ScalarExprEmitter::*f)(const BinOpInfo &));
 
  // TODO(cir): Candidate to be in a common AST helper between CIR and LLVM
  // codegen.
  QualType getPromotionType(QualType ty) {
    const clang::ASTContext &ctx = cgf.getContext();
    if (auto *complexTy = ty->getAs<ComplexType>()) {
      QualType elementTy = complexTy->getElementType();
      if (elementTy.UseExcessPrecision(ctx))
        return ctx.getComplexType(ctx.FloatTy);
    }
 
    if (ty.UseExcessPrecision(cgf.getContext())) {
      if (auto *vt = ty->getAs<VectorType>()) {
        unsigned numElements = vt->getNumElements();
        return ctx.getVectorType(ctx.FloatTy, numElements, vt->getVectorKind());
      }
      return cgf.getContext().FloatTy;
    }
 
    return QualType();
  }
 
// Binary operators and binary compound assignment operators.
#define HANDLEBINOP(OP)                                                        \
  mlir::Value VisitBin##OP(const BinaryOperator *e) {                          \
    QualType promotionTy = getPromotionType(e->getType());                     \
    auto result = emit##OP(emitBinOps(e, promotionTy));                        \
    if (result && !promotionTy.isNull())                                       \
      result = emitUnPromotedValue(result, e->getType());                      \
    return result;                                                             \
  }                                                                            \
  mlir::Value VisitBin##OP##Assign(const CompoundAssignOperator *e) {          \
    return emitCompoundAssign(e, &ScalarExprEmitter::emit##OP);                \
  }
 
  HANDLEBINOP(Mul)
  HANDLEBINOP(Div)
  HANDLEBINOP(Rem)
  HANDLEBINOP(Add)
  HANDLEBINOP(Sub)
  HANDLEBINOP(Shl)
  HANDLEBINOP(Shr)
  HANDLEBINOP(And)
  HANDLEBINOP(Xor)
  HANDLEBINOP(Or)
#undef HANDLEBINOP
 
  cir::CmpOpKind clangCmpToCIRCmp(clang::BinaryOperatorKind clangCmp) {
    switch (clangCmp) {
    case BO_LT:
      return cir::CmpOpKind::lt;
    case BO_GT:
      return cir::CmpOpKind::gt;
    case BO_LE:
      return cir::CmpOpKind::le;
    case BO_GE:
      return cir::CmpOpKind::ge;
    case BO_EQ:
      return cir::CmpOpKind::eq;
    case BO_NE:
      return cir::CmpOpKind::ne;
    default:
      llvm_unreachable("unsupported comparison kind for cir.cmp");
    }
  }
 
  mlir::Value emitCmp(const BinaryOperator *e) {
    ignoreResultAssign = false;
    const mlir::Location loc = cgf.getLoc(e->getExprLoc());
    mlir::Value result;
    QualType lhsTy = e->getLHS()->getType();
    QualType rhsTy = e->getRHS()->getType();
 
    cir::CmpOpKind kind = clangCmpToCIRCmp(e->getOpcode());
    if (lhsTy->getAs<MemberPointerType>()) {
      assert(!cir::MissingFeatures::dataMemberType());
      assert(e->getOpcode() == BO_EQ || e->getOpcode() == BO_NE);
      mlir::Value lhs = cgf.emitScalarExpr(e->getLHS());
      mlir::Value rhs = cgf.emitScalarExpr(e->getRHS());
      result = builder.createCompare(loc, kind, lhs, rhs);
    } else if (!lhsTy->isAnyComplexType() && !rhsTy->isAnyComplexType()) {
      BinOpInfo boInfo = emitBinOps(e);
      mlir::Value lhs = boInfo.lhs;
      mlir::Value rhs = boInfo.rhs;
 
      if (lhsTy->isVectorType()) {
        if (!e->getType()->isVectorType()) {
          // If AltiVec, the comparison results in a numeric type, so we use
          // intrinsics comparing vectors and giving 0 or 1 as a result
          cgf.cgm.errorNYI(loc, "AltiVec comparison");
        } else {
          // Other kinds of vectors. Element-wise comparison returning
          // a vector.
          result = cir::VecCmpOp::create(builder, cgf.getLoc(boInfo.loc),
                                         cgf.convertType(boInfo.fullType), kind,
                                         boInfo.lhs, boInfo.rhs);
        }
      } else if (boInfo.isFixedPointOp()) {
        result = emitFixedPointBinOp(boInfo);
      } else {
        // integers and pointers
        if (cgf.cgm.getCodeGenOpts().StrictVTablePointers &&
            mlir::isa<cir::PointerType>(lhs.getType()) &&
            mlir::isa<cir::PointerType>(rhs.getType())) {
          cgf.cgm.errorNYI(loc, "strict vtable pointer comparisons");
        }
        result = builder.createCompare(loc, kind, lhs, rhs);
      }
    } else {
      assert((e->getOpcode() == BO_EQ || e->getOpcode() == BO_NE) &&
             "Complex Comparison: can only be an equality comparison");
 
      mlir::Value lhs;
      if (lhsTy->isAnyComplexType()) {
        lhs = cgf.emitComplexExpr(e->getLHS());
      } else {
        mlir::Value lhsReal = Visit(e->getLHS());
        mlir::Value lhsImag = builder.getNullValue(convertType(lhsTy), loc);
        lhs = builder.createComplexCreate(loc, lhsReal, lhsImag);
      }
 
      mlir::Value rhs;
      if (rhsTy->isAnyComplexType()) {
        rhs = cgf.emitComplexExpr(e->getRHS());
      } else {
        mlir::Value rhsReal = Visit(e->getRHS());
        mlir::Value rhsImag = builder.getNullValue(convertType(rhsTy), loc);
        rhs = builder.createComplexCreate(loc, rhsReal, rhsImag);
      }
 
      result = builder.createCompare(loc, kind, lhs, rhs);
    }
 
    return emitScalarConversion(result, cgf.getContext().BoolTy, e->getType(),
                                e->getExprLoc());
  }
 
// Comparisons.
#define VISITCOMP(CODE)                                                        \
  mlir::Value VisitBin##CODE(const BinaryOperator *E) { return emitCmp(E); }
  VISITCOMP(LT)
  VISITCOMP(GT)
  VISITCOMP(LE)
  VISITCOMP(GE)
  VISITCOMP(EQ)
  VISITCOMP(NE)
#undef VISITCOMP
 
  mlir::Value VisitBinAssign(const BinaryOperator *e) {
    const bool ignore = std::exchange(ignoreResultAssign, false);
 
    mlir::Value rhs;
    LValue lhs;
 
    switch (e->getLHS()->getType().getObjCLifetime()) {
    case Qualifiers::OCL_Strong:
    case Qualifiers::OCL_Autoreleasing:
    case Qualifiers::OCL_ExplicitNone:
    case Qualifiers::OCL_Weak:
      assert(!cir::MissingFeatures::objCLifetime());
      break;
    case Qualifiers::OCL_None:
      // __block variables need to have the rhs evaluated first, plus this
      // should improve codegen just a little.
      rhs = Visit(e->getRHS());
      assert(!cir::MissingFeatures::sanitizers());
      // TODO(cir): This needs to be emitCheckedLValue() once we support
      // sanitizers
      lhs = cgf.emitLValue(e->getLHS());
 
      // Store the value into the LHS. Bit-fields are handled specially because
      // the result is altered by the store, i.e., [C99 6.5.16p1]
      // 'An assignment expression has the value of the left operand after the
      // assignment...'.
      if (lhs.isBitField()) {
        CIRGenFunction::SourceLocRAIIObject loc{cgf, e->getSourceRange()};
        rhs = cgf.emitStoreThroughBitfieldLValue(RValue::get(rhs), lhs);
      } else {
        cgf.emitNullabilityCheck(lhs, rhs, e->getExprLoc());
        CIRGenFunction::SourceLocRAIIObject loc{cgf, e->getSourceRange()};
        cgf.emitStoreThroughLValue(RValue::get(rhs), lhs);
      }
    }
 
    // If the result is clearly ignored, return now.
    if (ignore)
      return nullptr;
 
    // The result of an assignment in C is the assigned r-value.
    if (!cgf.getLangOpts().CPlusPlus)
      return rhs;
 
    // If the lvalue is non-volatile, return the computed value of the
    // assignment.
    if (!lhs.isVolatile())
      return rhs;
 
    // Otherwise, reload the value.
    return emitLoadOfLValue(lhs, e->getExprLoc());
  }
 
  mlir::Value VisitBinComma(const BinaryOperator *e) {
    cgf.emitIgnoredExpr(e->getLHS());
    // NOTE: We don't need to EnsureInsertPoint() like LLVM codegen.
    return Visit(e->getRHS());
  }
 
  mlir::Value VisitBinLAnd(const clang::BinaryOperator *e) {
    if (e->getType()->isVectorType()) {
      mlir::Location loc = cgf.getLoc(e->getExprLoc());
      mlir::Type lhsTy = cgf.convertType(e->getLHS()->getType());
      mlir::Value zeroVec = builder.getNullValue(lhsTy, loc);
 
      mlir::Value lhs = Visit(e->getLHS());
      mlir::Value rhs = Visit(e->getRHS());
 
      auto cmpOpKind = cir::CmpOpKind::ne;
      auto resultTy =
          mlir::cast<cir::VectorType>(cgf.convertType(e->getType()));
      lhs = builder.createVecCompare(loc, resultTy, cmpOpKind, lhs, zeroVec);
      rhs = builder.createVecCompare(loc, resultTy, cmpOpKind, rhs, zeroVec);
      return builder.createAnd(loc, lhs, rhs);
    }
 
    assert(!cir::MissingFeatures::instrumentation());
    mlir::Type resTy = cgf.convertType(e->getType());
    mlir::Location loc = cgf.getLoc(e->getExprLoc());
 
    mlir::Value lhsCondV = cgf.evaluateExprAsBool(e->getLHS());
 
    CIRGenFunction::ConditionalEvaluation eval(cgf, loc);
 
    auto resOp = cir::TernaryOp::create(
        builder, loc, lhsCondV, /*trueBuilder=*/
        [&](mlir::OpBuilder &b, mlir::Location loc) {
          CIRGenFunction::LexicalScope lexScope{cgf, loc,
                                                b.getInsertionBlock()};
          cgf.curLexScope->setAsTernary();
          eval.beginEvaluation();
          mlir::Value res = cgf.evaluateExprAsBool(e->getRHS());
          eval.endEvaluation();
          lexScope.forceCleanup({&res});
          cir::YieldOp::create(b, loc, res);
        },
        /*falseBuilder*/
        [&](mlir::OpBuilder &b, mlir::Location loc) {
          CIRGenFunction::LexicalScope lexScope{cgf, loc,
                                                b.getInsertionBlock()};
          cgf.curLexScope->setAsTernary();
          auto res = cir::ConstantOp::create(b, loc, builder.getFalseAttr());
          cir::YieldOp::create(b, loc, res.getRes());
        });
    return maybePromoteBoolResult(resOp.getResult(), resTy);
  }
 
  mlir::Value VisitBinLOr(const clang::BinaryOperator *e) {
    if (e->getType()->isVectorType()) {
      mlir::Location loc = cgf.getLoc(e->getExprLoc());
      mlir::Type lhsTy = cgf.convertType(e->getLHS()->getType());
      mlir::Value zeroVec = builder.getNullValue(lhsTy, loc);
 
      mlir::Value lhs = Visit(e->getLHS());
      mlir::Value rhs = Visit(e->getRHS());
 
      auto cmpOpKind = cir::CmpOpKind::ne;
      auto resultTy =
          mlir::cast<cir::VectorType>(cgf.convertType(e->getType()));
      lhs = builder.createVecCompare(loc, resultTy, cmpOpKind, lhs, zeroVec);
      rhs = builder.createVecCompare(loc, resultTy, cmpOpKind, rhs, zeroVec);
      return builder.createOr(loc, lhs, rhs);
    }
 
    assert(!cir::MissingFeatures::instrumentation());
    mlir::Type resTy = cgf.convertType(e->getType());
    mlir::Location loc = cgf.getLoc(e->getExprLoc());
 
    mlir::Value lhsCondV = cgf.evaluateExprAsBool(e->getLHS());
 
    CIRGenFunction::ConditionalEvaluation eval(cgf, loc);
 
    auto resOp = cir::TernaryOp::create(
        builder, loc, lhsCondV, /*trueBuilder=*/
        [&](mlir::OpBuilder &b, mlir::Location loc) {
          CIRGenFunction::LexicalScope lexScope{cgf, loc,
                                                b.getInsertionBlock()};
          cgf.curLexScope->setAsTernary();
          auto res = cir::ConstantOp::create(b, loc, builder.getTrueAttr());
          cir::YieldOp::create(b, loc, res.getRes());
        },
        /*falseBuilder*/
        [&](mlir::OpBuilder &b, mlir::Location loc) {
          CIRGenFunction::LexicalScope lexScope{cgf, loc,
                                                b.getInsertionBlock()};
          cgf.curLexScope->setAsTernary();
          eval.beginEvaluation();
          mlir::Value res = cgf.evaluateExprAsBool(e->getRHS());
          eval.endEvaluation();
          lexScope.forceCleanup({&res});
          cir::YieldOp::create(b, loc, res);
        });
 
    return maybePromoteBoolResult(resOp.getResult(), resTy);
  }
 
  mlir::Value VisitBinPtrMemD(const BinaryOperator *e) {
    return emitLoadOfLValue(e);
  }
 
  mlir::Value VisitBinPtrMemI(const BinaryOperator *e) {
    return emitLoadOfLValue(e);
  }
 
  // Other Operators.
  mlir::Value VisitBlockExpr(const BlockExpr *e) {
    cgf.cgm.errorNYI(e->getSourceRange(), "ScalarExprEmitter: block");
    return {};
  }
 
  mlir::Value VisitChooseExpr(ChooseExpr *e) {
    return Visit(e->getChosenSubExpr());
  }
 
  mlir::Value VisitObjCStringLiteral(const ObjCStringLiteral *e) {
    cgf.cgm.errorNYI(e->getSourceRange(),
                     "ScalarExprEmitter: objc string literal");
    return {};
  }
  mlir::Value VisitObjCBoxedExpr(ObjCBoxedExpr *e) {
    cgf.cgm.errorNYI(e->getSourceRange(), "ScalarExprEmitter: objc boxed");
    return {};
  }
  mlir::Value VisitObjCArrayLiteral(ObjCArrayLiteral *e) {
    cgf.cgm.errorNYI(e->getSourceRange(),
                     "ScalarExprEmitter: objc array literal");
    return {};
  }
  mlir::Value VisitObjCDictionaryLiteral(ObjCDictionaryLiteral *e) {
    cgf.cgm.errorNYI(e->getSourceRange(),
                     "ScalarExprEmitter: objc dictionary literal");
    return {};
  }
 
  mlir::Value convertVec3AndVec4(CIRGenBuilderTy &builder, mlir::Location loc,
                                 mlir::Value src, unsigned numElementsDst) {
    static constexpr int64_t mask[] = {0, 1, 2, -1};
    return builder.createVecShuffle(
        loc, src, llvm::ArrayRef<int64_t>(mask, numElementsDst));
  }
 
  // Create cast instructions for converting MLIR value \p Src to MLIR type \p
  // DstTy. \p Src has the same size as \p DstTy. Both are single value types
  // but could be scalar or vectors of different lengths, and either can be
  // pointer.
  //
  // There are 4 cases:
  // 1. non-pointer -> non-pointer  : needs 1 bitcast
  // 2. pointer -> pointer          : needs 1 bitcast or addrspacecast
  // 3. pointer -> non-pointer
  //   a) pointer -> intptr_t       : needs 1 ptrtoint
  //   b) pointer -> non-intptr_t   : needs 1 ptrtoint then 1 bitcast
  // 4. non-pointer -> pointer
  //   a) intptr_t -> pointer       : needs 1 inttoptr
  //   b) non-intptr_t -> pointer   : needs 1 bitcast then 1 inttoptr
  //
  // Note: for cases 3b and 4b two casts are required since LLVM casts do not
  // allow casting directly between pointer types and non-integer non-pointer
  // types.
  mlir::Value createCastsForTypeOfSameSize(mlir::Value src, mlir::Type dstTy) {
    mlir::Type srcTy = src.getType();
 
    // Case 1.
    if (!isa<cir::PointerType>(srcTy) && !isa<cir::PointerType>(dstTy))
      return builder.createBitcast(src, dstTy);
 
    // Case 2.
    if (isa<cir::PointerType>(srcTy) && isa<cir::PointerType>(dstTy)) {
      cgf.cgm.errorNYI(
          "ScalarExprEmitter: createCastsForTypeOfSameSize Case 2");
      return {};
    }
 
    // Case 3.
    if (isa<cir::PointerType>(srcTy) && !isa<cir::PointerType>(dstTy)) {
      if (!isa<cir::IntType>(dstTy)) {
        cgf.cgm.errorNYI(
            "ScalarExprEmitter: createCastsForTypeOfSameSize Case 3a");
      }
 
      cgf.cgm.errorNYI(
          "ScalarExprEmitter: createCastsForTypeOfSameSize Case 3a and 3b");
      return {};
    }
 
    // Case 4b.
    if (!isa<cir::IntType>(srcTy)) {
      cgf.cgm.errorNYI(
          "ScalarExprEmitter: createCastsForTypeOfSameSize Case 4a");
      return {};
    }
    // Cases 4a and 4b.
    return builder.createIntToPtr(src, dstTy);
  }
 
  mlir::Value VisitAsTypeExpr(AsTypeExpr *e) {
    mlir::Value src = cgf.emitScalarExpr(e->getSrcExpr());
    mlir::Type srcTy = src.getType();
    mlir::Type dstTy = cgf.convertType(e->getType());
 
    unsigned numElementsSrc = isa<cir::VectorType>(srcTy)
                                  ? cast<cir::VectorType>(srcTy).getFixedSize()
                                  : 0;
    unsigned numElementsDst = isa<cir::VectorType>(dstTy)
                                  ? cast<cir::VectorType>(dstTy).getFixedSize()
                                  : 0;
 
    // Use bit vector expansion for ext_vector_type boolean vectors.
    if (e->getType()->isExtVectorBoolType()) {
      cgf.cgm.errorNYI(e->getSourceRange(),
                       "ScalarExprEmitter: VisitAsTypeExpr ExtVectorBoolType");
      return {};
    }
 
    // Going from vec3 to non-vec3 is a special case and requires a shuffle
    // vector to get a vec4, then a bitcast if the target type is different.
    if (numElementsSrc == 3 && numElementsDst != 3) {
      cgf.cgm.errorNYI(e->getSourceRange(),
                       "ScalarExprEmitter: VisitAsTypeExpr numElemsSrc = 3, "
                       "numElemsDst != 3");
      return {};
    }
 
    // Going from non-vec3 to vec3 is a special case and requires a bitcast
    // to vec4 if the original type is not vec4, then a shuffle vector to
    // get a vec3.
    if (numElementsSrc != 3 && numElementsDst == 3) {
      mlir::Location loc = cgf.getLoc(e->getExprLoc());
      auto dstElemTy = cast<cir::VectorType>(dstTy).getElementType();
      auto dstVec4Ty = cir::VectorType::getFixed(dstElemTy, 4);
      src = createCastsForTypeOfSameSize(src, dstVec4Ty);
      src = convertVec3AndVec4(builder, loc, src, 3);
      return src;
    }
 
    return createCastsForTypeOfSameSize(src, dstTy);
  }
 
  mlir::Value VisitAtomicExpr(AtomicExpr *e) {
    return cgf.emitAtomicExpr(e).getValue();
  }
};
 
LValue ScalarExprEmitter::emitCompoundAssignLValue(
    const CompoundAssignOperator *e,
    mlir::Value (ScalarExprEmitter::*func)(const BinOpInfo &),
    mlir::Value &result) {
  if (e->getComputationResultType()->isAnyComplexType())
    return cgf.emitScalarCompoundAssignWithComplex(e, result);
 
  QualType lhsTy = e->getLHS()->getType();
  BinOpInfo opInfo;
 
  // Emit the RHS first.  __block variables need to have the rhs evaluated
  // first, plus this should improve codegen a little.
 
  QualType promotionTypeCR = getPromotionType(e->getComputationResultType());
  if (promotionTypeCR.isNull())
    promotionTypeCR = e->getComputationResultType();
 
  QualType promotionTypeLHS = getPromotionType(e->getComputationLHSType());
  QualType promotionTypeRHS = getPromotionType(e->getRHS()->getType());
 
  if (!promotionTypeRHS.isNull())
    opInfo.rhs = cgf.emitPromotedScalarExpr(e->getRHS(), promotionTypeRHS);
  else
    opInfo.rhs = Visit(e->getRHS());
 
  opInfo.fullType = promotionTypeCR;
  opInfo.compType = opInfo.fullType;
  if (const auto *vecType = opInfo.fullType->getAs<VectorType>())
    opInfo.compType = vecType->getElementType();
  opInfo.opcode = e->getOpcode();
  opInfo.fpFeatures = e->getFPFeaturesInEffect(cgf.getLangOpts());
  opInfo.e = e;
  opInfo.loc = e->getSourceRange();
 
  // Load/convert the LHS
  LValue lhsLV = cgf.emitLValue(e->getLHS());
 
  if (lhsTy->getAs<AtomicType>()) {
    cgf.cgm.errorNYI(result.getLoc(), "atomic lvalue assign");
    return LValue();
  }
 
  opInfo.lhs = emitLoadOfLValue(lhsLV, e->getExprLoc());
 
  CIRGenFunction::SourceLocRAIIObject sourceloc{cgf, e->getSourceRange()};
  SourceLocation loc = e->getExprLoc();
  if (!promotionTypeLHS.isNull())
    opInfo.lhs = emitScalarConversion(opInfo.lhs, lhsTy, promotionTypeLHS, loc);
  else
    opInfo.lhs = emitScalarConversion(opInfo.lhs, lhsTy,
                                      e->getComputationLHSType(), loc);
 
  // Expand the binary operator.
  result = (this->*func)(opInfo);
 
  // Convert the result back to the LHS type,
  // potentially with Implicit Conversion sanitizer check.
  result = emitScalarConversion(result, promotionTypeCR, lhsTy, loc,
                                ScalarConversionOpts(cgf.sanOpts));
 
  // Store the result value into the LHS lvalue. Bit-fields are handled
  // specially because the result is altered by the store, i.e., [C99 6.5.16p1]
  // 'An assignment expression has the value of the left operand after the
  // assignment...'.
  if (lhsLV.isBitField())
    cgf.emitStoreThroughBitfieldLValue(RValue::get(result), lhsLV);
  else
    cgf.emitStoreThroughLValue(RValue::get(result), lhsLV);
 
  assert(!cir::MissingFeatures::openMPLastprivateConditional());
 
  return lhsLV;
}
 
mlir::Value ScalarExprEmitter::emitComplexToScalarConversion(mlir::Location lov,
                                                             mlir::Value value,
                                                             CastKind kind,
                                                             QualType destTy) {
  cir::CastKind castOpKind;
  switch (kind) {
  case CK_FloatingComplexToReal:
    castOpKind = cir::CastKind::float_complex_to_real;
    break;
  case CK_IntegralComplexToReal:
    castOpKind = cir::CastKind::int_complex_to_real;
    break;
  case CK_FloatingComplexToBoolean:
    castOpKind = cir::CastKind::float_complex_to_bool;
    break;
  case CK_IntegralComplexToBoolean:
    castOpKind = cir::CastKind::int_complex_to_bool;
    break;
  default:
    llvm_unreachable("invalid complex-to-scalar cast kind");
  }
 
  return builder.createCast(lov, castOpKind, value, cgf.convertType(destTy));
}
 
mlir::Value ScalarExprEmitter::emitPromoted(const Expr *e,
                                            QualType promotionType) {
  e = e->IgnoreParens();
  if (const auto *bo = dyn_cast<BinaryOperator>(e)) {
    switch (bo->getOpcode()) {
#define HANDLE_BINOP(OP)                                                       \
  case BO_##OP:                                                                \
    return emit##OP(emitBinOps(bo, promotionType));
      HANDLE_BINOP(Add)
      HANDLE_BINOP(Sub)
      HANDLE_BINOP(Mul)
      HANDLE_BINOP(Div)
#undef HANDLE_BINOP
    default:
      break;
    }
  } else if (const auto *uo = dyn_cast<UnaryOperator>(e)) {
    switch (uo->getOpcode()) {
    case UO_Imag:
    case UO_Real:
      return VisitRealImag(uo, promotionType);
    case UO_Minus:
      return VisitUnaryMinus(uo, promotionType);
    case UO_Plus:
      return VisitUnaryPlus(uo, promotionType);
    default:
      break;
    }
  }
  mlir::Value result = Visit(const_cast<Expr *>(e));
  if (result) {
    if (!promotionType.isNull())
      return emitPromotedValue(result, promotionType);
    return emitUnPromotedValue(result, e->getType());
  }
  return result;
}
 
mlir::Value ScalarExprEmitter::emitCompoundAssign(
    const CompoundAssignOperator *e,
    mlir::Value (ScalarExprEmitter::*func)(const BinOpInfo &)) {
 
  bool ignore = std::exchange(ignoreResultAssign, false);
  mlir::Value rhs;
  LValue lhs = emitCompoundAssignLValue(e, func, rhs);
 
  // If the result is clearly ignored, return now.
  if (ignore)
    return {};
 
  // The result of an assignment in C is the assigned r-value.
  if (!cgf.getLangOpts().CPlusPlus)
    return rhs;
 
  // If the lvalue is non-volatile, return the computed value of the assignment.
  if (!lhs.isVolatile())
    return rhs;
 
  // Otherwise, reload the value.
  return emitLoadOfLValue(lhs, e->getExprLoc());
}
 
mlir::Value
ScalarExprEmitter::VisitSYCLUniqueStableNameExpr(SYCLUniqueStableNameExpr *e) {
  ASTContext &context = cgf.getContext();
  mlir::Location loc = cgf.getLoc(e->getSourceRange());
  cir::GlobalOp globalConstStr =
      cgf.cgm.createGlobalString(loc, e->ComputeName(context), "__usn_str");
 
  // Unlike LLVM IR, CIR pointers are typed, so the bitcast and the address
  // space cast are emitted as separate operations when both are needed.
  mlir::Value addr = builder.createGetGlobal(loc, globalConstStr);
  auto exprTy = mlir::cast<cir::PointerType>(convertType(e->getType()));
  auto addrTy = mlir::cast<cir::PointerType>(addr.getType());
  if (addrTy.getAddrSpace() != exprTy.getAddrSpace())
    addr = cgf.performAddrSpaceCast(
        addr, builder.getPointerTo(addrTy.getPointee(), exprTy.getAddrSpace()));
  return builder.createBitcast(loc, addr, exprTy);
}
 
mlir::Value ScalarExprEmitter::VisitExprWithCleanups(ExprWithCleanups *e) {
  CIRGenFunction::FullExprCleanupScope scope(cgf, e->getSubExpr());
  mlir::Value v = Visit(e->getSubExpr());
  // Defend against dominance problems caused by jumps out of expression
  // evaluation through the shared cleanup block.
  scope.exit({&v});
  return v;
}
 
} // namespace
 
LValue
CIRGenFunction::emitCompoundAssignmentLValue(const CompoundAssignOperator *e) {
  ScalarExprEmitter emitter(*this, builder);
  mlir::Value result;
  switch (e->getOpcode()) {
#define COMPOUND_OP(Op)                                                        \
  case BO_##Op##Assign:                                                        \
    return emitter.emitCompoundAssignLValue(e, &ScalarExprEmitter::emit##Op,   \
                                            result)
    COMPOUND_OP(Mul);
    COMPOUND_OP(Div);
    COMPOUND_OP(Rem);
    COMPOUND_OP(Add);
    COMPOUND_OP(Sub);
    COMPOUND_OP(Shl);
    COMPOUND_OP(Shr);
    COMPOUND_OP(And);
    COMPOUND_OP(Xor);
    COMPOUND_OP(Or);
#undef COMPOUND_OP
 
  case BO_PtrMemD:
  case BO_PtrMemI:
  case BO_Mul:
  case BO_Div:
  case BO_Rem:
  case BO_Add:
  case BO_Sub:
  case BO_Shl:
  case BO_Shr:
  case BO_LT:
  case BO_GT:
  case BO_LE:
  case BO_GE:
  case BO_EQ:
  case BO_NE:
  case BO_Cmp:
  case BO_And:
  case BO_Xor:
  case BO_Or:
  case BO_LAnd:
  case BO_LOr:
  case BO_Assign:
  case BO_Comma:
    llvm_unreachable("Not valid compound assignment operators");
  }
  llvm_unreachable("Unhandled compound assignment operator");
}
 
/// Emit the computation of the specified expression of scalar type.
mlir::Value CIRGenFunction::emitScalarExpr(const Expr *e,
                                           bool ignoreResultAssign) {
  assert(e && hasScalarEvaluationKind(e->getType()) &&
         "Invalid scalar expression to emit");
 
  return ScalarExprEmitter(*this, builder, ignoreResultAssign)
      .Visit(const_cast<Expr *>(e));
}
 
mlir::Value CIRGenFunction::emitPromotedScalarExpr(const Expr *e,
                                                   QualType promotionType) {
  if (!promotionType.isNull())
    return ScalarExprEmitter(*this, builder).emitPromoted(e, promotionType);
  return ScalarExprEmitter(*this, builder).Visit(const_cast<Expr *>(e));
}
 
[[maybe_unused]] static bool mustVisitNullValue(const Expr *e) {
  // If a null pointer expression's type is the C++0x nullptr_t and
  // the expression is not a simple literal, it must be evaluated
  // for its potential side effects.
  if (isa<IntegerLiteral>(e) || isa<CXXNullPtrLiteralExpr>(e))
    return false;
  return e->getType()->isNullPtrType();
}
 
/// If \p e is a widened promoted integer, get its base (unpromoted) type.
static std::optional<QualType>
getUnwidenedIntegerType(const ASTContext &astContext, const Expr *e) {
  const Expr *base = e->IgnoreImpCasts();
  if (e == base)
    return std::nullopt;
 
  QualType baseTy = base->getType();
  if (!astContext.isPromotableIntegerType(baseTy) ||
      astContext.getTypeSize(baseTy) >= astContext.getTypeSize(e->getType()))
    return std::nullopt;
 
  return baseTy;
}
 
/// Check if \p e is a widened promoted integer.
[[maybe_unused]] static bool isWidenedIntegerOp(const ASTContext &astContext,
                                                const Expr *e) {
  return getUnwidenedIntegerType(astContext, e).has_value();
}
 
/// Check if we can skip the overflow check for \p Op.
[[maybe_unused]] static bool canElideOverflowCheck(const ASTContext &astContext,
                                                   const BinOpInfo &op) {
  assert((isa<UnaryOperator>(op.e) || isa<BinaryOperator>(op.e)) &&
         "Expected a unary or binary operator");
 
  // If the binop has constant inputs and we can prove there is no overflow,
  // we can elide the overflow check.
  if (!op.mayHaveIntegerOverflow())
    return true;
 
  // If a unary op has a widened operand, the op cannot overflow.
  if (const auto *uo = dyn_cast<UnaryOperator>(op.e))
    return !uo->canOverflow();
 
  // We usually don't need overflow checks for binops with widened operands.
  // Multiplication with promoted unsigned operands is a special case.
  const auto *bo = cast<BinaryOperator>(op.e);
  std::optional<QualType> optionalLHSTy =
      getUnwidenedIntegerType(astContext, bo->getLHS());
  if (!optionalLHSTy)
    return false;
 
  std::optional<QualType> optionalRHSTy =
      getUnwidenedIntegerType(astContext, bo->getRHS());
  if (!optionalRHSTy)
    return false;
 
  QualType lhsTy = *optionalLHSTy;
  QualType rhsTy = *optionalRHSTy;
 
  // This is the simple case: binops without unsigned multiplication, and with
  // widened operands. No overflow check is needed here.
  if ((op.opcode != BO_Mul && op.opcode != BO_MulAssign) ||
      !lhsTy->isUnsignedIntegerType() || !rhsTy->isUnsignedIntegerType())
    return true;
 
  // For unsigned multiplication the overflow check can be elided if either one
  // of the unpromoted types are less than half the size of the promoted type.
  unsigned promotedSize = astContext.getTypeSize(op.e->getType());
  return (2 * astContext.getTypeSize(lhsTy)) < promotedSize ||
         (2 * astContext.getTypeSize(rhsTy)) < promotedSize;
}
 
/// Emit pointer + index arithmetic.
static mlir::Value emitPointerArithmetic(CIRGenFunction &cgf,
                                         const BinOpInfo &op,
                                         bool isSubtraction) {
  // Must have binary (not unary) expr here.  Unary pointer
  // increment/decrement doesn't use this path.
  const BinaryOperator *expr = cast<BinaryOperator>(op.e);
 
  mlir::Value pointer = op.lhs;
  Expr *pointerOperand = expr->getLHS();
  mlir::Value index = op.rhs;
  Expr *indexOperand = expr->getRHS();
 
  // In the case of subtraction, the FE has ensured that the LHS is always the
  // pointer. However, addition can have the pointer on either side. We will
  // always have a pointer operand and an integer operand, so if the LHS wasn't
  // a pointer, we need to swap our values.
  if (!isSubtraction && !mlir::isa<cir::PointerType>(pointer.getType())) {
    std::swap(pointer, index);
    std::swap(pointerOperand, indexOperand);
  }
  assert(mlir::isa<cir::PointerType>(pointer.getType()) &&
         "Need a pointer operand");
  assert(mlir::isa<cir::IntType>(index.getType()) && "Need an integer operand");
 
  // Some versions of glibc and gcc use idioms (particularly in their malloc
  // routines) that add a pointer-sized integer (known to be a pointer value)
  // to a null pointer in order to cast the value back to an integer or as
  // part of a pointer alignment algorithm.  This is undefined behavior, but
  // we'd like to be able to compile programs that use it.
  //
  // Normally, we'd generate a GEP with a null-pointer base here in response
  // to that code, but it's also UB to dereference a pointer created that
  // way.  Instead (as an acknowledged hack to tolerate the idiom) we will
  // generate a direct cast of the integer value to a pointer.
  //
  // The idiom (p = nullptr + N) is not met if any of the following are true:
  //
  //   The operation is subtraction.
  //   The index is not pointer-sized.
  //   The pointer type is not byte-sized.
  //
  if (BinaryOperator::isNullPointerArithmeticExtension(
          cgf.getContext(), op.opcode, expr->getLHS(), expr->getRHS()))
    return cgf.getBuilder().createIntToPtr(index, pointer.getType());
 
  // Differently from LLVM codegen, ABI bits for index sizes is handled during
  // LLVM lowering.
 
  // If this is subtraction, negate the index.
  if (isSubtraction)
    index = cgf.getBuilder().createNeg(cgf.getLoc(op.e->getExprLoc()), index);
 
  assert(!cir::MissingFeatures::sanitizers());
 
  const PointerType *pointerType =
      pointerOperand->getType()->getAs<PointerType>();
  if (!pointerType) {
    cgf.cgm.errorNYI("Objective-C:pointer arithmetic with non-pointer type");
    return nullptr;
  }
 
  QualType elementType = pointerType->getPointeeType();
  if (const VariableArrayType *vla =
          cgf.getContext().getAsVariableArrayType(elementType)) {
    mlir::Value numElements = cgf.getVLASize(vla).numElts;
    mlir::Location loc = cgf.getLoc(op.e->getExprLoc());
    index = cgf.getBuilder().createCast(cir::CastKind::integral, index,
                                        numElements.getType());
    // GEP indexes are signed, and scaling an index isn't permitted to
    // signed-overflow, so we use the same semantics for our explicit
    // multiply.  We suppress this if overflow is not undefined behavior.
    cir::OverflowBehavior overflowBehavior =
        cgf.getLangOpts().PointerOverflowDefined
            ? cir::OverflowBehavior::None
            : cir::OverflowBehavior::NoSignedWrap;
    index =
        cgf.getBuilder().createMul(loc, index, numElements, overflowBehavior);
    assert(!cir::MissingFeatures::sanitizers());
    return cir::PtrStrideOp::create(cgf.getBuilder(), loc, pointer.getType(),
                                    pointer, index);
  }
 
  assert(!cir::MissingFeatures::sanitizers());
  return cir::PtrStrideOp::create(cgf.getBuilder(),
                                  cgf.getLoc(op.e->getExprLoc()),
                                  pointer.getType(), pointer, index);
}
 
static bool isIntegerVectorBinOp(mlir::Type ty) {
  auto vecTy = mlir::dyn_cast<cir::VectorType>(ty);
  return vecTy && mlir::isa<cir::IntType>(vecTy.getElementType());
}
 
// Construct a cir.fmuladd op to represent a fused mul-add of `mulOp` and
// `addend`. Use negMul and negAdd to negate the first operand of the mul or
// the addend respectively. This allows fmuladd to represent a*b-c, or c-a*b.
// Patterns in LLVM should catch the negated forms and translate them to
// efficient operations.
static mlir::Value buildFMulAdd(mlir::Location addLoc, cir::FMulOp mulOp,
                                mlir::Value addend, CIRGenBuilderTy &builder,
                                bool negMul, bool negAdd) {
  mlir::Location loc = builder.getFusedLoc({mulOp.getLoc(), addLoc});
  mlir::Value mulOp0 = mulOp.getLhs();
  mlir::Value mulOp1 = mulOp.getRhs();
  if (negMul)
    mulOp0 = builder.createFNeg(loc, mulOp0);
  if (negAdd)
    addend = builder.createFNeg(loc, addend);
 
  // Carry the mul's fenv attribute so a constrained fmul yields a constrained
  // fmuladd; the builder is under the add's FP options, not the mul's.
  mlir::Value fmuladd =
      cir::FMulAddOp::create(builder, loc, addend.getType(), mulOp0, mulOp1,
                             addend, mulOp.getFenvAttr());
  mulOp.erase();
  return fmuladd;
}
 
// Check whether it would be legal to emit a cir.fmuladd op to represent op
// and if so, build it.
//
// Checks that (a) the operation is fusable, and (b) -ffp-contract=on.
// Does NOT check the type of the operation - it's assumed that this function
// will be called from contexts where it's known that the type is contractable.
static mlir::Value tryEmitFMulAdd(mlir::Location loc, const BinOpInfo &op,
                                  CIRGenBuilderTy &builder,
                                  bool isSub = false) {
  assert((op.opcode == BO_Add || op.opcode == BO_AddAssign ||
          op.opcode == BO_Sub || op.opcode == BO_SubAssign) &&
         "Only fadd/fsub can be the root of an fmuladd.");
 
  // Check whether this op is fusable, i.e. -ffp-contract=on. -ffp-contract=fast
  // is represented by the `contract` flag on the fmul/fadd instead.
  if (!op.fpFeatures.allowFPContractWithinStatement())
    return nullptr;
 
  mlir::Value lhs = op.lhs;
  mlir::Value rhs = op.rhs;
 
  // Peek through fneg to look for fmul. Make sure the fneg has no other users,
  // and that it is the only use of its operand.
  bool negLHS = false;
  if (auto lhsNeg = lhs.getDefiningOp<cir::FNegOp>()) {
    if (lhsNeg.getResult().use_empty() && lhsNeg.getInput().hasOneUse()) {
      lhs = lhsNeg.getInput();
      negLHS = true;
    }
  }
 
  bool negRHS = false;
  if (auto rhsNeg = rhs.getDefiningOp<cir::FNegOp>()) {
    if (rhsNeg.getResult().use_empty() && rhsNeg.getInput().hasOneUse()) {
      rhs = rhsNeg.getInput();
      negRHS = true;
    }
  }
 
  // We have a potentially fusable op. Look for a mul on one of the operands.
  // Also make sure that the mul result isn't used directly. In that case,
  // there's no point creating a muladd operation.
  if (auto lhsMul = lhs.getDefiningOp<cir::FMulOp>()) {
    if (lhsMul.getResult().use_empty() || negLHS) {
      // If we looked through fneg, erase it.
      if (negLHS)
        op.lhs.getDefiningOp<cir::FNegOp>().erase();
      return buildFMulAdd(loc, lhsMul, op.rhs, builder, negLHS, isSub);
    }
  }
  if (auto rhsMul = rhs.getDefiningOp<cir::FMulOp>()) {
    if (rhsMul.getResult().use_empty() || negRHS) {
      // If we looked through fneg, erase it.
      if (negRHS)
        op.rhs.getDefiningOp<cir::FNegOp>().erase();
      return buildFMulAdd(loc, rhsMul, op.lhs, builder, isSub ^ negRHS, false);
    }
  }
 
  return nullptr;
}
 
mlir::Value ScalarExprEmitter::emitFixedPointIncDec(const UnaryOperator *e,
                                                    mlir::Value value,
                                                    QualType type) {
  // Fixed-point types are tricky. In some cases, it isn't possible to
  // represent a 1 or a -1 in the type at all. Piggyback off of
  // EmitFixedPointBinOp to avoid having to reimplement saturation.
  BinOpInfo info;
  info.loc = e->getSourceRange();
  info.fpFeatures = e->getFPFeaturesInEffect(cgf.getLangOpts());
  info.e = e;
  info.compType = info.fullType = e->getType();
  info.opcode = e->isIncrementOp() ? BO_Add : BO_Sub;
  info.lhs = value;
  info.rhs = builder.getConstInt(cgf.getLoc(info.loc), value.getType(), 1);
  // If the type is signed, it's better to represent this as +(-1) or -(-1),
  // since -1 is guaranteed to be representable.
  // FIXME(cir): This is a carry-over from Classic codegen, we should probably
  // figure out if ++ -> -(-1) and -- -> +(-1) is REALLY what we want? For now
  // we are just doing what classic does here.
  if (type->isSignedFixedPointType()) {
    info.opcode = e->isIncrementOp() ? BO_Sub : BO_Add;
    info.rhs = builder.createNeg(cgf.getLoc(info.loc), info.rhs);
  }
  // Now, convert from our invented integer literal to the type of the unary
  // op. This will upscale and saturate if necessary. This value can become
  // undef in some cases.
  CIRGenFixedPointBuilder fpbuilder(builder, cgf.getLoc(info.loc));
  auto dstSema = cgf.getContext().getFixedPointSemantics(info.fullType);
  info.rhs =
      fpbuilder.createIntegerToFixed(info.rhs, /*srcIsSigned=*/true, dstSema);
  return emitFixedPointBinOp(info);
}
 
mlir::Value ScalarExprEmitter::emitFixedPointConversion(mlir::Value src,
                                                        QualType srcTy,
                                                        QualType dstTy,
                                                        mlir::Location loc) {
  assert(srcTy->isFixedPointType() || dstTy->isFixedPointType());
 
  CIRGenFixedPointBuilder fpBuilder(builder, loc);
  mlir::Value result;
  if (srcTy->isRealFloatingType()) {
    assert(dstTy->isFixedPointType());
    result = fpBuilder.createFloatingToFixed(
        src, cgf.getContext().getFixedPointSemantics(dstTy));
  } else if (dstTy->isRealFloatingType()) {
    assert(srcTy->isFixedPointType());
    result = fpBuilder.createFixedToFloating(
        src, cgf.getContext().getFixedPointSemantics(srcTy),
        cgf.convertType(dstTy));
  } else {
    llvm::FixedPointSemantics srcFPSema =
        cgf.getContext().getFixedPointSemantics(srcTy);
    llvm::FixedPointSemantics dstFPSema =
        cgf.getContext().getFixedPointSemantics(dstTy);
 
    if (dstTy->isIntegerType())
      result = fpBuilder.createFixedToInteger(
          src, srcFPSema, dstFPSema.getWidth(), dstFPSema.isSigned());
    else if (srcTy->isIntegerType())
      result =
          fpBuilder.createIntegerToFixed(src, srcFPSema.isSigned(), dstFPSema);
    else {
      assert(srcTy->isFixedPointType() && dstTy->isFixedPointType());
      result = fpBuilder.createFixedToFixed(src, srcFPSema, dstFPSema);
    }
  }
  return result;
}
 
mlir::Value ScalarExprEmitter::emitMul(const BinOpInfo &ops) {
  const mlir::Location loc = cgf.getLoc(ops.loc);
  if (!isIntegerVectorBinOp(ops.lhs.getType()) &&
      ops.compType->isSignedIntegerOrEnumerationType()) {
    switch (cgf.getLangOpts().getSignedOverflowBehavior()) {
    case LangOptions::SOB_Defined:
      if (!cgf.sanOpts.has(SanitizerKind::SignedIntegerOverflow))
        return builder.createMul(loc, ops.lhs, ops.rhs);
      [[fallthrough]];
    case LangOptions::SOB_Undefined:
      if (!cgf.sanOpts.has(SanitizerKind::SignedIntegerOverflow))
        return builder.createNSWMul(loc, ops.lhs, ops.rhs);
      [[fallthrough]];
    case LangOptions::SOB_Trapping:
      if (canElideOverflowCheck(cgf.getContext(), ops))
        return builder.createNSWMul(loc, ops.lhs, ops.rhs);
      cgf.cgm.errorNYI("sanitizers");
    }
  }
 
  if (ops.fullType->isConstantMatrixType()) {
    CIRGenFunction::CIRGenFPOptionsRAII fpOptsRAII(cgf, ops.fpFeatures);
 
    if (isa<cir::MatrixType>(ops.lhs.getType()) &&
        isa<cir::MatrixType>(ops.rhs.getType()))
      return builder.createMatrixMultiply(loc, ops.lhs, ops.rhs);
 
    auto [lhs, rhs] =
        builder.splatMatrixOpOperandsIfNecessary(loc, ops.lhs, ops.rhs);
    if (cir::isFPOrVectorOrMatrixOfFPType(lhs.getType()))
      return builder.createFMul(loc, lhs, rhs);
    return builder.createMul(loc, lhs, rhs);
  }
 
  if (ops.compType->isUnsignedIntegerType() &&
      cgf.sanOpts.has(SanitizerKind::UnsignedIntegerOverflow) &&
      !canElideOverflowCheck(cgf.getContext(), ops))
    cgf.cgm.errorNYI("unsigned int overflow sanitizer");
 
  if (cir::isFPOrVectorOfFPType(ops.lhs.getType())) {
    CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, ops.fpFeatures);
    return builder.createFMul(loc, ops.lhs, ops.rhs);
  }
 
  if (ops.isFixedPointOp())
    return emitFixedPointBinOp(ops);
 
  return cir::MulOp::create(builder, cgf.getLoc(ops.loc),
                            cgf.convertType(ops.fullType), ops.lhs, ops.rhs);
}
mlir::Value ScalarExprEmitter::emitDiv(const BinOpInfo &ops) {
  const mlir::Location loc = cgf.getLoc(ops.loc);
  if (cir::isFPOrVectorOfFPType(ops.lhs.getType())) {
    CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, ops.fpFeatures);
    return builder.createFDiv(loc, ops.lhs, ops.rhs);
  }
 
  if (ops.fullType->isConstantMatrixType()) {
    assert(mlir::isa<cir::MatrixType>(ops.lhs.getType()) &&
           "first operand must be a matrix");
    assert(cir::isAnyIntegerOrFloatingPointType(ops.rhs.getType()) &&
           "second operand must be an arithmetic type");
 
    auto [lhs, rhs] =
        builder.splatMatrixOpOperandsIfNecessary(loc, ops.lhs, ops.rhs);
 
    CIRGenFunction::CIRGenFPOptionsRAII fpOptsRAII(cgf, ops.fpFeatures);
    if (cir::isFPOrVectorOrMatrixOfFPType(lhs.getType()))
      return builder.createFDiv(loc, lhs, rhs);
    return builder.createDiv(loc, lhs, rhs);
  }
 
  if (ops.isFixedPointOp())
    return emitFixedPointBinOp(ops);
 
  return cir::DivOp::create(builder, loc, cgf.convertType(ops.fullType),
                            ops.lhs, ops.rhs);
}
mlir::Value ScalarExprEmitter::emitRem(const BinOpInfo &ops) {
  const mlir::Location loc = cgf.getLoc(ops.loc);
  if (cir::isFPOrVectorOfFPType(ops.lhs.getType())) {
    CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, ops.fpFeatures);
    return builder.createFRem(loc, ops.lhs, ops.rhs);
  }
  return cir::RemOp::create(builder, loc, cgf.convertType(ops.fullType),
                            ops.lhs, ops.rhs);
}
 
mlir::Value ScalarExprEmitter::emitFixedPointBinOp(const BinOpInfo &ops) {
  // This is either a binary operation where at least one of the operands is
  // a fixed-point type, or a unary operation where the operand is a fixed-point
  // type. The result type of a binary operation is determined by
  // Sema::handleFixedPointConversions().
  QualType resultTy = ops.compType;
  QualType lhsTy, rhsTy;
  if (const auto *binOp = dyn_cast<BinaryOperator>(ops.e)) {
    rhsTy = binOp->getRHS()->getType();
    if (const auto *cao = dyn_cast<CompoundAssignOperator>(binOp)) {
      // For compound assignment, the effective type of the LHS at this point
      // is the computation LHS type, not the actual LHS type, and the final
      // result type is not the type of the expression but rather the
      // computation result type.
      lhsTy = cao->getComputationLHSType();
      resultTy = cao->getComputationResultType();
    } else {
      lhsTy = binOp->getLHS()->getType();
    }
  } else if (const auto *unOp = dyn_cast<UnaryOperator>(ops.e)) {
    lhsTy = unOp->getSubExpr()->getType();
    rhsTy = unOp->getSubExpr()->getType();
  }
  ASTContext &ctx = cgf.getContext();
  mlir::Value lhs = ops.lhs;
  mlir::Value rhs = ops.rhs;
 
  auto lhsFixedSema = ctx.getFixedPointSemantics(lhsTy);
  auto rhsFixedSema = ctx.getFixedPointSemantics(rhsTy);
  auto resultFixedSema = ctx.getFixedPointSemantics(resultTy);
  auto commonFixedSema = lhsFixedSema.getCommonSemantics(rhsFixedSema);
 
  // Perform the actual operation.
  mlir::Value result;
  CIRGenFixedPointBuilder fpbuilder(builder, cgf.getLoc(ops.loc));
  switch (ops.opcode) {
  case BO_AddAssign:
  case BO_Add:
    result = fpbuilder.createAdd(lhs, lhsFixedSema, rhs, rhsFixedSema);
    break;
  case BO_SubAssign:
  case BO_Sub:
    result = fpbuilder.createSub(lhs, lhsFixedSema, rhs, rhsFixedSema);
    break;
  case BO_MulAssign:
  case BO_Mul:
    result = fpbuilder.createMul(lhs, lhsFixedSema, rhs, rhsFixedSema);
    break;
  case BO_DivAssign:
  case BO_Div:
    result = fpbuilder.createDiv(lhs, lhsFixedSema, rhs, rhsFixedSema);
    break;
  case BO_ShlAssign:
  case BO_Shl:
    result = fpbuilder.createShl(lhs, lhsFixedSema, rhs);
    break;
  case BO_ShrAssign:
  case BO_Shr:
    result = fpbuilder.createShr(lhs, rhs);
    break;
  case BO_LT:
  case BO_GT:
  case BO_LE:
  case BO_GE:
  case BO_EQ:
  case BO_NE:
    return fpbuilder.createCmp(lhs, lhsFixedSema, rhs, rhsFixedSema,
                               clangCmpToCIRCmp(ops.opcode));
  case BO_Cmp:
  case BO_LAnd:
  case BO_LOr:
    llvm_unreachable("Found unimplemented fixed point binary operation");
  case BO_PtrMemD:
  case BO_PtrMemI:
  case BO_Rem:
  case BO_Xor:
  case BO_And:
  case BO_Or:
  case BO_Assign:
  case BO_RemAssign:
  case BO_AndAssign:
  case BO_XorAssign:
  case BO_OrAssign:
  case BO_Comma:
    llvm_unreachable(
        "Found unsupported binary operation for fixed point types.");
  }
 
  bool isShift = BinaryOperator::isShiftOp(ops.opcode) ||
                 BinaryOperator::isShiftAssignOp(ops.opcode);
  // Convert to the result type.
  return fpbuilder.createFixedToFixed(
      result, isShift ? lhsFixedSema : commonFixedSema, resultFixedSema);
}
 
mlir::Value ScalarExprEmitter::emitAdd(const BinOpInfo &ops) {
  if (mlir::isa<cir::PointerType>(ops.lhs.getType()) ||
      mlir::isa<cir::PointerType>(ops.rhs.getType()))
    return emitPointerArithmetic(cgf, ops, /*isSubtraction=*/
                                 false);
 
  const mlir::Location loc = cgf.getLoc(ops.loc);
  if (!isIntegerVectorBinOp(ops.lhs.getType()) &&
      ops.compType->isSignedIntegerOrEnumerationType()) {
    switch (cgf.getLangOpts().getSignedOverflowBehavior()) {
    case LangOptions::SOB_Defined:
      if (!cgf.sanOpts.has(SanitizerKind::SignedIntegerOverflow))
        return builder.createAdd(loc, ops.lhs, ops.rhs);
      [[fallthrough]];
    case LangOptions::SOB_Undefined:
      if (!cgf.sanOpts.has(SanitizerKind::SignedIntegerOverflow))
        return builder.createNSWAdd(loc, ops.lhs, ops.rhs);
      [[fallthrough]];
    case LangOptions::SOB_Trapping:
      if (canElideOverflowCheck(cgf.getContext(), ops))
        return builder.createNSWAdd(loc, ops.lhs, ops.rhs);
      cgf.cgm.errorNYI("sanitizers");
    }
  }
  if (ops.fullType->isConstantMatrixType()) {
    // Like llvm::MatrixBuilder::CreateAdd, splat a scalar operand to the matrix
    // type before adding.
    auto [lhs, rhs] =
        builder.splatMatrixOpOperandsIfNecessary(loc, ops.lhs, ops.rhs);
 
    CIRGenFunction::CIRGenFPOptionsRAII fpOptsRAII(cgf, ops.fpFeatures);
    if (cir::isFPOrVectorOrMatrixOfFPType(lhs.getType()))
      return builder.createFAdd(loc, lhs, rhs);
    return builder.createAdd(loc, lhs, rhs);
  }
 
  if (ops.compType->isUnsignedIntegerType() &&
      cgf.sanOpts.has(SanitizerKind::UnsignedIntegerOverflow) &&
      !canElideOverflowCheck(cgf.getContext(), ops))
    cgf.cgm.errorNYI("unsigned int overflow sanitizer");
 
  if (cir::isFPOrVectorOfFPType(ops.lhs.getType())) {
    CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, ops.fpFeatures);
    // Try to form an fmuladd.
    if (mlir::Value fmuladd = tryEmitFMulAdd(loc, ops, builder))
      return fmuladd;
    return builder.createFAdd(loc, ops.lhs, ops.rhs);
  }
 
  if (ops.isFixedPointOp())
    return emitFixedPointBinOp(ops);
 
  return builder.createAdd(loc, ops.lhs, ops.rhs);
}
 
mlir::Value ScalarExprEmitter::emitSub(const BinOpInfo &ops) {
  const mlir::Location loc = cgf.getLoc(ops.loc);
  // The LHS is always a pointer if either side is.
  if (!mlir::isa<cir::PointerType>(ops.lhs.getType())) {
    if (!isIntegerVectorBinOp(ops.lhs.getType()) &&
        ops.compType->isSignedIntegerOrEnumerationType()) {
      switch (cgf.getLangOpts().getSignedOverflowBehavior()) {
      case LangOptions::SOB_Defined: {
        if (!cgf.sanOpts.has(SanitizerKind::SignedIntegerOverflow))
          return builder.createSub(loc, ops.lhs, ops.rhs);
        [[fallthrough]];
      }
      case LangOptions::SOB_Undefined:
        if (!cgf.sanOpts.has(SanitizerKind::SignedIntegerOverflow))
          return builder.createNSWSub(loc, ops.lhs, ops.rhs);
        [[fallthrough]];
      case LangOptions::SOB_Trapping:
        if (canElideOverflowCheck(cgf.getContext(), ops))
          return builder.createNSWSub(loc, ops.lhs, ops.rhs);
        cgf.cgm.errorNYI("sanitizers");
      }
    }
 
    if (ops.fullType->isConstantMatrixType()) {
      // Like llvm::MatrixBuilder::CreateSub, splat a scalar operand to the
      // matrix type before adding.
      auto [lhs, rhs] =
          builder.splatMatrixOpOperandsIfNecessary(loc, ops.lhs, ops.rhs);
 
      CIRGenFunction::CIRGenFPOptionsRAII fpOptsRAII(cgf, ops.fpFeatures);
      if (cir::isFPOrVectorOrMatrixOfFPType(lhs.getType()))
        return builder.createFSub(loc, lhs, rhs);
      return builder.createSub(loc, lhs, rhs);
    }
 
    if (ops.compType->isUnsignedIntegerType() &&
        cgf.sanOpts.has(SanitizerKind::UnsignedIntegerOverflow) &&
        !canElideOverflowCheck(cgf.getContext(), ops))
      cgf.cgm.errorNYI("unsigned int overflow sanitizer");
 
    if (cir::isFPOrVectorOfFPType(ops.lhs.getType())) {
      CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, ops.fpFeatures);
      // Try to form an fmuladd.
      if (mlir::Value fmuladd =
              tryEmitFMulAdd(loc, ops, builder, /*isSub=*/true))
        return fmuladd;
      return builder.createFSub(loc, ops.lhs, ops.rhs);
    }
 
    if (ops.isFixedPointOp())
      return emitFixedPointBinOp(ops);
 
    return builder.createSub(loc, ops.lhs, ops.rhs);
  }
 
  // If the RHS is not a pointer, then we have normal pointer
  // arithmetic.
  if (!mlir::isa<cir::PointerType>(ops.rhs.getType()))
    return emitPointerArithmetic(cgf, ops, /*isSubtraction=*/true);
 
  // Otherwise, this is a pointer subtraction
 
  // Do the raw subtraction part.
  //
  // TODO(cir): note for LLVM lowering out of this; when expanding this into
  // LLVM we shall take VLA's, division by element size, etc.
  //
  // See more in `EmitSub` in CGExprScalar.cpp.
  assert(!cir::MissingFeatures::llvmLoweringPtrDiffConsidersPointee());
  return cir::PtrDiffOp::create(builder, cgf.getLoc(ops.loc), cgf.ptrDiffTy,
                                ops.lhs, ops.rhs);
}
 
mlir::Value ScalarExprEmitter::emitShl(const BinOpInfo &ops) {
  // TODO: This misses out on the sanitizer check below.
  if (ops.isFixedPointOp())
    return emitFixedPointBinOp(ops);
 
  // CIR accepts shift between different types, meaning nothing special
  // to be done here. OTOH, LLVM requires the LHS and RHS to be the same type:
  // promote or truncate the RHS to the same size as the LHS.
 
  bool sanitizeSignedBase = cgf.sanOpts.has(SanitizerKind::ShiftBase) &&
                            ops.compType->hasSignedIntegerRepresentation() &&
                            !cgf.getLangOpts().isSignedOverflowDefined() &&
                            !cgf.getLangOpts().CPlusPlus20;
  bool sanitizeUnsignedBase =
      cgf.sanOpts.has(SanitizerKind::UnsignedShiftBase) &&
      ops.compType->hasUnsignedIntegerRepresentation();
  bool sanitizeBase = sanitizeSignedBase || sanitizeUnsignedBase;
  bool sanitizeExponent = cgf.sanOpts.has(SanitizerKind::ShiftExponent);
 
  // OpenCL 6.3j: shift values are effectively % word size of LHS.
  if (cgf.getLangOpts().OpenCL)
    cgf.cgm.errorNYI("opencl");
  else if ((sanitizeBase || sanitizeExponent) &&
           mlir::isa<cir::IntType>(ops.lhs.getType()))
    cgf.cgm.errorNYI("sanitizers");
 
  return builder.createShiftLeft(cgf.getLoc(ops.loc), ops.lhs, ops.rhs);
}
 
mlir::Value ScalarExprEmitter::emitShr(const BinOpInfo &ops) {
  // TODO: This misses out on the sanitizer check below.
  if (ops.isFixedPointOp())
    return emitFixedPointBinOp(ops);
 
  // CIR accepts shift between different types, meaning nothing special
  // to be done here. OTOH, LLVM requires the LHS and RHS to be the same type:
  // promote or truncate the RHS to the same size as the LHS.
 
  // OpenCL 6.3j: shift values are effectively % word size of LHS.
  if (cgf.getLangOpts().OpenCL)
    cgf.cgm.errorNYI("opencl");
  else if (cgf.sanOpts.has(SanitizerKind::ShiftExponent) &&
           mlir::isa<cir::IntType>(ops.lhs.getType()))
    cgf.cgm.errorNYI("sanitizers");
 
  // Note that we don't need to distinguish unsigned treatment at this
  // point since it will be handled later by LLVM lowering.
  return builder.createShiftRight(cgf.getLoc(ops.loc), ops.lhs, ops.rhs);
}
 
mlir::Value ScalarExprEmitter::emitAnd(const BinOpInfo &ops) {
  return cir::AndOp::create(builder, cgf.getLoc(ops.loc), ops.lhs, ops.rhs);
}
mlir::Value ScalarExprEmitter::emitXor(const BinOpInfo &ops) {
  return cir::XorOp::create(builder, cgf.getLoc(ops.loc), ops.lhs, ops.rhs);
}
mlir::Value ScalarExprEmitter::emitOr(const BinOpInfo &ops) {
  return cir::OrOp::create(builder, cgf.getLoc(ops.loc), ops.lhs, ops.rhs);
}
 
// Emit code for an explicit or implicit cast.  Implicit
// casts have to handle a more broad range of conversions than explicit
// casts, as they handle things like function to ptr-to-function decay
// etc.
mlir::Value ScalarExprEmitter::VisitCastExpr(CastExpr *ce) {
  Expr *subExpr = ce->getSubExpr();
  QualType destTy = ce->getType();
  CastKind kind = ce->getCastKind();
  CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, ce);
 
  // These cases are generally not written to ignore the result of evaluating
  // their sub-expressions, so we clear this now.
  ignoreResultAssign = false;
 
  switch (kind) {
  case clang::CK_Dependent:
    llvm_unreachable("dependent cast kind in CIR gen!");
  case clang::CK_BuiltinFnToFnPtr:
    llvm_unreachable("builtin functions are handled elsewhere");
  case CK_LValueBitCast:
  case CK_LValueToRValueBitCast: {
    LValue sourceLVal = cgf.emitLValue(subExpr);
    Address sourceAddr = sourceLVal.getAddress();
 
    mlir::Type destElemTy = cgf.convertTypeForMem(destTy);
    Address destAddr = sourceAddr.withElementType(cgf.getBuilder(), destElemTy);
    LValue destLVal = cgf.makeAddrLValue(destAddr, destTy);
    assert(!cir::MissingFeatures::opTBAA());
    return emitLoadOfLValue(destLVal, ce->getExprLoc());
  }
 
  case CK_CPointerToObjCPointerCast:
  case CK_BlockPointerToObjCPointerCast:
  case CK_AnyPointerToBlockPointerCast:
  case CK_BitCast: {
    mlir::Value src = Visit(const_cast<Expr *>(subExpr));
    mlir::Type dstTy = cgf.convertType(destTy);
 
    // Function and object pointers may differ in address space.
    auto srcPtrTy = mlir::dyn_cast<cir::PointerType>(src.getType());
    auto dstPtrTy = mlir::dyn_cast<cir::PointerType>(dstTy);
    if (srcPtrTy && dstPtrTy &&
        srcPtrTy.getAddrSpace() != dstPtrTy.getAddrSpace())
      src = cgf.performAddrSpaceCast(
          src,
          builder.getPointerTo(srcPtrTy.getPointee(), dstPtrTy.getAddrSpace()));
 
    if (cgf.sanOpts.has(SanitizerKind::CFIUnrelatedCast))
      cgf.getCIRGenModule().errorNYI(subExpr->getSourceRange(),
                                     "sanitizer support");
 
    if (cgf.cgm.getCodeGenOpts().StrictVTablePointers)
      cgf.getCIRGenModule().errorNYI(subExpr->getSourceRange(),
                                     "strict vtable pointers");
 
    // Update heapallocsite metadata when there is an explicit pointer cast.
    assert(!cir::MissingFeatures::addHeapAllocSiteMetadata());
 
    // If Src is a fixed vector and Dst is a scalable vector, and both have the
    // same element type, use the llvm.vector.insert intrinsic to perform the
    // bitcast.
    assert(!cir::MissingFeatures::scalableVectors());
 
    // If Src is a scalable vector and Dst is a fixed vector, and both have the
    // same element type, use the llvm.vector.extract intrinsic to perform the
    // bitcast.
    assert(!cir::MissingFeatures::scalableVectors());
 
    // Perform VLAT <-> VLST bitcast through memory.
    // TODO: since the llvm.experimental.vector.{insert,extract} intrinsics
    //       require the element types of the vectors to be the same, we
    //       need to keep this around for bitcasts between VLAT <-> VLST where
    //       the element types of the vectors are not the same, until we figure
    //       out a better way of doing these casts.
    assert(!cir::MissingFeatures::scalableVectors());
 
    return cgf.getBuilder().createBitcast(cgf.getLoc(subExpr->getSourceRange()),
                                          src, dstTy);
  }
  case CK_AddressSpaceConversion: {
    Expr::EvalResult result;
    if (subExpr->EvaluateAsRValue(result, cgf.getContext()) &&
        result.Val.isNullPointer()) {
      // If e has side effect, it is emitted even if its final result is a
      // null pointer. In that case, a DCE pass should be able to
      // eliminate the useless instructions emitted during translating E.
      if (result.HasSideEffects)
        Visit(subExpr);
      return cgf.cgm.getNullPointer(
          mlir::cast<cir::PointerType>(convertType(destTy)), destTy,
          cgf.getLoc(subExpr->getExprLoc()));
    }
    return cgf.performAddrSpaceCast(Visit(subExpr), convertType(destTy));
  }
 
  case CK_AtomicToNonAtomic:
  case CK_NonAtomicToAtomic:
  case CK_UserDefinedConversion:
    return Visit(const_cast<Expr *>(subExpr));
  case CK_NoOp:
    return ce->changesVolatileQualification() ? emitLoadOfLValue(ce)
                                              : Visit(subExpr);
  case CK_IntegralToPointer: {
    mlir::Type destCIRTy = cgf.convertType(destTy);
    mlir::Value src = Visit(const_cast<Expr *>(subExpr));
 
    // Properly resize by casting to an int of the same size as the pointer.
    // Clang's IntegralToPointer includes 'bool' as the source, but in CIR
    // 'bool' is not an integral type.  So check the source type to get the
    // correct CIR conversion.
    mlir::Type middleTy = cgf.cgm.getDataLayout().getIntPtrType(destCIRTy);
    mlir::Value middleVal = builder.createCast(
        subExpr->getType()->isBooleanType() ? cir::CastKind::bool_to_int
                                            : cir::CastKind::integral,
        src, middleTy);
 
    if (cgf.cgm.getCodeGenOpts().StrictVTablePointers) {
      cgf.cgm.errorNYI(subExpr->getSourceRange(),
                       "IntegralToPointer: strict vtable pointers");
      return {};
    }
 
    return builder.createIntToPtr(middleVal, destCIRTy);
  }
 
  case CK_BaseToDerived: {
    const CXXRecordDecl *derivedClassDecl = destTy->getPointeeCXXRecordDecl();
    assert(derivedClassDecl && "BaseToDerived arg isn't a C++ object pointer!");
    Address base = cgf.emitPointerWithAlignment(subExpr);
    Address derived = cgf.getAddressOfDerivedClass(
        cgf.getLoc(ce->getSourceRange()), base, derivedClassDecl, ce->path(),
        cgf.shouldNullCheckClassCastValue(ce));
 
    // C++11 [expr.static.cast]p11: Behavior is undefined if a downcast is
    // performed and the object is not of the derived type.
    assert(!cir::MissingFeatures::sanitizers());
 
    return cgf.getAsNaturalPointerTo(derived, ce->getType()->getPointeeType());
  }
  case CK_UncheckedDerivedToBase:
  case CK_DerivedToBase: {
    // The EmitPointerWithAlignment path does this fine; just discard
    // the alignment.
    return cgf.getAsNaturalPointerTo(cgf.emitPointerWithAlignment(ce),
                                     ce->getType()->getPointeeType());
  }
  case CK_Dynamic: {
    Address v = cgf.emitPointerWithAlignment(subExpr);
    const auto *dce = cast<CXXDynamicCastExpr>(ce);
    return cgf.emitDynamicCast(v, dce);
  }
  case CK_ArrayToPointerDecay:
    return cgf.emitArrayToPointerDecay(subExpr).getPointer();
 
  case CK_NullToPointer: {
    if (mustVisitNullValue(subExpr))
      cgf.emitIgnoredExpr(subExpr);
 
    // Note that DestTy is used as the MLIR type instead of a custom
    // nullptr type.
    auto ty = mlir::cast<cir::PointerType>(cgf.convertType(destTy));
    return cgf.cgm.getNullPointer(ty, destTy,
                                  cgf.getLoc(subExpr->getExprLoc()));
  }
 
  case CK_NullToMemberPointer: {
    if (mustVisitNullValue(subExpr))
      cgf.emitIgnoredExpr(subExpr);
 
    assert(!cir::MissingFeatures::cxxABI());
 
    const MemberPointerType *mpt = ce->getType()->getAs<MemberPointerType>();
    mlir::Location loc = cgf.getLoc(subExpr->getExprLoc());
    return cgf.getBuilder().getConstant(
        loc, cgf.cgm.emitNullMemberAttr(destTy, mpt));
  }
 
  case CK_ReinterpretMemberPointer: {
    mlir::Value src = Visit(subExpr);
    return builder.createBitcast(cgf.getLoc(subExpr->getExprLoc()), src,
                                 cgf.convertType(destTy));
  }
  case CK_BaseToDerivedMemberPointer:
  case CK_DerivedToBaseMemberPointer: {
    mlir::Value src = Visit(subExpr);
 
    assert(!cir::MissingFeatures::memberFuncPtrAuthInfo());
 
    QualType derivedTy =
        kind == CK_DerivedToBaseMemberPointer ? subExpr->getType() : destTy;
    const auto *mpType = derivedTy->castAs<MemberPointerType>();
    NestedNameSpecifier qualifier = mpType->getQualifier();
    assert(qualifier && "member pointer without class qualifier");
    const Type *qualifierType = qualifier.getAsType();
    assert(qualifierType && "member pointer qualifier is not a type");
    const CXXRecordDecl *derivedClass = qualifierType->getAsCXXRecordDecl();
    CharUnits offset =
        cgf.cgm.computeNonVirtualBaseClassOffset(derivedClass, ce->path());
 
    mlir::Location loc = cgf.getLoc(subExpr->getExprLoc());
    mlir::Type resultTy = cgf.convertType(destTy);
    mlir::IntegerAttr offsetAttr = builder.getIndexAttr(offset.getQuantity());
 
    if (subExpr->getType()->isMemberFunctionPointerType()) {
      if (kind == CK_BaseToDerivedMemberPointer)
        return cir::DerivedMethodOp::create(builder, loc, resultTy, src,
                                            offsetAttr);
      return cir::BaseMethodOp::create(builder, loc, resultTy, src, offsetAttr);
    }
 
    if (kind == CK_BaseToDerivedMemberPointer)
      return cir::DerivedDataMemberOp::create(builder, loc, resultTy, src,
                                              offsetAttr);
    return cir::BaseDataMemberOp::create(builder, loc, resultTy, src,
                                         offsetAttr);
  }
 
  case CK_LValueToRValue:
    assert(cgf.getContext().hasSameUnqualifiedType(subExpr->getType(), destTy));
    assert(subExpr->isGLValue() && "lvalue-to-rvalue applied to r-value!");
    return Visit(const_cast<Expr *>(subExpr));
 
  case CK_IntegralCast: {
    ScalarConversionOpts opts;
    if (auto *ice = dyn_cast<ImplicitCastExpr>(ce)) {
      if (!ice->isPartOfExplicitCast())
        opts = ScalarConversionOpts(cgf.sanOpts);
    }
    return emitScalarConversion(Visit(subExpr), subExpr->getType(), destTy,
                                ce->getExprLoc(), opts);
  }
 
  case CK_FloatingComplexToReal:
  case CK_IntegralComplexToReal:
  case CK_FloatingComplexToBoolean:
  case CK_IntegralComplexToBoolean: {
    mlir::Value value = cgf.emitComplexExpr(subExpr);
    return emitComplexToScalarConversion(cgf.getLoc(ce->getExprLoc()), value,
                                         kind, destTy);
  }
 
  case CK_FloatingRealToComplex:
  case CK_FloatingComplexCast:
  case CK_IntegralRealToComplex:
  case CK_IntegralComplexCast:
  case CK_IntegralComplexToFloatingComplex:
  case CK_FloatingComplexToIntegralComplex:
    llvm_unreachable("scalar cast to non-scalar value");
 
  case CK_PointerToIntegral: {
    assert(!destTy->isBooleanType() && "bool should use PointerToBool");
    if (cgf.cgm.getCodeGenOpts().StrictVTablePointers)
      cgf.getCIRGenModule().errorNYI(subExpr->getSourceRange(),
                                     "strict vtable pointers");
    return builder.createPtrToInt(Visit(subExpr), cgf.convertType(destTy));
  }
  case CK_ToVoid:
    cgf.emitIgnoredExpr(subExpr);
    return {};
 
  case CK_FixedPointCast:
    return emitScalarConversion(Visit(subExpr), subExpr->getType(), destTy,
                                ce->getExprLoc());
 
  case CK_FixedPointToBoolean:
    assert(subExpr->getType()->isFixedPointType() &&
           "Expected src type to be fixed point type");
    assert(destTy->isBooleanType() && "Expected dest type to be boolean type");
    return emitScalarConversion(Visit(subExpr), subExpr->getType(), destTy,
                                ce->getExprLoc());
 
  case CK_FixedPointToIntegral:
    assert(subExpr->getType()->isFixedPointType() &&
           "Expected src type to be fixed point type");
    assert(destTy->isIntegerType() && "Expected dest type to be an integer");
    return emitScalarConversion(Visit(subExpr), subExpr->getType(), destTy,
                                ce->getExprLoc());
 
  case CK_IntegralToFixedPoint:
    assert(subExpr->getType()->isIntegerType() &&
           "Expected src type to be an integer");
    assert(destTy->isFixedPointType() &&
           "Expected dest type to be fixed point type");
    return emitScalarConversion(Visit(subExpr), subExpr->getType(), destTy,
                                ce->getExprLoc());
 
  case CK_IntegralToFloating:
  case CK_FloatingToIntegral:
  case CK_FloatingCast:
  case CK_FixedPointToFloating:
  case CK_FloatingToFixedPoint: {
    CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, ce);
    return emitScalarConversion(Visit(subExpr), subExpr->getType(), destTy,
                                ce->getExprLoc());
  }
 
  case CK_IntegralToBoolean:
    return emitIntToBoolConversion(Visit(subExpr),
                                   cgf.getLoc(ce->getSourceRange()));
 
  case CK_PointerToBoolean:
    return emitPointerToBoolConversion(Visit(subExpr), subExpr->getType());
  case CK_FloatingToBoolean:
    return emitFloatToBoolConversion(Visit(subExpr),
                                     cgf.getLoc(subExpr->getExprLoc()));
  case CK_MemberPointerToBoolean: {
    mlir::Value memPtr = Visit(subExpr);
    return builder.createCast(cgf.getLoc(ce->getSourceRange()),
                              cir::CastKind::member_ptr_to_bool, memPtr,
                              cgf.convertType(destTy));
  }
 
  case CK_HLSLAggregateSplatCast:
  case CK_VectorSplat: {
    // Splat the element across to all elements
    mlir::Type destCIRTy = cgf.convertType(destTy);
    return cir::VecSplatOp::create(builder,
                                   cgf.getLoc(subExpr->getSourceRange()),
                                   destCIRTy, Visit(subExpr));
  }
  case CK_FunctionToPointerDecay:
    return cgf.emitLValue(subExpr).getPointer();
 
  case CK_HLSLVectorTruncation: {
    assert((destTy->isVectorType() || destTy->isBuiltinType()) &&
           "Destination type must be a vector or builtin type.");
    mlir::Value vec = Visit(subExpr);
    mlir::Location loc = cgf.getLoc(subExpr->getSourceRange());
    if (auto *vecTy = destTy->getAs<VectorType>()) {
      unsigned numElts = vecTy->getNumElements();
      llvm::SmallVector<int64_t, 4> mask(numElts);
      std::iota(mask.begin(), mask.end(), 0);
      return builder.createVecShuffle(loc, vec, mask);
    }
    return builder.createExtractElement(loc, vec, /*idx=*/0);
  }
 
  default:
    cgf.getCIRGenModule().errorNYI(subExpr->getSourceRange(),
                                   "CastExpr: ", ce->getCastKindName());
  }
  return {};
}
 
mlir::Value ScalarExprEmitter::VisitCallExpr(const CallExpr *e) {
  if (e->getCallReturnType(cgf.getContext())->isReferenceType())
    return emitLoadOfLValue(e);
 
  auto v = cgf.emitCallExpr(e).getValue();
  assert(!cir::MissingFeatures::emitLValueAlignmentAssumption());
  return v;
}
 
mlir::Value ScalarExprEmitter::VisitMemberExpr(MemberExpr *e) {
  if (CIRGenFunction::ConstantEmission constant = cgf.tryEmitAsConstant(e)) {
    cgf.emitIgnoredExpr(e->getBase());
    return cgf.emitScalarConstant(constant, e);
  } else {
    Expr::EvalResult result;
    if (e->EvaluateAsInt(result, cgf.getContext(), Expr::SE_AllowSideEffects)) {
      llvm::APSInt value = result.Val.getInt();
      cgf.emitIgnoredExpr(e->getBase());
      mlir::Location loc = cgf.getLoc(e->getExprLoc());
      // The constant is folded from an APSInt with the source-type's bit width
      // (1 for bool), but the AST's expression type is what later consumers of
      // this value see. For a bool member we have to emit a !cir.bool constant
      // -- otherwise downstream ops (cir.call into a bool parameter, cir.if /
      // cir.ternary on the value, ...) would all reject the !cir.int<u, 1> the
      // raw APSInt would produce.
      if (e->getType()->isBooleanType())
        return builder.getBool(value.getBoolValue(), loc);
      return builder.getConstInt(loc, value);
    }
  }
 
  return emitLoadOfLValue(e);
}
 
mlir::Value
ScalarExprEmitter::emitVectorInitWithSubvectors(mlir::Location loc,
                                                cir::VectorType vectorType,
                                                ArrayRef<mlir::Value> inits) {
  const unsigned resElts = vectorType.getFixedSize();
  mlir::Value result =
      builder.getConstant(loc, cir::PoisonAttr::get(vectorType));
  unsigned curIdx = 0;
  for (mlir::Value init : inits) {
    auto initType = mlir::dyn_cast<cir::VectorType>(init.getType());
    if (!initType) {
      result = builder.createInsertElement(loc, result, init, curIdx++);
      continue;
    }
 
    const unsigned initElts = initType.getFixedSize();
    assert(curIdx + initElts <= resElts &&
           "Sema should reject excess elements");
    SmallVector<int64_t, 16> mask(resElts, -1);
    std::iota(mask.begin(), mask.begin() + initElts, 0);
    mlir::Value ext = builder.createVecShuffle(loc, init, mask);
 
    // Poison stays on the RHS of the first shuffle, matching classic codegen.
    const unsigned offset = curIdx == 0 ? 0 : resElts;
    std::iota(mask.begin(), mask.begin() + curIdx, 0);
    std::iota(mask.begin() + curIdx, mask.begin() + curIdx + initElts, offset);
    if (curIdx == 0)
      std::swap(result, ext);
    result = builder.createVecShuffle(loc, result, ext, mask);
    curIdx += initElts;
  }
 
  if (curIdx < resElts) {
    const mlir::Value zeroValue =
        builder.getNullValue(vectorType.getElementType(), loc);
    for (; curIdx < resElts; ++curIdx)
      result = builder.createInsertElement(loc, result, zeroValue, curIdx);
  }
  return result;
}
 
mlir::Value ScalarExprEmitter::VisitInitListExpr(InitListExpr *e) {
  const unsigned numInitElements = e->getNumInits();
 
  [[maybe_unused]] const bool ignore = std::exchange(ignoreResultAssign, false);
  assert((ignore == false ||
          (numInitElements == 0 && e->getType()->isVoidType())) &&
         "init list ignored");
 
  if (e->hadArrayRangeDesignator()) {
    cgf.cgm.errorNYI(e->getSourceRange(), "ArrayRangeDesignator");
    return {};
  }
 
  if (e->getType()->isVectorType()) {
    const auto vectorType =
        mlir::cast<cir::VectorType>(cgf.convertType(e->getType()));
 
    SmallVector<mlir::Value, 16> elements;
    bool hasVectorInit = false;
    for (Expr *init : e->inits()) {
      elements.push_back(Visit(init));
      hasVectorInit |= mlir::isa<cir::VectorType>(elements.back().getType());
    }
 
    if (hasVectorInit)
      return emitVectorInitWithSubvectors(cgf.getLoc(e->getSourceRange()),
                                          vectorType, elements);
 
    // Zero-initialize any remaining values.
    const uint64_t vectorSize = vectorType.getFixedSize();
    if (numInitElements < vectorSize) {
      const mlir::Value zeroValue = cgf.getBuilder().getNullValue(
          vectorType.getElementType(), cgf.getLoc(e->getSourceRange()));
      std::fill_n(std::back_inserter(elements), vectorSize - numInitElements,
                  zeroValue);
    }
 
    return cir::VecCreateOp::create(cgf.getBuilder(),
                                    cgf.getLoc(e->getSourceRange()), vectorType,
                                    elements);
  }
 
  assert(!cir::MissingFeatures::scalableVectors());
 
  if (e->getType()->isVoidType())
    return {};
 
  // C++11 value-initialization for the scalar.
  if (numInitElements == 0)
    return emitNullValue(e->getType(), cgf.getLoc(e->getExprLoc()));
 
  return Visit(e->getInit(0));
}
 
mlir::Value CIRGenFunction::emitScalarConversion(mlir::Value src,
                                                 QualType srcTy, QualType dstTy,
                                                 SourceLocation loc) {
  assert(CIRGenFunction::hasScalarEvaluationKind(srcTy) &&
         CIRGenFunction::hasScalarEvaluationKind(dstTy) &&
         "Invalid scalar expression to emit");
  return ScalarExprEmitter(*this, builder)
      .emitScalarConversion(src, srcTy, dstTy, loc);
}
 
mlir::Value CIRGenFunction::emitComplexToScalarConversion(mlir::Value src,
                                                          QualType srcTy,
                                                          QualType dstTy,
                                                          SourceLocation loc) {
  assert(srcTy->isAnyComplexType() && hasScalarEvaluationKind(dstTy) &&
         "Invalid complex -> scalar conversion");
 
  QualType complexElemTy = srcTy->castAs<ComplexType>()->getElementType();
  if (dstTy->isBooleanType()) {
    auto kind = complexElemTy->isFloatingType()
                    ? cir::CastKind::float_complex_to_bool
                    : cir::CastKind::int_complex_to_bool;
    return builder.createCast(getLoc(loc), kind, src, convertType(dstTy));
  }
 
  auto kind = complexElemTy->isFloatingType()
                  ? cir::CastKind::float_complex_to_real
                  : cir::CastKind::int_complex_to_real;
  mlir::Value real =
      builder.createCast(getLoc(loc), kind, src, convertType(complexElemTy));
  return emitScalarConversion(real, complexElemTy, dstTy, loc);
}
 
mlir::Value ScalarExprEmitter::VisitUnaryLNot(const UnaryOperator *e) {
  // Perform vector logical not on comparison with zero vector.
  if (e->getType()->isVectorType() &&
      e->getType()->castAs<VectorType>()->getVectorKind() ==
          VectorKind::Generic) {
    mlir::Value oper = Visit(e->getSubExpr());
    mlir::Location loc = cgf.getLoc(e->getExprLoc());
    auto operVecTy = mlir::cast<cir::VectorType>(oper.getType());
    mlir::Value zeroVec = builder.getNullValue(operVecTy, loc);
    auto resultTy = mlir::cast<cir::VectorType>(cgf.convertType(e->getType()));
    return builder.createVecCompare(loc, resultTy, cir::CmpOpKind::eq, oper,
                                    zeroVec);
  }
 
  // Compare operand to zero.
  mlir::Value boolVal = cgf.evaluateExprAsBool(e->getSubExpr());
 
  // Invert value.
  boolVal = builder.createNot(boolVal);
 
  // ZExt result to the expr type.
  return maybePromoteBoolResult(boolVal, cgf.convertType(e->getType()));
}
 
mlir::Value ScalarExprEmitter::VisitOffsetOfExpr(OffsetOfExpr *e) {
  // Try folding the offsetof to a constant.
  Expr::EvalResult evalResult;
  if (e->EvaluateAsInt(evalResult, cgf.getContext())) {
    mlir::Type type = cgf.convertType(e->getType());
    llvm::APSInt value = evalResult.Val.getInt();
    return builder.getConstAPInt(cgf.getLoc(e->getExprLoc()), type, value);
  }
 
  cgf.getCIRGenModule().errorNYI(
      e->getSourceRange(),
      "ScalarExprEmitter::VisitOffsetOfExpr Can't eval expr as int");
  return {};
}
 
mlir::Value ScalarExprEmitter::VisitUnaryReal(const UnaryOperator *e) {
  QualType promotionTy = getPromotionType(e->getSubExpr()->getType());
  mlir::Value result = VisitRealImag(e, promotionTy);
  if (result && !promotionTy.isNull())
    result = emitUnPromotedValue(result, e->getType());
  return result;
}
 
mlir::Value ScalarExprEmitter::VisitUnaryImag(const UnaryOperator *e) {
  QualType promotionTy = getPromotionType(e->getSubExpr()->getType());
  mlir::Value result = VisitRealImag(e, promotionTy);
  if (result && !promotionTy.isNull())
    result = emitUnPromotedValue(result, e->getType());
  return result;
}
 
mlir::Value ScalarExprEmitter::VisitRealImag(const UnaryOperator *e,
                                             QualType promotionTy) {
  assert(
      (e->getOpcode() == clang::UO_Real || e->getOpcode() == clang::UO_Imag) &&
      "Invalid UnaryOp kind for ComplexType Real or Imag");
 
  Expr *op = e->getSubExpr();
  mlir::Location loc = cgf.getLoc(e->getExprLoc());
  if (op->getType()->isAnyComplexType()) {
    // If it's an l-value, load through the appropriate subobject l-value.
    // Note that we have to ask `e` because `op` might be an l-value that
    // this won't work for, e.g. an Obj-C property
    mlir::Value complex = cgf.emitComplexExpr(op);
    if (e->isGLValue() && !promotionTy.isNull()) {
      promotionTy = promotionTy->isAnyComplexType()
                        ? promotionTy
                        : cgf.getContext().getComplexType(promotionTy);
      complex = cgf.emitPromotedValue(complex, promotionTy);
    }
 
    return e->getOpcode() == clang::UO_Real
               ? builder.createComplexReal(loc, complex)
               : builder.createComplexImag(loc, complex);
  }
 
  if (e->getOpcode() == UO_Real) {
    mlir::Value operand = promotionTy.isNull()
                              ? Visit(op)
                              : cgf.emitPromotedScalarExpr(op, promotionTy);
    return builder.createComplexReal(loc, operand);
  }
 
  // __imag on a scalar returns zero. Emit the subexpr to ensure side
  // effects are evaluated, but not the actual value.
  mlir::Value operand;
  if (op->isGLValue()) {
    operand = cgf.emitLValue(op).getPointer();
    operand = cir::LoadOp::create(builder, loc, operand);
  } else if (!promotionTy.isNull()) {
    operand = cgf.emitPromotedScalarExpr(op, promotionTy);
  } else {
    operand = cgf.emitScalarExpr(op);
  }
  return builder.createComplexImag(loc, operand);
}
 
/// Return the size or alignment of the type of argument of the sizeof
/// expression as an integer.
mlir::Value ScalarExprEmitter::VisitUnaryExprOrTypeTraitExpr(
    const UnaryExprOrTypeTraitExpr *e) {
  const QualType typeToSize = e->getTypeOfArgument();
  const mlir::Location loc = cgf.getLoc(e->getSourceRange());
  if (auto kind = e->getKind();
      kind == UETT_SizeOf || kind == UETT_DataSizeOf || kind == UETT_CountOf) {
    if (const VariableArrayType *vat =
            cgf.getContext().getAsVariableArrayType(typeToSize)) {
      // For _Countof, we only want to evaluate if the extent is actually
      // variable as opposed to a multi-dimensional array whose extent is
      // constant but whose element type is variable.
      bool evaluateExtent = true;
      if (kind == UETT_CountOf && vat->getElementType()->isArrayType()) {
        evaluateExtent =
            !vat->getSizeExpr()->isIntegerConstantExpr(cgf.getContext());
      }
 
      if (evaluateExtent) {
        if (e->isArgumentType()) {
          // sizeof(type) - make sure to emit the VLA size.
          cgf.emitVariablyModifiedType(typeToSize);
        } else {
          // C99 6.5.3.4p2: If the argument is an expression of type
          // VLA, it is evaluated.
          cgf.emitIgnoredExpr(e->getArgumentExpr());
        }
 
        // For _Countof, we just want to return the size of a single dimension.
        if (kind == UETT_CountOf)
          return cgf.getVLAElements1D(vat).numElts;
 
        // For sizeof and __datasizeof, we need to scale the number of elements
        // by the size of the array element type.
        CIRGenFunction::VlaSizePair vlaSize = cgf.getVLASize(vat);
        mlir::Value numElts = vlaSize.numElts;
 
        // Scale the number of non-VLA elements by the non-VLA element size.
        CharUnits eltSize = cgf.getContext().getTypeSizeInChars(vlaSize.type);
        if (!eltSize.isOne()) {
          mlir::Location loc = cgf.getLoc(e->getSourceRange());
          mlir::Value eltSizeValue =
              builder.getConstAPInt(numElts.getLoc(), numElts.getType(),
                                    cgf.cgm.getSize(eltSize).getValue());
          return builder.createMul(loc, eltSizeValue, numElts,
                                   cir::OverflowBehavior::NoUnsignedWrap);
        }
 
        return numElts;
      }
    }
  } else if (e->getKind() == UETT_OpenMPRequiredSimdAlign) {
    clang::CharUnits::QuantityType alignment =
        cgf.getContext()
            .toCharUnitsFromBits(cgf.getContext().getOpenMPDefaultSimdAlign(
                e->getTypeOfArgument()->getPointeeType()))
            .getQuantity();
    return builder.getConstantInt(loc, cgf.cgm.sizeTy, alignment);
  } else if (e->getKind() == UETT_VectorElements) {
    auto vecTy = cast<cir::VectorType>(convertType(e->getTypeOfArgument()));
    if (vecTy.isScalable()) {
      cgf.getCIRGenModule().errorNYI(
          e->getSourceRange(),
          "VisitUnaryExprOrTypeTraitExpr: sizeOf scalable vector");
      return builder.getConstant(
          loc, cir::IntAttr::get(cgf.cgm.sizeTy,
                                 e->EvaluateKnownConstInt(cgf.getContext())));
    }
 
    return builder.getConstant(
        loc, cir::IntAttr::get(cgf.cgm.sizeTy, vecTy.getFixedSize()));
  }
 
  // The result type is size_t (target-dependent width); use it so the IntAttr
  // width matches the APInt from EvaluateKnownConstInt.
  return builder.getConstant(
      loc, cir::IntAttr::get(cgf.cgm.sizeTy,
                             e->EvaluateKnownConstInt(cgf.getContext())));
}
 
mlir::Value ScalarExprEmitter::VisitAbstractConditionalOperator(
    const AbstractConditionalOperator *e) {
  CIRGenBuilderTy &builder = cgf.getBuilder();
  mlir::Location loc = cgf.getLoc(e->getSourceRange());
  ignoreResultAssign = false;
 
  // Bind the common expression if necessary.
  CIRGenFunction::OpaqueValueMapping binding(cgf, e);
 
  Expr *condExpr = e->getCond();
  Expr *lhsExpr = e->getTrueExpr();
  Expr *rhsExpr = e->getFalseExpr();
 
  // If the condition constant folds and can be elided, try to avoid emitting
  // the condition and the dead arm.
  bool condExprBool;
  if (cgf.constantFoldsToBool(condExpr, condExprBool)) {
    Expr *live = lhsExpr, *dead = rhsExpr;
    if (!condExprBool)
      std::swap(live, dead);
 
    // If the dead side doesn't have labels we need, just emit the Live part.
    if (!cgf.containsLabel(dead)) {
      if (condExprBool)
        assert(!cir::MissingFeatures::incrementProfileCounter());
      mlir::Value result = Visit(live);
 
      // If the live part is a throw expression, it acts like it has a void
      // type, so evaluating it returns a null Value.  However, a conditional
      // with non-void type must return a non-null Value.
      if (!result && !e->getType()->isVoidType()) {
        result = builder.getConstant(
            loc, cir::PoisonAttr::get(builder.getContext(),
                                      cgf.convertType(e->getType())));
      }
 
      return result;
    }
  }
 
  QualType condType = condExpr->getType();
 
  // OpenCL: If the condition is a vector, we can treat this condition like
  // the select function.
  if (cgf.getLangOpts().OpenCL &&
      (condType->isVectorType() || condType->isExtVectorType())) {
    assert(!cir::MissingFeatures::incrementProfileCounter());
 
    mlir::Value condValue = cgf.emitScalarExpr(condExpr);
    mlir::Value lhsValue = Visit(lhsExpr);
    mlir::Value rhsValue = Visit(rhsExpr);
 
    // Test the MSB with a signed compare, as classic codegen does.
    mlir::Type vecTy =
        convertType(cgf.getContext().getCorrespondingSignedType(condType));
    condValue = builder.createBitcast(condValue, vecTy);
    mlir::Value zeroVec = builder.getNullValue(vecTy, loc);
    auto testMSB = cir::VecCmpOp::create(
        builder, loc, vecTy, cir::CmpOpKind::lt, condValue, zeroVec);
    mlir::Value tmp = builder.createIntCast(testMSB, vecTy);
    mlir::Value tmp2 = builder.createNot(tmp);
 
    // Bitcast float or differently signed operands to the condition type.
    mlir::Value rhsTmp = rhsValue;
    mlir::Value lhsTmp = lhsValue;
    bool wasCast = false;
    if (rhsValue.getType() != vecTy) {
      rhsTmp = builder.createBitcast(rhsValue, tmp2.getType());
      lhsTmp = builder.createBitcast(lhsValue, tmp.getType());
      wasCast = true;
    }
 
    mlir::Value tmp3 = builder.createAnd(loc, rhsTmp, tmp2);
    mlir::Value tmp4 = builder.createAnd(loc, lhsTmp, tmp);
    mlir::Value tmp5 = builder.createOr(loc, tmp3, tmp4);
    if (wasCast)
      tmp5 = builder.createBitcast(tmp5, rhsValue.getType());
    return tmp5;
  }
 
  if (condType->isVectorType() || condType->isSveVLSBuiltinType()) {
    if (!condType->isVectorType()) {
      assert(!cir::MissingFeatures::vecTernaryOp());
      cgf.cgm.errorNYI(loc, "TernaryOp for SVE vector");
      return {};
    }
 
    mlir::Value condValue = Visit(condExpr);
    mlir::Value lhsValue = Visit(lhsExpr);
    mlir::Value rhsValue = Visit(rhsExpr);
    return cir::VecTernaryOp::create(builder, loc, condValue, lhsValue,
                                     rhsValue);
  }
 
  // If this is a really simple expression (like x ? 4 : 5), emit this as a
  // select instead of as control flow.  We can only do this if it is cheap
  // and safe to evaluate the LHS and RHS unconditionally.
  if (CodeGenUtils::isCheapEnoughToEvaluateUnconditionally(lhsExpr,
                                                           cgf.getContext()) &&
      CodeGenUtils::isCheapEnoughToEvaluateUnconditionally(rhsExpr,
                                                           cgf.getContext())) {
    mlir::Value condV = cgf.evaluateExprAsBool(condExpr);
    assert(!cir::MissingFeatures::incrementProfileCounter());
 
    mlir::Value lhs = Visit(lhsExpr);
    mlir::Value rhs = Visit(rhsExpr);
    if (!lhs) {
      // If the conditional has void type, make sure we return a null Value.
      assert(!rhs && "lhs and rhs types must match");
      return {};
    }
 
    return builder.createSelect(loc, condV, lhs, rhs);
  }
 
  mlir::Value condV = cgf.emitOpOnBoolExpr(loc, condExpr);
  CIRGenFunction::ConditionalEvaluation eval(cgf, loc);
 
  auto emitBranch = [&](mlir::OpBuilder &b, mlir::Location loc, Expr *expr) {
    CIRGenFunction::LexicalScope lexScope{cgf, loc, b.getInsertionBlock()};
    cgf.curLexScope->setAsTernary();
 
    mlir::Value branch;
    {
      // Emit any cleanups that were needed on this branch so we can spill
      // and reload the return value.
      CIRGenFunction::RunCleanupsScope branchCleanups(cgf);
      assert(!cir::MissingFeatures::incrementProfileCounter());
      eval.beginEvaluation();
      branch = Visit(expr);
      eval.endEvaluation();
      branchCleanups.forceCleanup({&branch});
    }
 
    if (branch)
      cir::YieldOp::create(b, loc, branch);
  };
 
  cir::TernaryOp ternary = cir::TernaryOp::create(
      builder, loc, condV,
      /*trueBuilder=*/
      [&](mlir::OpBuilder &b, mlir::Location loc) {
        emitBranch(b, loc, lhsExpr);
      },
      /*falseBuilder=*/
      [&](mlir::OpBuilder &b, mlir::Location loc) {
        emitBranch(b, loc, rhsExpr);
      });
 
  // Only a void arm can be left unterminated (a noreturn arm already ends
  // in cir.unreachable); close it with an empty cir.yield.
  for (mlir::Region *region :
       {&ternary.getTrueRegion(), &ternary.getFalseRegion()}) {
    mlir::Block &lastBlock = region->back();
    if (lastBlock.empty() ||
        !lastBlock.back().hasTrait<mlir::OpTrait::IsTerminator>()) {
      mlir::OpBuilder::InsertionGuard guard(builder);
      builder.setInsertionPointToEnd(&lastBlock);
      cir::YieldOp::create(builder, loc);
    }
  }
 
  return ternary.getResult();
}
 
mlir::Value CIRGenFunction::emitScalarPrePostIncDec(const UnaryOperator *e,
                                                    LValue lv) {
  return ScalarExprEmitter(*this, builder).emitScalarPrePostIncDec(e, lv);
}