CIRGenExprScalar.cpp

Go to the documentation of this file.

//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// Emit Expr nodes with scalar CIR types as CIR code.
//
//===----------------------------------------------------------------------===//
 
#include "CIRGenConstantEmitter.h"
#include "CIRGenFunction.h"
#include "CIRGenValue.h"
 
#include "clang/AST/Expr.h"
#include "clang/AST/StmtVisitor.h"
#include "clang/CIR/Dialect/IR/CIRTypes.h"
#include "clang/CIR/MissingFeatures.h"
 
#include "mlir/Dialect/Ptr/IR/MemorySpaceInterfaces.h"
#include "mlir/IR/Location.h"
#include "mlir/IR/Value.h"
 
#include <cassert>
#include <utility>
 
using namespace clang;
using namespace clang::CIRGen;
 
namespace {
 
struct BinOpInfo {
  mlir::Value lhs;
  mlir::Value rhs;
  SourceRange loc;
  QualType fullType;             // Type of operands and result
  QualType compType;             // Type used for computations. Element type
                                 // for vectors, otherwise same as FullType.
  BinaryOperator::Opcode opcode; // Opcode of BinOp to perform
  FPOptions fpFeatures;
  const Expr *e; // Entire expr, for error unsupported.  May not be binop.
 
  /// Check if the binop computes a division or a remainder.
  bool isDivRemOp() const {
    return opcode == BO_Div || opcode == BO_Rem || opcode == BO_DivAssign ||
           opcode == BO_RemAssign;
  }
 
  /// Check if the binop can result in integer overflow.
  bool mayHaveIntegerOverflow() const {
    // Without constant input, we can't rule out overflow.
    auto lhsci = lhs.getDefiningOp<cir::ConstantOp>();
    auto rhsci = rhs.getDefiningOp<cir::ConstantOp>();
    if (!lhsci || !rhsci)
      return true;
 
    assert(!cir::MissingFeatures::mayHaveIntegerOverflow());
    // TODO(cir): For now we just assume that we might overflow
    return true;
  }
 
  /// Check if at least one operand is a fixed point type. In such cases,
  /// this operation did not follow usual arithmetic conversion and both
  /// operands might not be of the same type.
  bool isFixedPointOp() const {
    // We cannot simply check the result type since comparison operations
    // return an int.
    if (const auto *binOp = llvm::dyn_cast<BinaryOperator>(e)) {
      QualType lhstype = binOp->getLHS()->getType();
      QualType rhstype = binOp->getRHS()->getType();
      return lhstype->isFixedPointType() || rhstype->isFixedPointType();
    }
    if (const auto *unop = llvm::dyn_cast<UnaryOperator>(e))
      return unop->getSubExpr()->getType()->isFixedPointType();
    return false;
  }
};
 
class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
  CIRGenFunction &cgf;
  CIRGenBuilderTy &builder;
  // Unlike classic codegen we set this to false or use std::exchange to read
  // the value instead of calling TestAndClearIgnoreResultAssign to make it
  // explicit when the value is used
  bool ignoreResultAssign;
 
public:
  ScalarExprEmitter(CIRGenFunction &cgf, CIRGenBuilderTy &builder,
                    bool ignoreResultAssign = false)
      : cgf(cgf), builder(builder), ignoreResultAssign(ignoreResultAssign) {}
 
  //===--------------------------------------------------------------------===//
  //                               Utilities
  //===--------------------------------------------------------------------===//
  mlir::Type convertType(QualType ty) { return cgf.convertType(ty); }
 
  mlir::Value emitComplexToScalarConversion(mlir::Location loc,
                                            mlir::Value value, CastKind kind,
                                            QualType destTy);
 
  mlir::Value emitNullValue(QualType ty, mlir::Location loc) {
    return cgf.cgm.emitNullConstant(ty, loc);
  }
 
  mlir::Value emitPromotedValue(mlir::Value result, QualType promotionType) {
    return builder.createFloatingCast(result, cgf.convertType(promotionType));
  }
 
  mlir::Value emitUnPromotedValue(mlir::Value result, QualType exprType) {
    return builder.createFloatingCast(result, cgf.convertType(exprType));
  }
 
  mlir::Value emitPromoted(const Expr *e, QualType promotionType);
 
  mlir::Value maybePromoteBoolResult(mlir::Value value,
                                     mlir::Type dstTy) const {
    if (mlir::isa<cir::IntType>(dstTy))
      return builder.createBoolToInt(value, dstTy);
    if (mlir::isa<cir::BoolType>(dstTy))
      return value;
    llvm_unreachable("Can only promote integer or boolean types");
  }
 
  //===--------------------------------------------------------------------===//
  //                            Visitor Methods
  //===--------------------------------------------------------------------===//
 
  mlir::Value Visit(Expr *e) {
    return StmtVisitor<ScalarExprEmitter, mlir::Value>::Visit(e);
  }
 
  mlir::Value VisitStmt(Stmt *s) {
    llvm_unreachable("Statement passed to ScalarExprEmitter");
  }
 
  mlir::Value VisitExpr(Expr *e) {
    cgf.getCIRGenModule().errorNYI(
        e->getSourceRange(), "scalar expression kind: ", e->getStmtClassName());
    return {};
  }
 
  mlir::Value VisitConstantExpr(ConstantExpr *e) {
    // A constant expression of type 'void' generates no code and produces no
    // value.
    if (e->getType()->isVoidType())
      return {};
 
    if (mlir::Attribute result = ConstantEmitter(cgf).tryEmitConstantExpr(e)) {
      if (e->isGLValue()) {
        cgf.cgm.errorNYI(e->getSourceRange(),
                         "ScalarExprEmitter: constant expr GL Value");
        return {};
      }
 
      return builder.getConstant(cgf.getLoc(e->getSourceRange()),
                                 mlir::cast<mlir::TypedAttr>(result));
    }
 
    cgf.cgm.errorNYI(e->getSourceRange(), "ScalarExprEmitter: constant expr");
    return {};
  }
 
  mlir::Value VisitPackIndexingExpr(PackIndexingExpr *e) {
    return Visit(e->getSelectedExpr());
  }
 
  mlir::Value VisitParenExpr(ParenExpr *pe) { return Visit(pe->getSubExpr()); }
 
  mlir::Value VisitGenericSelectionExpr(GenericSelectionExpr *ge) {
    return Visit(ge->getResultExpr());
  }
 
  /// Emits the address of the l-value, then loads and returns the result.
  mlir::Value emitLoadOfLValue(const Expr *e) {
    LValue lv = cgf.emitLValue(e);
    // FIXME: add some akin to EmitLValueAlignmentAssumption(E, V);
    return cgf.emitLoadOfLValue(lv, e->getExprLoc()).getValue();
  }
 
  mlir::Value VisitCoawaitExpr(CoawaitExpr *s) {
    return cgf.emitCoawaitExpr(*s).getValue();
  }
 
  mlir::Value VisitCoyieldExpr(CoyieldExpr *e) {
    return cgf.emitCoyieldExpr(*e).getValue();
  }
 
  mlir::Value VisitUnaryCoawait(const UnaryOperator *e) {
    cgf.cgm.errorNYI(e->getSourceRange(), "ScalarExprEmitter: unary coawait");
    return {};
  }
 
  mlir::Value emitLoadOfLValue(LValue lv, SourceLocation loc) {
    return cgf.emitLoadOfLValue(lv, loc).getValue();
  }
 
  // l-values
  mlir::Value VisitDeclRefExpr(DeclRefExpr *e) {
    if (CIRGenFunction::ConstantEmission constant = cgf.tryEmitAsConstant(e))
      return cgf.emitScalarConstant(constant, e);
 
    return emitLoadOfLValue(e);
  }
 
  mlir::Value VisitAddrLabelExpr(const AddrLabelExpr *e) {
    auto func = cast<cir::FuncOp>(cgf.curFn);
    cir::BlockAddrInfoAttr blockInfoAttr = cir::BlockAddrInfoAttr::get(
        &cgf.getMLIRContext(), func.getSymName(), e->getLabel()->getName());
    cir::BlockAddressOp blockAddressOp = cir::BlockAddressOp::create(
        builder, cgf.getLoc(e->getSourceRange()), cgf.convertType(e->getType()),
        blockInfoAttr);
    cir::LabelOp resolvedLabel = cgf.cgm.lookupBlockAddressInfo(blockInfoAttr);
    if (!resolvedLabel) {
      cgf.cgm.mapUnresolvedBlockAddress(blockAddressOp);
      // Still add the op to maintain insertion order it will be resolved in
      // resolveBlockAddresses
      cgf.cgm.mapResolvedBlockAddress(blockAddressOp, nullptr);
    } else {
      cgf.cgm.mapResolvedBlockAddress(blockAddressOp, resolvedLabel);
    }
    cgf.instantiateIndirectGotoBlock();
    return blockAddressOp;
  }
 
  mlir::Value VisitIntegerLiteral(const IntegerLiteral *e) {
    mlir::Type type = cgf.convertType(e->getType());
    return cir::ConstantOp::create(builder, cgf.getLoc(e->getExprLoc()),
                                   cir::IntAttr::get(type, e->getValue()));
  }
 
  mlir::Value VisitFixedPointLiteral(const FixedPointLiteral *e) {
    mlir::Type type = cgf.convertType(e->getType());
    return cir::ConstantOp::create(builder, cgf.getLoc(e->getExprLoc()),
                                   cir::IntAttr::get(type, e->getValue()));
  }
 
  mlir::Value VisitFloatingLiteral(const FloatingLiteral *e) {
    mlir::Type type = cgf.convertType(e->getType());
    assert(mlir::isa<cir::FPTypeInterface>(type) &&
           "expect floating-point type");
    return cir::ConstantOp::create(builder, cgf.getLoc(e->getExprLoc()),
                                   cir::FPAttr::get(type, e->getValue()));
  }
 
  mlir::Value VisitCharacterLiteral(const CharacterLiteral *e) {
    mlir::Type ty = cgf.convertType(e->getType());
    // Character literals are always stored in an unsigned (even for signed
    // char), so allow implicit truncation here.
    auto intTy = mlir::cast<cir::IntTypeInterface>(ty);
    llvm::APInt apValue(intTy.getWidth(), e->getValue(),
                        /*isSigned=*/false, /*implicitTrunc=*/true);
    return cir::ConstantOp::create(builder, cgf.getLoc(e->getExprLoc()),
                                   cir::IntAttr::get(ty, apValue));
  }
 
  mlir::Value VisitCXXBoolLiteralExpr(const CXXBoolLiteralExpr *e) {
    return builder.getBool(e->getValue(), cgf.getLoc(e->getExprLoc()));
  }
 
  mlir::Value VisitCXXScalarValueInitExpr(const CXXScalarValueInitExpr *e) {
    if (e->getType()->isVoidType())
      return {};
 
    return emitNullValue(e->getType(), cgf.getLoc(e->getSourceRange()));
  }
 
  mlir::Value VisitGNUNullExpr(const GNUNullExpr *e) {
    return emitNullValue(e->getType(), cgf.getLoc(e->getSourceRange()));
  }
 
  mlir::Value VisitOffsetOfExpr(OffsetOfExpr *e);
 
  mlir::Value VisitSizeOfPackExpr(SizeOfPackExpr *e) {
    return builder.getConstInt(cgf.getLoc(e->getExprLoc()),
                               convertType(e->getType()), e->getPackLength());
  }
  mlir::Value VisitPseudoObjectExpr(PseudoObjectExpr *e) {
    return cgf.emitPseudoObjectRValue(e).getValue();
  }
  mlir::Value VisitSYCLUniqueStableNameExpr(SYCLUniqueStableNameExpr *e) {
    cgf.cgm.errorNYI(e->getSourceRange(),
                     "ScalarExprEmitter: sycl unique stable name");
    return {};
  }
  mlir::Value VisitEmbedExpr(EmbedExpr *e) {
    assert(e->getDataElementCount() == 1);
    auto it = e->begin();
    llvm::APInt value = (*it)->getValue();
    return builder.getConstInt(cgf.getLoc(e->getExprLoc()), value,
                               e->getType()->isUnsignedIntegerType());
  }
  mlir::Value VisitOpaqueValueExpr(OpaqueValueExpr *e) {
    if (e->isGLValue())
      return emitLoadOfLValue(cgf.getOrCreateOpaqueLValueMapping(e),
                              e->getExprLoc());
 
    // Otherwise, assume the mapping is the scalar directly.
    return cgf.getOrCreateOpaqueRValueMapping(e).getValue();
  }
 
  mlir::Value VisitObjCSelectorExpr(ObjCSelectorExpr *e) {
    cgf.cgm.errorNYI(e->getSourceRange(), "ScalarExprEmitter: objc selector");
    return {};
  }
  mlir::Value VisitObjCProtocolExpr(ObjCProtocolExpr *e) {
    cgf.cgm.errorNYI(e->getSourceRange(), "ScalarExprEmitter: objc protocol");
    return {};
  }
  mlir::Value VisitObjCIVarRefExpr(ObjCIvarRefExpr *e) {
    cgf.cgm.errorNYI(e->getSourceRange(), "ScalarExprEmitter: objc ivar ref");
    return {};
  }
  mlir::Value VisitObjCMessageExpr(ObjCMessageExpr *e) {
    cgf.cgm.errorNYI(e->getSourceRange(), "ScalarExprEmitter: objc message");
    return {};
  }
  mlir::Value VisitObjCIsaExpr(ObjCIsaExpr *e) {
    cgf.cgm.errorNYI(e->getSourceRange(), "ScalarExprEmitter: objc isa");
    return {};
  }
  mlir::Value VisitObjCAvailabilityCheckExpr(ObjCAvailabilityCheckExpr *e) {
    cgf.cgm.errorNYI(e->getSourceRange(),
                     "ScalarExprEmitter: objc availability check");
    return {};
  }
 
  mlir::Value VisitMatrixSubscriptExpr(MatrixSubscriptExpr *e) {
    cgf.cgm.errorNYI(e->getSourceRange(),
                     "ScalarExprEmitter: matrix subscript");
    return {};
  }
 
  mlir::Value VisitCastExpr(CastExpr *e);
  mlir::Value VisitCallExpr(const CallExpr *e);
 
  mlir::Value VisitStmtExpr(StmtExpr *e) {
    CIRGenFunction::StmtExprEvaluation eval(cgf);
    if (e->getType()->isVoidType()) {
      (void)cgf.emitCompoundStmt(*e->getSubStmt());
      return {};
    }
 
    Address retAlloca =
        cgf.createMemTemp(e->getType(), cgf.getLoc(e->getSourceRange()));
    (void)cgf.emitCompoundStmt(*e->getSubStmt(), &retAlloca);
 
    return cgf.emitLoadOfScalar(cgf.makeAddrLValue(retAlloca, e->getType()),
                                e->getExprLoc());
  }
 
  mlir::Value VisitArraySubscriptExpr(ArraySubscriptExpr *e) {
    ignoreResultAssign = false;
 
    if (e->getBase()->getType()->isVectorType()) {
      assert(!cir::MissingFeatures::scalableVectors());
 
      const mlir::Location loc = cgf.getLoc(e->getSourceRange());
      const mlir::Value vecValue = Visit(e->getBase());
      const mlir::Value indexValue = Visit(e->getIdx());
      return cir::VecExtractOp::create(cgf.builder, loc, vecValue, indexValue);
    }
    // Just load the lvalue formed by the subscript expression.
    return emitLoadOfLValue(e);
  }
 
  mlir::Value VisitShuffleVectorExpr(ShuffleVectorExpr *e) {
    if (e->getNumSubExprs() == 2) {
      // The undocumented form of __builtin_shufflevector.
      mlir::Value inputVec = Visit(e->getExpr(0));
      mlir::Value indexVec = Visit(e->getExpr(1));
      return cir::VecShuffleDynamicOp::create(
          cgf.builder, cgf.getLoc(e->getSourceRange()), inputVec, indexVec);
    }
 
    mlir::Value vec1 = Visit(e->getExpr(0));
    mlir::Value vec2 = Visit(e->getExpr(1));
 
    // The documented form of __builtin_shufflevector, where the indices are
    // a variable number of integer constants. The constants will be stored
    // in an ArrayAttr.
    SmallVector<mlir::Attribute, 8> indices;
    for (unsigned i = 2; i < e->getNumSubExprs(); ++i) {
      indices.push_back(
          cir::IntAttr::get(cgf.builder.getSInt64Ty(),
                            e->getExpr(i)
                                ->EvaluateKnownConstInt(cgf.getContext())
                                .getSExtValue()));
    }
 
    return cir::VecShuffleOp::create(cgf.builder,
                                     cgf.getLoc(e->getSourceRange()),
                                     cgf.convertType(e->getType()), vec1, vec2,
                                     cgf.builder.getArrayAttr(indices));
  }
 
  mlir::Value VisitConvertVectorExpr(ConvertVectorExpr *e) {
    // __builtin_convertvector is an element-wise cast, and is implemented as a
    // regular cast. The back end handles casts of vectors correctly.
    return emitScalarConversion(Visit(e->getSrcExpr()),
                                e->getSrcExpr()->getType(), e->getType(),
                                e->getSourceRange().getBegin());
  }
 
  mlir::Value VisitExtVectorElementExpr(Expr *e) { return emitLoadOfLValue(e); }
 
  mlir::Value VisitMatrixElementExpr(Expr *e) {
    cgf.cgm.errorNYI(e->getSourceRange(), "ScalarExprEmitter: matrix element");
    return {};
  }
 
  mlir::Value VisitMemberExpr(MemberExpr *e);
 
  mlir::Value VisitCompoundLiteralExpr(CompoundLiteralExpr *e) {
    return emitLoadOfLValue(e);
  }
 
  mlir::Value VisitInitListExpr(InitListExpr *e);
 
  mlir::Value VisitArrayInitIndexExpr(ArrayInitIndexExpr *e) {
    assert(cgf.getArrayInitIndex() &&
           "ArrayInitIndexExpr not inside an ArrayInitLoopExpr?");
    return cgf.getArrayInitIndex();
  }
 
  mlir::Value VisitImplicitValueInitExpr(const ImplicitValueInitExpr *e) {
    return emitNullValue(e->getType(), cgf.getLoc(e->getSourceRange()));
  }
 
  mlir::Value VisitExplicitCastExpr(ExplicitCastExpr *e) {
    return VisitCastExpr(e);
  }
 
  mlir::Value VisitCXXNullPtrLiteralExpr(CXXNullPtrLiteralExpr *e) {
    return cgf.cgm.emitNullConstant(e->getType(),
                                    cgf.getLoc(e->getSourceRange()));
  }
 
  /// Perform a pointer to boolean conversion.
  mlir::Value emitPointerToBoolConversion(mlir::Value v, QualType qt) {
    // TODO(cir): comparing the ptr to null is done when lowering CIR to LLVM.
    // We might want to have a separate pass for these types of conversions.
    return cgf.getBuilder().createPtrToBoolCast(v);
  }
 
  mlir::Value emitFloatToBoolConversion(mlir::Value src, mlir::Location loc) {
    cir::BoolType boolTy = builder.getBoolTy();
    return cir::CastOp::create(builder, loc, boolTy,
                               cir::CastKind::float_to_bool, src);
  }
 
  mlir::Value emitIntToBoolConversion(mlir::Value srcVal, mlir::Location loc) {
    // Because of the type rules of C, we often end up computing a
    // logical value, then zero extending it to int, then wanting it
    // as a logical value again.
    // TODO: optimize this common case here or leave it for later
    // CIR passes?
    cir::BoolType boolTy = builder.getBoolTy();
    return cir::CastOp::create(builder, loc, boolTy, cir::CastKind::int_to_bool,
                               srcVal);
  }
 
  /// Convert the specified expression value to a boolean (!cir.bool) truth
  /// value. This is equivalent to "Val != 0".
  mlir::Value emitConversionToBool(mlir::Value src, QualType srcType,
                                   mlir::Location loc) {
    assert(srcType.isCanonical() && "EmitScalarConversion strips typedefs");
 
    if (srcType->isRealFloatingType())
      return emitFloatToBoolConversion(src, loc);
 
    if (llvm::isa<MemberPointerType>(srcType)) {
      cgf.getCIRGenModule().errorNYI(loc, "member pointer to bool conversion");
      return builder.getFalse(loc);
    }
 
    if (srcType->isIntegerType())
      return emitIntToBoolConversion(src, loc);
 
    assert(::mlir::isa<cir::PointerType>(src.getType()));
    return emitPointerToBoolConversion(src, srcType);
  }
 
  // Emit a conversion from the specified type to the specified destination
  // type, both of which are CIR scalar types.
  struct ScalarConversionOpts {
    bool treatBooleanAsSigned;
    bool emitImplicitIntegerTruncationChecks;
    bool emitImplicitIntegerSignChangeChecks;
 
    ScalarConversionOpts()
        : treatBooleanAsSigned(false),
          emitImplicitIntegerTruncationChecks(false),
          emitImplicitIntegerSignChangeChecks(false) {}
 
    ScalarConversionOpts(clang::SanitizerSet sanOpts)
        : treatBooleanAsSigned(false),
          emitImplicitIntegerTruncationChecks(
              sanOpts.hasOneOf(SanitizerKind::ImplicitIntegerTruncation)),
          emitImplicitIntegerSignChangeChecks(
              sanOpts.has(SanitizerKind::ImplicitIntegerSignChange)) {}
  };
 
  // Conversion from bool, integral, or floating-point to integral or
  // floating-point. Conversions involving other types are handled elsewhere.
  // Conversion to bool is handled elsewhere because that's a comparison against
  // zero, not a simple cast. This handles both individual scalars and vectors.
  mlir::Value emitScalarCast(mlir::Value src, QualType srcType,
                             QualType dstType, mlir::Type srcTy,
                             mlir::Type dstTy, ScalarConversionOpts opts) {
    assert(!srcType->isMatrixType() && !dstType->isMatrixType() &&
           "Internal error: matrix types not handled by this function.");
    assert(!(mlir::isa<mlir::IntegerType>(srcTy) ||
             mlir::isa<mlir::IntegerType>(dstTy)) &&
           "Obsolete code. Don't use mlir::IntegerType with CIR.");
 
    mlir::Type fullDstTy = dstTy;
    if (mlir::isa<cir::VectorType>(srcTy) &&
        mlir::isa<cir::VectorType>(dstTy)) {
      // Use the element types of the vectors to figure out the CastKind.
      srcTy = mlir::dyn_cast<cir::VectorType>(srcTy).getElementType();
      dstTy = mlir::dyn_cast<cir::VectorType>(dstTy).getElementType();
    }
 
    std::optional<cir::CastKind> castKind;
 
    if (mlir::isa<cir::BoolType>(srcTy)) {
      if (opts.treatBooleanAsSigned)
        cgf.getCIRGenModule().errorNYI("signed bool");
      if (cgf.getBuilder().isInt(dstTy))
        castKind = cir::CastKind::bool_to_int;
      else if (mlir::isa<cir::FPTypeInterface>(dstTy))
        castKind = cir::CastKind::bool_to_float;
      else
        llvm_unreachable("Internal error: Cast to unexpected type");
    } else if (cgf.getBuilder().isInt(srcTy)) {
      if (cgf.getBuilder().isInt(dstTy))
        castKind = cir::CastKind::integral;
      else if (mlir::isa<cir::FPTypeInterface>(dstTy))
        castKind = cir::CastKind::int_to_float;
      else
        llvm_unreachable("Internal error: Cast to unexpected type");
    } else if (mlir::isa<cir::FPTypeInterface>(srcTy)) {
      if (cgf.getBuilder().isInt(dstTy)) {
        // If we can't recognize overflow as undefined behavior, assume that
        // overflow saturates. This protects against normal optimizations if we
        // are compiling with non-standard FP semantics.
        if (!cgf.cgm.getCodeGenOpts().StrictFloatCastOverflow)
          cgf.getCIRGenModule().errorNYI("strict float cast overflow");
        assert(!cir::MissingFeatures::fpConstraints());
        castKind = cir::CastKind::float_to_int;
      } else if (mlir::isa<cir::FPTypeInterface>(dstTy)) {
        // TODO: split this to createFPExt/createFPTrunc
        return builder.createFloatingCast(src, fullDstTy);
      } else {
        llvm_unreachable("Internal error: Cast to unexpected type");
      }
    } else {
      llvm_unreachable("Internal error: Cast from unexpected type");
    }
 
    assert(castKind.has_value() && "Internal error: CastKind not set.");
    return builder.createOrFold<cir::CastOp>(src.getLoc(), fullDstTy, *castKind,
                                             src);
  }
 
  mlir::Value
  VisitSubstNonTypeTemplateParmExpr(SubstNonTypeTemplateParmExpr *e) {
    return Visit(e->getReplacement());
  }
 
  mlir::Value VisitVAArgExpr(VAArgExpr *ve) {
    QualType ty = ve->getType();
 
    if (ty->isVariablyModifiedType()) {
      cgf.cgm.errorNYI(ve->getSourceRange(),
                       "variably modified types in varargs");
    }
 
    return cgf.emitVAArg(ve);
  }
 
  mlir::Value VisitCXXRewrittenBinaryOperator(CXXRewrittenBinaryOperator *e) {
    return Visit(e->getSemanticForm());
  }
 
  mlir::Value VisitUnaryExprOrTypeTraitExpr(const UnaryExprOrTypeTraitExpr *e);
  mlir::Value
  VisitAbstractConditionalOperator(const AbstractConditionalOperator *e);
 
  // Unary Operators.
  mlir::Value VisitUnaryPrePostIncDec(const UnaryOperator *e) {
    LValue lv = cgf.emitLValue(e->getSubExpr());
    return emitScalarPrePostIncDec(e, lv);
  }
  mlir::Value VisitUnaryPostDec(const UnaryOperator *e) {
    return VisitUnaryPrePostIncDec(e);
  }
  mlir::Value VisitUnaryPostInc(const UnaryOperator *e) {
    return VisitUnaryPrePostIncDec(e);
  }
  mlir::Value VisitUnaryPreDec(const UnaryOperator *e) {
    return VisitUnaryPrePostIncDec(e);
  }
  mlir::Value VisitUnaryPreInc(const UnaryOperator *e) {
    return VisitUnaryPrePostIncDec(e);
  }
  mlir::Value emitScalarPrePostIncDec(const UnaryOperator *e, LValue lv) {
    if (cgf.getLangOpts().OpenMP)
      cgf.cgm.errorNYI(e->getSourceRange(), "inc/dec OpenMP");
 
    QualType type = e->getSubExpr()->getType();
 
    mlir::Value value;
    mlir::Value input;
 
    if (type->getAs<AtomicType>()) {
      cgf.cgm.errorNYI(e->getSourceRange(), "Atomic inc/dec");
      // TODO(cir): This is not correct, but it will produce reasonable code
      // until atomic operations are implemented.
      value = cgf.emitLoadOfLValue(lv, e->getExprLoc()).getValue();
      input = value;
    } else {
      value = cgf.emitLoadOfLValue(lv, e->getExprLoc()).getValue();
      input = value;
    }
 
    // NOTE: When possible, more frequent cases are handled first.
 
    // Special case of integer increment that we have to check first: bool++.
    // Due to promotion rules, we get:
    //   bool++ -> bool = bool + 1
    //          -> bool = (int)bool + 1
    //          -> bool = ((int)bool + 1 != 0)
    // An interesting aspect of this is that increment is always true.
    // Decrement does not have this property.
    if (e->isIncrementOp() && type->isBooleanType()) {
      value = builder.getTrue(cgf.getLoc(e->getExprLoc()));
    } else if (type->isIntegerType()) {
      QualType promotedType;
      [[maybe_unused]] bool canPerformLossyDemotionCheck = false;
      if (cgf.getContext().isPromotableIntegerType(type)) {
        promotedType = cgf.getContext().getPromotedIntegerType(type);
        assert(promotedType != type && "Shouldn't promote to the same type.");
        canPerformLossyDemotionCheck = true;
        canPerformLossyDemotionCheck &=
            cgf.getContext().getCanonicalType(type) !=
            cgf.getContext().getCanonicalType(promotedType);
        canPerformLossyDemotionCheck &=
            type->isIntegerType() && promotedType->isIntegerType();
 
        // TODO(cir): Currently, we store bitwidths in CIR types only for
        // integers. This might also be required for other types.
 
        assert(
            (!canPerformLossyDemotionCheck ||
             type->isSignedIntegerOrEnumerationType() ||
             promotedType->isSignedIntegerOrEnumerationType() ||
             mlir::cast<cir::IntType>(cgf.convertType(type)).getWidth() ==
                 mlir::cast<cir::IntType>(cgf.convertType(type)).getWidth()) &&
            "The following check expects that if we do promotion to different "
            "underlying canonical type, at least one of the types (either "
            "base or promoted) will be signed, or the bitwidths will match.");
      }
 
      assert(!cir::MissingFeatures::sanitizers());
      if (e->canOverflow() && type->isSignedIntegerOrEnumerationType()) {
        value = emitIncDecConsiderOverflowBehavior(e, value);
      } else {
        // NOTE(CIR): clang calls CreateAdd but folds this to a unary op
        value = emitIncOrDec(e, input, /*nsw=*/false);
      }
    } else if (const PointerType *ptr = type->getAs<PointerType>()) {
      QualType type = ptr->getPointeeType();
      if (const VariableArrayType *vla =
              cgf.getContext().getAsVariableArrayType(type)) {
        mlir::Location loc = cgf.getLoc(e->getSourceRange());
        mlir::Value numElts = cgf.getVLASize(vla).numElts;
        if (!e->isIncrementOp())
          numElts = cgf.getBuilder().createNeg(loc, numElts, /*nsw=*/true);
        assert(!cir::MissingFeatures::sanitizers());
        value = cgf.getBuilder().createPtrStride(loc, value, numElts);
      } else {
        // For everything else, we can just do a simple increment.
        mlir::Location loc = cgf.getLoc(e->getSourceRange());
        CIRGenBuilderTy &builder = cgf.getBuilder();
        int amount = e->isIncrementOp() ? 1 : -1;
        mlir::Value amt = builder.getSInt32(amount, loc);
        assert(!cir::MissingFeatures::sanitizers());
        value = builder.createPtrStride(loc, value, amt);
      }
    } else if (type->isVectorType()) {
      cgf.cgm.errorNYI(e->getSourceRange(), "Unary inc/dec vector");
      return {};
    } else if (type->isRealFloatingType()) {
      CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, e);
 
      if (type->isHalfType() &&
          !cgf.getContext().getLangOpts().NativeHalfType) {
        cgf.cgm.errorNYI(e->getSourceRange(), "Unary inc/dec half");
        return {};
      }
 
      if (mlir::isa<cir::SingleType, cir::DoubleType, cir::LongDoubleType>(
              value.getType())) {
        mlir::Location loc = cgf.getLoc(e->getExprLoc());
        auto fpType = mlir::cast<cir::FPTypeInterface>(value.getType());
        mlir::Value amount = builder.getConstFP(
            loc, value.getType(), llvm::APFloat(fpType.getFloatSemantics(), 1));
        value = e->isIncrementOp() ? builder.createFAdd(loc, value, amount)
                                   : builder.createFSub(loc, value, amount);
      } else {
        cgf.cgm.errorNYI(e->getSourceRange(), "Unary inc/dec other fp type");
        return {};
      }
    } else if (type->isFixedPointType()) {
      cgf.cgm.errorNYI(e->getSourceRange(), "Unary inc/dec other fixed point");
      return {};
    } else {
      assert(type->castAs<ObjCObjectPointerType>());
      cgf.cgm.errorNYI(e->getSourceRange(), "Unary inc/dec ObjectiveC pointer");
      return {};
    }
 
    CIRGenFunction::SourceLocRAIIObject sourceloc{
        cgf, cgf.getLoc(e->getSourceRange())};
 
    // Store the updated result through the lvalue
    if (lv.isBitField())
      return cgf.emitStoreThroughBitfieldLValue(RValue::get(value), lv);
    else
      cgf.emitStoreThroughLValue(RValue::get(value), lv);
 
    // If this is a postinc, return the value read from memory, otherwise use
    // the updated value.
    return e->isPrefix() ? value : input;
  }
 
  mlir::Value emitIncDecConsiderOverflowBehavior(const UnaryOperator *e,
                                                 mlir::Value inVal) {
    switch (cgf.getLangOpts().getSignedOverflowBehavior()) {
    case LangOptions::SOB_Defined:
      return emitIncOrDec(e, inVal, /*nsw=*/false);
    case LangOptions::SOB_Undefined:
      assert(!cir::MissingFeatures::sanitizers());
      return emitIncOrDec(e, inVal, /*nsw=*/true);
    case LangOptions::SOB_Trapping:
      if (!e->canOverflow())
        return emitIncOrDec(e, inVal, /*nsw=*/true);
      cgf.cgm.errorNYI(e->getSourceRange(), "inc/def overflow SOB_Trapping");
      return {};
    }
    llvm_unreachable("Unexpected signed overflow behavior kind");
  }
 
  mlir::Value VisitUnaryAddrOf(const UnaryOperator *e) {
    if (llvm::isa<MemberPointerType>(e->getType()))
      return cgf.cgm.emitMemberPointerConstant(e);
 
    return cgf.emitLValue(e->getSubExpr()).getPointer();
  }
 
  mlir::Value VisitUnaryDeref(const UnaryOperator *e) {
    if (e->getType()->isVoidType())
      return Visit(e->getSubExpr()); // the actual value should be unused
    return emitLoadOfLValue(e);
  }
 
  mlir::Value VisitUnaryPlus(const UnaryOperator *e) {
    QualType promotionType = getPromotionType(e->getSubExpr()->getType());
    mlir::Value result = VisitUnaryPlus(e, promotionType);
    if (result && !promotionType.isNull())
      return emitUnPromotedValue(result, e->getType());
    return result;
  }
 
  mlir::Value VisitUnaryPlus(const UnaryOperator *e, QualType promotionType) {
    ignoreResultAssign = false;
    if (!promotionType.isNull())
      return cgf.emitPromotedScalarExpr(e->getSubExpr(), promotionType);
    return Visit(e->getSubExpr());
  }
 
  mlir::Value VisitUnaryMinus(const UnaryOperator *e) {
    QualType promotionType = getPromotionType(e->getSubExpr()->getType());
    mlir::Value result = VisitUnaryMinus(e, promotionType);
    if (result && !promotionType.isNull())
      return emitUnPromotedValue(result, e->getType());
    return result;
  }
 
  mlir::Value VisitUnaryMinus(const UnaryOperator *e, QualType promotionType) {
    ignoreResultAssign = false;
    mlir::Value operand;
    if (!promotionType.isNull())
      operand = cgf.emitPromotedScalarExpr(e->getSubExpr(), promotionType);
    else
      operand = Visit(e->getSubExpr());
 
    mlir::Location loc = cgf.getLoc(e->getSourceRange().getBegin());
 
    if (cir::isFPOrVectorOfFPType(operand.getType()))
      return builder.createOrFold<cir::FNegOp>(loc, operand);
 
    // TODO(cir): We might have to change this to support overflow trapping.
    //            Classic codegen routes unary minus through emitSub to ensure
    //            that the overflow behavior is handled correctly.
    bool nsw = e->getType()->isSignedIntegerType() &&
               cgf.getLangOpts().getSignedOverflowBehavior() !=
                   LangOptions::SOB_Defined;
 
    return builder.createOrFold<cir::MinusOp>(loc, operand, nsw);
  }
 
  mlir::Value emitIncOrDec(const UnaryOperator *e, mlir::Value input,
                           bool nsw = false) {
    mlir::Location loc = cgf.getLoc(e->getSourceRange().getBegin());
    return e->isIncrementOp()
               ? builder.createOrFold<cir::IncOp>(loc, input, nsw)
               : builder.createOrFold<cir::DecOp>(loc, input, nsw);
  }
 
  mlir::Value VisitUnaryNot(const UnaryOperator *e) {
    ignoreResultAssign = false;
    mlir::Value op = Visit(e->getSubExpr());
    return builder.createOrFold<cir::NotOp>(
        cgf.getLoc(e->getSourceRange().getBegin()), op);
  }
 
  mlir::Value VisitUnaryLNot(const UnaryOperator *e);
 
  mlir::Value VisitUnaryReal(const UnaryOperator *e);
  mlir::Value VisitUnaryImag(const UnaryOperator *e);
  mlir::Value VisitRealImag(const UnaryOperator *e,
                            QualType promotionType = QualType());
 
  mlir::Value VisitUnaryExtension(const UnaryOperator *e) {
    return Visit(e->getSubExpr());
  }
 
  // C++
  mlir::Value VisitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *e) {
    cgf.cgm.errorNYI(e->getSourceRange(),
                     "ScalarExprEmitter: materialize temporary");
    return {};
  }
  mlir::Value VisitSourceLocExpr(SourceLocExpr *e) {
    ASTContext &ctx = cgf.getContext();
    APValue evaluated =
        e->EvaluateInContext(ctx, cgf.curSourceLocExprScope.getDefaultExpr());
    mlir::Attribute attribute = ConstantEmitter(cgf).emitAbstract(
        e->getLocation(), evaluated, e->getType());
    mlir::TypedAttr typedAttr = mlir::cast<mlir::TypedAttr>(attribute);
    return cir::ConstantOp::create(builder, cgf.getLoc(e->getExprLoc()),
                                   typedAttr);
  }
  mlir::Value VisitCXXDefaultArgExpr(CXXDefaultArgExpr *dae) {
    CIRGenFunction::CXXDefaultArgExprScope scope(cgf, dae);
    return Visit(dae->getExpr());
  }
  mlir::Value VisitCXXDefaultInitExpr(CXXDefaultInitExpr *die) {
    CIRGenFunction::CXXDefaultInitExprScope scope(cgf, die);
    return Visit(die->getExpr());
  }
 
  mlir::Value VisitCXXThisExpr(CXXThisExpr *te) { return cgf.loadCXXThis(); }
 
  mlir::Value VisitExprWithCleanups(ExprWithCleanups *e);
  mlir::Value VisitCXXNewExpr(const CXXNewExpr *e) {
    return cgf.emitCXXNewExpr(e);
  }
  mlir::Value VisitCXXDeleteExpr(const CXXDeleteExpr *e) {
    cgf.emitCXXDeleteExpr(e);
    return {};
  }
  mlir::Value VisitTypeTraitExpr(const TypeTraitExpr *e) {
    // We diverge slightly from classic codegen here because CIR has stricter
    // typing. In LLVM IR, constant folding covers up some potential type
    // mismatches such as bool-to-int conversions that would fail the verifier
    // in CIR. To make things work, we need to be sure we only emit a bool value
    // if the expression type is bool.
    mlir::Location loc = cgf.getLoc(e->getExprLoc());
    if (e->isStoredAsBoolean()) {
      if (e->getType()->isBooleanType())
        return builder.getBool(e->getBoolValue(), loc);
      assert(e->getType()->isIntegerType() &&
             "Expected int type for TypeTraitExpr");
      return builder.getConstInt(loc, cgf.convertType(e->getType()),
                                 (uint64_t)e->getBoolValue());
    }
    return builder.getConstInt(loc, e->getAPValue().getInt());
  }
  mlir::Value
  VisitConceptSpecializationExpr(const ConceptSpecializationExpr *e) {
    return builder.getBool(e->isSatisfied(), cgf.getLoc(e->getExprLoc()));
  }
  mlir::Value VisitRequiresExpr(const RequiresExpr *e) {
    return builder.getBool(e->isSatisfied(), cgf.getLoc(e->getExprLoc()));
  }
  mlir::Value VisitArrayTypeTraitExpr(const ArrayTypeTraitExpr *e) {
    mlir::Type type = cgf.convertType(e->getType());
    mlir::Location loc = cgf.getLoc(e->getExprLoc());
    return builder.getConstInt(loc, type, e->getValue());
  }
  mlir::Value VisitExpressionTraitExpr(const ExpressionTraitExpr *e) {
    return builder.getBool(e->getValue(), cgf.getLoc(e->getExprLoc()));
  }
  mlir::Value VisitCXXPseudoDestructorExpr(const CXXPseudoDestructorExpr *e) {
    cgf.cgm.errorNYI(e->getSourceRange(),
                     "ScalarExprEmitter: cxx pseudo destructor");
    return {};
  }
  mlir::Value VisitCXXThrowExpr(const CXXThrowExpr *e) {
    cgf.emitCXXThrowExpr(e);
    return {};
  }
 
  mlir::Value VisitCXXNoexceptExpr(CXXNoexceptExpr *e) {
    return builder.getBool(e->getValue(), cgf.getLoc(e->getExprLoc()));
  }
 
  /// Emit a conversion from the specified type to the specified destination
  /// type, both of which are CIR scalar types.
  /// TODO: do we need ScalarConversionOpts here? Should be done in another
  /// pass.
  mlir::Value
  emitScalarConversion(mlir::Value src, QualType srcType, QualType dstType,
                       SourceLocation loc,
                       ScalarConversionOpts opts = ScalarConversionOpts()) {
    // All conversions involving fixed point types should be handled by the
    // emitFixedPoint family functions. This is done to prevent bloating up
    // this function more, and although fixed point numbers are represented by
    // integers, we do not want to follow any logic that assumes they should be
    // treated as integers.
    // TODO(leonardchan): When necessary, add another if statement checking for
    // conversions to fixed point types from other types.
    // conversions to fixed point types from other types.
    if (srcType->isFixedPointType() || dstType->isFixedPointType()) {
      cgf.getCIRGenModule().errorNYI(loc, "fixed point conversions");
      return {};
    }
 
    srcType = srcType.getCanonicalType();
    dstType = dstType.getCanonicalType();
    if (srcType == dstType) {
      if (opts.emitImplicitIntegerSignChangeChecks)
        cgf.getCIRGenModule().errorNYI(loc,
                                       "implicit integer sign change checks");
      return src;
    }
 
    if (dstType->isVoidType())
      return {};
 
    mlir::Type mlirSrcType = src.getType();
 
    // Handle conversions to bool first, they are special: comparisons against
    // 0.
    if (dstType->isBooleanType())
      return emitConversionToBool(src, srcType, cgf.getLoc(loc));
 
    mlir::Type mlirDstType = cgf.convertType(dstType);
 
    if (srcType->isHalfType() &&
        !cgf.getContext().getLangOpts().NativeHalfType) {
      // Cast to FP using the intrinsic if the half type itself isn't supported.
      if (mlir::isa<cir::FPTypeInterface>(mlirDstType)) {
        if (cgf.getContext().getTargetInfo().useFP16ConversionIntrinsics())
          cgf.getCIRGenModule().errorNYI(loc,
                                         "cast via llvm.convert.from.fp16");
      } else {
        // Cast to other types through float, using either the intrinsic or
        // FPExt, depending on whether the half type itself is supported (as
        // opposed to operations on half, available with NativeHalfType).
        if (cgf.getContext().getTargetInfo().useFP16ConversionIntrinsics())
          cgf.getCIRGenModule().errorNYI(loc,
                                         "cast via llvm.convert.from.fp16");
        // FIXME(cir): For now lets pretend we shouldn't use the conversion
        // intrinsics and insert a cast here unconditionally.
        src = builder.createCast(cgf.getLoc(loc), cir::CastKind::floating, src,
                                 cgf.floatTy);
        srcType = cgf.getContext().FloatTy;
        mlirSrcType = cgf.floatTy;
      }
    }
 
    // TODO(cir): LLVM codegen ignore conversions like int -> uint,
    // is there anything to be done for CIR here?
    if (mlirSrcType == mlirDstType) {
      if (opts.emitImplicitIntegerSignChangeChecks)
        cgf.getCIRGenModule().errorNYI(loc,
                                       "implicit integer sign change checks");
      return src;
    }
 
    // Handle pointer conversions next: pointers can only be converted to/from
    // other pointers and integers. Check for pointer types in terms of LLVM, as
    // some native types (like Obj-C id) may map to a pointer type.
    if (auto dstPT = dyn_cast<cir::PointerType>(mlirDstType)) {
      cgf.getCIRGenModule().errorNYI(loc, "pointer casts");
      return builder.getNullPtr(dstPT, src.getLoc());
    }
 
    if (isa<cir::PointerType>(mlirSrcType)) {
      // Must be an ptr to int cast.
      assert(isa<cir::IntType>(mlirDstType) && "not ptr->int?");
      return builder.createPtrToInt(src, mlirDstType);
    }
 
    // A scalar can be splatted to an extended vector of the same element type
    if (dstType->isExtVectorType() && !srcType->isVectorType()) {
      // Sema should add casts to make sure that the source expression's type
      // is the same as the vector's element type (sans qualifiers)
      assert(dstType->castAs<ExtVectorType>()->getElementType().getTypePtr() ==
                 srcType.getTypePtr() &&
             "Splatted expr doesn't match with vector element type?");
 
      cgf.getCIRGenModule().errorNYI(loc, "vector splatting");
      return {};
    }
 
    if (srcType->isMatrixType() && dstType->isMatrixType()) {
      cgf.getCIRGenModule().errorNYI(loc,
                                     "matrix type to matrix type conversion");
      return {};
    }
    assert(!srcType->isMatrixType() && !dstType->isMatrixType() &&
           "Internal error: conversion between matrix type and scalar type");
 
    // Finally, we have the arithmetic types or vectors of arithmetic types.
    mlir::Value res = nullptr;
    mlir::Type resTy = mlirDstType;
 
    res = emitScalarCast(src, srcType, dstType, mlirSrcType, mlirDstType, opts);
 
    if (mlirDstType != resTy) {
      if (cgf.getContext().getTargetInfo().useFP16ConversionIntrinsics()) {
        cgf.getCIRGenModule().errorNYI(loc, "cast via llvm.convert.to.fp16");
      }
      // FIXME(cir): For now we never use FP16 conversion intrinsics even if
      // required by the target. Change that once this is implemented
      res = builder.createCast(cgf.getLoc(loc), cir::CastKind::floating, res,
                               resTy);
    }
 
    if (opts.emitImplicitIntegerTruncationChecks)
      cgf.getCIRGenModule().errorNYI(loc, "implicit integer truncation checks");
 
    if (opts.emitImplicitIntegerSignChangeChecks)
      cgf.getCIRGenModule().errorNYI(loc,
                                     "implicit integer sign change checks");
 
    return res;
  }
 
  BinOpInfo emitBinOps(const BinaryOperator *e,
                       QualType promotionType = QualType()) {
    ignoreResultAssign = false;
    BinOpInfo result;
    result.lhs = cgf.emitPromotedScalarExpr(e->getLHS(), promotionType);
    result.rhs = cgf.emitPromotedScalarExpr(e->getRHS(), promotionType);
    if (!promotionType.isNull())
      result.fullType = promotionType;
    else
      result.fullType = e->getType();
    result.compType = result.fullType;
    if (const auto *vecType = result.fullType->getAs<VectorType>())
      result.compType = vecType->getElementType();
    result.opcode = e->getOpcode();
    result.loc = e->getSourceRange();
    // TODO(cir): Result.FPFeatures
    CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, e);
    result.e = e;
    return result;
  }
 
  mlir::Value emitMul(const BinOpInfo &ops);
  mlir::Value emitDiv(const BinOpInfo &ops);
  mlir::Value emitRem(const BinOpInfo &ops);
  mlir::Value emitAdd(const BinOpInfo &ops);
  mlir::Value emitSub(const BinOpInfo &ops);
  mlir::Value emitShl(const BinOpInfo &ops);
  mlir::Value emitShr(const BinOpInfo &ops);
  mlir::Value emitAnd(const BinOpInfo &ops);
  mlir::Value emitXor(const BinOpInfo &ops);
  mlir::Value emitOr(const BinOpInfo &ops);
 
  LValue emitCompoundAssignLValue(
      const CompoundAssignOperator *e,
      mlir::Value (ScalarExprEmitter::*f)(const BinOpInfo &),
      mlir::Value &result);
  mlir::Value
  emitCompoundAssign(const CompoundAssignOperator *e,
                     mlir::Value (ScalarExprEmitter::*f)(const BinOpInfo &));
 
  // TODO(cir): Candidate to be in a common AST helper between CIR and LLVM
  // codegen.
  QualType getPromotionType(QualType ty) {
    const clang::ASTContext &ctx = cgf.getContext();
    if (auto *complexTy = ty->getAs<ComplexType>()) {
      QualType elementTy = complexTy->getElementType();
      if (elementTy.UseExcessPrecision(ctx))
        return ctx.getComplexType(ctx.FloatTy);
    }
 
    if (ty.UseExcessPrecision(cgf.getContext())) {
      if (auto *vt = ty->getAs<VectorType>()) {
        unsigned numElements = vt->getNumElements();
        return ctx.getVectorType(ctx.FloatTy, numElements, vt->getVectorKind());
      }
      return cgf.getContext().FloatTy;
    }
 
    return QualType();
  }
 
// Binary operators and binary compound assignment operators.
#define HANDLEBINOP(OP)                                                        \
  mlir::Value VisitBin##OP(const BinaryOperator *e) {                          \
    QualType promotionTy = getPromotionType(e->getType());                     \
    auto result = emit##OP(emitBinOps(e, promotionTy));                        \
    if (result && !promotionTy.isNull())                                       \
      result = emitUnPromotedValue(result, e->getType());                      \
    return result;                                                             \
  }                                                                            \
  mlir::Value VisitBin##OP##Assign(const CompoundAssignOperator *e) {          \
    return emitCompoundAssign(e, &ScalarExprEmitter::emit##OP);                \
  }
 
  HANDLEBINOP(Mul)
  HANDLEBINOP(Div)
  HANDLEBINOP(Rem)
  HANDLEBINOP(Add)
  HANDLEBINOP(Sub)
  HANDLEBINOP(Shl)
  HANDLEBINOP(Shr)
  HANDLEBINOP(And)
  HANDLEBINOP(Xor)
  HANDLEBINOP(Or)
#undef HANDLEBINOP
 
  mlir::Value emitCmp(const BinaryOperator *e) {
    ignoreResultAssign = false;
    const mlir::Location loc = cgf.getLoc(e->getExprLoc());
    mlir::Value result;
    QualType lhsTy = e->getLHS()->getType();
    QualType rhsTy = e->getRHS()->getType();
 
    auto clangCmpToCIRCmp =
        [](clang::BinaryOperatorKind clangCmp) -> cir::CmpOpKind {
      switch (clangCmp) {
      case BO_LT:
        return cir::CmpOpKind::lt;
      case BO_GT:
        return cir::CmpOpKind::gt;
      case BO_LE:
        return cir::CmpOpKind::le;
      case BO_GE:
        return cir::CmpOpKind::ge;
      case BO_EQ:
        return cir::CmpOpKind::eq;
      case BO_NE:
        return cir::CmpOpKind::ne;
      default:
        llvm_unreachable("unsupported comparison kind for cir.cmp");
      }
    };
 
    cir::CmpOpKind kind = clangCmpToCIRCmp(e->getOpcode());
    if (lhsTy->getAs<MemberPointerType>()) {
      assert(!cir::MissingFeatures::dataMemberType());
      assert(e->getOpcode() == BO_EQ || e->getOpcode() == BO_NE);
      mlir::Value lhs = cgf.emitScalarExpr(e->getLHS());
      mlir::Value rhs = cgf.emitScalarExpr(e->getRHS());
      result = builder.createCompare(loc, kind, lhs, rhs);
    } else if (!lhsTy->isAnyComplexType() && !rhsTy->isAnyComplexType()) {
      BinOpInfo boInfo = emitBinOps(e);
      mlir::Value lhs = boInfo.lhs;
      mlir::Value rhs = boInfo.rhs;
 
      if (lhsTy->isVectorType()) {
        if (!e->getType()->isVectorType()) {
          // If AltiVec, the comparison results in a numeric type, so we use
          // intrinsics comparing vectors and giving 0 or 1 as a result
          cgf.cgm.errorNYI(loc, "AltiVec comparison");
        } else {
          // Other kinds of vectors. Element-wise comparison returning
          // a vector.
          result = cir::VecCmpOp::create(builder, cgf.getLoc(boInfo.loc),
                                         cgf.convertType(boInfo.fullType), kind,
                                         boInfo.lhs, boInfo.rhs);
        }
      } else if (boInfo.isFixedPointOp()) {
        assert(!cir::MissingFeatures::fixedPointType());
        cgf.cgm.errorNYI(loc, "fixed point comparisons");
        result = builder.getBool(false, loc);
      } else {
        // integers and pointers
        if (cgf.cgm.getCodeGenOpts().StrictVTablePointers &&
            mlir::isa<cir::PointerType>(lhs.getType()) &&
            mlir::isa<cir::PointerType>(rhs.getType())) {
          cgf.cgm.errorNYI(loc, "strict vtable pointer comparisons");
        }
        result = builder.createCompare(loc, kind, lhs, rhs);
      }
    } else {
      assert((e->getOpcode() == BO_EQ || e->getOpcode() == BO_NE) &&
             "Complex Comparison: can only be an equality comparison");
 
      mlir::Value lhs;
      if (lhsTy->isAnyComplexType()) {
        lhs = cgf.emitComplexExpr(e->getLHS());
      } else {
        mlir::Value lhsReal = Visit(e->getLHS());
        mlir::Value lhsImag = builder.getNullValue(convertType(lhsTy), loc);
        lhs = builder.createComplexCreate(loc, lhsReal, lhsImag);
      }
 
      mlir::Value rhs;
      if (rhsTy->isAnyComplexType()) {
        rhs = cgf.emitComplexExpr(e->getRHS());
      } else {
        mlir::Value rhsReal = Visit(e->getRHS());
        mlir::Value rhsImag = builder.getNullValue(convertType(rhsTy), loc);
        rhs = builder.createComplexCreate(loc, rhsReal, rhsImag);
      }
 
      result = builder.createCompare(loc, kind, lhs, rhs);
    }
 
    return emitScalarConversion(result, cgf.getContext().BoolTy, e->getType(),
                                e->getExprLoc());
  }
 
// Comparisons.
#define VISITCOMP(CODE)                                                        \
  mlir::Value VisitBin##CODE(const BinaryOperator *E) { return emitCmp(E); }
  VISITCOMP(LT)
  VISITCOMP(GT)
  VISITCOMP(LE)
  VISITCOMP(GE)
  VISITCOMP(EQ)
  VISITCOMP(NE)
#undef VISITCOMP
 
  mlir::Value VisitBinAssign(const BinaryOperator *e) {
    const bool ignore = std::exchange(ignoreResultAssign, false);
 
    mlir::Value rhs;
    LValue lhs;
 
    switch (e->getLHS()->getType().getObjCLifetime()) {
    case Qualifiers::OCL_Strong:
    case Qualifiers::OCL_Autoreleasing:
    case Qualifiers::OCL_ExplicitNone:
    case Qualifiers::OCL_Weak:
      assert(!cir::MissingFeatures::objCLifetime());
      break;
    case Qualifiers::OCL_None:
      // __block variables need to have the rhs evaluated first, plus this
      // should improve codegen just a little.
      rhs = Visit(e->getRHS());
      assert(!cir::MissingFeatures::sanitizers());
      // TODO(cir): This needs to be emitCheckedLValue() once we support
      // sanitizers
      lhs = cgf.emitLValue(e->getLHS());
 
      // Store the value into the LHS. Bit-fields are handled specially because
      // the result is altered by the store, i.e., [C99 6.5.16p1]
      // 'An assignment expression has the value of the left operand after the
      // assignment...'.
      if (lhs.isBitField()) {
        CIRGenFunction::SourceLocRAIIObject loc{
            cgf, cgf.getLoc(e->getSourceRange())};
        rhs = cgf.emitStoreThroughBitfieldLValue(RValue::get(rhs), lhs);
      } else {
        cgf.emitNullabilityCheck(lhs, rhs, e->getExprLoc());
        CIRGenFunction::SourceLocRAIIObject loc{
            cgf, cgf.getLoc(e->getSourceRange())};
        cgf.emitStoreThroughLValue(RValue::get(rhs), lhs);
      }
    }
 
    // If the result is clearly ignored, return now.
    if (ignore)
      return nullptr;
 
    // The result of an assignment in C is the assigned r-value.
    if (!cgf.getLangOpts().CPlusPlus)
      return rhs;
 
    // If the lvalue is non-volatile, return the computed value of the
    // assignment.
    if (!lhs.isVolatile())
      return rhs;
 
    // Otherwise, reload the value.
    return emitLoadOfLValue(lhs, e->getExprLoc());
  }
 
  mlir::Value VisitBinComma(const BinaryOperator *e) {
    cgf.emitIgnoredExpr(e->getLHS());
    // NOTE: We don't need to EnsureInsertPoint() like LLVM codegen.
    return Visit(e->getRHS());
  }
 
  mlir::Value VisitBinLAnd(const clang::BinaryOperator *e) {
    if (e->getType()->isVectorType()) {
      mlir::Location loc = cgf.getLoc(e->getExprLoc());
      mlir::Type lhsTy = cgf.convertType(e->getLHS()->getType());
      mlir::Value zeroVec = builder.getNullValue(lhsTy, loc);
 
      mlir::Value lhs = Visit(e->getLHS());
      mlir::Value rhs = Visit(e->getRHS());
 
      auto cmpOpKind = cir::CmpOpKind::ne;
      mlir::Type resTy = cgf.convertType(e->getType());
      lhs = cir::VecCmpOp::create(builder, loc, resTy, cmpOpKind, lhs, zeroVec);
      rhs = cir::VecCmpOp::create(builder, loc, resTy, cmpOpKind, rhs, zeroVec);
      mlir::Value vecOr = builder.createAnd(loc, lhs, rhs);
      return builder.createIntCast(vecOr, resTy);
    }
 
    assert(!cir::MissingFeatures::instrumentation());
    mlir::Type resTy = cgf.convertType(e->getType());
    mlir::Location loc = cgf.getLoc(e->getExprLoc());
 
    CIRGenFunction::ConditionalEvaluation eval(cgf);
 
    mlir::Value lhsCondV = cgf.evaluateExprAsBool(e->getLHS());
    auto resOp = cir::TernaryOp::create(
        builder, loc, lhsCondV, /*trueBuilder=*/
        [&](mlir::OpBuilder &b, mlir::Location loc) {
          CIRGenFunction::LexicalScope lexScope{cgf, loc,
                                                b.getInsertionBlock()};
          cgf.curLexScope->setAsTernary();
          mlir::Value res = cgf.evaluateExprAsBool(e->getRHS());
          lexScope.forceCleanup({&res});
          cir::YieldOp::create(b, loc, res);
        },
        /*falseBuilder*/
        [&](mlir::OpBuilder &b, mlir::Location loc) {
          CIRGenFunction::LexicalScope lexScope{cgf, loc,
                                                b.getInsertionBlock()};
          cgf.curLexScope->setAsTernary();
          auto res = cir::ConstantOp::create(b, loc, builder.getFalseAttr());
          cir::YieldOp::create(b, loc, res.getRes());
        });
    return maybePromoteBoolResult(resOp.getResult(), resTy);
  }
 
  mlir::Value VisitBinLOr(const clang::BinaryOperator *e) {
    if (e->getType()->isVectorType()) {
      mlir::Location loc = cgf.getLoc(e->getExprLoc());
      mlir::Type lhsTy = cgf.convertType(e->getLHS()->getType());
      mlir::Value zeroVec = builder.getNullValue(lhsTy, loc);
 
      mlir::Value lhs = Visit(e->getLHS());
      mlir::Value rhs = Visit(e->getRHS());
 
      auto cmpOpKind = cir::CmpOpKind::ne;
      mlir::Type resTy = cgf.convertType(e->getType());
      lhs = cir::VecCmpOp::create(builder, loc, resTy, cmpOpKind, lhs, zeroVec);
      rhs = cir::VecCmpOp::create(builder, loc, resTy, cmpOpKind, rhs, zeroVec);
      mlir::Value vecOr = builder.createOr(loc, lhs, rhs);
      return builder.createIntCast(vecOr, resTy);
    }
 
    assert(!cir::MissingFeatures::instrumentation());
    mlir::Type resTy = cgf.convertType(e->getType());
    mlir::Location loc = cgf.getLoc(e->getExprLoc());
 
    CIRGenFunction::ConditionalEvaluation eval(cgf);
 
    mlir::Value lhsCondV = cgf.evaluateExprAsBool(e->getLHS());
    auto resOp = cir::TernaryOp::create(
        builder, loc, lhsCondV, /*trueBuilder=*/
        [&](mlir::OpBuilder &b, mlir::Location loc) {
          CIRGenFunction::LexicalScope lexScope{cgf, loc,
                                                b.getInsertionBlock()};
          cgf.curLexScope->setAsTernary();
          auto res = cir::ConstantOp::create(b, loc, builder.getTrueAttr());
          cir::YieldOp::create(b, loc, res.getRes());
        },
        /*falseBuilder*/
        [&](mlir::OpBuilder &b, mlir::Location loc) {
          CIRGenFunction::LexicalScope lexScope{cgf, loc,
                                                b.getInsertionBlock()};
          cgf.curLexScope->setAsTernary();
          mlir::Value res = cgf.evaluateExprAsBool(e->getRHS());
          lexScope.forceCleanup({&res});
          cir::YieldOp::create(b, loc, res);
        });
 
    return maybePromoteBoolResult(resOp.getResult(), resTy);
  }
 
  mlir::Value VisitBinPtrMemD(const BinaryOperator *e) {
    return emitLoadOfLValue(e);
  }
 
  mlir::Value VisitBinPtrMemI(const BinaryOperator *e) {
    return emitLoadOfLValue(e);
  }
 
  // Other Operators.
  mlir::Value VisitBlockExpr(const BlockExpr *e) {
    cgf.cgm.errorNYI(e->getSourceRange(), "ScalarExprEmitter: block");
    return {};
  }
 
  mlir::Value VisitChooseExpr(ChooseExpr *e) {
    return Visit(e->getChosenSubExpr());
  }
 
  mlir::Value VisitObjCStringLiteral(const ObjCStringLiteral *e) {
    cgf.cgm.errorNYI(e->getSourceRange(),
                     "ScalarExprEmitter: objc string literal");
    return {};
  }
  mlir::Value VisitObjCBoxedExpr(ObjCBoxedExpr *e) {
    cgf.cgm.errorNYI(e->getSourceRange(), "ScalarExprEmitter: objc boxed");
    return {};
  }
  mlir::Value VisitObjCArrayLiteral(ObjCArrayLiteral *e) {
    cgf.cgm.errorNYI(e->getSourceRange(),
                     "ScalarExprEmitter: objc array literal");
    return {};
  }
  mlir::Value VisitObjCDictionaryLiteral(ObjCDictionaryLiteral *e) {
    cgf.cgm.errorNYI(e->getSourceRange(),
                     "ScalarExprEmitter: objc dictionary literal");
    return {};
  }
 
  mlir::Value convertVec3AndVec4(CIRGenBuilderTy &builder, mlir::Location loc,
                                 mlir::Value src, unsigned numElementsDst) {
    static constexpr int64_t mask[] = {0, 1, 2, -1};
    return builder.createVecShuffle(
        loc, src, llvm::ArrayRef<int64_t>(mask, numElementsDst));
  }
 
  // Create cast instructions for converting MLIR value \p Src to MLIR type \p
  // DstTy. \p Src has the same size as \p DstTy. Both are single value types
  // but could be scalar or vectors of different lengths, and either can be
  // pointer.
  //
  // There are 4 cases:
  // 1. non-pointer -> non-pointer  : needs 1 bitcast
  // 2. pointer -> pointer          : needs 1 bitcast or addrspacecast
  // 3. pointer -> non-pointer
  //   a) pointer -> intptr_t       : needs 1 ptrtoint
  //   b) pointer -> non-intptr_t   : needs 1 ptrtoint then 1 bitcast
  // 4. non-pointer -> pointer
  //   a) intptr_t -> pointer       : needs 1 inttoptr
  //   b) non-intptr_t -> pointer   : needs 1 bitcast then 1 inttoptr
  //
  // Note: for cases 3b and 4b two casts are required since LLVM casts do not
  // allow casting directly between pointer types and non-integer non-pointer
  // types.
  mlir::Value createCastsForTypeOfSameSize(mlir::Value src, mlir::Type dstTy) {
    mlir::Type srcTy = src.getType();
 
    // Case 1.
    if (!isa<cir::PointerType>(srcTy) && !isa<cir::PointerType>(dstTy))
      return builder.createBitcast(src, dstTy);
 
    // Case 2.
    if (isa<cir::PointerType>(srcTy) && isa<cir::PointerType>(dstTy)) {
      cgf.cgm.errorNYI(
          "ScalarExprEmitter: createCastsForTypeOfSameSize Case 2");
      return {};
    }
 
    // Case 3.
    if (isa<cir::PointerType>(srcTy) && !isa<cir::PointerType>(dstTy)) {
      if (!isa<cir::IntType>(dstTy)) {
        cgf.cgm.errorNYI(
            "ScalarExprEmitter: createCastsForTypeOfSameSize Case 3a");
      }
 
      cgf.cgm.errorNYI(
          "ScalarExprEmitter: createCastsForTypeOfSameSize Case 3a and 3b");
      return {};
    }
 
    // Case 4b.
    if (!isa<cir::IntType>(srcTy)) {
      cgf.cgm.errorNYI(
          "ScalarExprEmitter: createCastsForTypeOfSameSize Case 4a");
      return {};
    }
    // Cases 4a and 4b.
    return builder.createIntToPtr(src, dstTy);
  }
 
  mlir::Value VisitAsTypeExpr(AsTypeExpr *e) {
    mlir::Value src = cgf.emitScalarExpr(e->getSrcExpr());
    mlir::Type srcTy = src.getType();
    mlir::Type dstTy = cgf.convertType(e->getType());
 
    unsigned numElementsSrc = isa<cir::VectorType>(srcTy)
                                  ? cast<cir::VectorType>(srcTy).getSize()
                                  : 0;
    unsigned numElementsDst = isa<cir::VectorType>(dstTy)
                                  ? cast<cir::VectorType>(dstTy).getSize()
                                  : 0;
 
    // Use bit vector expansion for ext_vector_type boolean vectors.
    if (e->getType()->isExtVectorBoolType()) {
      cgf.cgm.errorNYI(e->getSourceRange(),
                       "ScalarExprEmitter: VisitAsTypeExpr ExtVectorBoolType");
      return {};
    }
 
    // Going from vec3 to non-vec3 is a special case and requires a shuffle
    // vector to get a vec4, then a bitcast if the target type is different.
    if (numElementsSrc == 3 && numElementsDst != 3) {
      cgf.cgm.errorNYI(e->getSourceRange(),
                       "ScalarExprEmitter: VisitAsTypeExpr numElemsSrc = 3, "
                       "numElemsDst != 3");
      return {};
    }
 
    // Going from non-vec3 to vec3 is a special case and requires a bitcast
    // to vec4 if the original type is not vec4, then a shuffle vector to
    // get a vec3.
    if (numElementsSrc != 3 && numElementsDst == 3) {
      mlir::Location loc = cgf.getLoc(e->getExprLoc());
      auto dstElemTy = cast<cir::VectorType>(dstTy).getElementType();
      auto dstVec4Ty = cir::VectorType::get(dstElemTy, 4);
      src = createCastsForTypeOfSameSize(src, dstVec4Ty);
      src = convertVec3AndVec4(builder, loc, src, 3);
      return src;
    }
 
    return createCastsForTypeOfSameSize(src, dstTy);
  }
 
  mlir::Value VisitAtomicExpr(AtomicExpr *e) {
    return cgf.emitAtomicExpr(e).getValue();
  }
};
 
LValue ScalarExprEmitter::emitCompoundAssignLValue(
    const CompoundAssignOperator *e,
    mlir::Value (ScalarExprEmitter::*func)(const BinOpInfo &),
    mlir::Value &result) {
  if (e->getComputationResultType()->isAnyComplexType())
    return cgf.emitScalarCompoundAssignWithComplex(e, result);
 
  QualType lhsTy = e->getLHS()->getType();
  BinOpInfo opInfo;
 
  // Emit the RHS first.  __block variables need to have the rhs evaluated
  // first, plus this should improve codegen a little.
 
  QualType promotionTypeCR = getPromotionType(e->getComputationResultType());
  if (promotionTypeCR.isNull())
    promotionTypeCR = e->getComputationResultType();
 
  QualType promotionTypeLHS = getPromotionType(e->getComputationLHSType());
  QualType promotionTypeRHS = getPromotionType(e->getRHS()->getType());
 
  if (!promotionTypeRHS.isNull())
    opInfo.rhs = cgf.emitPromotedScalarExpr(e->getRHS(), promotionTypeRHS);
  else
    opInfo.rhs = Visit(e->getRHS());
 
  opInfo.fullType = promotionTypeCR;
  opInfo.compType = opInfo.fullType;
  if (const auto *vecType = opInfo.fullType->getAs<VectorType>())
    opInfo.compType = vecType->getElementType();
  opInfo.opcode = e->getOpcode();
  opInfo.fpFeatures = e->getFPFeaturesInEffect(cgf.getLangOpts());
  opInfo.e = e;
  opInfo.loc = e->getSourceRange();
 
  // Load/convert the LHS
  LValue lhsLV = cgf.emitLValue(e->getLHS());
 
  if (lhsTy->getAs<AtomicType>()) {
    cgf.cgm.errorNYI(result.getLoc(), "atomic lvalue assign");
    return LValue();
  }
 
  opInfo.lhs = emitLoadOfLValue(lhsLV, e->getExprLoc());
 
  CIRGenFunction::SourceLocRAIIObject sourceloc{
      cgf, cgf.getLoc(e->getSourceRange())};
  SourceLocation loc = e->getExprLoc();
  if (!promotionTypeLHS.isNull())
    opInfo.lhs = emitScalarConversion(opInfo.lhs, lhsTy, promotionTypeLHS, loc);
  else
    opInfo.lhs = emitScalarConversion(opInfo.lhs, lhsTy,
                                      e->getComputationLHSType(), loc);
 
  // Expand the binary operator.
  result = (this->*func)(opInfo);
 
  // Convert the result back to the LHS type,
  // potentially with Implicit Conversion sanitizer check.
  result = emitScalarConversion(result, promotionTypeCR, lhsTy, loc,
                                ScalarConversionOpts(cgf.sanOpts));
 
  // Store the result value into the LHS lvalue. Bit-fields are handled
  // specially because the result is altered by the store, i.e., [C99 6.5.16p1]
  // 'An assignment expression has the value of the left operand after the
  // assignment...'.
  if (lhsLV.isBitField())
    cgf.emitStoreThroughBitfieldLValue(RValue::get(result), lhsLV);
  else
    cgf.emitStoreThroughLValue(RValue::get(result), lhsLV);
 
  if (cgf.getLangOpts().OpenMP)
    cgf.cgm.errorNYI(e->getSourceRange(), "openmp");
 
  return lhsLV;
}
 
mlir::Value ScalarExprEmitter::emitComplexToScalarConversion(mlir::Location lov,
                                                             mlir::Value value,
                                                             CastKind kind,
                                                             QualType destTy) {
  cir::CastKind castOpKind;
  switch (kind) {
  case CK_FloatingComplexToReal:
    castOpKind = cir::CastKind::float_complex_to_real;
    break;
  case CK_IntegralComplexToReal:
    castOpKind = cir::CastKind::int_complex_to_real;
    break;
  case CK_FloatingComplexToBoolean:
    castOpKind = cir::CastKind::float_complex_to_bool;
    break;
  case CK_IntegralComplexToBoolean:
    castOpKind = cir::CastKind::int_complex_to_bool;
    break;
  default:
    llvm_unreachable("invalid complex-to-scalar cast kind");
  }
 
  return builder.createCast(lov, castOpKind, value, cgf.convertType(destTy));
}
 
mlir::Value ScalarExprEmitter::emitPromoted(const Expr *e,
                                            QualType promotionType) {
  e = e->IgnoreParens();
  if (const auto *bo = dyn_cast<BinaryOperator>(e)) {
    switch (bo->getOpcode()) {
#define HANDLE_BINOP(OP)                                                       \
  case BO_##OP:                                                                \
    return emit##OP(emitBinOps(bo, promotionType));
      HANDLE_BINOP(Add)
      HANDLE_BINOP(Sub)
      HANDLE_BINOP(Mul)
      HANDLE_BINOP(Div)
#undef HANDLE_BINOP
    default:
      break;
    }
  } else if (const auto *uo = dyn_cast<UnaryOperator>(e)) {
    switch (uo->getOpcode()) {
    case UO_Imag:
    case UO_Real:
      return VisitRealImag(uo, promotionType);
    case UO_Minus:
      return VisitUnaryMinus(uo, promotionType);
    case UO_Plus:
      return VisitUnaryPlus(uo, promotionType);
    default:
      break;
    }
  }
  mlir::Value result = Visit(const_cast<Expr *>(e));
  if (result) {
    if (!promotionType.isNull())
      return emitPromotedValue(result, promotionType);
    return emitUnPromotedValue(result, e->getType());
  }
  return result;
}
 
mlir::Value ScalarExprEmitter::emitCompoundAssign(
    const CompoundAssignOperator *e,
    mlir::Value (ScalarExprEmitter::*func)(const BinOpInfo &)) {
 
  bool ignore = std::exchange(ignoreResultAssign, false);
  mlir::Value rhs;
  LValue lhs = emitCompoundAssignLValue(e, func, rhs);
 
  // If the result is clearly ignored, return now.
  if (ignore)
    return {};
 
  // The result of an assignment in C is the assigned r-value.
  if (!cgf.getLangOpts().CPlusPlus)
    return rhs;
 
  // If the lvalue is non-volatile, return the computed value of the assignment.
  if (!lhs.isVolatile())
    return rhs;
 
  // Otherwise, reload the value.
  return emitLoadOfLValue(lhs, e->getExprLoc());
}
 
mlir::Value ScalarExprEmitter::VisitExprWithCleanups(ExprWithCleanups *e) {
  CIRGenFunction::FullExprCleanupScope scope(cgf, e->getSubExpr());
  mlir::Value v = Visit(e->getSubExpr());
  // Defend against dominance problems caused by jumps out of expression
  // evaluation through the shared cleanup block.
  scope.exit({&v});
  return v;
}
 
} // namespace
 
LValue
CIRGenFunction::emitCompoundAssignmentLValue(const CompoundAssignOperator *e) {
  ScalarExprEmitter emitter(*this, builder);
  mlir::Value result;
  switch (e->getOpcode()) {
#define COMPOUND_OP(Op)                                                        \
  case BO_##Op##Assign:                                                        \
    return emitter.emitCompoundAssignLValue(e, &ScalarExprEmitter::emit##Op,   \
                                            result)
    COMPOUND_OP(Mul);
    COMPOUND_OP(Div);
    COMPOUND_OP(Rem);
    COMPOUND_OP(Add);
    COMPOUND_OP(Sub);
    COMPOUND_OP(Shl);
    COMPOUND_OP(Shr);
    COMPOUND_OP(And);
    COMPOUND_OP(Xor);
    COMPOUND_OP(Or);
#undef COMPOUND_OP
 
  case BO_PtrMemD:
  case BO_PtrMemI:
  case BO_Mul:
  case BO_Div:
  case BO_Rem:
  case BO_Add:
  case BO_Sub:
  case BO_Shl:
  case BO_Shr:
  case BO_LT:
  case BO_GT:
  case BO_LE:
  case BO_GE:
  case BO_EQ:
  case BO_NE:
  case BO_Cmp:
  case BO_And:
  case BO_Xor:
  case BO_Or:
  case BO_LAnd:
  case BO_LOr:
  case BO_Assign:
  case BO_Comma:
    llvm_unreachable("Not valid compound assignment operators");
  }
  llvm_unreachable("Unhandled compound assignment operator");
}
 
/// Emit the computation of the specified expression of scalar type.
mlir::Value CIRGenFunction::emitScalarExpr(const Expr *e,
                                           bool ignoreResultAssign) {
  assert(e && hasScalarEvaluationKind(e->getType()) &&
         "Invalid scalar expression to emit");
 
  return ScalarExprEmitter(*this, builder, ignoreResultAssign)
      .Visit(const_cast<Expr *>(e));
}
 
mlir::Value CIRGenFunction::emitPromotedScalarExpr(const Expr *e,
                                                   QualType promotionType) {
  if (!promotionType.isNull())
    return ScalarExprEmitter(*this, builder).emitPromoted(e, promotionType);
  return ScalarExprEmitter(*this, builder).Visit(const_cast<Expr *>(e));
}
 
[[maybe_unused]] static bool mustVisitNullValue(const Expr *e) {
  // If a null pointer expression's type is the C++0x nullptr_t and
  // the expression is not a simple literal, it must be evaluated
  // for its potential side effects.
  if (isa<IntegerLiteral>(e) || isa<CXXNullPtrLiteralExpr>(e))
    return false;
  return e->getType()->isNullPtrType();
}
 
/// If \p e is a widened promoted integer, get its base (unpromoted) type.
static std::optional<QualType>
getUnwidenedIntegerType(const ASTContext &astContext, const Expr *e) {
  const Expr *base = e->IgnoreImpCasts();
  if (e == base)
    return std::nullopt;
 
  QualType baseTy = base->getType();
  if (!astContext.isPromotableIntegerType(baseTy) ||
      astContext.getTypeSize(baseTy) >= astContext.getTypeSize(e->getType()))
    return std::nullopt;
 
  return baseTy;
}
 
/// Check if \p e is a widened promoted integer.
[[maybe_unused]] static bool isWidenedIntegerOp(const ASTContext &astContext,
                                                const Expr *e) {
  return getUnwidenedIntegerType(astContext, e).has_value();
}
 
/// Check if we can skip the overflow check for \p Op.
[[maybe_unused]] static bool canElideOverflowCheck(const ASTContext &astContext,
                                                   const BinOpInfo &op) {
  assert((isa<UnaryOperator>(op.e) || isa<BinaryOperator>(op.e)) &&
         "Expected a unary or binary operator");
 
  // If the binop has constant inputs and we can prove there is no overflow,
  // we can elide the overflow check.
  if (!op.mayHaveIntegerOverflow())
    return true;
 
  // If a unary op has a widened operand, the op cannot overflow.
  if (const auto *uo = dyn_cast<UnaryOperator>(op.e))
    return !uo->canOverflow();
 
  // We usually don't need overflow checks for binops with widened operands.
  // Multiplication with promoted unsigned operands is a special case.
  const auto *bo = cast<BinaryOperator>(op.e);
  std::optional<QualType> optionalLHSTy =
      getUnwidenedIntegerType(astContext, bo->getLHS());
  if (!optionalLHSTy)
    return false;
 
  std::optional<QualType> optionalRHSTy =
      getUnwidenedIntegerType(astContext, bo->getRHS());
  if (!optionalRHSTy)
    return false;
 
  QualType lhsTy = *optionalLHSTy;
  QualType rhsTy = *optionalRHSTy;
 
  // This is the simple case: binops without unsigned multiplication, and with
  // widened operands. No overflow check is needed here.
  if ((op.opcode != BO_Mul && op.opcode != BO_MulAssign) ||
      !lhsTy->isUnsignedIntegerType() || !rhsTy->isUnsignedIntegerType())
    return true;
 
  // For unsigned multiplication the overflow check can be elided if either one
  // of the unpromoted types are less than half the size of the promoted type.
  unsigned promotedSize = astContext.getTypeSize(op.e->getType());
  return (2 * astContext.getTypeSize(lhsTy)) < promotedSize ||
         (2 * astContext.getTypeSize(rhsTy)) < promotedSize;
}
 
/// Emit pointer + index arithmetic.
static mlir::Value emitPointerArithmetic(CIRGenFunction &cgf,
                                         const BinOpInfo &op,
                                         bool isSubtraction) {
  // Must have binary (not unary) expr here.  Unary pointer
  // increment/decrement doesn't use this path.
  const BinaryOperator *expr = cast<BinaryOperator>(op.e);
 
  mlir::Value pointer = op.lhs;
  Expr *pointerOperand = expr->getLHS();
  mlir::Value index = op.rhs;
  Expr *indexOperand = expr->getRHS();
 
  // In the case of subtraction, the FE has ensured that the LHS is always the
  // pointer. However, addition can have the pointer on either side. We will
  // always have a pointer operand and an integer operand, so if the LHS wasn't
  // a pointer, we need to swap our values.
  if (!isSubtraction && !mlir::isa<cir::PointerType>(pointer.getType())) {
    std::swap(pointer, index);
    std::swap(pointerOperand, indexOperand);
  }
  assert(mlir::isa<cir::PointerType>(pointer.getType()) &&
         "Need a pointer operand");
  assert(mlir::isa<cir::IntType>(index.getType()) && "Need an integer operand");
 
  // Some versions of glibc and gcc use idioms (particularly in their malloc
  // routines) that add a pointer-sized integer (known to be a pointer value)
  // to a null pointer in order to cast the value back to an integer or as
  // part of a pointer alignment algorithm.  This is undefined behavior, but
  // we'd like to be able to compile programs that use it.
  //
  // Normally, we'd generate a GEP with a null-pointer base here in response
  // to that code, but it's also UB to dereference a pointer created that
  // way.  Instead (as an acknowledged hack to tolerate the idiom) we will
  // generate a direct cast of the integer value to a pointer.
  //
  // The idiom (p = nullptr + N) is not met if any of the following are true:
  //
  //   The operation is subtraction.
  //   The index is not pointer-sized.
  //   The pointer type is not byte-sized.
  //
  if (BinaryOperator::isNullPointerArithmeticExtension(
          cgf.getContext(), op.opcode, expr->getLHS(), expr->getRHS()))
    return cgf.getBuilder().createIntToPtr(index, pointer.getType());
 
  // Differently from LLVM codegen, ABI bits for index sizes is handled during
  // LLVM lowering.
 
  // If this is subtraction, negate the index.
  if (isSubtraction)
    index = cgf.getBuilder().createNeg(cgf.getLoc(op.e->getExprLoc()), index);
 
  assert(!cir::MissingFeatures::sanitizers());
 
  const PointerType *pointerType =
      pointerOperand->getType()->getAs<PointerType>();
  if (!pointerType) {
    cgf.cgm.errorNYI("Objective-C:pointer arithmetic with non-pointer type");
    return nullptr;
  }
 
  QualType elementType = pointerType->getPointeeType();
  if (const VariableArrayType *vla =
          cgf.getContext().getAsVariableArrayType(elementType)) {
    mlir::Value numElements = cgf.getVLASize(vla).numElts;
    mlir::Location loc = cgf.getLoc(op.e->getExprLoc());
    index = cgf.getBuilder().createCast(cir::CastKind::integral, index,
                                        numElements.getType());
    // GEP indexes are signed, and scaling an index isn't permitted to
    // signed-overflow, so we use the same semantics for our explicit
    // multiply.  We suppress this if overflow is not undefined behavior.
    cir::OverflowBehavior overflowBehavior =
        cgf.getLangOpts().PointerOverflowDefined
            ? cir::OverflowBehavior::None
            : cir::OverflowBehavior::NoSignedWrap;
    index =
        cgf.getBuilder().createMul(loc, index, numElements, overflowBehavior);
    assert(!cir::MissingFeatures::sanitizers());
    return cir::PtrStrideOp::create(cgf.getBuilder(), loc, pointer.getType(),
                                    pointer, index);
  }
 
  assert(!cir::MissingFeatures::sanitizers());
  return cir::PtrStrideOp::create(cgf.getBuilder(),
                                  cgf.getLoc(op.e->getExprLoc()),
                                  pointer.getType(), pointer, index);
}
 
static bool isIntegerVectorBinOp(mlir::Type ty) {
  auto vecTy = mlir::dyn_cast<cir::VectorType>(ty);
  return vecTy && mlir::isa<cir::IntType>(vecTy.getElementType());
}
 
mlir::Value ScalarExprEmitter::emitMul(const BinOpInfo &ops) {
  const mlir::Location loc = cgf.getLoc(ops.loc);
  if (!isIntegerVectorBinOp(ops.lhs.getType()) &&
      ops.compType->isSignedIntegerOrEnumerationType()) {
    switch (cgf.getLangOpts().getSignedOverflowBehavior()) {
    case LangOptions::SOB_Defined:
      if (!cgf.sanOpts.has(SanitizerKind::SignedIntegerOverflow))
        return builder.createMul(loc, ops.lhs, ops.rhs);
      [[fallthrough]];
    case LangOptions::SOB_Undefined:
      if (!cgf.sanOpts.has(SanitizerKind::SignedIntegerOverflow))
        return builder.createNSWMul(loc, ops.lhs, ops.rhs);
      [[fallthrough]];
    case LangOptions::SOB_Trapping:
      if (canElideOverflowCheck(cgf.getContext(), ops))
        return builder.createNSWMul(loc, ops.lhs, ops.rhs);
      cgf.cgm.errorNYI("sanitizers");
    }
  }
  if (ops.fullType->isConstantMatrixType()) {
    assert(!cir::MissingFeatures::matrixType());
    cgf.cgm.errorNYI("matrix types");
    return nullptr;
  }
  if (ops.compType->isUnsignedIntegerType() &&
      cgf.sanOpts.has(SanitizerKind::UnsignedIntegerOverflow) &&
      !canElideOverflowCheck(cgf.getContext(), ops))
    cgf.cgm.errorNYI("unsigned int overflow sanitizer");
 
  if (cir::isFPOrVectorOfFPType(ops.lhs.getType())) {
    CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, ops.fpFeatures);
    return builder.createFMul(loc, ops.lhs, ops.rhs);
  }
 
  if (ops.isFixedPointOp()) {
    assert(!cir::MissingFeatures::fixedPointType());
    cgf.cgm.errorNYI("fixed point");
    return nullptr;
  }
 
  return cir::MulOp::create(builder, cgf.getLoc(ops.loc),
                            cgf.convertType(ops.fullType), ops.lhs, ops.rhs);
}
mlir::Value ScalarExprEmitter::emitDiv(const BinOpInfo &ops) {
  const mlir::Location loc = cgf.getLoc(ops.loc);
  if (cir::isFPOrVectorOfFPType(ops.lhs.getType())) {
    CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, ops.fpFeatures);
    return builder.createFDiv(loc, ops.lhs, ops.rhs);
  }
  return cir::DivOp::create(builder, loc, cgf.convertType(ops.fullType),
                            ops.lhs, ops.rhs);
}
mlir::Value ScalarExprEmitter::emitRem(const BinOpInfo &ops) {
  const mlir::Location loc = cgf.getLoc(ops.loc);
  if (cir::isFPOrVectorOfFPType(ops.lhs.getType())) {
    CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, ops.fpFeatures);
    return builder.createFRem(loc, ops.lhs, ops.rhs);
  }
  return cir::RemOp::create(builder, loc, cgf.convertType(ops.fullType),
                            ops.lhs, ops.rhs);
}
 
mlir::Value ScalarExprEmitter::emitAdd(const BinOpInfo &ops) {
  if (mlir::isa<cir::PointerType>(ops.lhs.getType()) ||
      mlir::isa<cir::PointerType>(ops.rhs.getType()))
    return emitPointerArithmetic(cgf, ops, /*isSubtraction=*/
                                 false);
 
  const mlir::Location loc = cgf.getLoc(ops.loc);
  if (!isIntegerVectorBinOp(ops.lhs.getType()) &&
      ops.compType->isSignedIntegerOrEnumerationType()) {
    switch (cgf.getLangOpts().getSignedOverflowBehavior()) {
    case LangOptions::SOB_Defined:
      if (!cgf.sanOpts.has(SanitizerKind::SignedIntegerOverflow))
        return builder.createAdd(loc, ops.lhs, ops.rhs);
      [[fallthrough]];
    case LangOptions::SOB_Undefined:
      if (!cgf.sanOpts.has(SanitizerKind::SignedIntegerOverflow))
        return builder.createNSWAdd(loc, ops.lhs, ops.rhs);
      [[fallthrough]];
    case LangOptions::SOB_Trapping:
      if (canElideOverflowCheck(cgf.getContext(), ops))
        return builder.createNSWAdd(loc, ops.lhs, ops.rhs);
      cgf.cgm.errorNYI("sanitizers");
    }
  }
  if (ops.fullType->isConstantMatrixType()) {
    assert(!cir::MissingFeatures::matrixType());
    cgf.cgm.errorNYI("matrix types");
    return nullptr;
  }
 
  if (ops.compType->isUnsignedIntegerType() &&
      cgf.sanOpts.has(SanitizerKind::UnsignedIntegerOverflow) &&
      !canElideOverflowCheck(cgf.getContext(), ops))
    cgf.cgm.errorNYI("unsigned int overflow sanitizer");
 
  if (cir::isFPOrVectorOfFPType(ops.lhs.getType())) {
    CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, ops.fpFeatures);
    return builder.createFAdd(loc, ops.lhs, ops.rhs);
  }
 
  if (ops.isFixedPointOp()) {
    assert(!cir::MissingFeatures::fixedPointType());
    cgf.cgm.errorNYI("fixed point");
    return {};
  }
 
  return builder.createAdd(loc, ops.lhs, ops.rhs);
}
 
mlir::Value ScalarExprEmitter::emitSub(const BinOpInfo &ops) {
  const mlir::Location loc = cgf.getLoc(ops.loc);
  // The LHS is always a pointer if either side is.
  if (!mlir::isa<cir::PointerType>(ops.lhs.getType())) {
    if (!isIntegerVectorBinOp(ops.lhs.getType()) &&
        ops.compType->isSignedIntegerOrEnumerationType()) {
      switch (cgf.getLangOpts().getSignedOverflowBehavior()) {
      case LangOptions::SOB_Defined: {
        if (!cgf.sanOpts.has(SanitizerKind::SignedIntegerOverflow))
          return builder.createSub(loc, ops.lhs, ops.rhs);
        [[fallthrough]];
      }
      case LangOptions::SOB_Undefined:
        if (!cgf.sanOpts.has(SanitizerKind::SignedIntegerOverflow))
          return builder.createNSWSub(loc, ops.lhs, ops.rhs);
        [[fallthrough]];
      case LangOptions::SOB_Trapping:
        if (canElideOverflowCheck(cgf.getContext(), ops))
          return builder.createNSWSub(loc, ops.lhs, ops.rhs);
        cgf.cgm.errorNYI("sanitizers");
      }
    }
 
    if (ops.fullType->isConstantMatrixType()) {
      assert(!cir::MissingFeatures::matrixType());
      cgf.cgm.errorNYI("matrix types");
      return nullptr;
    }
 
    if (ops.compType->isUnsignedIntegerType() &&
        cgf.sanOpts.has(SanitizerKind::UnsignedIntegerOverflow) &&
        !canElideOverflowCheck(cgf.getContext(), ops))
      cgf.cgm.errorNYI("unsigned int overflow sanitizer");
 
    if (cir::isFPOrVectorOfFPType(ops.lhs.getType())) {
      CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, ops.fpFeatures);
      return builder.createFSub(loc, ops.lhs, ops.rhs);
    }
 
    if (ops.isFixedPointOp()) {
      assert(!cir::MissingFeatures::fixedPointType());
      cgf.cgm.errorNYI("fixed point");
      return {};
    }
 
    return builder.createSub(loc, ops.lhs, ops.rhs);
  }
 
  // If the RHS is not a pointer, then we have normal pointer
  // arithmetic.
  if (!mlir::isa<cir::PointerType>(ops.rhs.getType()))
    return emitPointerArithmetic(cgf, ops, /*isSubtraction=*/true);
 
  // Otherwise, this is a pointer subtraction
 
  // Do the raw subtraction part.
  //
  // TODO(cir): note for LLVM lowering out of this; when expanding this into
  // LLVM we shall take VLA's, division by element size, etc.
  //
  // See more in `EmitSub` in CGExprScalar.cpp.
  assert(!cir::MissingFeatures::llvmLoweringPtrDiffConsidersPointee());
  return cir::PtrDiffOp::create(builder, cgf.getLoc(ops.loc), cgf.ptrDiffTy,
                                ops.lhs, ops.rhs);
}
 
mlir::Value ScalarExprEmitter::emitShl(const BinOpInfo &ops) {
  // TODO: This misses out on the sanitizer check below.
  if (ops.isFixedPointOp()) {
    assert(!cir::MissingFeatures::fixedPointType());
    cgf.cgm.errorNYI("fixed point");
    return {};
  }
 
  // CIR accepts shift between different types, meaning nothing special
  // to be done here. OTOH, LLVM requires the LHS and RHS to be the same type:
  // promote or truncate the RHS to the same size as the LHS.
 
  bool sanitizeSignedBase = cgf.sanOpts.has(SanitizerKind::ShiftBase) &&
                            ops.compType->hasSignedIntegerRepresentation() &&
                            !cgf.getLangOpts().isSignedOverflowDefined() &&
                            !cgf.getLangOpts().CPlusPlus20;
  bool sanitizeUnsignedBase =
      cgf.sanOpts.has(SanitizerKind::UnsignedShiftBase) &&
      ops.compType->hasUnsignedIntegerRepresentation();
  bool sanitizeBase = sanitizeSignedBase || sanitizeUnsignedBase;
  bool sanitizeExponent = cgf.sanOpts.has(SanitizerKind::ShiftExponent);
 
  // OpenCL 6.3j: shift values are effectively % word size of LHS.
  if (cgf.getLangOpts().OpenCL)
    cgf.cgm.errorNYI("opencl");
  else if ((sanitizeBase || sanitizeExponent) &&
           mlir::isa<cir::IntType>(ops.lhs.getType()))
    cgf.cgm.errorNYI("sanitizers");
 
  return builder.createShiftLeft(cgf.getLoc(ops.loc), ops.lhs, ops.rhs);
}
 
mlir::Value ScalarExprEmitter::emitShr(const BinOpInfo &ops) {
  // TODO: This misses out on the sanitizer check below.
  if (ops.isFixedPointOp()) {
    assert(!cir::MissingFeatures::fixedPointType());
    cgf.cgm.errorNYI("fixed point");
    return {};
  }
 
  // CIR accepts shift between different types, meaning nothing special
  // to be done here. OTOH, LLVM requires the LHS and RHS to be the same type:
  // promote or truncate the RHS to the same size as the LHS.
 
  // OpenCL 6.3j: shift values are effectively % word size of LHS.
  if (cgf.getLangOpts().OpenCL)
    cgf.cgm.errorNYI("opencl");
  else if (cgf.sanOpts.has(SanitizerKind::ShiftExponent) &&
           mlir::isa<cir::IntType>(ops.lhs.getType()))
    cgf.cgm.errorNYI("sanitizers");
 
  // Note that we don't need to distinguish unsigned treatment at this
  // point since it will be handled later by LLVM lowering.
  return builder.createShiftRight(cgf.getLoc(ops.loc), ops.lhs, ops.rhs);
}
 
mlir::Value ScalarExprEmitter::emitAnd(const BinOpInfo &ops) {
  return cir::AndOp::create(builder, cgf.getLoc(ops.loc), ops.lhs, ops.rhs);
}
mlir::Value ScalarExprEmitter::emitXor(const BinOpInfo &ops) {
  return cir::XorOp::create(builder, cgf.getLoc(ops.loc), ops.lhs, ops.rhs);
}
mlir::Value ScalarExprEmitter::emitOr(const BinOpInfo &ops) {
  return cir::OrOp::create(builder, cgf.getLoc(ops.loc), ops.lhs, ops.rhs);
}
 
// Emit code for an explicit or implicit cast.  Implicit
// casts have to handle a more broad range of conversions than explicit
// casts, as they handle things like function to ptr-to-function decay
// etc.
mlir::Value ScalarExprEmitter::VisitCastExpr(CastExpr *ce) {
  Expr *subExpr = ce->getSubExpr();
  QualType destTy = ce->getType();
  CastKind kind = ce->getCastKind();
 
  // These cases are generally not written to ignore the result of evaluating
  // their sub-expressions, so we clear this now.
  ignoreResultAssign = false;
 
  switch (kind) {
  case clang::CK_Dependent:
    llvm_unreachable("dependent cast kind in CIR gen!");
  case clang::CK_BuiltinFnToFnPtr:
    llvm_unreachable("builtin functions are handled elsewhere");
  case CK_LValueBitCast:
  case CK_LValueToRValueBitCast: {
    LValue sourceLVal = cgf.emitLValue(subExpr);
    Address sourceAddr = sourceLVal.getAddress();
 
    mlir::Type destElemTy = cgf.convertTypeForMem(destTy);
    Address destAddr = sourceAddr.withElementType(cgf.getBuilder(), destElemTy);
    LValue destLVal = cgf.makeAddrLValue(destAddr, destTy);
    assert(!cir::MissingFeatures::opTBAA());
    return emitLoadOfLValue(destLVal, ce->getExprLoc());
  }
 
  case CK_CPointerToObjCPointerCast:
  case CK_BlockPointerToObjCPointerCast:
  case CK_AnyPointerToBlockPointerCast:
  case CK_BitCast: {
    mlir::Value src = Visit(const_cast<Expr *>(subExpr));
    mlir::Type dstTy = cgf.convertType(destTy);
 
    assert(!cir::MissingFeatures::addressSpace());
 
    if (cgf.sanOpts.has(SanitizerKind::CFIUnrelatedCast))
      cgf.getCIRGenModule().errorNYI(subExpr->getSourceRange(),
                                     "sanitizer support");
 
    if (cgf.cgm.getCodeGenOpts().StrictVTablePointers)
      cgf.getCIRGenModule().errorNYI(subExpr->getSourceRange(),
                                     "strict vtable pointers");
 
    // Update heapallocsite metadata when there is an explicit pointer cast.
    assert(!cir::MissingFeatures::addHeapAllocSiteMetadata());
 
    // If Src is a fixed vector and Dst is a scalable vector, and both have the
    // same element type, use the llvm.vector.insert intrinsic to perform the
    // bitcast.
    assert(!cir::MissingFeatures::scalableVectors());
 
    // If Src is a scalable vector and Dst is a fixed vector, and both have the
    // same element type, use the llvm.vector.extract intrinsic to perform the
    // bitcast.
    assert(!cir::MissingFeatures::scalableVectors());
 
    // Perform VLAT <-> VLST bitcast through memory.
    // TODO: since the llvm.experimental.vector.{insert,extract} intrinsics
    //       require the element types of the vectors to be the same, we
    //       need to keep this around for bitcasts between VLAT <-> VLST where
    //       the element types of the vectors are not the same, until we figure
    //       out a better way of doing these casts.
    assert(!cir::MissingFeatures::scalableVectors());
 
    return cgf.getBuilder().createBitcast(cgf.getLoc(subExpr->getSourceRange()),
                                          src, dstTy);
  }
  case CK_AddressSpaceConversion: {
    Expr::EvalResult result;
    if (subExpr->EvaluateAsRValue(result, cgf.getContext()) &&
        result.Val.isNullPointer()) {
      // If e has side effect, it is emitted even if its final result is a
      // null pointer. In that case, a DCE pass should be able to
      // eliminate the useless instructions emitted during translating E.
      if (result.HasSideEffects)
        Visit(subExpr);
      return cgf.cgm.emitNullConstant(destTy,
                                      cgf.getLoc(subExpr->getExprLoc()));
    }
    return cgf.performAddrSpaceCast(Visit(subExpr), convertType(destTy));
  }
 
  case CK_AtomicToNonAtomic:
  case CK_NonAtomicToAtomic:
  case CK_UserDefinedConversion:
    return Visit(const_cast<Expr *>(subExpr));
  case CK_NoOp:
    return ce->changesVolatileQualification() ? emitLoadOfLValue(ce)
                                              : Visit(subExpr);
  case CK_IntegralToPointer: {
    mlir::Type destCIRTy = cgf.convertType(destTy);
    mlir::Value src = Visit(const_cast<Expr *>(subExpr));
 
    // Properly resize by casting to an int of the same size as the pointer.
    // Clang's IntegralToPointer includes 'bool' as the source, but in CIR
    // 'bool' is not an integral type.  So check the source type to get the
    // correct CIR conversion.
    mlir::Type middleTy = cgf.cgm.getDataLayout().getIntPtrType(destCIRTy);
    mlir::Value middleVal = builder.createCast(
        subExpr->getType()->isBooleanType() ? cir::CastKind::bool_to_int
                                            : cir::CastKind::integral,
        src, middleTy);
 
    if (cgf.cgm.getCodeGenOpts().StrictVTablePointers) {
      cgf.cgm.errorNYI(subExpr->getSourceRange(),
                       "IntegralToPointer: strict vtable pointers");
      return {};
    }
 
    return builder.createIntToPtr(middleVal, destCIRTy);
  }
 
  case CK_BaseToDerived: {
    const CXXRecordDecl *derivedClassDecl = destTy->getPointeeCXXRecordDecl();
    assert(derivedClassDecl && "BaseToDerived arg isn't a C++ object pointer!");
    Address base = cgf.emitPointerWithAlignment(subExpr);
    Address derived = cgf.getAddressOfDerivedClass(
        cgf.getLoc(ce->getSourceRange()), base, derivedClassDecl, ce->path(),
        cgf.shouldNullCheckClassCastValue(ce));
 
    // C++11 [expr.static.cast]p11: Behavior is undefined if a downcast is
    // performed and the object is not of the derived type.
    assert(!cir::MissingFeatures::sanitizers());
 
    return cgf.getAsNaturalPointerTo(derived, ce->getType()->getPointeeType());
  }
  case CK_UncheckedDerivedToBase:
  case CK_DerivedToBase: {
    // The EmitPointerWithAlignment path does this fine; just discard
    // the alignment.
    return cgf.getAsNaturalPointerTo(cgf.emitPointerWithAlignment(ce),
                                     ce->getType()->getPointeeType());
  }
  case CK_Dynamic: {
    Address v = cgf.emitPointerWithAlignment(subExpr);
    const auto *dce = cast<CXXDynamicCastExpr>(ce);
    return cgf.emitDynamicCast(v, dce);
  }
  case CK_ArrayToPointerDecay:
    return cgf.emitArrayToPointerDecay(subExpr).getPointer();
 
  case CK_NullToPointer: {
    if (mustVisitNullValue(subExpr))
      cgf.emitIgnoredExpr(subExpr);
 
    // Note that DestTy is used as the MLIR type instead of a custom
    // nullptr type.
    mlir::Type ty = cgf.convertType(destTy);
    return builder.getNullPtr(ty, cgf.getLoc(subExpr->getExprLoc()));
  }
 
  case CK_NullToMemberPointer: {
    if (mustVisitNullValue(subExpr))
      cgf.emitIgnoredExpr(subExpr);
 
    assert(!cir::MissingFeatures::cxxABI());
 
    const MemberPointerType *mpt = ce->getType()->getAs<MemberPointerType>();
    mlir::Location loc = cgf.getLoc(subExpr->getExprLoc());
    return cgf.getBuilder().getConstant(
        loc, cgf.cgm.emitNullMemberAttr(destTy, mpt));
  }
 
  case CK_ReinterpretMemberPointer: {
    mlir::Value src = Visit(subExpr);
    return builder.createBitcast(cgf.getLoc(subExpr->getExprLoc()), src,
                                 cgf.convertType(destTy));
  }
  case CK_BaseToDerivedMemberPointer:
  case CK_DerivedToBaseMemberPointer: {
    mlir::Value src = Visit(subExpr);
 
    assert(!cir::MissingFeatures::memberFuncPtrAuthInfo());
 
    QualType derivedTy =
        kind == CK_DerivedToBaseMemberPointer ? subExpr->getType() : destTy;
    const auto *mpType = derivedTy->castAs<MemberPointerType>();
    NestedNameSpecifier qualifier = mpType->getQualifier();
    assert(qualifier && "member pointer without class qualifier");
    const Type *qualifierType = qualifier.getAsType();
    assert(qualifierType && "member pointer qualifier is not a type");
    const CXXRecordDecl *derivedClass = qualifierType->getAsCXXRecordDecl();
    CharUnits offset =
        cgf.cgm.computeNonVirtualBaseClassOffset(derivedClass, ce->path());
 
    mlir::Location loc = cgf.getLoc(subExpr->getExprLoc());
    mlir::Type resultTy = cgf.convertType(destTy);
    mlir::IntegerAttr offsetAttr = builder.getIndexAttr(offset.getQuantity());
 
    if (subExpr->getType()->isMemberFunctionPointerType()) {
      if (kind == CK_BaseToDerivedMemberPointer)
        return cir::DerivedMethodOp::create(builder, loc, resultTy, src,
                                            offsetAttr);
      return cir::BaseMethodOp::create(builder, loc, resultTy, src, offsetAttr);
    }
 
    if (kind == CK_BaseToDerivedMemberPointer)
      return cir::DerivedDataMemberOp::create(builder, loc, resultTy, src,
                                              offsetAttr);
    return cir::BaseDataMemberOp::create(builder, loc, resultTy, src,
                                         offsetAttr);
  }
 
  case CK_LValueToRValue:
    assert(cgf.getContext().hasSameUnqualifiedType(subExpr->getType(), destTy));
    assert(subExpr->isGLValue() && "lvalue-to-rvalue applied to r-value!");
    return Visit(const_cast<Expr *>(subExpr));
 
  case CK_IntegralCast: {
    ScalarConversionOpts opts;
    if (auto *ice = dyn_cast<ImplicitCastExpr>(ce)) {
      if (!ice->isPartOfExplicitCast())
        opts = ScalarConversionOpts(cgf.sanOpts);
    }
    return emitScalarConversion(Visit(subExpr), subExpr->getType(), destTy,
                                ce->getExprLoc(), opts);
  }
 
  case CK_FloatingComplexToReal:
  case CK_IntegralComplexToReal:
  case CK_FloatingComplexToBoolean:
  case CK_IntegralComplexToBoolean: {
    mlir::Value value = cgf.emitComplexExpr(subExpr);
    return emitComplexToScalarConversion(cgf.getLoc(ce->getExprLoc()), value,
                                         kind, destTy);
  }
 
  case CK_FloatingRealToComplex:
  case CK_FloatingComplexCast:
  case CK_IntegralRealToComplex:
  case CK_IntegralComplexCast:
  case CK_IntegralComplexToFloatingComplex:
  case CK_FloatingComplexToIntegralComplex:
    llvm_unreachable("scalar cast to non-scalar value");
 
  case CK_PointerToIntegral: {
    assert(!destTy->isBooleanType() && "bool should use PointerToBool");
    if (cgf.cgm.getCodeGenOpts().StrictVTablePointers)
      cgf.getCIRGenModule().errorNYI(subExpr->getSourceRange(),
                                     "strict vtable pointers");
    return builder.createPtrToInt(Visit(subExpr), cgf.convertType(destTy));
  }
  case CK_ToVoid:
    cgf.emitIgnoredExpr(subExpr);
    return {};
 
  case CK_IntegralToFloating:
  case CK_FloatingToIntegral:
  case CK_FloatingCast:
  case CK_FixedPointToFloating:
  case CK_FloatingToFixedPoint: {
    if (kind == CK_FixedPointToFloating || kind == CK_FloatingToFixedPoint) {
      cgf.getCIRGenModule().errorNYI(subExpr->getSourceRange(),
                                     "fixed point casts");
      return {};
    }
    CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, ce);
    return emitScalarConversion(Visit(subExpr), subExpr->getType(), destTy,
                                ce->getExprLoc());
  }
 
  case CK_IntegralToBoolean:
    return emitIntToBoolConversion(Visit(subExpr),
                                   cgf.getLoc(ce->getSourceRange()));
 
  case CK_PointerToBoolean:
    return emitPointerToBoolConversion(Visit(subExpr), subExpr->getType());
  case CK_FloatingToBoolean:
    return emitFloatToBoolConversion(Visit(subExpr),
                                     cgf.getLoc(subExpr->getExprLoc()));
  case CK_MemberPointerToBoolean: {
    mlir::Value memPtr = Visit(subExpr);
    return builder.createCast(cgf.getLoc(ce->getSourceRange()),
                              cir::CastKind::member_ptr_to_bool, memPtr,
                              cgf.convertType(destTy));
  }
 
  case CK_VectorSplat: {
    // Create a vector object and fill all elements with the same scalar value.
    assert(destTy->isVectorType() && "CK_VectorSplat to non-vector type");
    return cir::VecSplatOp::create(builder,
                                   cgf.getLoc(subExpr->getSourceRange()),
                                   cgf.convertType(destTy), Visit(subExpr));
  }
  case CK_FunctionToPointerDecay:
    return cgf.emitLValue(subExpr).getPointer();
 
  default:
    cgf.getCIRGenModule().errorNYI(subExpr->getSourceRange(),
                                   "CastExpr: ", ce->getCastKindName());
  }
  return {};
}
 
mlir::Value ScalarExprEmitter::VisitCallExpr(const CallExpr *e) {
  if (e->getCallReturnType(cgf.getContext())->isReferenceType())
    return emitLoadOfLValue(e);
 
  auto v = cgf.emitCallExpr(e).getValue();
  assert(!cir::MissingFeatures::emitLValueAlignmentAssumption());
  return v;
}
 
mlir::Value ScalarExprEmitter::VisitMemberExpr(MemberExpr *e) {
  // TODO(cir): The classic codegen calls tryEmitAsConstant() here. Folding
  // constants sound like work for MLIR optimizers, but we'll keep an assertion
  // for now.
  assert(!cir::MissingFeatures::tryEmitAsConstant());
  Expr::EvalResult result;
  if (e->EvaluateAsInt(result, cgf.getContext(), Expr::SE_AllowSideEffects)) {
    llvm::APSInt value = result.Val.getInt();
    cgf.emitIgnoredExpr(e->getBase());
    mlir::Location loc = cgf.getLoc(e->getExprLoc());
    // The constant is folded from an APSInt with the source-type's bit width
    // (1 for bool), but the AST's expression type is what later consumers of
    // this value see. For a bool member we have to emit a !cir.bool constant
    // -- otherwise downstream ops (cir.call into a bool parameter, cir.if /
    // cir.ternary on the value, ...) would all reject the !cir.int<u, 1> the
    // raw APSInt would produce.
    if (e->getType()->isBooleanType())
      return builder.getBool(value.getBoolValue(), loc);
    return builder.getConstInt(loc, value);
  }
  return emitLoadOfLValue(e);
}
 
mlir::Value ScalarExprEmitter::VisitInitListExpr(InitListExpr *e) {
  const unsigned numInitElements = e->getNumInits();
 
  [[maybe_unused]] const bool ignore = std::exchange(ignoreResultAssign, false);
  assert((ignore == false ||
          (numInitElements == 0 && e->getType()->isVoidType())) &&
         "init list ignored");
 
  if (e->hadArrayRangeDesignator()) {
    cgf.cgm.errorNYI(e->getSourceRange(), "ArrayRangeDesignator");
    return {};
  }
 
  if (e->getType()->isVectorType()) {
    const auto vectorType =
        mlir::cast<cir::VectorType>(cgf.convertType(e->getType()));
 
    SmallVector<mlir::Value, 16> elements;
    for (Expr *init : e->inits()) {
      elements.push_back(Visit(init));
    }
 
    // Zero-initialize any remaining values.
    if (numInitElements < vectorType.getSize()) {
      const mlir::Value zeroValue = cgf.getBuilder().getNullValue(
          vectorType.getElementType(), cgf.getLoc(e->getSourceRange()));
      std::fill_n(std::back_inserter(elements),
                  vectorType.getSize() - numInitElements, zeroValue);
    }
 
    return cir::VecCreateOp::create(cgf.getBuilder(),
                                    cgf.getLoc(e->getSourceRange()), vectorType,
                                    elements);
  }
 
  // C++11 value-initialization for the scalar.
  if (numInitElements == 0)
    return emitNullValue(e->getType(), cgf.getLoc(e->getExprLoc()));
 
  return Visit(e->getInit(0));
}
 
mlir::Value CIRGenFunction::emitScalarConversion(mlir::Value src,
                                                 QualType srcTy, QualType dstTy,
                                                 SourceLocation loc) {
  assert(CIRGenFunction::hasScalarEvaluationKind(srcTy) &&
         CIRGenFunction::hasScalarEvaluationKind(dstTy) &&
         "Invalid scalar expression to emit");
  return ScalarExprEmitter(*this, builder)
      .emitScalarConversion(src, srcTy, dstTy, loc);
}
 
mlir::Value CIRGenFunction::emitComplexToScalarConversion(mlir::Value src,
                                                          QualType srcTy,
                                                          QualType dstTy,
                                                          SourceLocation loc) {
  assert(srcTy->isAnyComplexType() && hasScalarEvaluationKind(dstTy) &&
         "Invalid complex -> scalar conversion");
 
  QualType complexElemTy = srcTy->castAs<ComplexType>()->getElementType();
  if (dstTy->isBooleanType()) {
    auto kind = complexElemTy->isFloatingType()
                    ? cir::CastKind::float_complex_to_bool
                    : cir::CastKind::int_complex_to_bool;
    return builder.createCast(getLoc(loc), kind, src, convertType(dstTy));
  }
 
  auto kind = complexElemTy->isFloatingType()
                  ? cir::CastKind::float_complex_to_real
                  : cir::CastKind::int_complex_to_real;
  mlir::Value real =
      builder.createCast(getLoc(loc), kind, src, convertType(complexElemTy));
  return emitScalarConversion(real, complexElemTy, dstTy, loc);
}
 
mlir::Value ScalarExprEmitter::VisitUnaryLNot(const UnaryOperator *e) {
  // Perform vector logical not on comparison with zero vector.
  if (e->getType()->isVectorType() &&
      e->getType()->castAs<VectorType>()->getVectorKind() ==
          VectorKind::Generic) {
    mlir::Value oper = Visit(e->getSubExpr());
    mlir::Location loc = cgf.getLoc(e->getExprLoc());
    auto operVecTy = mlir::cast<cir::VectorType>(oper.getType());
    auto exprVecTy = mlir::cast<cir::VectorType>(cgf.convertType(e->getType()));
    mlir::Value zeroVec = builder.getNullValue(operVecTy, loc);
    return cir::VecCmpOp::create(builder, loc, exprVecTy, cir::CmpOpKind::eq,
                                 oper, zeroVec);
  }
 
  // Compare operand to zero.
  mlir::Value boolVal = cgf.evaluateExprAsBool(e->getSubExpr());
 
  // Invert value.
  boolVal = builder.createNot(boolVal);
 
  // ZExt result to the expr type.
  return maybePromoteBoolResult(boolVal, cgf.convertType(e->getType()));
}
 
mlir::Value ScalarExprEmitter::VisitOffsetOfExpr(OffsetOfExpr *e) {
  // Try folding the offsetof to a constant.
  Expr::EvalResult evalResult;
  if (e->EvaluateAsInt(evalResult, cgf.getContext())) {
    mlir::Type type = cgf.convertType(e->getType());
    llvm::APSInt value = evalResult.Val.getInt();
    return builder.getConstAPInt(cgf.getLoc(e->getExprLoc()), type, value);
  }
 
  cgf.getCIRGenModule().errorNYI(
      e->getSourceRange(),
      "ScalarExprEmitter::VisitOffsetOfExpr Can't eval expr as int");
  return {};
}
 
mlir::Value ScalarExprEmitter::VisitUnaryReal(const UnaryOperator *e) {
  QualType promotionTy = getPromotionType(e->getSubExpr()->getType());
  mlir::Value result = VisitRealImag(e, promotionTy);
  if (result && !promotionTy.isNull())
    result = emitUnPromotedValue(result, e->getType());
  return result;
}
 
mlir::Value ScalarExprEmitter::VisitUnaryImag(const UnaryOperator *e) {
  QualType promotionTy = getPromotionType(e->getSubExpr()->getType());
  mlir::Value result = VisitRealImag(e, promotionTy);
  if (result && !promotionTy.isNull())
    result = emitUnPromotedValue(result, e->getType());
  return result;
}
 
mlir::Value ScalarExprEmitter::VisitRealImag(const UnaryOperator *e,
                                             QualType promotionTy) {
  assert(
      (e->getOpcode() == clang::UO_Real || e->getOpcode() == clang::UO_Imag) &&
      "Invalid UnaryOp kind for ComplexType Real or Imag");
 
  Expr *op = e->getSubExpr();
  mlir::Location loc = cgf.getLoc(e->getExprLoc());
  if (op->getType()->isAnyComplexType()) {
    // If it's an l-value, load through the appropriate subobject l-value.
    // Note that we have to ask `e` because `op` might be an l-value that
    // this won't work for, e.g. an Obj-C property
    mlir::Value complex = cgf.emitComplexExpr(op);
    if (e->isGLValue() && !promotionTy.isNull()) {
      promotionTy = promotionTy->isAnyComplexType()
                        ? promotionTy
                        : cgf.getContext().getComplexType(promotionTy);
      complex = cgf.emitPromotedValue(complex, promotionTy);
    }
 
    return e->getOpcode() == clang::UO_Real
               ? builder.createComplexReal(loc, complex)
               : builder.createComplexImag(loc, complex);
  }
 
  if (e->getOpcode() == UO_Real) {
    mlir::Value operand = promotionTy.isNull()
                              ? Visit(op)
                              : cgf.emitPromotedScalarExpr(op, promotionTy);
    return builder.createComplexReal(loc, operand);
  }
 
  // __imag on a scalar returns zero. Emit the subexpr to ensure side
  // effects are evaluated, but not the actual value.
  mlir::Value operand;
  if (op->isGLValue()) {
    operand = cgf.emitLValue(op).getPointer();
    operand = cir::LoadOp::create(builder, loc, operand);
  } else if (!promotionTy.isNull()) {
    operand = cgf.emitPromotedScalarExpr(op, promotionTy);
  } else {
    operand = cgf.emitScalarExpr(op);
  }
  return builder.createComplexImag(loc, operand);
}
 
/// Return the size or alignment of the type of argument of the sizeof
/// expression as an integer.
mlir::Value ScalarExprEmitter::VisitUnaryExprOrTypeTraitExpr(
    const UnaryExprOrTypeTraitExpr *e) {
  const QualType typeToSize = e->getTypeOfArgument();
  const mlir::Location loc = cgf.getLoc(e->getSourceRange());
  if (auto kind = e->getKind();
      kind == UETT_SizeOf || kind == UETT_DataSizeOf || kind == UETT_CountOf) {
    if (const VariableArrayType *vat =
            cgf.getContext().getAsVariableArrayType(typeToSize)) {
      // For _Countof, we only want to evaluate if the extent is actually
      // variable as opposed to a multi-dimensional array whose extent is
      // constant but whose element type is variable.
      bool evaluateExtent = true;
      if (kind == UETT_CountOf && vat->getElementType()->isArrayType()) {
        evaluateExtent =
            !vat->getSizeExpr()->isIntegerConstantExpr(cgf.getContext());
      }
 
      if (evaluateExtent) {
        if (e->isArgumentType()) {
          // sizeof(type) - make sure to emit the VLA size.
          cgf.emitVariablyModifiedType(typeToSize);
        } else {
          // C99 6.5.3.4p2: If the argument is an expression of type
          // VLA, it is evaluated.
          cgf.emitIgnoredExpr(e->getArgumentExpr());
        }
 
        // For _Countof, we just want to return the size of a single dimension.
        if (kind == UETT_CountOf)
          return cgf.getVLAElements1D(vat).numElts;
 
        // For sizeof and __datasizeof, we need to scale the number of elements
        // by the size of the array element type.
        CIRGenFunction::VlaSizePair vlaSize = cgf.getVLASize(vat);
        mlir::Value numElts = vlaSize.numElts;
 
        // Scale the number of non-VLA elements by the non-VLA element size.
        CharUnits eltSize = cgf.getContext().getTypeSizeInChars(vlaSize.type);
        if (!eltSize.isOne()) {
          mlir::Location loc = cgf.getLoc(e->getSourceRange());
          mlir::Value eltSizeValue =
              builder.getConstAPInt(numElts.getLoc(), numElts.getType(),
                                    cgf.cgm.getSize(eltSize).getValue());
          return builder.createMul(loc, eltSizeValue, numElts,
                                   cir::OverflowBehavior::NoUnsignedWrap);
        }
 
        return numElts;
      }
    }
  } else if (e->getKind() == UETT_OpenMPRequiredSimdAlign) {
    clang::CharUnits::QuantityType alignment =
        cgf.getContext()
            .toCharUnitsFromBits(cgf.getContext().getOpenMPDefaultSimdAlign(
                e->getTypeOfArgument()->getPointeeType()))
            .getQuantity();
    return builder.getConstantInt(loc, cgf.cgm.sizeTy, alignment);
  } else if (e->getKind() == UETT_VectorElements) {
    auto vecTy = cast<cir::VectorType>(convertType(e->getTypeOfArgument()));
    if (vecTy.getIsScalable()) {
      cgf.getCIRGenModule().errorNYI(
          e->getSourceRange(),
          "VisitUnaryExprOrTypeTraitExpr: sizeOf scalable vector");
      return builder.getConstant(
          loc, cir::IntAttr::get(cgf.cgm.uInt64Ty,
                                 e->EvaluateKnownConstInt(cgf.getContext())));
    }
 
    return builder.getConstant(
        loc, cir::IntAttr::get(cgf.cgm.uInt64Ty, vecTy.getSize()));
  }
 
  return builder.getConstant(
      loc, cir::IntAttr::get(cgf.cgm.uInt64Ty,
                             e->EvaluateKnownConstInt(cgf.getContext())));
}
 
/// Return true if the specified expression is cheap enough and side-effect-free
/// enough to evaluate unconditionally instead of conditionally.  This is used
/// to convert control flow into selects in some cases.
/// TODO(cir): can be shared with LLVM codegen.
static bool isCheapEnoughToEvaluateUnconditionally(const Expr *e,
                                                   CIRGenFunction &cgf) {
  // Anything that is an integer or floating point constant is fine.
  return e->IgnoreParens()->isEvaluatable(cgf.getContext());
 
  // Even non-volatile automatic variables can't be evaluated unconditionally.
  // Referencing a thread_local may cause non-trivial initialization work to
  // occur. If we're inside a lambda and one of the variables is from the scope
  // outside the lambda, that function may have returned already. Reading its
  // locals is a bad idea. Also, these reads may introduce races there didn't
  // exist in the source-level program.
}
 
mlir::Value ScalarExprEmitter::VisitAbstractConditionalOperator(
    const AbstractConditionalOperator *e) {
  CIRGenBuilderTy &builder = cgf.getBuilder();
  mlir::Location loc = cgf.getLoc(e->getSourceRange());
  ignoreResultAssign = false;
 
  // Bind the common expression if necessary.
  CIRGenFunction::OpaqueValueMapping binding(cgf, e);
 
  Expr *condExpr = e->getCond();
  Expr *lhsExpr = e->getTrueExpr();
  Expr *rhsExpr = e->getFalseExpr();
 
  // If the condition constant folds and can be elided, try to avoid emitting
  // the condition and the dead arm.
  bool condExprBool;
  if (cgf.constantFoldsToBool(condExpr, condExprBool)) {
    Expr *live = lhsExpr, *dead = rhsExpr;
    if (!condExprBool)
      std::swap(live, dead);
 
    // If the dead side doesn't have labels we need, just emit the Live part.
    if (!cgf.containsLabel(dead)) {
      if (condExprBool)
        assert(!cir::MissingFeatures::incrementProfileCounter());
      mlir::Value result = Visit(live);
 
      // If the live part is a throw expression, it acts like it has a void
      // type, so evaluating it returns a null Value.  However, a conditional
      // with non-void type must return a non-null Value.
      if (!result && !e->getType()->isVoidType()) {
        result = builder.getConstant(
            loc, cir::PoisonAttr::get(builder.getContext(),
                                      cgf.convertType(e->getType())));
      }
 
      return result;
    }
  }
 
  QualType condType = condExpr->getType();
 
  // OpenCL: If the condition is a vector, we can treat this condition like
  // the select function.
  if (cgf.getLangOpts().OpenCL &&
      (condType->isVectorType() || condType->isExtVectorType())) {
    assert(!cir::MissingFeatures::incrementProfileCounter());
 
    mlir::Value condValue = cgf.emitScalarExpr(condExpr);
    mlir::Value lhsValue = Visit(lhsExpr);
    mlir::Value rhsValue = Visit(rhsExpr);
 
    mlir::Type vecTy = convertType(condType);
    mlir::Value zeroVec = builder.getNullValue(vecTy, loc);
    auto testMSB = cir::VecCmpOp::create(
        builder, loc, vecTy, cir::CmpOpKind::lt, condValue, zeroVec);
    mlir::Value tmp = builder.createIntCast(testMSB, vecTy);
    mlir::Value tmp2 = builder.createNot(tmp);
 
    // Cast float to int to perform ANDs if necessary.
    mlir::Value rhsTmp = rhsValue;
    mlir::Value lhsTmp = lhsValue;
    bool wasCast = false;
    auto rhsVecTy = cast<cir::VectorType>(rhsValue.getType());
    if (cir::isAnyFloatingPointType(rhsVecTy.getElementType())) {
      rhsTmp = builder.createBitcast(rhsValue, tmp2.getType());
      lhsTmp = builder.createBitcast(lhsValue, tmp.getType());
      wasCast = true;
    }
 
    mlir::Value tmp3 = builder.createAnd(loc, rhsTmp, tmp2);
    mlir::Value tmp4 = builder.createAnd(loc, lhsTmp, tmp);
    mlir::Value tmp5 = builder.createOr(loc, tmp3, tmp4);
    if (wasCast)
      tmp5 = builder.createBitcast(tmp5, rhsValue.getType());
    return tmp5;
  }
 
  if (condType->isVectorType() || condType->isSveVLSBuiltinType()) {
    if (!condType->isVectorType()) {
      assert(!cir::MissingFeatures::vecTernaryOp());
      cgf.cgm.errorNYI(loc, "TernaryOp for SVE vector");
      return {};
    }
 
    mlir::Value condValue = Visit(condExpr);
    mlir::Value lhsValue = Visit(lhsExpr);
    mlir::Value rhsValue = Visit(rhsExpr);
    return cir::VecTernaryOp::create(builder, loc, condValue, lhsValue,
                                     rhsValue);
  }
 
  // If this is a really simple expression (like x ? 4 : 5), emit this as a
  // select instead of as control flow.  We can only do this if it is cheap
  // and safe to evaluate the LHS and RHS unconditionally.
  if (isCheapEnoughToEvaluateUnconditionally(lhsExpr, cgf) &&
      isCheapEnoughToEvaluateUnconditionally(rhsExpr, cgf)) {
    bool lhsIsVoid = false;
    mlir::Value condV = cgf.evaluateExprAsBool(condExpr);
    assert(!cir::MissingFeatures::incrementProfileCounter());
 
    mlir::Value lhs = Visit(lhsExpr);
    if (!lhs) {
      lhs = builder.getNullValue(cgf.voidTy, loc);
      lhsIsVoid = true;
    }
 
    mlir::Value rhs = Visit(rhsExpr);
    if (lhsIsVoid) {
      assert(!rhs && "lhs and rhs types must match");
      rhs = builder.getNullValue(cgf.voidTy, loc);
    }
 
    return builder.createSelect(loc, condV, lhs, rhs);
  }
 
  mlir::Value condV = cgf.emitOpOnBoolExpr(loc, condExpr);
  CIRGenFunction::ConditionalEvaluation eval(cgf);
  SmallVector<mlir::OpBuilder::InsertPoint, 2> insertPoints{};
  mlir::Type yieldTy{};
 
  auto emitBranch = [&](mlir::OpBuilder &b, mlir::Location loc, Expr *expr) {
    CIRGenFunction::LexicalScope lexScope{cgf, loc, b.getInsertionBlock()};
    cgf.curLexScope->setAsTernary();
 
    mlir::Value branch;
    {
      // Emit any cleanups that were needed on this branch so we can spill
      // and reload the return value.
      CIRGenFunction::RunCleanupsScope branchCleanups(cgf);
      assert(!cir::MissingFeatures::incrementProfileCounter());
      eval.beginEvaluation();
      branch = Visit(expr);
      eval.endEvaluation();
      branchCleanups.forceCleanup({&branch});
    }
 
    if (branch) {
      yieldTy = branch.getType();
      cir::YieldOp::create(b, loc, branch);
    } else {
      // If LHS or RHS is a throw or void expression we need to patch
      // arms as to properly match yield types.
      insertPoints.push_back(b.saveInsertionPoint());
    }
  };
 
  mlir::Value result = cir::TernaryOp::create(
                           builder, loc, condV,
                           /*trueBuilder=*/
                           [&](mlir::OpBuilder &b, mlir::Location loc) {
                             emitBranch(b, loc, lhsExpr);
                           },
                           /*falseBuilder=*/
                           [&](mlir::OpBuilder &b, mlir::Location loc) {
                             emitBranch(b, loc, rhsExpr);
                           })
                           .getResult();
 
  if (!insertPoints.empty()) {
    // If both arms are void, so be it.
    if (!yieldTy)
      yieldTy = cgf.voidTy;
 
    // Insert required yields.
    for (mlir::OpBuilder::InsertPoint &toInsert : insertPoints) {
      mlir::OpBuilder::InsertionGuard guard(builder);
      builder.restoreInsertionPoint(toInsert);
 
      // Block does not return: build empty yield.
      if (mlir::isa<cir::VoidType>(yieldTy)) {
        cir::YieldOp::create(builder, loc);
      } else { // Block returns: set null yield value.
        mlir::Value op0 = builder.getNullValue(yieldTy, loc);
        cir::YieldOp::create(builder, loc, op0);
      }
    }
  }
 
  return result;
}
 
mlir::Value CIRGenFunction::emitScalarPrePostIncDec(const UnaryOperator *e,
                                                    LValue lv) {
  return ScalarExprEmitter(*this, builder).emitScalarPrePostIncDec(e, lv);
}