CIRGenExpr.cpp

Go to the documentation of this file.

//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This contains code to emit Expr nodes as CIR code.
//
//===----------------------------------------------------------------------===//
 
#include "Address.h"
#include "CIRGenConstantEmitter.h"
#include "CIRGenFunction.h"
#include "CIRGenModule.h"
#include "CIRGenValue.h"
#include "TargetInfo.h"
#include "mlir/Dialect/Ptr/IR/MemorySpaceInterfaces.h"
#include "mlir/IR/BuiltinAttributes.h"
#include "mlir/IR/Value.h"
#include "clang/AST/Attr.h"
#include "clang/AST/CharUnits.h"
#include "clang/AST/Decl.h"
#include "clang/AST/Expr.h"
#include "clang/AST/ExprCXX.h"
#include "clang/Basic/AddressSpaces.h"
#include "clang/Basic/TargetInfo.h"
#include "clang/CIR/Dialect/IR/CIRAttrs.h"
#include "clang/CIR/Dialect/IR/CIRDialect.h"
#include "clang/CIR/Dialect/IR/CIRTypes.h"
#include "clang/CIR/MissingFeatures.h"
#include "clang/CodeGenUtils/CodeGenUtils.h"
#include "clang/CodeGenUtils/ExprUtils.h"
#include <optional>
 
using namespace clang;
using namespace clang::CIRGen;
using namespace cir;
 
/// Get the address of a zero-sized field within a record. Zero-sized fields
/// (e.g. empty bases with [[no_unique_address]]) don't appear in the CIR
/// record layout, so we compute their address using the ASTContext field
/// offset and byte-level pointer arithmetic instead of cir.get_member.
static Address emitAddrOfZeroSizeField(CIRGenFunction &cgf, Address base,
                                       const FieldDecl *field) {
  CIRGenBuilderTy &builder = cgf.getBuilder();
  CharUnits offset = cgf.getContext().toCharUnitsFromBits(
      cgf.getContext().getFieldOffset(field));
  mlir::Type fieldType = cgf.convertType(field->getType());
 
  if (offset.isZero()) {
    return Address(builder.createPtrBitcast(base.getPointer(), fieldType),
                   base.getAlignment());
  }
 
  // Cast to byte pointer, stride by the field offset, then cast to the
  // field pointer type (CIR pointers are typed, so we need explicit casts
  // unlike OG's opaque-pointer GEP).
  mlir::Location loc = cgf.getLoc(field->getLocation());
  mlir::Value addr =
      builder.createPtrBitcast(base.getPointer(), builder.getUInt8Ty());
  addr = builder.createPtrStride(loc, addr,
                                 builder.getUInt64(offset.getQuantity(), loc));
  addr = builder.createPtrBitcast(addr, fieldType);
  return Address(addr, base.getAlignment().alignmentAtOffset(offset));
}
 
Address CIRGenFunction::emitAddrOfFieldStorage(Address base,
                                               const FieldDecl *field,
                                               llvm::StringRef fieldName,
                                               unsigned fieldIndex) {
  // Retrieve layout information for both type resolution and alignment.
  const RecordDecl *rec = field->getParent();
  const CIRGenRecordLayout &layout = cgm.getTypes().getCIRGenRecordLayout(rec);
 
  bool addressedByFieldIndex =
      field->isPotentiallyOverlapping()
          ? layout.hasCIRField(field)
          : !isEmptyFieldForLayout(getContext(), field);
  if (!addressedByFieldIndex)
    return emitAddrOfZeroSizeField(*this, base, field);
 
  mlir::Location loc = getLoc(field->getLocation());
  unsigned idx = layout.getCIRFieldNo(field);
 
  // For potentially-overlapping fields (e.g. [[no_unique_address]]), the
  // record stores the base subobject type (without tail padding) rather than
  // the complete object type. Use the record's member type for get_member,
  // then bitcast to the complete type for downstream use.
  //
  // For unions, all fields map to index 0, so we use the field's declared type
  // directly instead of looking up the member type from the layout.
  mlir::Type fieldType = convertType(field->getType());
  auto fieldPtr = cir::PointerType::get(fieldType);
  bool needsBitcast = false;
 
  if (!rec->isUnion() && field->isPotentiallyOverlapping()) {
    mlir::Type memberType = layout.getCIRType().getMembers()[idx];
    fieldPtr = cir::PointerType::get(memberType);
    needsBitcast = true;
  }
 
  // For most cases fieldName is the same as field->getName() but for lambdas,
  // which do not currently carry the name, so it can be passed down from the
  // CaptureStmt.
  mlir::Value addr = builder.createGetMember(loc, fieldPtr, base.getPointer(),
                                             fieldName, fieldIndex);
 
  // If the field is potentially overlapping, the record member uses the base
  // subobject type. Cast to the complete object pointer type expected by
  // callers (analogous to OG's opaque pointer behavior).
  if (needsBitcast)
    addr = builder.createPtrBitcast(addr, fieldType);
 
  CharUnits offset = CharUnits::fromQuantity(
      layout.getCIRType().getElementOffset(cgm.getDataLayout().layout, idx));
  return Address(addr, base.getAlignment().alignmentAtOffset(offset));
}
 
/// Given an expression of pointer type, try to
/// derive a more accurate bound on the alignment of the pointer.
Address CIRGenFunction::emitPointerWithAlignment(const Expr *expr,
                                                 LValueBaseInfo *baseInfo) {
  // We allow this with ObjC object pointers because of fragile ABIs.
  assert(expr->getType()->isPointerType() ||
         expr->getType()->isObjCObjectPointerType());
  expr = expr->IgnoreParens();
 
  // Casts:
  if (auto const *ce = dyn_cast<CastExpr>(expr)) {
    if (const auto *ece = dyn_cast<ExplicitCastExpr>(ce))
      cgm.emitExplicitCastExprType(ece);
 
    switch (ce->getCastKind()) {
    // Non-converting casts (but not C's implicit conversion from void*).
    case CK_BitCast:
    case CK_NoOp:
    case CK_AddressSpaceConversion: {
      if (const auto *ptrTy =
              ce->getSubExpr()->getType()->getAs<PointerType>()) {
        if (ptrTy->getPointeeType()->isVoidType())
          break;
 
        LValueBaseInfo innerBaseInfo;
        assert(!cir::MissingFeatures::opTBAA());
        Address addr =
            emitPointerWithAlignment(ce->getSubExpr(), &innerBaseInfo);
        if (baseInfo)
          *baseInfo = innerBaseInfo;
 
        if (isa<ExplicitCastExpr>(ce)) {
          LValueBaseInfo targetTypeBaseInfo;
 
          const QualType pointeeType = expr->getType()->getPointeeType();
          const CharUnits align =
              cgm.getNaturalTypeAlignment(pointeeType, &targetTypeBaseInfo);
 
          // If the source l-value is opaque, honor the alignment of the
          // casted-to type.
          if (innerBaseInfo.getAlignmentSource() != AlignmentSource::Decl) {
            if (baseInfo)
              baseInfo->mergeForCast(targetTypeBaseInfo);
            addr = Address(addr.getPointer(), addr.getElementType(), align);
          }
        }
 
        assert(!cir::MissingFeatures::sanitizers());
 
        const mlir::Type eltTy =
            convertTypeForMem(expr->getType()->getPointeeType());
        addr = getBuilder().createElementBitCast(getLoc(expr->getSourceRange()),
                                                 addr, eltTy);
        assert(!cir::MissingFeatures::addressSpace());
 
        return addr;
      }
      break;
    }
 
    // Array-to-pointer decay. TODO(cir): BaseInfo and TBAAInfo.
    case CK_ArrayToPointerDecay:
      return emitArrayToPointerDecay(ce->getSubExpr(), baseInfo);
 
    case CK_UncheckedDerivedToBase:
    case CK_DerivedToBase: {
      assert(!cir::MissingFeatures::opTBAA());
      assert(!cir::MissingFeatures::addressIsKnownNonNull());
      Address addr = emitPointerWithAlignment(ce->getSubExpr(), baseInfo);
      const CXXRecordDecl *derived =
          ce->getSubExpr()->getType()->getPointeeCXXRecordDecl();
      return getAddressOfBaseClass(addr, derived, ce->path(),
                                   shouldNullCheckClassCastValue(ce),
                                   ce->getExprLoc());
    }
 
    case CK_AnyPointerToBlockPointerCast:
    case CK_BaseToDerived:
    case CK_BaseToDerivedMemberPointer:
    case CK_BlockPointerToObjCPointerCast:
    case CK_BuiltinFnToFnPtr:
    case CK_CPointerToObjCPointerCast:
    case CK_DerivedToBaseMemberPointer:
    case CK_Dynamic:
    case CK_FunctionToPointerDecay:
    case CK_IntegralToPointer:
    case CK_LValueToRValue:
    case CK_LValueToRValueBitCast:
    case CK_NullToMemberPointer:
    case CK_NullToPointer:
    case CK_ReinterpretMemberPointer:
    case CK_UserDefinedConversion:
      // Common pointer conversions, nothing to do here.
      // TODO: Is there any reason to treat base-to-derived conversions
      // specially?
      break;
 
    case CK_ARCConsumeObject:
    case CK_ARCExtendBlockObject:
    case CK_ARCProduceObject:
    case CK_ARCReclaimReturnedObject:
    case CK_AtomicToNonAtomic:
    case CK_BooleanToSignedIntegral:
    case CK_ConstructorConversion:
    case CK_CopyAndAutoreleaseBlockObject:
    case CK_Dependent:
    case CK_FixedPointCast:
    case CK_FixedPointToBoolean:
    case CK_FixedPointToFloating:
    case CK_FixedPointToIntegral:
    case CK_FloatingCast:
    case CK_FloatingComplexCast:
    case CK_FloatingComplexToBoolean:
    case CK_FloatingComplexToIntegralComplex:
    case CK_FloatingComplexToReal:
    case CK_FloatingRealToComplex:
    case CK_FloatingToBoolean:
    case CK_FloatingToFixedPoint:
    case CK_FloatingToIntegral:
    case CK_HLSLAggregateSplatCast:
    case CK_HLSLArrayRValue:
    case CK_HLSLElementwiseCast:
    case CK_HLSLVectorTruncation:
    case CK_HLSLMatrixTruncation:
    case CK_IntToOCLSampler:
    case CK_IntegralCast:
    case CK_IntegralComplexCast:
    case CK_IntegralComplexToBoolean:
    case CK_IntegralComplexToFloatingComplex:
    case CK_IntegralComplexToReal:
    case CK_IntegralRealToComplex:
    case CK_IntegralToBoolean:
    case CK_IntegralToFixedPoint:
    case CK_IntegralToFloating:
    case CK_LValueBitCast:
    case CK_MatrixCast:
    case CK_MemberPointerToBoolean:
    case CK_NonAtomicToAtomic:
    case CK_ObjCObjectLValueCast:
    case CK_PointerToBoolean:
    case CK_PointerToIntegral:
    case CK_ToUnion:
    case CK_ToVoid:
    case CK_VectorSplat:
    case CK_ZeroToOCLOpaqueType:
      // Classic codegen has a default that does nothing. In CIR, we are issuing
      // a diagnostic so we can examine casts that are reached here to be sure
      // no action is needed. If nothing is needed, the cast can be moved to the
      // group above that does nothing.
      cgm.errorNYI(ce->getSourceRange(),
                   "unexpected cast for emitPointerWithAlignment: ",
                   ce->getCastKindName());
      break;
    }
  }
 
  // Unary &
  if (const UnaryOperator *uo = dyn_cast<UnaryOperator>(expr)) {
    // TODO(cir): maybe we should use a CIR unary op for pointers here instead.
    if (uo->getOpcode() == UO_AddrOf) {
      LValue lv = emitLValue(uo->getSubExpr());
      if (baseInfo)
        *baseInfo = lv.getBaseInfo();
      assert(!cir::MissingFeatures::opTBAA());
      return lv.getAddress();
    }
  }
 
  // std::addressof and variants.
  if (auto const *call = dyn_cast<CallExpr>(expr)) {
    switch (call->getBuiltinCallee()) {
    default:
      break;
    case Builtin::BIaddressof:
    case Builtin::BI__addressof:
    case Builtin::BI__builtin_addressof: {
      LValue lv = emitLValue(call->getArg(0));
      if (baseInfo)
        *baseInfo = lv.getBaseInfo();
      assert(!cir::MissingFeatures::opTBAA());
      return lv.getAddress();
    }
    }
  }
 
  // Otherwise, use the alignment of the type.
  return makeNaturalAddressForPointer(
      emitScalarExpr(expr), expr->getType()->getPointeeType(), CharUnits(),
      /*forPointeeType=*/true, baseInfo);
}
 
void CIRGenFunction::emitStoreThroughExtVectorComponentLValue(RValue src,
                                                              LValue dst) {
  auto getScalarSizeInBits = [&](mlir::Type ty) -> unsigned {
    mlir::Type scalarTy = mlir::isa<cir::VectorType>(ty)
                              ? mlir::cast<cir::VectorType>(ty).getElementType()
                              : ty;
    cir::CIRDataLayout dl = cgm.getDataLayout();
    return dl.getTypeSizeInBits(scalarTy).getFixedValue();
  };
 
  mlir::Value srcVal = src.getValue();
  Address dstAddr = dst.getExtVectorAddress();
  if (getScalarSizeInBits(dstAddr.getElementType()) >
      getScalarSizeInBits(srcVal.getType())) {
    cgm.errorNYI(
        dst.getPointer().getLoc(),
        "emitStoreThroughExtVectorComponentLValue: dstTySize > srcTysize");
    return;
  }
 
  if (getLangOpts().HLSL) {
    cgm.errorNYI(dst.getPointer().getLoc(),
                 "emitStoreThroughExtVectorComponentLValue: HLSL");
    return;
  }
 
  // This access turns into a read/modify/write of the vector.  Load the input
  // value now.
  mlir::Location loc = dst.getExtVectorPointer().getLoc();
 
  mlir::ArrayAttr elts = dst.getExtVectorElts();
 
  mlir::Value vec = builder.createLoad(loc, dstAddr, dst.isVolatile());
  if (const auto *vecTy = dst.getType()->getAs<clang::VectorType>()) {
    unsigned numSrcElts = vecTy->getNumElements();
    unsigned numDstElts = cast<cir::VectorType>(vec.getType()).getSize();
    if (numDstElts == numSrcElts) {
      // Use shuffle vector is the src and destination are the same number of
      // elements and restore the vector mask since it is on the side it will be
      // stored.
      SmallVector<int64_t> mask(numDstElts);
      for (unsigned i = 0; i != numDstElts; ++i)
        mask[getAccessedFieldNo(i, elts)] = i;
 
      vec = builder.createVecShuffle(loc, srcVal, mask);
    } else if (numDstElts > numSrcElts) {
      // Extended the source vector to the same length and then shuffle it
      // into the destination.
      // FIXME: since we're shuffling with undef, can we just use the indices
      //        into that?  This could be simpler.
      SmallVector<int64_t> extMask(numDstElts, -1);
      std::iota(extMask.begin(), extMask.begin() + numSrcElts, 0);
 
      mlir::Value extSrcVal = builder.createVecShuffle(loc, srcVal, extMask);
 
      // build identity
      SmallVector<int64_t> mask(numDstElts);
      std::iota(mask.begin(), mask.begin() + numDstElts, 0);
 
      // When the vector size is odd and .odd or .hi is used, the last element
      // of the Elts constant array will be one past the size of the vector.
      // Ignore the last element here, if it is greater than the mask size.
      if ((unsigned)getAccessedFieldNo(numSrcElts - 1, elts) == mask.size())
        numSrcElts--;
 
      // modify when what gets shuffled in
      for (unsigned i = 0; i != numSrcElts; ++i)
        mask[getAccessedFieldNo(i, elts)] = i + numDstElts;
 
      vec = builder.createVecShuffle(loc, vec, extSrcVal, mask);
    } else {
      // We should never shorten the vector
      llvm_unreachable("unexpected shorten vector length");
    }
  } else {
    // If the Src is a scalar (not a vector), and the target is a vector it
    // must be updating one element.
    unsigned inIdx = getAccessedFieldNo(0, elts);
    cir::ConstantOp elt = builder.getSInt64(inIdx, loc);
    vec = cir::VecInsertOp::create(builder, loc, vec, srcVal, elt);
  }
 
  builder.createStore(loc, vec, dst.getExtVectorAddress(),
                      dst.isVolatileQualified());
}
 
void CIRGenFunction::emitStoreThroughLValue(RValue src, LValue dst,
                                            bool isInit) {
  if (!dst.isSimple()) {
    if (dst.isVectorElt()) {
      // Read/modify/write the vector, inserting the new element
      const mlir::Location loc = dst.getVectorPointer().getLoc();
      const mlir::Value vector =
          builder.createLoad(loc, dst.getVectorAddress());
      const mlir::Value newVector = cir::VecInsertOp::create(
          builder, loc, vector, src.getValue(), dst.getVectorIdx());
      builder.createStore(loc, newVector, dst.getVectorAddress());
      return;
    }
 
    if (dst.isExtVectorElt())
      return emitStoreThroughExtVectorComponentLValue(src, dst);
 
    if (dst.isMatrixElt()) {
      cgm.errorNYI("emitStoreThroughLValue: !dst.isSimple() && isMatrixElt");
      return;
    }
 
    if (dst.isMatrixRow()) {
      cgm.errorNYI("emitStoreThroughLValue: !dst.isSimple() && isMatrixRow");
      return;
    }
 
    assert(dst.isBitField() && "Unknown LValue type");
    emitStoreThroughBitfieldLValue(src, dst);
    return;
  }
 
  assert(!cir::MissingFeatures::objCLifetime());
  assert(!cir::MissingFeatures::opLoadStoreObjC());
 
  assert(src.isScalar() && "Can't emit an aggregate store with this method");
  emitStoreOfScalar(src.getValue(), dst, isInit);
}
 
static LValue emitGlobalVarDeclLValue(CIRGenFunction &cgf, const Expr *e,
                                      const VarDecl *vd) {
  QualType t = e->getType();
 
  // In classic codegen, thread-locals get a wrapper function here. Rather than
  // doing that, we instead treat this as a normal 'global', and leave it to
  // lowerng-prepare to correctly generate the wrapper/etc.
 
  // Check if the variable is marked as declare target with link clause in
  // device codegen.
  if (cgf.getLangOpts().OpenMP)
    cgf.cgm.errorNYI(e->getSourceRange(), "emitGlobalVarDeclLValue: OpenMP");
 
  // Traditional LLVM codegen handles thread local separately, CIR handles
  // as part of getAddrOfGlobalVar.
  mlir::Value v = cgf.cgm.getAddrOfGlobalVar(vd);
 
  mlir::Type realVarTy = cgf.convertTypeForMem(vd->getType());
  cir::PointerType realPtrTy = cir::PointerType::get(
      realVarTy, mlir::cast<cir::PointerType>(v.getType()).getAddrSpace());
  if (realPtrTy != v.getType())
    v = cgf.getBuilder().createBitcast(v.getLoc(), v, realPtrTy);
 
  CharUnits alignment = cgf.getContext().getDeclAlign(vd);
  Address addr(v, realVarTy, alignment);
  LValue lv;
  if (vd->getType()->isReferenceType())
    lv = cgf.emitLoadOfReferenceLValue(addr, cgf.getLoc(e->getSourceRange()),
                                       vd->getType(), AlignmentSource::Decl);
  else
    lv = cgf.makeAddrLValue(addr, t, AlignmentSource::Decl);
  assert(!cir::MissingFeatures::setObjCGCLValueClass());
  return lv;
}
 
void CIRGenFunction::emitStoreOfScalar(mlir::Value value, Address addr,
                                       bool isVolatile, QualType ty,
                                       LValueBaseInfo baseInfo, bool isInit,
                                       bool isNontemporal) {
 
  if (const auto *clangVecTy = ty->getAs<clang::VectorType>()) {
    // Handle vectors of size 3 like size 4 for better performance.
    // TODO(CIR): Use `ABIInfo::getOptimalVectorMemoryType` once it upstreamed
    assert(!cir::MissingFeatures::cirgenABIInfo());
    if (clangVecTy->getNumElements() == 3 && !getLangOpts().PreserveVec3Type)
      cgm.errorNYI(addr.getPointer().getLoc(),
                   "emitStoreOfScalar Vec3 & PreserveVec3Type disabled");
  }
 
  value = emitToMemory(value, ty);
 
  assert(!cir::MissingFeatures::opLoadStoreTbaa());
  LValue atomicLValue = LValue::makeAddr(addr, ty, baseInfo);
  if (ty->isAtomicType() ||
      (!isInit && isLValueSuitableForInlineAtomic(atomicLValue))) {
    emitAtomicStore(RValue::get(value), atomicLValue, isInit);
    return;
  }
 
  // Update the alloca with more info on initialization.
  assert(addr.getPointer() && "expected pointer to exist");
  cir::AllocaOp srcAlloca = addr.getUnderlyingAllocaOp();
  if (currVarDecl && srcAlloca) {
    const VarDecl *vd = currVarDecl;
    assert(vd && "VarDecl expected");
    if (vd->hasInit())
      srcAlloca.setInitAttr(mlir::UnitAttr::get(&getMLIRContext()));
  }
 
  assert(currSrcLoc && "must pass in source location");
  builder.createStore(getLoc(*currSrcLoc), value, addr, isVolatile,
                      isNontemporal);
 
  assert(!cir::MissingFeatures::opTBAA());
}
 
mlir::Value CIRGenFunction::emitStoreThroughBitfieldLValue(RValue src,
                                                           LValue dst) {
 
  const CIRGenBitFieldInfo &info = dst.getBitFieldInfo();
  mlir::Type resLTy = convertTypeForMem(dst.getType());
  Address ptr = dst.getBitFieldAddress();
 
  bool useVoaltile =
      cgm.getCodeGenOpts().AAPCSBitfieldWidth && dst.isVolatileQualified() &&
      info.volatileStorageSize != 0 && CodeGenUtils::isAAPCS(cgm.getTarget());
 
  assert(currSrcLoc && "must pass in source location");
 
  return builder.createSetBitfield(getLoc(*currSrcLoc), resLTy, ptr,
                                   ptr.getElementType(), src.getValue(), info,
                                   dst.isVolatileQualified(), useVoaltile);
}
 
RValue CIRGenFunction::emitLoadOfBitfieldLValue(LValue lv, SourceLocation loc) {
  const CIRGenBitFieldInfo &info = lv.getBitFieldInfo();
 
  // Get the output type.
  mlir::Type resLTy = convertType(lv.getType());
  Address ptr = lv.getBitFieldAddress();
 
  bool useVoaltile = lv.isVolatileQualified() && info.volatileOffset != 0 &&
                     CodeGenUtils::isAAPCS(cgm.getTarget());
 
  mlir::Value field =
      builder.createGetBitfield(getLoc(loc), resLTy, ptr, ptr.getElementType(),
                                info, lv.isVolatile(), useVoaltile);
  assert(!cir::MissingFeatures::opLoadEmitScalarRangeCheck() && "NYI");
  return RValue::get(field);
}
 
Address CIRGenFunction::getAddrOfBitFieldStorage(LValue base,
                                                 const FieldDecl *field,
                                                 mlir::Type fieldType,
                                                 unsigned index) {
  mlir::Location loc = getLoc(field->getLocation());
  cir::PointerType fieldPtr = cir::PointerType::get(fieldType);
  auto rec = cast<cir::RecordType>(base.getAddress().getElementType());
  cir::GetMemberOp sea = getBuilder().createGetMember(
      loc, fieldPtr, base.getPointer(), field->getName(),
      mlir::isa<cir::UnionType>(rec) ? field->getFieldIndex() : index);
  CharUnits offset = CharUnits::fromQuantity(
      rec.getElementOffset(cgm.getDataLayout().layout, index));
  return Address(sea, base.getAlignment().alignmentAtOffset(offset));
}
 
LValue CIRGenFunction::emitLValueForBitField(LValue base,
                                             const FieldDecl *field) {
  LValueBaseInfo baseInfo = base.getBaseInfo();
  const CIRGenRecordLayout &layout =
      cgm.getTypes().getCIRGenRecordLayout(field->getParent());
  const CIRGenBitFieldInfo &info = layout.getBitFieldInfo(field);
 
  assert(!cir::MissingFeatures::preservedAccessIndexRegion());
 
  unsigned idx = layout.getCIRFieldNo(field);
  Address addr = getAddrOfBitFieldStorage(base, field, info.storageType, idx);
 
  mlir::Location loc = getLoc(field->getLocation());
  if (addr.getElementType() != info.storageType)
    addr = builder.createElementBitCast(loc, addr, info.storageType);
 
  QualType fieldType =
      field->getType().withCVRQualifiers(base.getVRQualifiers());
  // TODO(cir): Support TBAA for bit fields.
  assert(!cir::MissingFeatures::opTBAA());
  LValueBaseInfo fieldBaseInfo(baseInfo.getAlignmentSource());
  return LValue::makeBitfield(addr, info, fieldType, fieldBaseInfo);
}
 
LValue CIRGenFunction::emitLValueForField(LValue base, const FieldDecl *field) {
  LValueBaseInfo baseInfo = base.getBaseInfo();
 
  if (field->isBitField())
    return emitLValueForBitField(base, field);
 
  QualType fieldType = field->getType();
  const RecordDecl *rec = field->getParent();
  AlignmentSource baseAlignSource = baseInfo.getAlignmentSource();
  LValueBaseInfo fieldBaseInfo(getFieldAlignmentSource(baseAlignSource));
  assert(!cir::MissingFeatures::opTBAA());
 
  Address addr = base.getAddress();
  if (auto *classDecl = dyn_cast<CXXRecordDecl>(rec)) {
    if (cgm.getCodeGenOpts().StrictVTablePointers &&
        classDecl->isDynamicClass()) {
      cgm.errorNYI(field->getSourceRange(),
                   "emitLValueForField: strict vtable for dynamic class");
    }
  }
 
  unsigned recordCVR = base.getVRQualifiers();
 
  llvm::StringRef fieldName = field->getName();
  unsigned fieldIndex;
  if (cgm.lambdaFieldToName.count(field))
    fieldName = cgm.lambdaFieldToName[field];
 
  // Fields with no CIR field index (e.g. a [[no_unique_address]] field that
  // is empty for both layout and the ABI) aren't in the record layout, so
  // handle them separately: use the base address directly with the right
  // type instead of get_member.
  const CIRGenRecordLayout &layout =
      cgm.getTypes().getCIRGenRecordLayout(field->getParent());
  if (!rec->isUnion() && !layout.hasCIRField(field)) {
    addr = emitAddrOfZeroSizeField(*this, addr, field);
    LValue lv = makeAddrLValue(addr, fieldType, fieldBaseInfo);
    lv.getQuals().addCVRQualifiers(recordCVR);
    return lv;
  }
 
  if (rec->isUnion())
    fieldIndex = field->getFieldIndex();
  else
    fieldIndex = layout.getCIRFieldNo(field);
 
  addr = emitAddrOfFieldStorage(addr, field, fieldName, fieldIndex);
  assert(!cir::MissingFeatures::preservedAccessIndexRegion());
 
  // If this is a reference field, load the reference right now.
  if (fieldType->isReferenceType()) {
    assert(!cir::MissingFeatures::opTBAA());
    LValue refLVal = makeAddrLValue(addr, fieldType, fieldBaseInfo);
    if (recordCVR & Qualifiers::Volatile)
      refLVal.getQuals().addVolatile();
    addr = emitLoadOfReference(refLVal, getLoc(field->getSourceRange()),
                               &fieldBaseInfo);
 
    // Qualifiers on the struct don't apply to the referencee.
    recordCVR = 0;
    fieldType = fieldType->getPointeeType();
  }
 
  if (field->hasAttr<AnnotateAttr>()) {
    cgm.errorNYI(field->getSourceRange(), "emitLValueForField: AnnotateAttr");
    return LValue();
  }
 
  LValue lv = makeAddrLValue(addr, fieldType, fieldBaseInfo);
  lv.getQuals().addCVRQualifiers(recordCVR);
 
  // __weak attribute on a field is ignored.
  if (lv.getQuals().getObjCGCAttr() == Qualifiers::Weak) {
    cgm.errorNYI(field->getSourceRange(),
                 "emitLValueForField: __weak attribute");
    return LValue();
  }
 
  return lv;
}
 
LValue CIRGenFunction::emitLValueForFieldInitialization(
    LValue base, const clang::FieldDecl *field, llvm::StringRef fieldName) {
  QualType fieldType = field->getType();
 
  if (!fieldType->isReferenceType())
    return emitLValueForField(base, field);
 
  Address v = base.getAddress();
  const CIRGenRecordLayout &layout =
      cgm.getTypes().getCIRGenRecordLayout(field->getParent());
  if (!layout.hasCIRField(field)) {
    v = emitAddrOfZeroSizeField(*this, v, field);
  } else {
    unsigned fieldIndex = layout.getCIRFieldNo(field);
    v = emitAddrOfFieldStorage(v, field, fieldName, fieldIndex);
  }
 
  // Make sure that the address is pointing to the right type.
  mlir::Type memTy = convertTypeForMem(fieldType);
  v = builder.createElementBitCast(getLoc(field->getSourceRange()), v, memTy);
 
  // TODO: Generate TBAA information that describes this access as a structure
  // member access and not just an access to an object of the field's type. This
  // should be similar to what we do in EmitLValueForField().
  LValueBaseInfo baseInfo = base.getBaseInfo();
  AlignmentSource fieldAlignSource = baseInfo.getAlignmentSource();
  LValueBaseInfo fieldBaseInfo(getFieldAlignmentSource(fieldAlignSource));
  assert(!cir::MissingFeatures::opTBAA());
  return makeAddrLValue(v, fieldType, fieldBaseInfo);
}
 
/// Converts a scalar value from its primary IR type (as returned
/// by ConvertType) to its load/store type.
mlir::Value CIRGenFunction::emitToMemory(mlir::Value value, QualType ty) {
  if (auto *atomicTy = ty->getAs<AtomicType>())
    ty = atomicTy->getValueType();
 
  if (ty->isConstantMatrixBoolType()) {
    cgm.errorNYI("emitToMemory: ConstantMatrixBoolType");
    return {};
  }
 
  // Unlike in classic codegen CIR, bools are kept as `cir.bool` and BitInts are
  // kept as `cir.int<N>` until further lowering
  return value;
}
 
mlir::Value CIRGenFunction::emitFromMemory(mlir::Value value, QualType ty) {
  if (auto *atomicTy = ty->getAs<AtomicType>())
    ty = atomicTy->getValueType();
 
  if (ty->isPackedVectorBoolType(getContext())) {
    cgm.errorNYI("emitFromMemory: PackedVectorBoolType");
  }
 
  return value;
}
 
void CIRGenFunction::emitStoreOfScalar(mlir::Value value, LValue lvalue,
                                       bool isInit) {
  if (lvalue.getType()->isConstantMatrixType()) {
    cgm.errorNYI("emitStoreOfScalar constant matrix type");
    return;
  }
 
  emitStoreOfScalar(value, lvalue.getAddress(), lvalue.isVolatile(),
                    lvalue.getType(), lvalue.getBaseInfo(), isInit,
                    lvalue.isNontemporal());
}
 
mlir::Value CIRGenFunction::emitLoadOfScalar(Address addr, bool isVolatile,
                                             QualType ty, SourceLocation loc,
                                             LValueBaseInfo baseInfo,
                                             bool isNontemporal) {
  // Traditional LLVM codegen handles thread local separately, CIR handles
  // as part of getAddrOfGlobalVar (GetGlobalOp).
  mlir::Type eltTy = addr.getElementType();
  if (const auto *clangVecTy = ty->getAs<clang::VectorType>()) {
    // Handle vectors of size 3 like size 4 for better performance.
    assert(!cir::MissingFeatures::cirgenABIInfo());
    if (clangVecTy->getNumElements() == 3 && !getLangOpts().PreserveVec3Type)
      cgm.errorNYI(addr.getPointer().getLoc(),
                   "emitLoadOfScalar Vec3 & PreserveVec3Type disabled");
  }
 
  assert(!cir::MissingFeatures::opLoadStoreTbaa());
  LValue atomicLValue = LValue::makeAddr(addr, ty, baseInfo);
  if (ty->isAtomicType() || isLValueSuitableForInlineAtomic(atomicLValue))
    return emitAtomicLoad(atomicLValue, loc).getValue();
 
  if (mlir::isa<cir::VoidType>(eltTy))
    cgm.errorNYI(loc, "emitLoadOfScalar: void type");
 
  assert(!cir::MissingFeatures::opLoadEmitScalarRangeCheck());
 
  mlir::Value loadOp =
      builder.createLoad(getLoc(loc), addr, isVolatile, isNontemporal);
 
  // Types with a boolean representation that are not the builtin bool (an enum
  // whose underlying type is bool, or a _BitInt(1)) need no register/memory
  // conversion here: like bool and _BitInt(N), CIR keeps them in their literal
  // type until LowerToLLVM widens them to the in-memory integer type (see
  // emitToMemory).
  return loadOp;
}
 
mlir::Value CIRGenFunction::emitLoadOfScalar(LValue lvalue,
                                             SourceLocation loc) {
  assert(!cir::MissingFeatures::opLoadStoreTbaa());
  return emitLoadOfScalar(lvalue.getAddress(), lvalue.isVolatile(),
                          lvalue.getType(), loc, lvalue.getBaseInfo(),
                          lvalue.isNontemporal());
}
 
/// Given an expression that represents a value lvalue, this
/// method emits the address of the lvalue, then loads the result as an rvalue,
/// returning the rvalue.
RValue CIRGenFunction::emitLoadOfLValue(LValue lv, SourceLocation loc) {
  assert(!lv.getType()->isFunctionType());
 
  if (lv.isBitField())
    return emitLoadOfBitfieldLValue(lv, loc);
 
  if (lv.isSimple()) {
    if (lv.getType()->isConstantMatrixType()) {
      cgm.errorNYI(loc, "emitLoadOfLValue: constant matrix type");
      return RValue::get(nullptr);
    }
 
    return RValue::get(emitLoadOfScalar(lv, loc));
  }
 
  if (lv.isVectorElt()) {
    const mlir::Value load =
        builder.createLoad(getLoc(loc), lv.getVectorAddress());
    return RValue::get(cir::VecExtractOp::create(builder, getLoc(loc), load,
                                                 lv.getVectorIdx()));
  }
 
  if (lv.isExtVectorElt())
    return emitLoadOfExtVectorElementLValue(lv);
 
  cgm.errorNYI(loc, "emitLoadOfLValue");
  return RValue::get(nullptr);
}
 
int64_t CIRGenFunction::getAccessedFieldNo(unsigned int idx,
                                           const mlir::ArrayAttr elts) {
  auto elt = mlir::cast<mlir::IntegerAttr>(elts[idx]);
  return elt.getInt();
}
 
// If this is a reference to a subset of the elements of a vector, create an
// appropriate shufflevector.
RValue CIRGenFunction::emitLoadOfExtVectorElementLValue(LValue lv) {
  mlir::Location loc = lv.getExtVectorPointer().getLoc();
  mlir::Value vec = builder.createLoad(loc, lv.getExtVectorAddress());
 
  // HLSL allows treating scalars as one-element vectors. Converting the scalar
  // IR value to a vector here allows the rest of codegen to behave as normal.
  if (getLangOpts().HLSL && !mlir::isa<cir::VectorType>(vec.getType())) {
    cgm.errorNYI(loc, "emitLoadOfExtVectorElementLValue: HLSL");
    return {};
  }
 
  const mlir::ArrayAttr elts = lv.getExtVectorElts();
 
  // If the result of the expression is a non-vector type, we must be extracting
  // a single element. Just codegen as an extractelement.
  const auto *exprVecTy = lv.getType()->getAs<clang::VectorType>();
  if (!exprVecTy) {
    int64_t indexValue = getAccessedFieldNo(0, elts);
    cir::ConstantOp index =
        builder.getConstInt(loc, builder.getSInt64Ty(), indexValue);
    return RValue::get(cir::VecExtractOp::create(builder, loc, vec, index));
  }
 
  // Always use shuffle vector to try to retain the original program structure
  SmallVector<int64_t> mask;
  for (auto i : llvm::seq<unsigned>(0, exprVecTy->getNumElements()))
    mask.push_back(getAccessedFieldNo(i, elts));
 
  cir::VecShuffleOp resultVec = builder.createVecShuffle(loc, vec, mask);
  if (lv.getType()->isExtVectorBoolType()) {
    cgm.errorNYI(loc, "emitLoadOfExtVectorElementLValue: ExtVectorBoolType");
    return {};
  }
 
  return RValue::get(resultVec);
}
 
LValue
CIRGenFunction::emitPointerToDataMemberBinaryExpr(const BinaryOperator *e) {
  assert((e->getOpcode() == BO_PtrMemD || e->getOpcode() == BO_PtrMemI) &&
         "unexpected binary operator opcode");
 
  Address baseAddr = Address::invalid();
  if (e->getOpcode() == BO_PtrMemD)
    baseAddr = emitLValue(e->getLHS()).getAddress();
  else
    baseAddr = emitPointerWithAlignment(e->getLHS());
 
  const auto *memberPtrTy = e->getRHS()->getType()->castAs<MemberPointerType>();
 
  mlir::Value memberPtr = emitScalarExpr(e->getRHS());
 
  LValueBaseInfo baseInfo;
  assert(!cir::MissingFeatures::opTBAA());
  Address memberAddr = emitCXXMemberDataPointerAddress(e, baseAddr, memberPtr,
                                                       memberPtrTy, &baseInfo);
 
  return makeAddrLValue(memberAddr, memberPtrTy->getPointeeType(), baseInfo);
}
 
/// Generates lvalue for partial ext_vector access.
Address CIRGenFunction::emitExtVectorElementLValue(LValue lv,
                                                   mlir::Location loc) {
  Address vectorAddress = lv.getExtVectorAddress();
  QualType elementTy = lv.getType()->castAs<VectorType>()->getElementType();
  mlir::Type vectorElementTy = cgm.getTypes().convertType(elementTy);
  Address castToPointerElement =
      vectorAddress.withElementType(builder, vectorElementTy);
 
  mlir::ArrayAttr extVecElts = lv.getExtVectorElts();
  unsigned idx = getAccessedFieldNo(0, extVecElts);
  mlir::Value idxValue =
      builder.getConstInt(loc, mlir::cast<cir::IntType>(ptrDiffTy), idx);
 
  mlir::Value elementValue = builder.getArrayElement(
      loc, loc, castToPointerElement.getPointer(), vectorElementTy, idxValue,
      /*shouldDecay=*/false);
 
  const CharUnits eltSize = getContext().getTypeSizeInChars(elementTy);
  const CharUnits alignment =
      castToPointerElement.getAlignment().alignmentAtOffset(idx * eltSize);
  return Address(elementValue, vectorElementTy, alignment);
}
 
static cir::FuncOp emitFunctionDeclPointer(CIRGenModule &cgm, GlobalDecl gd) {
  assert(!cir::MissingFeatures::weakRefReference());
  return cgm.getAddrOfFunction(gd);
}
 
static LValue emitCapturedFieldLValue(CIRGenFunction &cgf, const FieldDecl *fd,
                                      mlir::Value thisValue) {
  return cgf.emitLValueForLambdaField(fd, thisValue);
}
 
/// Given that we are currently emitting a lambda, emit an l-value for
/// one of its members.
///
LValue CIRGenFunction::emitLValueForLambdaField(const FieldDecl *field,
                                                mlir::Value thisValue) {
  bool hasExplicitObjectParameter = false;
  const auto *md = dyn_cast_if_present<CXXMethodDecl>(curCodeDecl);
  LValue lambdaLV;
  if (md) {
    hasExplicitObjectParameter = md->isExplicitObjectMemberFunction();
    assert(md->getParent()->isLambda());
    assert(md->getParent() == field->getParent());
  }
 
  if (hasExplicitObjectParameter) {
    const VarDecl *d = cast<CXXMethodDecl>(curCodeDecl)->getParamDecl(0);
    auto it = localDeclMap.find(d);
    assert(it != localDeclMap.end() && "explicit parameter not loaded?");
    Address addrOfExplicitObject = it->second;
    if (d->getType()->isReferenceType())
      lambdaLV = emitLoadOfReferenceLValue(addrOfExplicitObject,
                                           getLoc(field->getSourceRange()),
                                           d->getType(), AlignmentSource::Decl);
    else
      lambdaLV = makeAddrLValue(addrOfExplicitObject,
                                d->getType().getNonReferenceType());
 
    // Make sure we have an lvalue to the lambda itself and not a derived class.
    auto *thisTy = d->getType().getNonReferenceType()->getAsCXXRecordDecl();
    auto *lambdaTy = cast<CXXRecordDecl>(field->getParent());
    if (thisTy != lambdaTy) {
      const CXXCastPath &basePathArray = getContext().LambdaCastPaths.at(md);
      Address base = getAddressOfBaseClass(
          lambdaLV.getAddress(), thisTy,
          llvm::make_range(basePathArray.begin(), basePathArray.end()),
          /*nullCheckValue=*/false, SourceLocation());
      CanQualType t = getContext().getCanonicalTagType(lambdaTy);
      lambdaLV = makeAddrLValue(base, t);
    }
  } else {
    CanQualType lambdaTagType =
        getContext().getCanonicalTagType(field->getParent());
    lambdaLV = makeNaturalAlignAddrLValue(thisValue, lambdaTagType);
  }
  return emitLValueForField(lambdaLV, field);
}
 
LValue CIRGenFunction::emitLValueForLambdaField(const FieldDecl *field) {
  return emitLValueForLambdaField(field, cxxabiThisValue);
}
 
static LValue emitFunctionDeclLValue(CIRGenFunction &cgf, const Expr *e,
                                     GlobalDecl gd) {
  const FunctionDecl *fd = cast<FunctionDecl>(gd.getDecl());
  cir::FuncOp funcOp = emitFunctionDeclPointer(cgf.cgm, gd);
  mlir::Location loc = cgf.getLoc(e->getSourceRange());
  CharUnits align = cgf.getContext().getDeclAlign(fd);
 
  assert(!cir::MissingFeatures::sanitizers());
 
  mlir::Type fnTy = funcOp.getFunctionType();
  mlir::Type ptrTy = cir::PointerType::get(fnTy);
  mlir::Value addr = cir::GetGlobalOp::create(cgf.getBuilder(), loc, ptrTy,
                                              funcOp.getSymName());
 
  if (funcOp.getFunctionType() != cgf.convertType(fd->getType())) {
    fnTy = cgf.convertType(fd->getType());
    ptrTy = cir::PointerType::get(fnTy);
 
    addr = cir::CastOp::create(cgf.getBuilder(), addr.getLoc(), ptrTy,
                               cir::CastKind::bitcast, addr);
  }
 
  return cgf.makeAddrLValue(Address(addr, fnTy, align), e->getType(),
                            AlignmentSource::Decl);
}
 
/// Determine whether we can emit a reference to \p vd from the current
/// context, despite not necessarily having seen an odr-use of the variable in
/// this context.
/// TODO(cir): This could be shared with classic codegen.
static bool canEmitSpuriousReferenceToVariable(CIRGenFunction &cgf,
                                               const DeclRefExpr *e,
                                               const VarDecl *vd) {
  // For a variable declared in an enclosing scope, do not emit a spurious
  // reference even if we have a capture, as that will emit an unwarranted
  // reference to our capture state, and will likely generate worse code than
  // emitting a local copy.
  if (e->refersToEnclosingVariableOrCapture())
    return false;
 
  // For a local declaration declared in this function, we can always reference
  // it even if we don't have an odr-use.
  if (vd->hasLocalStorage()) {
    return vd->getDeclContext() ==
           dyn_cast_or_null<DeclContext>(cgf.curCodeDecl);
  }
 
  // For a global declaration, we can emit a reference to it if we know
  // for sure that we are able to emit a definition of it.
  vd = vd->getDefinition(cgf.getContext());
  if (!vd)
    return false;
 
  // Don't emit a spurious reference if it might be to a variable that only
  // exists on a different device / target.
  // FIXME: This is unnecessarily broad. Check whether this would actually be a
  // cross-target reference.
  if (cgf.getLangOpts().OpenMP || cgf.getLangOpts().CUDA ||
      cgf.getLangOpts().OpenCL) {
    return false;
  }
 
  // We can emit a spurious reference only if the linkage implies that we'll
  // be emitting a non-interposable symbol that will be retained until link
  // time.
  switch (cgf.cgm.getCIRLinkageVarDefinition(vd)) {
  case cir::GlobalLinkageKind::ExternalLinkage:
  case cir::GlobalLinkageKind::LinkOnceODRLinkage:
  case cir::GlobalLinkageKind::WeakODRLinkage:
  case cir::GlobalLinkageKind::InternalLinkage:
  case cir::GlobalLinkageKind::PrivateLinkage:
    return true;
  default:
    return false;
  }
}
 
LValue CIRGenFunction::emitDeclRefLValue(const DeclRefExpr *e) {
  const NamedDecl *nd = e->getDecl();
  QualType ty = e->getType();
 
  assert(e->isNonOdrUse() != NOUR_Unevaluated &&
         "should not emit an unevaluated operand");
 
  if (const auto *vd = dyn_cast<VarDecl>(nd)) {
    // Global Named registers access via intrinsics only
    if (vd->getStorageClass() == SC_Register && vd->hasAttr<AsmLabelAttr>() &&
        !vd->isLocalVarDecl()) {
      cgm.errorNYI(e->getSourceRange(),
                   "emitDeclRefLValue: Global Named registers access");
      return LValue();
    }
 
    if (e->isNonOdrUse() == NOUR_Constant &&
        (vd->getType()->isReferenceType() ||
         !canEmitSpuriousReferenceToVariable(*this, e, vd))) {
      vd->getAnyInitializer(vd);
      mlir::Attribute val = ConstantEmitter(*this).emitAbstract(
          e->getLocation(), *vd->evaluateValue(), vd->getType());
      assert(val && "failed to emit constant expression");
 
      Address addr = Address::invalid();
      if (!vd->getType()->isReferenceType()) {
        // Spill the constant value to a global.
        addr = cgm.createUnnamedGlobalFrom(*vd, val,
                                           getContext().getDeclAlign(vd));
        mlir::Type varTy = getTypes().convertTypeForMem(vd->getType());
        auto ptrTy = mlir::cast<cir::PointerType>(addr.getPointer().getType());
        if (ptrTy.getPointee() != varTy) {
          addr = addr.withElementType(builder, varTy);
        }
      } else {
        // Should we be using the alignment of the constant pointer we emitted?
        CharUnits alignment = cgm.getNaturalTypeAlignment(
            e->getType(), /*baseInfo=*/nullptr, /*forPointeeType=*/true);
        // Classic codegen passes TBAA as null-ptr to the above function, so it
        // probably needs to deal with that.
        assert(!cir::MissingFeatures::opTBAA());
        mlir::Value ptrVal = getBuilder().getConstant(
            getLoc(e->getSourceRange()), mlir::cast<mlir::TypedAttr>(val));
        addr = makeNaturalAddressForPointer(ptrVal, ty, alignment);
      }
      return makeAddrLValue(addr, ty, AlignmentSource::Decl);
    }
 
    // Check for captured variables.
    if (e->refersToEnclosingVariableOrCapture()) {
      vd = vd->getCanonicalDecl();
      if (FieldDecl *fd = lambdaCaptureFields.lookup(vd))
        return emitCapturedFieldLValue(*this, fd, cxxabiThisValue);
      assert(!cir::MissingFeatures::cgCapturedStmtInfo());
      assert(!cir::MissingFeatures::openMP());
    }
  }
 
  // FIXME: We should be able to assert this for FunctionDecls as well!
  // FIXME: We should be able to assert this for all DeclRefExprs, not just
  // those with a valid source location.
  assert((nd->isUsed(false) || !isa<VarDecl>(nd) || e->isNonOdrUse() ||
          !e->getLocation().isValid()) &&
         "Should not use decl without marking it used!");
 
  if (nd->hasAttr<WeakRefAttr>())
    cgm.errorNYI(nd->getSourceRange(), "emitGlobal: WeakRefAttr");
 
  if (const auto *vd = dyn_cast<VarDecl>(nd)) {
    // Checks for omitted feature handling
    assert(!cir::MissingFeatures::opAllocaStaticLocal());
    assert(!cir::MissingFeatures::opAllocaNonGC());
    assert(!cir::MissingFeatures::opAllocaImpreciseLifetime());
    assert(!cir::MissingFeatures::opAllocaTLS());
    assert(!cir::MissingFeatures::opAllocaOpenMPThreadPrivate());
    assert(!cir::MissingFeatures::opAllocaEscapeByReference());
 
    // Check if this is a global variable
    if (vd->hasLinkage() || vd->isStaticDataMember())
      return emitGlobalVarDeclLValue(*this, e, vd);
 
    Address addr = Address::invalid();
 
    // The variable should generally be present in the local decl map.
    auto iter = localDeclMap.find(vd);
    if (iter != localDeclMap.end()) {
      addr = iter->second;
 
    } else if (vd->isStaticLocal()) {
      // Otherwise, it might be static local we haven't emitted yet for some
      // reason; most likely, because it's in an outer function.
      cir::GlobalOp var =
          cgm.getOrCreateStaticVarDecl(*vd, cgm.getCIRLinkageVarDefinition(vd));
      mlir::Value getGlobVal = builder.createGetGlobal(var);
      auto getGlob = getGlobVal.getDefiningOp<cir::GetGlobalOp>();
      getGlob.setStaticLocal(var.getStaticLocalGuard().has_value());
      getGlob.setTls(vd->getTLSKind() != VarDecl::TLS_None);
      addr = Address(getGlob, convertTypeForMem(vd->getType()),
                     getContext().getDeclAlign(vd));
    } else {
      llvm_unreachable("DeclRefExpr for Decl not entered in localDeclMap?");
    }
 
    // Drill into reference types.
    LValue lv =
        vd->getType()->isReferenceType()
            ? emitLoadOfReferenceLValue(addr, getLoc(e->getSourceRange()),
                                        vd->getType(), AlignmentSource::Decl)
            : makeAddrLValue(addr, ty, AlignmentSource::Decl);
 
    // Statics are defined as globals, so they are not include in the function's
    // symbol table.
    assert((vd->isStaticLocal() || symbolTable.count(vd)) &&
           "non-static locals should be already mapped");
 
    return lv;
  }
 
  if (const auto *bd = dyn_cast<BindingDecl>(nd)) {
    if (e->refersToEnclosingVariableOrCapture()) {
      FieldDecl *fd = lambdaCaptureFields.lookup(bd);
      return emitCapturedFieldLValue(*this, fd, cxxabiThisValue);
    }
    return emitLValue(bd->getBinding());
  }
 
  if (const auto *fd = dyn_cast<FunctionDecl>(nd)) {
    LValue lv = emitFunctionDeclLValue(*this, e, fd);
 
    // Emit debuginfo for the function declaration if the target wants to.
    if (getContext().getTargetInfo().allowDebugInfoForExternalRef())
      assert(!cir::MissingFeatures::generateDebugInfo());
 
    return lv;
  }
  if (isa<MSGuidDecl>(nd))
    cgm.errorNYI(e->getSourceRange(),
                 "emitDeclRefLValue: unhandled MS Guid Decl");
 
  if (const auto *tpo = dyn_cast<TemplateParamObjectDecl>(nd)) {
    CharUnits alignment = cgm.getNaturalTypeAlignment(tpo->getType());
    cir::GetGlobalOp atpo =
        builder.createGetGlobal(cgm.getAddrOfTemplateParamObject(tpo));
    assert(!MissingFeatures::addressSpace() &&
           "Do an address space conversion if necessary");
 
    return makeAddrLValue(
        Address(atpo, convertTypeForMem(tpo->getType()), alignment), ty,
        AlignmentSource::Decl);
  }
 
  llvm_unreachable("Unhandled DeclRefExpr");
}
 
mlir::Value CIRGenFunction::evaluateExprAsBool(const Expr *e) {
  QualType boolTy = getContext().BoolTy;
  SourceLocation loc = e->getExprLoc();
 
  assert(!cir::MissingFeatures::pgoUse());
  if (e->getType()->getAs<MemberPointerType>()) {
    cgm.errorNYI(e->getSourceRange(),
                 "evaluateExprAsBool: member pointer type");
    return createDummyValue(getLoc(loc), boolTy);
  }
 
  CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(*this, e);
  if (!e->getType()->isAnyComplexType())
    return emitScalarConversion(emitScalarExpr(e), e->getType(), boolTy, loc);
 
  return emitComplexToScalarConversion(emitComplexExpr(e), e->getType(), boolTy,
                                       loc);
}
 
LValue CIRGenFunction::emitUnaryOpLValue(const UnaryOperator *e) {
  UnaryOperatorKind op = e->getOpcode();
 
  // __extension__ doesn't affect lvalue-ness.
  if (op == UO_Extension)
    return emitLValue(e->getSubExpr());
 
  switch (op) {
  case UO_Deref: {
    QualType t = e->getSubExpr()->getType()->getPointeeType();
    assert(!t.isNull() && "CodeGenFunction::EmitUnaryOpLValue: Illegal type");
 
    assert(!cir::MissingFeatures::opTBAA());
    LValueBaseInfo baseInfo;
    Address addr = emitPointerWithAlignment(e->getSubExpr(), &baseInfo);
 
    // Tag 'load' with deref attribute.
    // FIXME: This misses some derefence cases and has problematic interactions
    // with other operators.
    if (auto loadOp = addr.getDefiningOp<cir::LoadOp>())
      loadOp.setIsDerefAttr(mlir::UnitAttr::get(&getMLIRContext()));
 
    LValue lv = makeAddrLValue(addr, t, baseInfo);
    assert(!cir::MissingFeatures::addressSpace());
    assert(!cir::MissingFeatures::setNonGC());
    return lv;
  }
  case UO_Real:
  case UO_Imag: {
    LValue lv = emitLValue(e->getSubExpr());
    assert(lv.isSimple() && "real/imag on non-ordinary l-value");
 
    // __real is valid on scalars. This is a faster way of testing that.
    // __imag can only produce an rvalue on scalars.
    if (e->getOpcode() == UO_Real &&
        !mlir::isa<cir::ComplexType>(lv.getAddress().getElementType())) {
      assert(e->getSubExpr()->getType()->isArithmeticType());
      return lv;
    }
 
    QualType exprTy = getContext().getCanonicalType(e->getSubExpr()->getType());
    QualType elemTy = exprTy->castAs<clang::ComplexType>()->getElementType();
    mlir::Location loc = getLoc(e->getExprLoc());
    Address component =
        e->getOpcode() == UO_Real
            ? builder.createComplexRealPtr(loc, lv.getAddress())
            : builder.createComplexImagPtr(loc, lv.getAddress());
    assert(!cir::MissingFeatures::opTBAA());
    LValue elemLV = makeAddrLValue(component, elemTy);
    elemLV.getQuals().addQualifiers(lv.getQuals());
    return elemLV;
  }
  case UO_PreInc:
  case UO_PreDec: {
    LValue lv = emitLValue(e->getSubExpr());
 
    assert(e->isPrefix() && "Prefix operator in unexpected state!");
 
    if (e->getType()->isAnyComplexType())
      emitComplexPrePostIncDec(e, lv);
    else
      emitScalarPrePostIncDec(e, lv);
 
    return lv;
  }
  case UO_Extension:
    llvm_unreachable("UnaryOperator extension should be handled above!");
  case UO_Plus:
  case UO_Minus:
  case UO_Not:
  case UO_LNot:
  case UO_AddrOf:
  case UO_PostInc:
  case UO_PostDec:
  case UO_Coawait:
    llvm_unreachable("UnaryOperator of non-lvalue kind!");
  }
  llvm_unreachable("Unknown unary operator kind!");
}
 
/// If the specified expr is a simple decay from an array to pointer,
/// return the array subexpression.
/// FIXME: this could be abstracted into a common AST helper.
static const Expr *getSimpleArrayDecayOperand(const Expr *e) {
  // If this isn't just an array->pointer decay, bail out.
  const auto *castExpr = dyn_cast<CastExpr>(e);
  if (!castExpr || castExpr->getCastKind() != CK_ArrayToPointerDecay)
    return nullptr;
 
  // If this is a decay from variable width array, bail out.
  const Expr *subExpr = castExpr->getSubExpr();
  if (subExpr->getType()->isVariableArrayType())
    return nullptr;
 
  return subExpr;
}
 
static cir::IntAttr getConstantIndexOrNull(mlir::Value idx) {
  // TODO(cir): should we consider using MLIRs IndexType instead of IntegerAttr?
  if (auto constantOp = idx.getDefiningOp<cir::ConstantOp>())
    return constantOp.getValueAttr<cir::IntAttr>();
  return {};
}
 
static CharUnits getArrayElementAlign(CharUnits arrayAlign, mlir::Value idx,
                                      CharUnits eltSize) {
  // If we have a constant index, we can use the exact offset of the
  // element we're accessing.
  if (const cir::IntAttr constantIdx = getConstantIndexOrNull(idx)) {
    const CharUnits offset = constantIdx.getValue().getZExtValue() * eltSize;
    return arrayAlign.alignmentAtOffset(offset);
  }
  // Otherwise, use the worst-case alignment for any element.
  return arrayAlign.alignmentOfArrayElement(eltSize);
}
 
static mlir::Value emitArraySubscriptPtr(CIRGenFunction &cgf,
                                         mlir::Location beginLoc,
                                         mlir::Location endLoc, mlir::Value ptr,
                                         mlir::Type eltTy, mlir::Value idx,
                                         bool shouldDecay) {
  CIRGenModule &cgm = cgf.getCIRGenModule();
  // TODO(cir): LLVM codegen emits in bound gep check here, is there anything
  // that would enhance tracking this later in CIR?
  assert(!cir::MissingFeatures::emitCheckedInBoundsGEP());
  return cgm.getBuilder().getArrayElement(beginLoc, endLoc, ptr, eltTy, idx,
                                          shouldDecay);
}
 
static Address emitArraySubscriptPtr(CIRGenFunction &cgf,
                                     mlir::Location beginLoc,
                                     mlir::Location endLoc, Address addr,
                                     QualType eltType, mlir::Value idx,
                                     mlir::Location loc, bool shouldDecay) {
 
  // Determine the element size of the statically-sized base.  This is
  // the thing that the indices are expressed in terms of.
  if (const VariableArrayType *vla =
          cgf.getContext().getAsVariableArrayType(eltType)) {
    eltType = CodeGenUtils::getFixedSizeElementType(cgf.getContext(), vla);
  }
 
  // We can use that to compute the best alignment of the element.
  const CharUnits eltSize = cgf.getContext().getTypeSizeInChars(eltType);
  const CharUnits eltAlign =
      getArrayElementAlign(addr.getAlignment(), idx, eltSize);
 
  assert(!cir::MissingFeatures::preservedAccessIndexRegion());
  const mlir::Value eltPtr =
      emitArraySubscriptPtr(cgf, beginLoc, endLoc, addr.getPointer(),
                            addr.getElementType(), idx, shouldDecay);
  const mlir::Type elementType = cgf.convertTypeForMem(eltType);
  return Address(eltPtr, elementType, eltAlign);
}
 
LValue
CIRGenFunction::emitArraySubscriptExpr(const clang::ArraySubscriptExpr *e) {
  if (e->getType()->getAs<ObjCObjectType>()) {
    cgm.errorNYI(e->getSourceRange(), "emitArraySubscriptExpr: ObjCObjectType");
    return LValue::makeAddr(Address::invalid(), e->getType(), LValueBaseInfo());
  }
 
  // The index must always be an integer, which is not an aggregate.  Emit it
  // in lexical order (this complexity is, sadly, required by C++17).
  assert((e->getIdx() == e->getLHS() || e->getIdx() == e->getRHS()) &&
         "index was neither LHS nor RHS");
 
  auto emitIdxAfterBase = [&](bool promote) -> mlir::Value {
    mlir::Value idx = emitScalarExpr(e->getIdx());
 
    assert(!cir::MissingFeatures::sanitizers());
 
    // Extend or truncate the index type to pointer-sized integer.
    if (promote) {
      // Choose the type we extend or truncate to based on the signedness of the
      // index type.
      mlir::Type desiredIdxTy =
          e->getIdx()->getType()->isSignedIntegerOrEnumerationType()
              ? ptrDiffTy
              : uIntPtrTy;
 
      if (idx.getType() != desiredIdxTy) {
        cir::CastKind kind = mlir::isa<cir::BoolType>(idx.getType())
                                 ? cir::CastKind::bool_to_int
                                 : cir::CastKind::integral;
        idx = builder.createOrFold<cir::CastOp>(idx.getLoc(), desiredIdxTy,
                                                kind, idx);
      }
    }
 
    return idx;
  };
 
  // If the base is a vector type, then we are forming a vector element
  // with this subscript.
  if (e->getBase()->getType()->isSubscriptableVectorType() &&
      !isa<ExtVectorElementExpr>(e->getBase())) {
    const mlir::Value idx = emitIdxAfterBase(/*promote=*/false);
    const LValue lv = emitLValue(e->getBase());
    return LValue::makeVectorElt(lv.getAddress(), idx, e->getBase()->getType(),
                                 lv.getBaseInfo());
  }
 
  // The HLSL runtime handles subscript expressions on global resource arrays
  // and objects with HLSL buffer layouts.
  if (getLangOpts().HLSL) {
    cgm.errorNYI(e->getSourceRange(), "emitArraySubscriptExpr: HLSL");
    return {};
  }
 
  mlir::Value idx = emitIdxAfterBase(/*promote=*/true);
 
  // Handle the extvector case we ignored above.
  if (isa<ExtVectorElementExpr>(e->getBase())) {
    const LValue lv = emitLValue(e->getBase());
    Address addr = emitExtVectorElementLValue(lv, cgm.getLoc(e->getExprLoc()));
 
    QualType elementType = lv.getType()->castAs<VectorType>()->getElementType();
    addr = emitArraySubscriptPtr(*this, cgm.getLoc(e->getBeginLoc()),
                                 cgm.getLoc(e->getEndLoc()), addr, e->getType(),
                                 idx, cgm.getLoc(e->getExprLoc()),
                                 /*shouldDecay=*/false);
 
    return makeAddrLValue(addr, elementType, lv.getBaseInfo());
  }
 
  if (const VariableArrayType *vla =
          getContext().getAsVariableArrayType(e->getType())) {
    // The base must be a pointer, which is not an aggregate.  Emit
    // it.  It needs to be emitted first in case it's what captures
    // the VLA bounds.
    Address addr = emitPointerWithAlignment(e->getBase());
 
    // The element count here is the total number of non-VLA elements.
    mlir::Value numElements = getVLASize(vla).numElts;
    idx = builder.createIntCast(idx, numElements.getType());
 
    // Effectively, the multiply by the VLA size is part of the GEP.
    // GEP indexes are signed, and scaling an index isn't permitted to
    // signed-overflow, so we use the same semantics for our explicit
    // multiply.  We suppress this if overflow is not undefined behavior.
    OverflowBehavior overflowBehavior = getLangOpts().PointerOverflowDefined
                                            ? OverflowBehavior::None
                                            : OverflowBehavior::NoSignedWrap;
    idx = builder.createMul(cgm.getLoc(e->getExprLoc()), idx, numElements,
                            overflowBehavior);
 
    addr = emitArraySubscriptPtr(*this, cgm.getLoc(e->getBeginLoc()),
                                 cgm.getLoc(e->getEndLoc()), addr, e->getType(),
                                 idx, cgm.getLoc(e->getExprLoc()),
                                 /*shouldDecay=*/false);
 
    return makeAddrLValue(addr, vla->getElementType(), LValueBaseInfo());
  }
 
  if (const Expr *array = getSimpleArrayDecayOperand(e->getBase())) {
    LValue arrayLV;
    if (const auto *ase = dyn_cast<ArraySubscriptExpr>(array))
      arrayLV = emitArraySubscriptExpr(ase);
    else
      arrayLV = emitLValue(array);
 
    // Propagate the alignment from the array itself to the result.
    const Address addr = emitArraySubscriptPtr(
        *this, cgm.getLoc(array->getBeginLoc()), cgm.getLoc(array->getEndLoc()),
        arrayLV.getAddress(), e->getType(), idx, cgm.getLoc(e->getExprLoc()),
        /*shouldDecay=*/true);
 
    const LValue lv = LValue::makeAddr(addr, e->getType(), LValueBaseInfo());
 
    if (getLangOpts().ObjC && getLangOpts().getGC() != LangOptions::NonGC) {
      cgm.errorNYI(e->getSourceRange(), "emitArraySubscriptExpr: ObjC with GC");
    }
 
    return lv;
  }
 
  // The base must be a pointer; emit it with an estimate of its alignment.
  assert(e->getBase()->getType()->isPointerType() &&
         "The base must be a pointer");
 
  LValueBaseInfo eltBaseInfo;
  const Address ptrAddr = emitPointerWithAlignment(e->getBase(), &eltBaseInfo);
  // Propagate the alignment from the array itself to the result.
  const Address addxr = emitArraySubscriptPtr(
      *this, cgm.getLoc(e->getBeginLoc()), cgm.getLoc(e->getEndLoc()), ptrAddr,
      e->getType(), idx, cgm.getLoc(e->getExprLoc()),
      /*shouldDecay=*/false);
 
  const LValue lv = LValue::makeAddr(addxr, e->getType(), eltBaseInfo);
 
  if (getLangOpts().ObjC && getLangOpts().getGC() != LangOptions::NonGC) {
    cgm.errorNYI(e->getSourceRange(), "emitArraySubscriptExpr: ObjC with GC");
  }
 
  return lv;
}
 
LValue CIRGenFunction::emitExtVectorElementExpr(const ExtVectorElementExpr *e) {
  // Emit the base vector as an l-value.
  LValue base;
 
  // ExtVectorElementExpr's base can either be a vector or pointer to vector.
  if (e->isArrow()) {
    // If it is a pointer to a vector, emit the address and form an lvalue with
    // it.
    LValueBaseInfo baseInfo;
    Address ptr = emitPointerWithAlignment(e->getBase(), &baseInfo);
    const auto *clangPtrTy =
        e->getBase()->getType()->castAs<clang::PointerType>();
    base = makeAddrLValue(ptr, clangPtrTy->getPointeeType(), baseInfo);
    base.getQuals().removeObjCGCAttr();
  } else if (e->getBase()->isGLValue()) {
    // Otherwise, if the base is an lvalue ( as in the case of foo.x.x),
    // emit the base as an lvalue.
    assert(e->getBase()->getType()->isVectorType());
    base = emitLValue(e->getBase());
  } else {
    // Otherwise, the base is a normal rvalue (as in (V+V).x), emit it as such.
    assert(e->getBase()->getType()->isVectorType() &&
           "Result must be a vector");
    mlir::Value vec = emitScalarExpr(e->getBase());
 
    // Store the vector to memory (because LValue wants an address).
    QualType baseTy = e->getBase()->getType();
    Address vecMem = createMemTemp(baseTy, vec.getLoc(), "tmp");
    if (!getLangOpts().HLSL && baseTy->isExtVectorBoolType()) {
      cgm.errorNYI(e->getSourceRange(),
                   "emitExtVectorElementExpr: ExtVectorBoolType & !HLSL");
      return {};
    }
    builder.createStore(vec.getLoc(), vec, vecMem);
    base = makeAddrLValue(vecMem, baseTy, AlignmentSource::Decl);
  }
 
  QualType type =
      e->getType().withCVRQualifiers(base.getQuals().getCVRQualifiers());
 
  // Encode the element access list into a vector of unsigned indices.
  SmallVector<uint32_t, 4> indices;
  e->getEncodedElementAccess(indices);
 
  if (base.isSimple()) {
    SmallVector<int64_t> attrElts(indices.begin(), indices.end());
    mlir::ArrayAttr elts = builder.getI64ArrayAttr(attrElts);
    return LValue::makeExtVectorElt(base.getAddress(), elts, type,
                                    base.getBaseInfo());
  }
 
  if (base.isMatrixRow()) {
    cgm.errorNYI(e->getSourceRange(), "emitExtVectorElementExpr: isMatrixRow");
    return {};
  }
 
  assert(base.isExtVectorElt() && "Can only subscript lvalue vec elts here!");
  mlir::ArrayAttr baseElts = base.getExtVectorElts();
  SmallVector<int64_t> elts;
  for (unsigned idx : indices)
    elts.push_back(getAccessedFieldNo(idx, baseElts));
  mlir::ArrayAttr cv = builder.getI64ArrayAttr(elts);
  return LValue::makeExtVectorElt(base.getAddress(), cv, type,
                                  base.getBaseInfo());
}
 
LValue CIRGenFunction::emitStringLiteralLValue(const StringLiteral *e,
                                               llvm::StringRef name) {
  cir::GlobalOp globalOp = cgm.getGlobalForStringLiteral(e, name);
  assert(globalOp.getAlignment() && "expected alignment for string literal");
  unsigned align = *(globalOp.getAlignment());
  mlir::Value addr =
      builder.createGetGlobal(getLoc(e->getSourceRange()), globalOp);
  return makeAddrLValue(
      Address(addr, globalOp.getSymType(), CharUnits::fromQuantity(align)),
      e->getType(), AlignmentSource::Decl);
}
 
/// Casts are never lvalues unless that cast is to a reference type. If the cast
/// is to a reference, we can have the usual lvalue result, otherwise if a cast
/// is needed by the code generator in an lvalue context, then it must mean that
/// we need the address of an aggregate in order to access one of its members.
/// This can happen for all the reasons that casts are permitted with aggregate
/// result, including noop aggregate casts, and cast from scalar to union.
LValue CIRGenFunction::emitCastLValue(const CastExpr *e) {
  switch (e->getCastKind()) {
  case CK_ToVoid:
  case CK_BitCast:
  case CK_LValueToRValueBitCast:
  case CK_ArrayToPointerDecay:
  case CK_FunctionToPointerDecay:
  case CK_NullToMemberPointer:
  case CK_NullToPointer:
  case CK_IntegralToPointer:
  case CK_PointerToIntegral:
  case CK_PointerToBoolean:
  case CK_IntegralCast:
  case CK_BooleanToSignedIntegral:
  case CK_IntegralToBoolean:
  case CK_IntegralToFloating:
  case CK_FloatingToIntegral:
  case CK_FloatingToBoolean:
  case CK_FloatingCast:
  case CK_FloatingRealToComplex:
  case CK_FloatingComplexToReal:
  case CK_FloatingComplexToBoolean:
  case CK_FloatingComplexCast:
  case CK_FloatingComplexToIntegralComplex:
  case CK_IntegralRealToComplex:
  case CK_IntegralComplexToReal:
  case CK_IntegralComplexToBoolean:
  case CK_IntegralComplexCast:
  case CK_IntegralComplexToFloatingComplex:
  case CK_DerivedToBaseMemberPointer:
  case CK_BaseToDerivedMemberPointer:
  case CK_MemberPointerToBoolean:
  case CK_ReinterpretMemberPointer:
  case CK_AnyPointerToBlockPointerCast:
  case CK_ARCProduceObject:
  case CK_ARCConsumeObject:
  case CK_ARCReclaimReturnedObject:
  case CK_ARCExtendBlockObject:
  case CK_CopyAndAutoreleaseBlockObject:
  case CK_IntToOCLSampler:
  case CK_FloatingToFixedPoint:
  case CK_FixedPointToFloating:
  case CK_FixedPointCast:
  case CK_FixedPointToBoolean:
  case CK_FixedPointToIntegral:
  case CK_IntegralToFixedPoint:
  case CK_MatrixCast:
  case CK_HLSLVectorTruncation:
  case CK_HLSLMatrixTruncation:
  case CK_HLSLArrayRValue:
  case CK_HLSLElementwiseCast:
  case CK_HLSLAggregateSplatCast:
    llvm_unreachable("unexpected cast lvalue");
 
  case CK_Dependent:
    llvm_unreachable("dependent cast kind in IR gen!");
 
  case CK_BuiltinFnToFnPtr:
    llvm_unreachable("builtin functions are handled elsewhere");
 
  case CK_Dynamic: {
    LValue lv = emitLValue(e->getSubExpr());
    Address v = lv.getAddress();
    const auto *dce = cast<CXXDynamicCastExpr>(e);
    return makeNaturalAlignAddrLValue(emitDynamicCast(v, dce), e->getType());
  }
 
  // These are never l-values; just use the aggregate emission code.
  case CK_ToUnion:
    return emitAggExprToLValue(e);
 
  case CK_ConstructorConversion:
  case CK_UserDefinedConversion:
  case CK_CPointerToObjCPointerCast:
  case CK_BlockPointerToObjCPointerCast:
  case CK_LValueToRValue:
    return emitLValue(e->getSubExpr());
 
  case CK_NonAtomicToAtomic:
  case CK_AtomicToNonAtomic:
  case CK_ObjCObjectLValueCast:
  case CK_VectorSplat: {
    cgm.errorNYI(e->getSourceRange(),
                 std::string("emitCastLValue for unhandled cast kind: ") +
                     e->getCastKindName());
 
    return {};
  }
 
  case CK_AddressSpaceConversion: {
    LValue lv = emitLValue(e->getSubExpr());
    QualType destTy = getContext().getPointerType(e->getType());
 
    mlir::Value v = performAddrSpaceCast(lv.getPointer(), convertType(destTy));
 
    return makeAddrLValue(Address(v, convertTypeForMem(e->getType()),
                                  lv.getAddress().getAlignment()),
                          e->getType(), lv.getBaseInfo());
  }
 
  case CK_LValueBitCast: {
    // This must be a reinterpret_cast (or c-style equivalent).
    const auto *ce = cast<ExplicitCastExpr>(e);
 
    cgm.emitExplicitCastExprType(ce, this);
    LValue LV = emitLValue(e->getSubExpr());
    Address V = LV.getAddress().withElementType(
        builder, convertTypeForMem(ce->getTypeAsWritten()->getPointeeType()));
 
    return makeAddrLValue(V, e->getType(), LV.getBaseInfo());
  }
 
  case CK_NoOp: {
    // CK_NoOp can model a qualification conversion, which can remove an array
    // bound and change the IR type.
    LValue lv = emitLValue(e->getSubExpr());
    // Propagate the volatile qualifier to LValue, if exists in e.
    if (e->changesVolatileQualification())
      lv.getQuals() = e->getType().getQualifiers();
    if (lv.isSimple()) {
      Address v = lv.getAddress();
      if (v.isValid()) {
        mlir::Type ty = convertTypeForMem(e->getType());
        if (v.getElementType() != ty) {
          // We have only inspected/reproduced this with complete to incomplete
          // array types, so we do an NYI for other cases, so we can make sure
          // we're doing a conversion we want to be making.
          auto fromTy = dyn_cast<cir::ArrayType>(v.getElementType());
          auto toTy = dyn_cast<cir::ArrayType>(ty);
          if (!fromTy || !toTy ||
              fromTy.getElementType() != toTy.getElementType() ||
              toTy.getSize() != 0)
            cgm.errorNYI(e->getSourceRange(),
                         "emitCastLValue NoOp not array-shrink case");
 
          lv = makeAddrLValue(v.withElementType(builder, ty), e->getType(),
                              lv.getBaseInfo());
        }
      }
    }
    return lv;
  }
 
  case CK_UncheckedDerivedToBase:
  case CK_DerivedToBase: {
    auto *derivedClassDecl = e->getSubExpr()->getType()->castAsCXXRecordDecl();
 
    LValue lv = emitLValue(e->getSubExpr());
    Address thisAddr = lv.getAddress();
 
    // Perform the derived-to-base conversion
    Address baseAddr =
        getAddressOfBaseClass(thisAddr, derivedClassDecl, e->path(),
                              /*NullCheckValue=*/false, e->getExprLoc());
 
    // TODO: Support accesses to members of base classes in TBAA. For now, we
    // conservatively pretend that the complete object is of the base class
    // type.
    assert(!cir::MissingFeatures::opTBAA());
    return makeAddrLValue(baseAddr, e->getType(), lv.getBaseInfo());
  }
 
  case CK_BaseToDerived: {
    const auto *derivedClassDecl = e->getType()->castAsCXXRecordDecl();
    LValue lv = emitLValue(e->getSubExpr());
 
    // Perform the base-to-derived conversion
    Address derived = getAddressOfDerivedClass(
        getLoc(e->getSourceRange()), lv.getAddress(), derivedClassDecl,
        e->path(), /*NullCheckValue=*/false);
    // C++11 [expr.static.cast]p2: Behavior is undefined if a downcast is
    // performed and the object is not of the derived type.
    assert(!cir::MissingFeatures::sanitizers());
 
    assert(!cir::MissingFeatures::opTBAA());
    return makeAddrLValue(derived, e->getType(), lv.getBaseInfo());
  }
 
  case CK_ZeroToOCLOpaqueType:
    llvm_unreachable("NULL to OpenCL opaque type lvalue cast is not valid");
  }
 
  llvm_unreachable("Invalid cast kind");
}
 
static DeclRefExpr *tryToConvertMemberExprToDeclRefExpr(CIRGenFunction &cgf,
                                                        const MemberExpr *me) {
  if (auto *vd = dyn_cast<VarDecl>(me->getMemberDecl())) {
    // Try to emit static variable member expressions as DREs.
    return DeclRefExpr::Create(
        cgf.getContext(), NestedNameSpecifierLoc(), SourceLocation(), vd,
        /*RefersToEnclosingVariableOrCapture=*/false, me->getExprLoc(),
        me->getType(), me->getValueKind(), nullptr, nullptr, me->isNonOdrUse());
  }
  return nullptr;
}
 
LValue CIRGenFunction::emitMemberExpr(const MemberExpr *e) {
  if (DeclRefExpr *dre = tryToConvertMemberExprToDeclRefExpr(*this, e)) {
    emitIgnoredExpr(e->getBase());
    return emitDeclRefLValue(dre);
  }
 
  Expr *baseExpr = e->getBase();
  // If this is s.x, emit s as an lvalue.  If it is s->x, emit s as a scalar.
  LValue baseLV;
  if (e->isArrow()) {
    LValueBaseInfo baseInfo;
    assert(!cir::MissingFeatures::opTBAA());
    Address addr = emitPointerWithAlignment(baseExpr, &baseInfo);
    QualType ptrTy = baseExpr->getType()->getPointeeType();
    assert(!cir::MissingFeatures::typeChecks());
    baseLV = makeAddrLValue(addr, ptrTy, baseInfo);
  } else {
    assert(!cir::MissingFeatures::typeChecks());
    baseLV = emitLValue(baseExpr);
  }
 
  const NamedDecl *nd = e->getMemberDecl();
  if (auto *field = dyn_cast<FieldDecl>(nd)) {
    LValue lv = emitLValueForField(baseLV, field);
    assert(!cir::MissingFeatures::setObjCGCLValueClass());
    if (getLangOpts().OpenMP) {
      // If the member was explicitly marked as nontemporal, mark it as
      // nontemporal. If the base lvalue is marked as nontemporal, mark access
      // to children as nontemporal too.
      cgm.errorNYI(e->getSourceRange(), "emitMemberExpr: OpenMP");
    }
    return lv;
  }
 
  if (const auto *fd = dyn_cast<FunctionDecl>(nd))
    return emitFunctionDeclLValue(*this, e, fd);
 
  llvm_unreachable("Unhandled member declaration!");
}
 
/// Evaluate an expression into a given memory location.
void CIRGenFunction::emitAnyExprToMem(const Expr *e, Address location,
                                      Qualifiers quals, bool isInit) {
  // FIXME: This function should take an LValue as an argument.
  switch (getEvaluationKind(e->getType())) {
  case cir::TEK_Complex: {
    LValue lv = makeAddrLValue(location, e->getType());
    emitComplexExprIntoLValue(e, lv, isInit);
    return;
  }
 
  case cir::TEK_Aggregate: {
    emitAggExpr(e, AggValueSlot::forAddr(location, quals,
                                         AggValueSlot::IsDestructed_t(isInit),
                                         AggValueSlot::IsAliased_t(!isInit),
                                         AggValueSlot::MayOverlap));
    return;
  }
 
  case cir::TEK_Scalar: {
    RValue rv = RValue::get(emitScalarExpr(e));
    LValue lv = makeAddrLValue(location, e->getType());
    emitStoreThroughLValue(rv, lv);
    return;
  }
  }
 
  llvm_unreachable("bad evaluation kind");
}
 
static Address createReferenceTemporary(CIRGenFunction &cgf,
                                        const MaterializeTemporaryExpr *m,
                                        const Expr *inner) {
  // TODO(cir): cgf.getTargetHooks();
  switch (m->getStorageDuration()) {
  case SD_FullExpression:
  case SD_Automatic: {
    QualType ty = inner->getType();
 
    assert(!cir::MissingFeatures::mergeAllConstants());
 
    // The temporary memory should be created in the same scope as the extending
    // declaration of the temporary materialization expression.
    cir::AllocaOp extDeclAlloca;
    if (const ValueDecl *extDecl = m->getExtendingDecl()) {
      auto extDeclAddrIter = cgf.localDeclMap.find(extDecl);
      if (extDeclAddrIter != cgf.localDeclMap.end())
        extDeclAlloca = extDeclAddrIter->second.getUnderlyingAllocaOp();
    }
    mlir::OpBuilder::InsertPoint ip;
    if (extDeclAlloca) {
      ip = {extDeclAlloca->getBlock(), extDeclAlloca->getIterator()};
    } else if (cgf.isInConditionalBranch() &&
               m->getStorageDuration() == SD_FullExpression) {
      // Place in the function entry block so the alloca dominates both
      // regions of any enclosing cir.cleanup.scope.  The default path
      // would use curLexScope which may be a ternary branch.
      ip = cgf.getBuilder().getBestAllocaInsertPoint(
          cgf.getCurFunctionEntryBlock());
    }
    return cgf.createMemTemp(ty, cgf.getLoc(m->getSourceRange()),
                             cgf.getCounterRefTmpAsString(), /*alloca=*/nullptr,
                             ip);
  }
  case SD_Thread:
  case SD_Static: {
    auto addr =
        mlir::cast<cir::GlobalOp>(cgf.cgm.getAddrOfGlobalTemporary(m, inner));
    auto getGlobal = cgf.cgm.getBuilder().createGetGlobal(addr);
    assert(addr.getAlignment().has_value() &&
           "This should always have an alignment");
    return Address(getGlobal,
                   clang::CharUnits::fromQuantity(addr.getAlignment().value()));
  }
 
  case SD_Dynamic:
    llvm_unreachable("temporary can't have dynamic storage duration");
  }
  llvm_unreachable("unknown storage duration");
}
 
static void pushTemporaryCleanup(CIRGenFunction &cgf,
                                 const MaterializeTemporaryExpr *m,
                                 const Expr *e, Address referenceTemporary) {
  // Objective-C++ ARC:
  //   If we are binding a reference to a temporary that has ownership, we
  //   need to perform retain/release operations on the temporary.
  //
  // FIXME(ogcg): This should be looking at e, not m.
  if (m->getType().getObjCLifetime()) {
    cgf.cgm.errorNYI(e->getSourceRange(), "pushTemporaryCleanup: ObjCLifetime");
    return;
  }
 
  const QualType::DestructionKind dk = e->getType().isDestructedType();
  if (dk == QualType::DK_none)
    return;
 
  switch (m->getStorageDuration()) {
  case SD_Static:
  case SD_Thread: {
    CXXDestructorDecl *referenceTemporaryDtor = nullptr;
    if (const auto *classDecl =
            e->getType()->getBaseElementTypeUnsafe()->getAsCXXRecordDecl();
        classDecl && !classDecl->hasTrivialDestructor())
      // Get the destructor for the reference temporary.
      referenceTemporaryDtor = classDecl->getDestructor();
 
    if (!referenceTemporaryDtor)
      return;
 
    // Classic codegen calls registerGlobalDtor here, passing either the
    // destructor or a generated array-destroy helper. CIR instead emits the
    // destruction into the dtor region of whatever destroys the variable that
    // extended the temporary: the cir.global's own region at namespace scope,
    // and the enclosing cir.local_init's for a function-local static, which the
    // verifier requires to be destroyed in-function under its guard.
    CIRGenModule &cgm = cgf.cgm;
    auto globalOp =
        mlir::cast<cir::GlobalOp>(cgm.getAddrOfGlobalTemporary(m, e));
 
    mlir::Region *dtorRegion = cgf.curStaticVarDtorRegion;
    assert(dtorRegion && "temporary extended outside a static initializer");
 
    CIRGenBuilderTy &builder = cgm.getBuilder();
    mlir::OpBuilder::InsertionGuard guard(builder);
    mlir::Location loc = cgm.getLoc(m->getSourceRange());
 
    // Temporaries are destroyed in reverse order of construction, so each one
    // goes in front of those registered before it.
    if (dtorRegion->empty()) {
      builder.setInsertionPointToStart(builder.createBlock(dtorRegion));
      cir::YieldOp::create(builder, loc);
    }
    builder.setInsertionPointToStart(&dtorRegion->front());
 
    mlir::Value tempAddr = builder.createGetGlobal(globalOp);
 
    if (e->getType()->isArrayType()) {
      // emitDestroy will produce a cir.array.dtor here. LoweringPrepare's
      // getOrCreateDtorFunc recognizes the non-trivial dtor region and
      // hoists it into a __cxx_global_array_dtor helper.
      Address addr{tempAddr, cgf.convertTypeForMem(e->getType()),
                   referenceTemporary.getAlignment()};
      cgf.emitDestroy(addr, e->getType(), CIRGenFunction::destroyCXXObject);
    } else {
      GlobalDecl gd(referenceTemporaryDtor, Dtor_Complete);
      cir::FuncOp dtorFn = cgm.getAddrAndTypeOfCXXStructor(gd).second;
      builder.createCallOp(loc, dtorFn, mlir::ValueRange{tempAddr});
    }
    break;
  }
 
  case SD_FullExpression:
    cgf.pushDestroy(NormalAndEHCleanup, referenceTemporary, e->getType(),
                    CIRGenFunction::destroyCXXObject);
    break;
 
  case SD_Automatic:
    cgf.pushLifetimeExtendedDestroy(
        NormalAndEHCleanup, referenceTemporary, e->getType(),
        CIRGenFunction::destroyCXXObject, cgf.getLangOpts().Exceptions);
    break;
 
  case SD_Dynamic:
    llvm_unreachable("temporary cannot have dynamic storage duration");
  }
}
 
LValue CIRGenFunction::emitMaterializeTemporaryExpr(
    const MaterializeTemporaryExpr *m) {
  const Expr *e = m->getSubExpr();
 
  assert((!m->getExtendingDecl() || !isa<VarDecl>(m->getExtendingDecl()) ||
          !cast<VarDecl>(m->getExtendingDecl())->isARCPseudoStrong()) &&
         "Reference should never be pseudo-strong!");
 
  // FIXME: ideally this would use emitAnyExprToMem, however, we cannot do so
  // as that will cause the lifetime adjustment to be lost for ARC
  auto ownership = m->getType().getObjCLifetime();
  if (ownership != Qualifiers::OCL_None &&
      ownership != Qualifiers::OCL_ExplicitNone) {
    cgm.errorNYI(e->getSourceRange(),
                 "emitMaterializeTemporaryExpr: ObjCLifetime");
    return {};
  }
 
  SmallVector<const Expr *, 2> commaLHSs;
  SmallVector<SubobjectAdjustment, 2> adjustments;
  e = e->skipRValueSubobjectAdjustments(commaLHSs, adjustments);
 
  for (const Expr *ignored : commaLHSs)
    emitIgnoredExpr(ignored);
 
  if (isa<OpaqueValueExpr>(e)) {
    cgm.errorNYI(e->getSourceRange(),
                 "emitMaterializeTemporaryExpr: OpaqueValueExpr");
    return {};
  }
 
  // Create and initialize the reference temporary.
  Address object = createReferenceTemporary(*this, m, e);
  cir::GlobalOp var = nullptr;
  if (auto getGlobalOp = object.getPointer().getDefiningOp<cir::GetGlobalOp>())
    var = mlir::dyn_cast_or_null<cir::GlobalOp>(
        cgm.getGlobalValue(getGlobalOp.getName()));
 
  if (var) {
    if (!var.getInitialValue().has_value()) {
      var.setInitialValueAttr(builder.getZeroInitAttr(var.getSymType()));
      assert(!cir::MissingFeatures::pointerAuthentication());
      emitAnyExprToMem(e, object, Qualifiers(), /*isInitializer=*/true);
    }
  } else {
    assert(!cir::MissingFeatures::emitLifetimeMarkers());
    emitAnyExprToMem(e, object, Qualifiers(), /*isInitializer=*/true);
  }
  pushTemporaryCleanup(*this, m, e, object);
 
  // Perform derived-to-base casts and/or field accesses, to get from the
  // temporary object we created (and, potentially, for which we extended
  // the lifetime) to the subobject we're binding the reference to.
  for (SubobjectAdjustment &adjustment : llvm::reverse(adjustments)) {
    switch (adjustment.Kind) {
    case SubobjectAdjustment::DerivedToBaseAdjustment:
      object =
          getAddressOfBaseClass(object, adjustment.DerivedToBase.DerivedClass,
                                adjustment.DerivedToBase.BasePath->path(),
                                /*nullCheckValue=*/false, e->getExprLoc());
      break;
    case SubobjectAdjustment::FieldAdjustment: {
      LValue lv = makeAddrLValue(object, e->getType(), AlignmentSource::Decl);
      lv = emitLValueForField(lv, adjustment.Field);
      assert(lv.isSimple() &&
             "materialized temporary field is not a simple lvalue");
      object = lv.getAddress();
      break;
    }
    case SubobjectAdjustment::MemberPointerAdjustment: {
      mlir::Value ptr = emitScalarExpr(adjustment.Ptr.RHS);
      object = emitCXXMemberDataPointerAddress(
          e, object, ptr, adjustment.Ptr.MPT, /*baseInfo=*/nullptr);
      break;
    }
    }
  }
 
  return makeAddrLValue(object, m->getType(), AlignmentSource::Decl);
}
 
LValue
CIRGenFunction::getOrCreateOpaqueLValueMapping(const OpaqueValueExpr *e) {
  assert(OpaqueValueMapping::shouldBindAsLValue(e));
 
  auto it = opaqueLValues.find(e);
  if (it != opaqueLValues.end())
    return it->second;
 
  assert(e->isUnique() && "LValue for a nonunique OVE hasn't been emitted");
  return emitLValue(e->getSourceExpr());
}
 
RValue
CIRGenFunction::getOrCreateOpaqueRValueMapping(const OpaqueValueExpr *e) {
  assert(!OpaqueValueMapping::shouldBindAsLValue(e));
 
  auto it = opaqueRValues.find(e);
  if (it != opaqueRValues.end())
    return it->second;
 
  assert(e->isUnique() && "RValue for a nonunique OVE hasn't been emitted");
  return emitAnyExpr(e->getSourceExpr());
}
 
LValue CIRGenFunction::emitCompoundLiteralLValue(const CompoundLiteralExpr *e) {
  if (e->isFileScope()) {
    cgm.errorNYI(e->getSourceRange(), "emitCompoundLiteralLValue: FileScope");
    return {};
  }
 
  if (e->getType()->isVariablyModifiedType())
    emitVariablyModifiedType(e->getType());
 
  Address declPtr = createMemTemp(e->getType(), getLoc(e->getSourceRange()),
                                  ".compoundliteral");
  const Expr *initExpr = e->getInitializer();
  LValue result = makeAddrLValue(declPtr, e->getType(), AlignmentSource::Decl);
 
  emitAnyExprToMem(initExpr, declPtr, e->getType().getQualifiers(),
                   /*Init*/ true);
 
  // Block-scope compound literals are destroyed at the end of the enclosing
  // scope in C.
  if (!getLangOpts().CPlusPlus && e->getType().isDestructedType()) {
    cgm.errorNYI(e->getSourceRange(),
                 "emitCompoundLiteralLValue: non C++ DestructedType");
    return {};
  }
 
  return result;
}
 
LValue CIRGenFunction::emitCallExprLValue(const CallExpr *e) {
  RValue rv = emitCallExpr(e);
 
  if (!rv.isScalar())
    return makeAddrLValue(rv.getAggregateAddress(), e->getType(),
                          AlignmentSource::Decl);
 
  assert(e->getCallReturnType(getContext())->isReferenceType() &&
         "Can't have a scalar return unless the return type is a "
         "reference type!");
 
  return makeNaturalAlignPointeeAddrLValue(rv.getValue(), e->getType());
}
 
LValue
CIRGenFunction::emitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *e) {
  AggValueSlot slot =
      createAggTemp(e->getType(), getLoc(e->getSourceRange()), "temp.lvalue");
  slot.setExternallyDestructed();
  emitAggExpr(e->getSubExpr(), slot);
  emitCXXTemporary(e->getTemporary(), e->getType(), slot.getAddress());
  return makeAddrLValue(slot.getAddress(), e->getType(), AlignmentSource::Decl);
}
 
LValue CIRGenFunction::emitCXXConstructLValue(const CXXConstructExpr *e) {
  assert(e->getType()->getAsCXXRecordDecl()->hasTrivialDestructor() &&
         "binding l-value to type which needs a temporary");
  AggValueSlot slot = createAggTemp(e->getType(), getLoc(e->getSourceRange()));
  emitCXXConstructExpr(e, slot);
  return makeAddrLValue(slot.getAddress(), e->getType(), AlignmentSource::Decl);
}
 
LValue CIRGenFunction::emitBinaryOperatorLValue(const BinaryOperator *e) {
  // Comma expressions just emit their LHS then their RHS as an l-value.
  if (e->getOpcode() == BO_Comma) {
    emitIgnoredExpr(e->getLHS());
    return emitLValue(e->getRHS());
  }
 
  if (e->getOpcode() == BO_PtrMemD || e->getOpcode() == BO_PtrMemI)
    return emitPointerToDataMemberBinaryExpr(e);
 
  assert(e->getOpcode() == BO_Assign && "unexpected binary l-value");
 
  // Note that in all of these cases, __block variables need the RHS
  // evaluated first just in case the variable gets moved by the RHS.
 
  switch (CIRGenFunction::getEvaluationKind(e->getType())) {
  case cir::TEK_Scalar: {
    assert(!cir::MissingFeatures::objCLifetime());
    if (e->getLHS()->getType().getObjCLifetime() !=
        clang::Qualifiers::ObjCLifetime::OCL_None) {
      cgm.errorNYI(e->getSourceRange(), "objc lifetimes");
      return {};
    }
 
    RValue rv = emitAnyExpr(e->getRHS());
    LValue lv = emitLValue(e->getLHS());
 
    SourceLocRAIIObject loc{*this, e->getSourceRange()};
    if (lv.isBitField())
      emitStoreThroughBitfieldLValue(rv, lv);
    else
      emitStoreThroughLValue(rv, lv);
 
    if (getLangOpts().OpenMP) {
      cgm.errorNYI(e->getSourceRange(), "openmp");
      return {};
    }
 
    return lv;
  }
 
  case cir::TEK_Complex: {
    return emitComplexAssignmentLValue(e);
  }
 
  case cir::TEK_Aggregate:
    cgm.errorNYI(e->getSourceRange(), "aggregate lvalues");
    return {};
  }
  llvm_unreachable("bad evaluation kind");
}
 
/// Emit code to compute the specified expression which
/// can have any type.  The result is returned as an RValue struct.
RValue CIRGenFunction::emitAnyExpr(const Expr *e, AggValueSlot aggSlot,
                                   bool ignoreResult) {
  switch (CIRGenFunction::getEvaluationKind(e->getType())) {
  case cir::TEK_Scalar:
    return RValue::get(emitScalarExpr(e, ignoreResult));
  case cir::TEK_Complex:
    return RValue::getComplex(emitComplexExpr(e));
  case cir::TEK_Aggregate: {
    if (!ignoreResult && aggSlot.isIgnored())
      aggSlot = createAggTemp(e->getType(), getLoc(e->getSourceRange()),
                              getCounterAggTmpAsString());
    emitAggExpr(e, aggSlot);
    return aggSlot.asRValue();
  }
  }
  llvm_unreachable("bad evaluation kind");
}
 
CIRGenCallee CIRGenFunction::emitDirectCallee(const GlobalDecl &gd) {
  const auto *fd = cast<FunctionDecl>(gd.getDecl());
 
  if (unsigned builtinID = fd->getBuiltinID()) {
    StringRef ident = cgm.getMangledName(gd);
    std::string fdInlineName = (ident + ".inline").str();
 
    bool isPredefinedLibFunction =
        cgm.getASTContext().BuiltinInfo.isPredefinedLibFunction(builtinID);
    // TODO: Read no-builtin function attribute and set this accordingly.
    // Using false here matches OGCG's default behavior - builtins are called
    // as builtins unless explicitly disabled. The previous value of true was
    // overly conservative and caused functions to be marked as no_inline when
    // they shouldn't be.
    bool hasAttributeNoBuiltin = false;
    assert(!cir::MissingFeatures::attributeNoBuiltin());
 
    // When directly calling an inline builtin, call it through it's mangled
    // name to make it clear it's not the actual builtin.
    if (auto fn = dyn_cast<cir::FuncOp>(curFn);
        (!fn || fn.getName() != fdInlineName) &&
        CodeGenUtils::onlyHasInlineBuiltinDeclaration(fd)) {
      cir::FuncOp clone =
          mlir::cast_or_null<cir::FuncOp>(cgm.getGlobalValue(fdInlineName));
 
      if (!clone) {
        // Create a forward declaration - the body will be generated in
        // generateCode when the function definition is processed
        cir::FuncOp calleeFunc = emitFunctionDeclPointer(cgm, gd);
        mlir::OpBuilder::InsertionGuard guard(builder);
        builder.setInsertionPointToStart(cgm.getModule().getBody());
 
        clone = cir::FuncOp::create(builder, calleeFunc.getLoc(), fdInlineName,
                                    calleeFunc.getFunctionType());
        cgm.insertGlobalSymbol(clone);
        clone.setLinkageAttr(cir::GlobalLinkageKindAttr::get(
            &cgm.getMLIRContext(), cir::GlobalLinkageKind::InternalLinkage));
        clone.setSymVisibility("private");
        clone.setInlineKind(cir::InlineKind::AlwaysInline);
      }
      return CIRGenCallee::forDirect(clone, gd);
    }
 
    // Replaceable builtins provide their own implementation of a builtin. If we
    // are in an inline builtin implementation, avoid trivial infinite
    // recursion. Honor __attribute__((no_builtin("foo"))) or
    // __attribute__((no_builtin)) on the current function unless foo is
    // not a predefined library function which means we must generate the
    // builtin no matter what.
    else if (!isPredefinedLibFunction || !hasAttributeNoBuiltin)
      return CIRGenCallee::forBuiltin(builtinID, fd);
  }
 
  cir::FuncOp callee = emitFunctionDeclPointer(cgm, gd);
 
  if ((cgm.getLangOpts().CUDA || cgm.getLangOpts().HIP) &&
      !cgm.getLangOpts().CUDAIsDevice && fd->hasAttr<CUDAGlobalAttr>()) {
    mlir::Operation *handle = cgm.getCUDARuntime().getKernelHandle(callee, gd);
    callee =
        mlir::cast<cir::FuncOp>(*cgm.getCUDARuntime().getKernelStub(handle));
  }
 
  return CIRGenCallee::forDirect(callee, gd);
}
 
mlir::Value CIRGenFunction::getUndefConstant(mlir::Location loc,
                                             mlir::Type cirTy) {
  return builder.getConstant(loc, cir::UndefAttr::get(cirTy));
}
 
RValue CIRGenFunction::getUndefRValue(QualType ty) {
  if (ty->isVoidType())
    return RValue::get(nullptr);
 
  mlir::Location loc = builder.getUnknownLoc();
 
  switch (getEvaluationKind(ty)) {
  case cir::TEK_Complex: {
    QualType elemTy = ty->castAs<ComplexType>()->getElementType();
    mlir::Type elemCirTy = convertType(elemTy);
    mlir::Value undefElem = getUndefConstant(loc, elemCirTy);
    mlir::Value v = builder.createComplexCreate(loc, undefElem, undefElem);
    return RValue::getComplex(v);
  }
 
  // If this is a use of an undefined aggregate type, the aggregate must have
  // an identifiable address.  Just because the contents of the value are
  // undefined doesn't mean that the address can't be taken and compared.
  case cir::TEK_Aggregate: {
    Address destPtr = createMemTempWithoutCast(ty, loc, "undef.agg.tmp");
    return RValue::getAggregate(destPtr);
  }
 
  case cir::TEK_Scalar:
    return RValue::get(getUndefConstant(loc, convertType(ty)));
  }
  llvm_unreachable("bad evaluation kind");
}
 
RValue CIRGenFunction::emitCall(clang::QualType calleeTy,
                                const CIRGenCallee &origCallee,
                                const clang::CallExpr *e,
                                ReturnValueSlot returnValue) {
  // Get the actual function type. The callee type will always be a pointer to
  // function type or a block pointer type.
  assert(calleeTy->isFunctionPointerType() &&
         "Callee must have function pointer type!");
 
  calleeTy = getContext().getCanonicalType(calleeTy);
  auto pointeeTy = cast<PointerType>(calleeTy)->getPointeeType();
 
  CIRGenCallee callee = origCallee;
 
  if (getLangOpts().CPlusPlus)
    assert(!cir::MissingFeatures::sanitizers());
 
  const auto *fnType = cast<FunctionType>(pointeeTy);
 
  assert(!cir::MissingFeatures::sanitizers());
 
  CallArgList args;
  assert(!cir::MissingFeatures::opCallArgEvaluationOrder());
 
  // C++23 static-member operators (`static operator()` /
  // `static operator[]`) produce a CXXOperatorCallExpr whose first argument
  // is the object expression even though the operator is static.  Emit the
  // object for its side effects and drop it before walking the parameter
  // arguments.
  auto arguments = e->arguments();
  if (const auto *oce = dyn_cast<CXXOperatorCallExpr>(e)) {
    if (const auto *md =
            dyn_cast_if_present<CXXMethodDecl>(oce->getCalleeDecl());
        md && md->isStatic()) {
      emitIgnoredExpr(e->getArg(0));
      arguments = llvm::drop_begin(arguments, 1);
    }
  }
 
  emitCallArgs(args, dyn_cast<FunctionProtoType>(fnType), arguments,
               e->getDirectCallee());
 
  const CIRGenFunctionInfo &funcInfo =
      cgm.getTypes().arrangeFreeFunctionCall(args, fnType);
 
  // C99 6.5.2.2p6:
  //   If the expression that denotes the called function has a type that does
  //   not include a prototype, [the default argument promotions are performed].
  //   If the number of arguments does not equal the number of parameters, the
  //   behavior is undefined. If the function is defined with a type that
  //   includes a prototype, and either the prototype ends with an ellipsis (,
  //   ...) or the types of the arguments after promotion are not compatible
  //   with the types of the parameters, the behavior is undefined. If the
  //   function is defined with a type that does not include a prototype, and
  //   the types of the arguments after promotion are not compatible with those
  //   of the parameters after promotion, the behavior is undefined [except in
  //   some trivial cases].
  // That is, in the general case, we should assume that a call through an
  // unprototyped function type works like a *non-variadic* call. The way we
  // make this work is to cast to the exxact type fo the promoted arguments.
  if (isa<FunctionNoProtoType>(fnType)) {
    assert(!cir::MissingFeatures::opCallChain());
    assert(!cir::MissingFeatures::addressSpace());
    cir::FuncType calleeTy = getTypes().getFunctionType(funcInfo);
    // get non-variadic function type
    calleeTy = cir::FuncType::get(calleeTy.getInputs(),
                                  calleeTy.getReturnType(), false);
    auto calleePtrTy = cir::PointerType::get(calleeTy);
 
    mlir::Operation *fn = callee.getFunctionPointer();
    mlir::Value addr;
    if (auto funcOp = mlir::dyn_cast<cir::FuncOp>(fn)) {
      addr = cir::GetGlobalOp::create(
          builder, getLoc(e->getSourceRange()),
          cir::PointerType::get(funcOp.getFunctionType()), funcOp.getSymName());
    } else {
      addr = fn->getResult(0);
    }
 
    fn = builder.createBitcast(addr, calleePtrTy).getDefiningOp();
    callee.setFunctionPointer(fn);
  }
 
  assert(!cir::MissingFeatures::opCallFnInfoOpts());
  assert(!cir::MissingFeatures::hip());
 
  cir::CIRCallOpInterface callOp;
  RValue callResult = emitCall(funcInfo, callee, returnValue, args, &callOp,
                               e == mustTailCall, e->getSourceRange());
 
  assert(!cir::MissingFeatures::generateDebugInfo());
 
  return callResult;
}
 
CIRGenCallee CIRGenFunction::emitCallee(const clang::Expr *e) {
  e = e->IgnoreParens();
 
  // Look through function-to-pointer decay.
  if (const auto *implicitCast = dyn_cast<ImplicitCastExpr>(e)) {
    if (implicitCast->getCastKind() == CK_FunctionToPointerDecay ||
        implicitCast->getCastKind() == CK_BuiltinFnToFnPtr) {
      return emitCallee(implicitCast->getSubExpr());
    }
    // Classic codegen has some handling here for ptr-auth (as a part of the
    // large ptr-auth-qualifier PR (#100830)). In the meantime, other cast kinds
    // can fall-through and be handled by the indirect call work below,
    // including L-to-R value conversions and atomic conversions.
    assert(!MissingFeatures::pointerAuthentication());
 
  } else if (const auto *declRef = dyn_cast<DeclRefExpr>(e)) {
    // Resolve direct calls.
    if (const auto *funcDecl = dyn_cast<FunctionDecl>(declRef->getDecl()))
      return emitDirectCallee(funcDecl);
  } else if (auto me = dyn_cast<MemberExpr>(e)) {
    if (const auto *fd = dyn_cast<FunctionDecl>(me->getMemberDecl())) {
      emitIgnoredExpr(me->getBase());
      return emitDirectCallee(fd);
    }
    // Else fall through to the indirect reference handling below.
  } else if (auto *pde = dyn_cast<CXXPseudoDestructorExpr>(e)) {
    return CIRGenCallee::forPseudoDestructor(pde);
  }
 
  // Otherwise, we have an indirect reference.
  mlir::Value calleePtr;
  QualType functionType;
  if (const auto *ptrType = e->getType()->getAs<clang::PointerType>()) {
    calleePtr = emitScalarExpr(e);
    functionType = ptrType->getPointeeType();
  } else {
    functionType = e->getType();
    calleePtr = emitLValue(e).getPointer();
  }
  assert(functionType->isFunctionType());
 
  GlobalDecl gd;
  if (const auto *vd =
          dyn_cast_or_null<VarDecl>(e->getReferencedDeclOfCallee()))
    gd = GlobalDecl(vd);
 
  CIRGenCalleeInfo calleeInfo(functionType->getAs<FunctionProtoType>(), gd);
  CIRGenCallee callee(calleeInfo, calleePtr.getDefiningOp());
  return callee;
}
 
RValue CIRGenFunction::emitCallExpr(const clang::CallExpr *e,
                                    ReturnValueSlot returnValue) {
  assert(!cir::MissingFeatures::objCBlocks());
 
  if (const auto *ce = dyn_cast<CXXMemberCallExpr>(e))
    return emitCXXMemberCallExpr(ce, returnValue);
 
  if (const auto *cudaKernelCallExpr = dyn_cast<CUDAKernelCallExpr>(e))
    return emitCUDAKernelCallExpr(cudaKernelCallExpr, returnValue);
 
  // A CXXOperatorCallExpr is created even for explicit-object methods or
  // static member operators (C++23 `static operator()` / `static
  // operator[]`), but those should be treated like ordinary static function
  // calls.  Only route through the member-call path for ordinary instance
  // operators.
  if (const auto *operatorCall = dyn_cast<CXXOperatorCallExpr>(e)) {
    if (const auto *md =
            dyn_cast_if_present<CXXMethodDecl>(operatorCall->getCalleeDecl());
        md && md->isImplicitObjectMemberFunction())
      return emitCXXOperatorMemberCallExpr(operatorCall, md, returnValue);
  }
 
  CIRGenCallee callee = emitCallee(e->getCallee());
 
  if (callee.isBuiltin())
    return emitBuiltinExpr(callee.getBuiltinDecl(), callee.getBuiltinID(), e,
                           returnValue);
 
  if (callee.isPseudoDestructor())
    return emitCXXPseudoDestructorExpr(callee.getPseudoDestructorExpr());
 
  return emitCall(e->getCallee()->getType(), callee, e, returnValue);
}
 
/// Emit code to compute the specified expression, ignoring the result.
void CIRGenFunction::emitIgnoredExpr(const Expr *e) {
  if (e->isPRValue()) {
    emitAnyExpr(e, AggValueSlot::ignored(), /*ignoreResult=*/true);
    return;
  }
 
  // Just emit it as an l-value and drop the result.
  emitLValue(e);
}
 
Address CIRGenFunction::emitArrayToPointerDecay(const Expr *e,
                                                LValueBaseInfo *baseInfo) {
  assert(!cir::MissingFeatures::opTBAA());
  assert(e->getType()->isArrayType() &&
         "Array to pointer decay must have array source type!");
 
  // Expressions of array type can't be bitfields or vector elements.
  LValue lv = emitLValue(e);
  Address addr = lv.getAddress();
 
  // If the array type was an incomplete type, we need to make sure
  // the decay ends up being the right type.
  auto lvalueAddrTy = mlir::cast<cir::PointerType>(addr.getPointer().getType());
 
  if (e->getType()->isVariableArrayType())
    return addr;
 
  [[maybe_unused]] auto pointeeTy =
      mlir::cast<cir::ArrayType>(lvalueAddrTy.getPointee());
 
  [[maybe_unused]] mlir::Type arrayTy = convertType(e->getType());
  assert(mlir::isa<cir::ArrayType>(arrayTy) && "expected array");
  assert(pointeeTy == arrayTy);
 
  // The result of this decay conversion points to an array element within the
  // base lvalue. However, since TBAA currently does not support representing
  // accesses to elements of member arrays, we conservatively represent accesses
  // to the pointee object as if it had no any base lvalue specified.
  // TODO: Support TBAA for member arrays.
  QualType eltType = e->getType()->castAsArrayTypeUnsafe()->getElementType();
  assert(!cir::MissingFeatures::opTBAA());
 
  mlir::Value ptr = builder.maybeBuildArrayDecay(
      cgm.getLoc(e->getSourceRange()), addr.getPointer(),
      convertTypeForMem(eltType));
  return Address(ptr, addr.getAlignment());
}
 
/// Given the address of a temporary variable, produce an r-value of its type.
RValue CIRGenFunction::convertTempToRValue(Address addr, clang::QualType type,
                                           clang::SourceLocation loc) {
  LValue lvalue = makeAddrLValue(addr, type, AlignmentSource::Decl);
  switch (getEvaluationKind(type)) {
  case cir::TEK_Complex:
    return RValue::getComplex(emitLoadOfComplex(lvalue, loc));
  case cir::TEK_Aggregate:
    return lvalue.asAggregateRValue();
  case cir::TEK_Scalar:
    return RValue::get(emitLoadOfScalar(lvalue, loc));
  }
  llvm_unreachable("bad evaluation kind");
}
 
/// Emit an `if` on a boolean condition, filling `then` and `else` into
/// appropriated regions.
mlir::LogicalResult CIRGenFunction::emitIfOnBoolExpr(const Expr *cond,
                                                     const Stmt *thenS,
                                                     const Stmt *elseS) {
  mlir::Location thenLoc = getLoc(thenS->getSourceRange());
  std::optional<mlir::Location> elseLoc;
  if (elseS)
    elseLoc = getLoc(elseS->getSourceRange());
 
  mlir::LogicalResult resThen = mlir::success(), resElse = mlir::success();
  emitIfOnBoolExpr(
      cond, /*thenBuilder=*/
      [&](mlir::OpBuilder &, mlir::Location) {
        LexicalScope lexScope{*this, thenLoc, builder.getInsertionBlock()};
        resThen = emitStmt(thenS, /*useCurrentScope=*/true);
      },
      thenLoc,
      /*elseBuilder=*/
      [&](mlir::OpBuilder &, mlir::Location) {
        assert(elseLoc && "Invalid location for elseS.");
        LexicalScope lexScope{*this, *elseLoc, builder.getInsertionBlock()};
        resElse = emitStmt(elseS, /*useCurrentScope=*/true);
      },
      elseLoc);
 
  return mlir::LogicalResult::success(resThen.succeeded() &&
                                      resElse.succeeded());
}
 
/// Emit an `if` on a boolean condition, filling `then` and `else` into
/// appropriated regions.
cir::IfOp CIRGenFunction::emitIfOnBoolExpr(
    const clang::Expr *cond, BuilderCallbackRef thenBuilder,
    mlir::Location thenLoc, BuilderCallbackRef elseBuilder,
    std::optional<mlir::Location> elseLoc) {
  // Attempt to be as accurate as possible with IfOp location, generate
  // one fused location that has either 2 or 4 total locations, depending
  // on else's availability.
  SmallVector<mlir::Location, 2> ifLocs{thenLoc};
  if (elseLoc)
    ifLocs.push_back(*elseLoc);
  mlir::Location loc = mlir::FusedLoc::get(&getMLIRContext(), ifLocs);
 
  // Emit the code with the fully general case.
  mlir::Value condV = emitOpOnBoolExpr(loc, cond);
  return emitIfOnBoolValue(condV, loc, thenBuilder, thenLoc, elseBuilder,
                           elseLoc);
}
 
cir::IfOp CIRGenFunction::emitIfOnBoolValue(
    mlir::Value condV, mlir::Location loc, BuilderCallbackRef thenBuilder,
    mlir::Location thenLoc, BuilderCallbackRef elseBuilder,
    std::optional<mlir::Location> elseLoc) {
  cir::IfOp ifOp = cir::IfOp::create(builder, loc, condV, elseLoc.has_value(),
                                     /*thenBuilder=*/thenBuilder,
                                     /*elseBuilder=*/elseBuilder);
  terminateStructuredRegionBody(ifOp.getThenRegion(), thenLoc);
  assert((elseLoc.has_value() || ifOp.getElseRegion().empty()) &&
         "else region created with no else location");
  if (elseLoc.has_value())
    terminateStructuredRegionBody(ifOp.getElseRegion(), *elseLoc);
  return ifOp;
}
 
/// TODO(cir): see EmitBranchOnBoolExpr for extra ideas).
mlir::Value CIRGenFunction::emitOpOnBoolExpr(mlir::Location loc,
                                             const Expr *cond) {
  assert(!cir::MissingFeatures::pgoUse());
  assert(!cir::MissingFeatures::generateDebugInfo());
  cond = cond->IgnoreParens();
 
  // In LLVM the condition is reversed here for efficient codegen.
  // This should be done in CIR prior to LLVM lowering, if we do now
  // we can make CIR based diagnostics misleading.
  //  cir.ternary(!x, t, f) -> cir.ternary(x, f, t)
  assert(!cir::MissingFeatures::shouldReverseUnaryCondOnBoolExpr());
 
  if (const ConditionalOperator *condOp = dyn_cast<ConditionalOperator>(cond)) {
    Expr *trueExpr = condOp->getTrueExpr();
    Expr *falseExpr = condOp->getFalseExpr();
    mlir::Value condV = emitOpOnBoolExpr(loc, condOp->getCond());
 
    mlir::Value ternaryOpRes =
        cir::TernaryOp::create(
            builder, loc, condV, /*thenBuilder=*/
            [this, trueExpr](mlir::OpBuilder &b, mlir::Location loc) {
              mlir::Value lhs = emitScalarExpr(trueExpr);
              cir::YieldOp::create(b, loc, lhs);
            },
            /*elseBuilder=*/
            [this, falseExpr](mlir::OpBuilder &b, mlir::Location loc) {
              mlir::Value rhs = emitScalarExpr(falseExpr);
              cir::YieldOp::create(b, loc, rhs);
            })
            .getResult();
 
    return emitScalarConversion(ternaryOpRes, condOp->getType(),
                                getContext().BoolTy, condOp->getExprLoc());
  }
 
  if (isa<CXXThrowExpr>(cond)) {
    cgm.errorNYI("NYI");
    return createDummyValue(loc, cond->getType());
  }
 
  // If the branch has a condition wrapped by __builtin_unpredictable,
  // create metadata that specifies that the branch is unpredictable.
  // Don't bother if not optimizing because that metadata would not be used.
  assert(!cir::MissingFeatures::insertBuiltinUnpredictable());
 
  // Emit the code with the fully general case.
  return evaluateExprAsBool(cond);
}
 
mlir::Value CIRGenFunction::emitAlloca(StringRef name, mlir::Type ty,
                                       mlir::Location loc, CharUnits alignment,
                                       bool insertIntoFnEntryBlock,
                                       mlir::Value arraySize) {
  mlir::Block *entryBlock = insertIntoFnEntryBlock
                                ? getCurFunctionEntryBlock()
                                : curLexScope->getEntryBlock();
 
  // If this is an alloca in the entry basic block of a cir.try and there's
  // a surrounding cir.scope, make sure the alloca ends up in the surrounding
  // scope instead. This is necessary in order to guarantee all SSA values are
  // reachable during cleanups.
  if (auto tryOp =
          llvm::dyn_cast_if_present<cir::TryOp>(entryBlock->getParentOp())) {
    if (auto scopeOp = llvm::dyn_cast<cir::ScopeOp>(tryOp->getParentOp()))
      entryBlock = &scopeOp.getScopeRegion().front();
  }
 
  return emitAlloca(name, ty, loc, alignment,
                    builder.getBestAllocaInsertPoint(entryBlock), arraySize);
}
 
mlir::Value CIRGenFunction::emitAlloca(StringRef name, mlir::Type ty,
                                       mlir::Location loc, CharUnits alignment,
                                       mlir::OpBuilder::InsertPoint ip,
                                       mlir::Value arraySize) {
  // CIR uses its own alloca address space rather than follow the target data
  // layout like original CodeGen. The data layout awareness should be done in
  // the lowering pass instead.
  cir::PointerType localVarPtrTy =
      builder.getPointerTo(ty, getCIRAllocaAddressSpace());
  mlir::IntegerAttr alignIntAttr = cgm.getSize(alignment);
 
  mlir::Value addr;
  {
    mlir::OpBuilder::InsertionGuard guard(builder);
    builder.restoreInsertionPoint(ip);
    addr = builder.createAlloca(loc, /*addr type*/ localVarPtrTy, name,
                                alignIntAttr, arraySize);
    assert(!cir::MissingFeatures::astVarDeclInterface());
  }
  return addr;
}
 
// Note: this function also emit constructor calls to support a MSVC extensions
// allowing explicit constructor function call.
RValue CIRGenFunction::emitCXXMemberCallExpr(const CXXMemberCallExpr *ce,
                                             ReturnValueSlot returnValue) {
  const Expr *callee = ce->getCallee()->IgnoreParens();
 
  if (isa<BinaryOperator>(callee))
    return emitCXXMemberPointerCallExpr(ce, returnValue);
 
  const auto *me = cast<MemberExpr>(callee);
  const auto *md = cast<CXXMethodDecl>(me->getMemberDecl());
 
  if (md->isStatic()) {
    cgm.errorNYI(ce->getSourceRange(), "emitCXXMemberCallExpr: static method");
    return RValue::get(nullptr);
  }
 
  bool hasQualifier = me->hasQualifier();
  NestedNameSpecifier qualifier = me->getQualifier();
  bool isArrow = me->isArrow();
  const Expr *base = me->getBase();
 
  return emitCXXMemberOrOperatorMemberCallExpr(
      ce, md, returnValue, hasQualifier, qualifier, isArrow, base);
}
 
RValue CIRGenFunction::emitReferenceBindingToExpr(const Expr *e) {
  // Emit the expression as an lvalue.
  LValue lv = emitLValue(e);
  assert(lv.isSimple());
  mlir::Value value = lv.getPointer();
 
  assert(!cir::MissingFeatures::sanitizers());
 
  return RValue::get(value);
}
 
Address CIRGenFunction::emitLoadOfReference(LValue refLVal, mlir::Location loc,
                                            LValueBaseInfo *pointeeBaseInfo) {
  if (refLVal.isVolatile())
    cgm.errorNYI(loc, "load of volatile reference");
 
  cir::LoadOp load =
      cir::LoadOp::create(builder, loc, refLVal.getAddress().getElementType(),
                          refLVal.getAddress().getPointer());
 
  assert(!cir::MissingFeatures::opTBAA());
 
  QualType pointeeType = refLVal.getType()->getPointeeType();
  CharUnits align = cgm.getNaturalTypeAlignment(pointeeType, pointeeBaseInfo);
  return Address(load, convertTypeForMem(pointeeType), align);
}
 
LValue CIRGenFunction::emitLoadOfReferenceLValue(Address refAddr,
                                                 mlir::Location loc,
                                                 QualType refTy,
                                                 AlignmentSource source) {
  LValue refLVal = makeAddrLValue(refAddr, refTy, LValueBaseInfo(source));
  LValueBaseInfo pointeeBaseInfo;
  assert(!cir::MissingFeatures::opTBAA());
  Address pointeeAddr = emitLoadOfReference(refLVal, loc, &pointeeBaseInfo);
  return makeAddrLValue(pointeeAddr, refLVal.getType()->getPointeeType(),
                        pointeeBaseInfo);
}
 
void CIRGenFunction::emitTrap(mlir::Location loc, bool createNewBlock) {
  cir::TrapOp::create(builder, loc);
  if (createNewBlock)
    builder.createBlock(builder.getBlock()->getParent());
}
 
void CIRGenFunction::emitUnreachable(clang::SourceLocation loc,
                                     bool createNewBlock) {
  assert(!cir::MissingFeatures::sanitizers());
  cir::UnreachableOp::create(builder, getLoc(loc));
  if (createNewBlock)
    builder.createBlock(builder.getBlock()->getParent());
}
 
mlir::Value CIRGenFunction::createDummyValue(mlir::Location loc,
                                             clang::QualType qt) {
  mlir::Type t = convertType(qt);
  CharUnits alignment = getContext().getTypeAlignInChars(qt);
  return builder.createDummyValue(loc, t, alignment);
}
 
//===----------------------------------------------------------------------===//
// CIR builder helpers
//===----------------------------------------------------------------------===//
 
Address CIRGenFunction::createMemTempWithoutCast(QualType ty,
                                                 mlir::Location loc,
                                                 const Twine &name) {
  return createTempAllocaWithoutCast(
      convertTypeForMem(ty), getContext().getTypeAlignInChars(ty), loc, name);
}
 
Address CIRGenFunction::createMemTemp(QualType ty, mlir::Location loc,
                                      const Twine &name, Address *alloca,
                                      mlir::OpBuilder::InsertPoint ip) {
  // FIXME: Should we prefer the preferred type alignment here?
  return createMemTemp(ty, getContext().getTypeAlignInChars(ty), loc, name,
                       alloca, ip);
}
 
Address CIRGenFunction::createMemTemp(QualType ty, CharUnits align,
                                      mlir::Location loc, const Twine &name,
                                      Address *alloca,
                                      mlir::OpBuilder::InsertPoint ip) {
  Address result =
      createTempAlloca(convertTypeForMem(ty), /*destAddrSpace=*/{}, align, loc,
                       name, /*arraySize=*/nullptr, alloca, ip);
  if (ty->isConstantMatrixType()) {
    assert(!cir::MissingFeatures::matrixType());
    cgm.errorNYI(loc, "createMemTemp constant matrix type");
    return Address::invalid();
  }
 
  return result;
}
 
/// This creates a alloca and inserts it into the entry block of the
/// current region.
Address CIRGenFunction::createTempAllocaWithoutCast(
    mlir::Type ty, CharUnits align, mlir::Location loc, const Twine &name,
    mlir::Value arraySize, mlir::OpBuilder::InsertPoint ip) {
  cir::AllocaOp alloca = ip.isSet()
                             ? createTempAlloca(ty, loc, name, ip, arraySize)
                             : createTempAlloca(ty, loc, name, arraySize);
  alloca.setAlignmentAttr(cgm.getSize(align));
  return Address(alloca, ty, align);
}
 
Address CIRGenFunction::createTempAlloca(mlir::Type ty, CharUnits align,
                                         mlir::Location loc, const Twine &name,
                                         mlir::Value arraySize,
                                         Address *allocaAddr,
                                         mlir::OpBuilder::InsertPoint ip) {
  return createTempAlloca(ty, /*destAddrSpace=*/{}, align, loc, name, arraySize,
                          allocaAddr, ip);
}
 
Address CIRGenFunction::maybeCastStackAddressSpace(
    Address alloca, mlir::ptr::MemorySpaceAttrInterface destAddrSpace,
    mlir::Value arraySize) {
  if (!destAddrSpace)
    destAddrSpace = cir::toCIRAddressSpaceAttr(
        getMLIRContext(), cgm.getLangTempAllocaAddressSpace());
 
  mlir::ptr::MemorySpaceAttrInterface srcAddrSpace = getCIRAllocaAddressSpace();
  // Alloca always returns a pointer in alloca address space, which may
  // be different from the type defined by the language. For example,
  // in C++ the auto variables are in the default address space. Therefore
  // cast alloca to the default address space when necessary.
  if (srcAddrSpace == destAddrSpace)
    return alloca;
 
  mlir::OpBuilder::InsertionGuard guard(builder);
  if (cir::AllocaOp allocaOp = alloca.getUnderlyingAllocaOp()) {
    builder.setInsertionPointAfter(allocaOp);
  } else if (!arraySize) {
    mlir::Block *entryBlock = getCurFunctionEntryBlock();
    builder.restoreInsertionPoint(builder.getBestAllocaInsertPoint(entryBlock));
  }
 
  mlir::Type destPtrTy =
      builder.getPointerTo(alloca.getElementType(), destAddrSpace);
  mlir::Value casted = performAddrSpaceCast(alloca.getPointer(), destPtrTy);
  return Address(casted, alloca.getElementType(), alloca.getAlignment(),
                 /*isKnownNonNull=*/true);
}
 
/// This creates a alloca and inserts it into the entry block. The alloca is
/// casted to the requested language address space if necessary.
Address CIRGenFunction::createTempAlloca(
    mlir::Type ty, mlir::ptr::MemorySpaceAttrInterface destAddrSpace,
    CharUnits align, mlir::Location loc, const Twine &name,
    mlir::Value arraySize, Address *allocaAddr,
    mlir::OpBuilder::InsertPoint ip) {
  Address alloca =
      createTempAllocaWithoutCast(ty, align, loc, name, arraySize, ip);
  if (allocaAddr)
    *allocaAddr = alloca;
  return maybeCastStackAddressSpace(alloca, destAddrSpace, arraySize);
}
 
/// This creates an alloca and inserts it into the entry block if \p ArraySize
/// is nullptr, otherwise inserts it at the current insertion point of the
/// builder.
cir::AllocaOp CIRGenFunction::createTempAlloca(mlir::Type ty,
                                               mlir::Location loc,
                                               const Twine &name,
                                               mlir::Value arraySize,
                                               bool insertIntoFnEntryBlock) {
  return mlir::cast<cir::AllocaOp>(emitAlloca(name.str(), ty, loc, CharUnits(),
                                              insertIntoFnEntryBlock, arraySize)
                                       .getDefiningOp());
}
 
/// This creates an alloca and inserts it into the provided insertion point
cir::AllocaOp CIRGenFunction::createTempAlloca(mlir::Type ty,
                                               mlir::Location loc,
                                               const Twine &name,
                                               mlir::OpBuilder::InsertPoint ip,
                                               mlir::Value arraySize) {
  assert(ip.isSet() && "Insertion point is not set");
  return mlir::cast<cir::AllocaOp>(
      emitAlloca(name.str(), ty, loc, CharUnits(), ip, arraySize)
          .getDefiningOp());
}
 
/// CreateDefaultAlignTempAlloca - This creates an alloca with the
/// default alignment of the corresponding LLVM type, which is *not*
/// guaranteed to be related in any way to the expected alignment of
/// an AST type that might have been lowered to Ty.
Address CIRGenFunction::createDefaultAlignTempAlloca(mlir::Type ty,
                                                     mlir::Location loc,
                                                     const Twine &name) {
  CharUnits align =
      CharUnits::fromQuantity(cgm.getDataLayout().getABITypeAlign(ty));
  return createTempAlloca(ty, align, loc, name);
}
 
/// Try to emit a reference to the given value without producing it as
/// an l-value.  For many cases, this is just an optimization, but it avoids
/// us needing to emit global copies of variables if they're named without
/// triggering a formal use in a context where we can't emit a direct
/// reference to them, for instance if a block or lambda or a member of a
/// local class uses a const int variable or constexpr variable from an
/// enclosing function.
///
/// For named members of enums, this is the only way they are emitted.
CIRGenFunction::ConstantEmission
CIRGenFunction::tryEmitAsConstant(const DeclRefExpr *refExpr) {
  const ValueDecl *value = refExpr->getDecl();
 
  // There is a lot more to do here, but for now only EnumConstantDecl is
  // supported.
  assert(!cir::MissingFeatures::tryEmitAsConstant());
 
  // The value needs to be an enum constant or a constant variable.
  if (!isa<EnumConstantDecl>(value))
    return ConstantEmission();
 
  Expr::EvalResult result;
  if (!refExpr->EvaluateAsRValue(result, getContext()))
    return ConstantEmission();
 
  QualType resultType = refExpr->getType();
 
  // As long as we're only handling EnumConstantDecl, there should be no
  // side-effects.
  assert(!result.HasSideEffects);
 
  // Emit as a constant.
  // FIXME(cir): have emitAbstract build a TypedAttr instead (this requires
  // somewhat heavy refactoring...)
  mlir::Attribute c = ConstantEmitter(*this).emitAbstract(
      refExpr->getLocation(), result.Val, resultType);
  mlir::TypedAttr cstToEmit = mlir::dyn_cast_if_present<mlir::TypedAttr>(c);
  assert(cstToEmit && "expected a typed attribute");
 
  assert(!cir::MissingFeatures::generateDebugInfo());
 
  return ConstantEmission::forValue(cstToEmit);
}
 
CIRGenFunction::ConstantEmission
CIRGenFunction::tryEmitAsConstant(const MemberExpr *me) {
  if (DeclRefExpr *dre = tryToConvertMemberExprToDeclRefExpr(*this, me))
    return tryEmitAsConstant(dre);
  return ConstantEmission();
}
 
mlir::Value CIRGenFunction::emitScalarConstant(
    const CIRGenFunction::ConstantEmission &constant, Expr *e) {
  assert(constant && "not a constant");
  if (constant.isReference()) {
    cgm.errorNYI(e->getSourceRange(), "emitScalarConstant: reference");
    return {};
  }
  return builder.getConstant(getLoc(e->getSourceRange()), constant.getValue());
}
 
LValue CIRGenFunction::emitPredefinedLValue(const PredefinedExpr *e) {
  const StringLiteral *sl = e->getFunctionName();
  assert(sl != nullptr && "No StringLiteral name in PredefinedExpr");
  auto fn = cast<cir::FuncOp>(curFn);
  StringRef fnName = fn.getName();
  fnName.consume_front("\01");
  std::array<StringRef, 2> nameItems = {
      PredefinedExpr::getIdentKindName(e->getIdentKind()), fnName};
  std::string gvName = llvm::join(nameItems, ".");
  if (isa_and_nonnull<BlockDecl>(curCodeDecl))
    cgm.errorNYI(e->getSourceRange(), "predefined lvalue in block");
 
  return emitStringLiteralLValue(sl, gvName);
}
 
LValue CIRGenFunction::emitOpaqueValueLValue(const OpaqueValueExpr *e) {
  assert(OpaqueValueMappingData::shouldBindAsLValue(e));
  return getOrCreateOpaqueLValueMapping(e);
}
 
namespace {
// Handle the case where the condition is a constant evaluatable simple integer,
// which means we don't have to separately handle the true/false blocks.
std::optional<LValue> handleConditionalOperatorLValueSimpleCase(
    CIRGenFunction &cgf, const AbstractConditionalOperator *e) {
  const Expr *condExpr = e->getCond();
  llvm::APSInt condExprVal;
  if (!cgf.constantFoldsToSimpleInteger(condExpr, condExprVal))
    return std::nullopt;
 
  const Expr *live = e->getTrueExpr(), *dead = e->getFalseExpr();
  if (!condExprVal.getBoolValue())
    std::swap(live, dead);
 
  if (cgf.containsLabel(dead))
    return std::nullopt;
 
  // If the true case is live, we need to track its region.
  assert(!cir::MissingFeatures::incrementProfileCounter());
  assert(!cir::MissingFeatures::pgoUse());
  // If a throw expression we emit it and return an undefined lvalue
  // because it can't be used.
  if (auto *throwExpr = dyn_cast<CXXThrowExpr>(live->IgnoreParens())) {
    cgf.emitCXXThrowExpr(throwExpr);
    // Return an undefined lvalue - the throw terminates execution
    // so this value will never actually be used
    mlir::Type elemTy = cgf.convertType(dead->getType());
    mlir::Value undefPtr =
        cgf.getBuilder().getNullPtr(cgf.getBuilder().getPointerTo(elemTy),
                                    cgf.getLoc(throwExpr->getSourceRange()));
    return cgf.makeAddrLValue(Address(undefPtr, elemTy, CharUnits::One()),
                              dead->getType());
  }
  return cgf.emitLValue(live);
}
 
/// Emit the operand of a glvalue conditional operator. This is either a glvalue
/// or a (possibly-parenthesized) throw-expression. If this is a throw, no
/// LValue is returned and the current block has been terminated.
static std::optional<LValue> emitLValueOrThrowExpression(CIRGenFunction &cgf,
                                                         const Expr *operand) {
  if (auto *throwExpr = dyn_cast<CXXThrowExpr>(operand->IgnoreParens())) {
    cgf.emitCXXThrowExpr(throwExpr);
    return std::nullopt;
  }
 
  return cgf.emitLValue(operand);
}
} // namespace
 
// Create and generate the 3 blocks for a conditional operator.
// Leaves the 'current block' in the continuation basic block.
template <typename FuncTy>
CIRGenFunction::ConditionalInfo
CIRGenFunction::emitConditionalBlocks(const AbstractConditionalOperator *e,
                                      const FuncTy &branchGenFunc) {
  ConditionalInfo info;
  mlir::Location loc = getLoc(e->getSourceRange());
  CIRGenBuilderTy &builder = getBuilder();
 
  mlir::Value condV = emitOpOnBoolExpr(loc, e->getCond());
 
  ConditionalEvaluation eval(*this, loc);
 
  auto emitBranch = [&](mlir::OpBuilder &b, mlir::Location loc,
                        const Expr *expr, std::optional<LValue> &resultLV) {
    CIRGenFunction::LexicalScope lexScope{*this, loc, b.getInsertionBlock()};
    curLexScope->setAsTernary();
 
    mlir::Value resultPtr;
    {
      // Emit any cleanups that were needed on this branch so we can spill
      // and reload the return value.
      CIRGenFunction::RunCleanupsScope branchCleanups(*this);
      assert(!cir::MissingFeatures::incrementProfileCounter());
      eval.beginEvaluation();
      resultLV = branchGenFunc(*this, expr);
      resultPtr = resultLV ? resultLV->getPointer() : mlir::Value();
      eval.endEvaluation();
      branchCleanups.forceCleanup({&resultPtr});
    }
 
    // A branch that produced no result is a throw-expression; its region is
    // already terminated by cir.unreachable and needs no yield.
    if (resultPtr)
      cir::YieldOp::create(b, loc, resultPtr);
  };
 
  cir::TernaryOp ternary = cir::TernaryOp::create(
      builder, loc, condV,
      /*trueBuilder=*/
      [&](mlir::OpBuilder &b, mlir::Location loc) {
        emitBranch(b, loc, e->getTrueExpr(), info.lhs);
      },
      /*falseBuilder=*/
      [&](mlir::OpBuilder &b, mlir::Location loc) {
        emitBranch(b, loc, e->getFalseExpr(), info.rhs);
      });
 
  // Arms end with a yield or cir.unreachable (throw); this is a backstop
  // for error (NYI) paths that leave a region open.
  terminateStructuredRegionBody(ternary.getTrueRegion(), loc);
  terminateStructuredRegionBody(ternary.getFalseRegion(), loc);
 
  info.result = ternary.getResult();
  return info;
}
 
LValue CIRGenFunction::emitConditionalOperatorLValue(
    const AbstractConditionalOperator *expr) {
  if (!expr->isGLValue()) {
    // ?: here should be an aggregate.
    assert(hasAggregateEvaluationKind(expr->getType()) &&
           "Unexpected conditional operator!");
    return emitAggExprToLValue(expr);
  }
 
  OpaqueValueMapping binding(*this, expr);
  if (std::optional<LValue> res =
          handleConditionalOperatorLValueSimpleCase(*this, expr))
    return *res;
 
  ConditionalInfo info =
      emitConditionalBlocks(expr, [](CIRGenFunction &cgf, const Expr *e) {
        return emitLValueOrThrowExpression(cgf, e);
      });
 
  if ((info.lhs && !info.lhs->isSimple()) ||
      (info.rhs && !info.rhs->isSimple())) {
    cgm.errorNYI(expr->getSourceRange(),
                 "unsupported conditional operator with non-simple lvalue");
    return LValue();
  }
 
  if (info.lhs && info.rhs) {
    Address lhsAddr = info.lhs->getAddress();
    Address rhsAddr = info.rhs->getAddress();
    Address result(info.result, lhsAddr.getElementType(),
                   std::min(lhsAddr.getAlignment(), rhsAddr.getAlignment()));
    AlignmentSource alignSource =
        std::max(info.lhs->getBaseInfo().getAlignmentSource(),
                 info.rhs->getBaseInfo().getAlignmentSource());
    assert(!cir::MissingFeatures::opTBAA());
    return makeAddrLValue(result, expr->getType(), LValueBaseInfo(alignSource));
  }
 
  assert((info.lhs || info.rhs) &&
         "both operands of glvalue conditional are throw-expressions?");
 
  // One arm threw; the surviving arm's pointer is local to the ternary's
  // region, so address the result through the ternary's result value.
  const LValue &survivingLV = info.lhs ? *info.lhs : *info.rhs;
  Address survivingAddr = survivingLV.getAddress();
  Address result(info.result, survivingAddr.getElementType(),
                 survivingAddr.getAlignment());
  assert(!cir::MissingFeatures::opTBAA());
  return makeAddrLValue(result, expr->getType(), survivingLV.getBaseInfo());
}
 
/// An LValue is a candidate for having its loads and stores be made atomic if
/// we are operating under /volatile:ms *and* the LValue itself is volatile and
/// performing such an operation can be performed without a libcall.
bool CIRGenFunction::isLValueSuitableForInlineAtomic(LValue lv) {
  if (!cgm.getLangOpts().MSVolatile)
    return false;
 
  cgm.errorNYI("LValueSuitableForInlineAtomic LangOpts MSVolatile");
  return false;
}
 
LValue CIRGenFunction::emitCXXTypeidLValue(const CXXTypeidExpr *e) {
  return makeNaturalAlignAddrLValue(emitCXXTypeidExpr(e), e->getType());
}