CIRGenExprAggregate.cpp

Go to the documentation of this file.

//===- CIRGenExprAggregrate.cpp - Emit CIR Code from Aggregate Expressions ===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This contains code to emit Aggregate Expr nodes as CIR code.
//
//===----------------------------------------------------------------------===//
 
#include "CIRGenBuilder.h"
#include "CIRGenConstantEmitter.h"
#include "CIRGenFunction.h"
#include "CIRGenValue.h"
#include "mlir/IR/Builders.h"
#include "clang/CIR/Dialect/IR/CIRAttrs.h"
 
#include "clang/AST/Expr.h"
#include "clang/AST/RecordLayout.h"
#include "clang/AST/StmtVisitor.h"
#include "llvm/IR/Value.h"
#include <cstdint>
 
using namespace clang;
using namespace clang::CIRGen;
 
namespace {
// FIXME(cir): This should be a common helper between CIRGen
// and traditional CodeGen
/// Is the value of the given expression possibly a reference to or
/// into a __block variable?
static bool isBlockVarRef(const Expr *e) {
  // Make sure we look through parens.
  e = e->IgnoreParens();
 
  // Check for a direct reference to a __block variable.
  if (const DeclRefExpr *dre = dyn_cast<DeclRefExpr>(e)) {
    const VarDecl *var = dyn_cast<VarDecl>(dre->getDecl());
    return (var && var->hasAttr<BlocksAttr>());
  }
 
  // More complicated stuff.
 
  // Binary operators.
  if (const BinaryOperator *op = dyn_cast<BinaryOperator>(e)) {
    // For an assignment or pointer-to-member operation, just care
    // about the LHS.
    if (op->isAssignmentOp() || op->isPtrMemOp())
      return isBlockVarRef(op->getLHS());
 
    // For a comma, just care about the RHS.
    if (op->getOpcode() == BO_Comma)
      return isBlockVarRef(op->getRHS());
 
    // FIXME: pointer arithmetic?
    return false;
 
    // Check both sides of a conditional operator.
  } else if (const AbstractConditionalOperator *op =
                 dyn_cast<AbstractConditionalOperator>(e)) {
    return isBlockVarRef(op->getTrueExpr()) ||
           isBlockVarRef(op->getFalseExpr());
 
    // OVEs are required to support BinaryConditionalOperators.
  } else if (const OpaqueValueExpr *op = dyn_cast<OpaqueValueExpr>(e)) {
    if (const Expr *src = op->getSourceExpr())
      return isBlockVarRef(src);
 
    // Casts are necessary to get things like (*(int*)&var) = foo().
    // We don't really care about the kind of cast here, except
    // we don't want to look through l2r casts, because it's okay
    // to get the *value* in a __block variable.
  } else if (const CastExpr *cast = dyn_cast<CastExpr>(e)) {
    if (cast->getCastKind() == CK_LValueToRValue)
      return false;
    return isBlockVarRef(cast->getSubExpr());
 
    // Handle unary operators.  Again, just aggressively look through
    // it, ignoring the operation.
  } else if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(e)) {
    return isBlockVarRef(uop->getSubExpr());
 
    // Look into the base of a field access.
  } else if (const MemberExpr *mem = dyn_cast<MemberExpr>(e)) {
    return isBlockVarRef(mem->getBase());
 
    // Look into the base of a subscript.
  } else if (const ArraySubscriptExpr *sub = dyn_cast<ArraySubscriptExpr>(e)) {
    return isBlockVarRef(sub->getBase());
  }
 
  return false;
}
 
class AggExprEmitter : public StmtVisitor<AggExprEmitter> {
 
  CIRGenFunction &cgf;
  AggValueSlot dest;
 
  // Calls `fn` with a valid return value slot, potentially creating a temporary
  // to do so. If a temporary is created, an appropriate copy into `Dest` will
  // be emitted, as will lifetime markers.
  //
  // The given function should take a ReturnValueSlot, and return an RValue that
  // points to said slot.
  void withReturnValueSlot(const Expr *e,
                           llvm::function_ref<RValue(ReturnValueSlot)> fn);
 
  AggValueSlot ensureSlot(mlir::Location loc, QualType t) {
    if (!dest.isIgnored())
      return dest;
    return cgf.createAggTemp(t, loc, "agg.tmp.ensured");
  }
 
  void ensureDest(mlir::Location loc, QualType ty) {
    if (!dest.isIgnored())
      return;
    dest = cgf.createAggTemp(ty, loc, "agg.tmp.ensured");
  }
 
public:
  AggExprEmitter(CIRGenFunction &cgf, AggValueSlot dest)
      : cgf(cgf), dest(dest) {}
 
  /// Given an expression with aggregate type that represents a value lvalue,
  /// this method emits the address of the lvalue, then loads the result into
  /// DestPtr.
  void emitAggLoadOfLValue(const Expr *e);
 
  void emitArrayInit(Address destPtr, cir::ArrayType arrayTy, QualType arrayQTy,
                     Expr *exprToVisit, ArrayRef<Expr *> args,
                     Expr *arrayFiller);
 
  void emitFinalDestCopy(QualType type, RValue src);
 
  /// Perform the final copy to DestPtr, if desired.
  void emitFinalDestCopy(QualType type, const LValue &src,
                         CIRGenFunction::ExprValueKind srcValueKind =
                             CIRGenFunction::EVK_NonRValue);
 
  void emitCopy(QualType type, const AggValueSlot &dest,
                const AggValueSlot &src);
 
  void emitInitializationToLValue(Expr *e, LValue lv);
 
  void emitNullInitializationToLValue(mlir::Location loc, LValue lv);
 
  void emitComparisonResult(const Expr *e, mlir::Location loc,
                            const ComparisonCategoryInfo &cmpInfo,
                            mlir::Value resultValue);
 
  void Visit(Expr *e) { StmtVisitor<AggExprEmitter>::Visit(e); }
 
  void VisitArraySubscriptExpr(ArraySubscriptExpr *e) {
    emitAggLoadOfLValue(e);
  }
 
  void VisitCallExpr(const CallExpr *e);
  void VisitStmtExpr(const StmtExpr *e) {
    CIRGenFunction::StmtExprEvaluation eval(cgf);
    Address retAlloca =
        cgf.createMemTemp(e->getType(), cgf.getLoc(e->getSourceRange()));
    (void)cgf.emitCompoundStmt(*e->getSubStmt(), &retAlloca, dest);
  }
 
  void VisitBinAssign(const BinaryOperator *e) {
    // For an assignment to work, the value on the right has
    // to be compatible with the value on the left.
    assert(cgf.getContext().hasSameUnqualifiedType(e->getLHS()->getType(),
                                                   e->getRHS()->getType()) &&
           "Invalid assignment");
 
    if (isBlockVarRef(e->getLHS()) &&
        e->getRHS()->HasSideEffects(cgf.getContext())) {
      cgf.cgm.errorNYI(e->getSourceRange(),
                       "block var reference with side effects");
      return;
    }
 
    LValue lhs = cgf.emitLValue(e->getLHS());
 
    // If we have an atomic type, evaluate into the destination and then
    // do an atomic copy.
    if (lhs.getType()->isAtomicType() ||
        cgf.isLValueSuitableForInlineAtomic(lhs)) {
      ensureDest(cgf.getLoc(e->getExprLoc()), e->getRHS()->getType());
      Visit(e->getRHS());
      cgf.emitAtomicStore(dest.asRValue(), lhs, /*isInit=*/false);
      return;
    }
 
    // Codegen the RHS so that it stores directly into the LHS.
    assert(!cir::MissingFeatures::aggValueSlotGC());
    AggValueSlot lhsSlot = AggValueSlot::forLValue(
        lhs, AggValueSlot::IsDestructed, AggValueSlot::IsAliased,
        AggValueSlot::MayOverlap);
 
    // A non-volatile aggregate destination might have volatile member.
    if (!lhsSlot.isVolatile() && cgf.hasVolatileMember(e->getLHS()->getType()))
      lhsSlot.setVolatile(true);
 
    cgf.emitAggExpr(e->getRHS(), lhsSlot);
 
    // Copy into the destination if the assignment isn't ignored.
    emitFinalDestCopy(e->getType(), lhs);
 
    if (!dest.isIgnored() && !dest.isExternallyDestructed() &&
        e->getType().isDestructedType() == QualType::DK_nontrivial_c_struct)
      cgf.pushDestroy(QualType::DK_nontrivial_c_struct, dest.getAddress(),
                      e->getType());
  }
 
  void VisitDeclRefExpr(DeclRefExpr *e) { emitAggLoadOfLValue(e); }
 
  void VisitInitListExpr(InitListExpr *e);
  void VisitCXXConstructExpr(const CXXConstructExpr *e);
 
  void visitCXXParenListOrInitListExpr(Expr *e, ArrayRef<Expr *> args,
                                       FieldDecl *initializedFieldInUnion,
                                       Expr *arrayFiller);
  void VisitCXXDefaultInitExpr(CXXDefaultInitExpr *die) {
    CIRGenFunction::CXXDefaultInitExprScope Scope(cgf, die);
    Visit(die->getExpr());
  }
  void VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *e) {
    // Ensure that we have a slot, but if we already do, remember
    // whether it was externally destructed.
    bool wasExternallyDestructed = dest.isExternallyDestructed();
    ensureDest(cgf.getLoc(e->getSourceRange()), e->getType());
 
    // We're going to push a destructor if there isn't already one.
    dest.setExternallyDestructed();
 
    Visit(e->getSubExpr());
 
    // Push that destructor we promised.
    if (!wasExternallyDestructed)
      cgf.emitCXXTemporary(e->getTemporary(), e->getType(), dest.getAddress());
  }
  void VisitLambdaExpr(LambdaExpr *e);
  void VisitExprWithCleanups(ExprWithCleanups *e);
 
  /// Attempt to look through various unimportant expressions to find a
  /// cast of the given kind.
  static Expr *findPeephole(Expr *op, CastKind kind, const ASTContext &ctx) {
    op = op->IgnoreParenNoopCasts(ctx);
    if (auto *castE = dyn_cast<CastExpr>(op)) {
      if (castE->getCastKind() == kind)
        return castE->getSubExpr();
    }
    return nullptr;
  }
 
  // Stubs -- These should be moved up when they are implemented.
  void VisitCastExpr(CastExpr *e) {
    switch (e->getCastKind()) {
    case CK_LValueToRValueBitCast: {
      if (dest.isIgnored()) {
        cgf.emitAnyExpr(e->getSubExpr(), AggValueSlot::ignored(),
                        /*ignoreResult=*/true);
        break;
      }
 
      LValue sourceLV = cgf.emitLValue(e->getSubExpr());
      Address sourceAddress =
          sourceLV.getAddress().withElementType(cgf.getBuilder(), cgf.voidTy);
      Address destAddress =
          dest.getAddress().withElementType(cgf.getBuilder(), cgf.voidTy);
 
      mlir::Location loc = cgf.getLoc(e->getExprLoc());
 
      mlir::Value sizeVal = cgf.getBuilder().getConstInt(
          loc, cgf.sizeTy,
          cgf.getContext().getTypeSizeInChars(e->getType()).getQuantity());
      cgf.getBuilder().createMemCpy(loc, destAddress.getPointer(),
                                    sourceAddress.getPointer(), sizeVal);
 
      break;
    }
 
    case CK_NonAtomicToAtomic:
    case CK_AtomicToNonAtomic: {
      bool isToAtomic = (e->getCastKind() == CK_NonAtomicToAtomic);
 
      // Determine the atomic and value types.
      QualType atomicType = e->getSubExpr()->getType();
      QualType valueType = e->getType();
      if (isToAtomic)
        std::swap(atomicType, valueType);
 
      assert(atomicType->isAtomicType());
      assert(cgf.getContext().hasSameUnqualifiedType(
          valueType, atomicType->castAs<AtomicType>()->getValueType()));
 
      // Just recurse normally if we're ignoring the result or the
      // atomic type doesn't change representation.
      if (dest.isIgnored() || !cgf.cgm.isPaddedAtomicType(atomicType))
        return Visit(e->getSubExpr());
 
      // These two cases are reverses of each other; try to peephole them.
      CastKind peepholeTarget =
          (isToAtomic ? CK_AtomicToNonAtomic : CK_NonAtomicToAtomic);
 
      // These two cases are reverses of each other; try to peephole them.
      if (Expr *op =
              findPeephole(e->getSubExpr(), peepholeTarget, cgf.getContext())) {
        assert(cgf.getContext().hasSameUnqualifiedType(op->getType(),
                                                       e->getType()) &&
               "peephole significantly changed types?");
        return Visit(op);
      }
 
      // If we're converting an r-value of non-atomic type to an r-value
      // of atomic type, just emit directly into the relevant sub-object.
      if (isToAtomic) {
        AggValueSlot valueDest = dest;
        if (!valueDest.isIgnored() && cgf.cgm.isPaddedAtomicType(atomicType)) {
          // Zero-initialize.  (Strictly speaking, we only need to initialize
          // the padding at the end, but this is simpler.)
          mlir::Location loc = cgf.getLoc(e->getExprLoc());
          if (!dest.isZeroed())
            cgf.emitNullInitialization(loc, dest.getAddress(), atomicType);
 
          Address valueAddr = cgf.getBuilder().createGetMember(
              loc, valueDest.getAddress(), "value_addr", 0);
 
          assert(!cir::MissingFeatures::aggValueSlotGC());
          valueDest = AggValueSlot::forAddr(
              valueAddr, valueDest.getQualifiers(),
              valueDest.isExternallyDestructed(),
              valueDest.isPotentiallyAliased(), AggValueSlot::DoesNotOverlap,
              AggValueSlot::IsZeroed);
        }
 
        cgf.emitAggExpr(e->getSubExpr(), valueDest);
        return;
      }
 
      mlir::Location loc = cgf.getLoc(e->getExprLoc());
      AggValueSlot atomicSlot = cgf.createAggTemp(atomicType, loc);
      cgf.emitAggExpr(e->getSubExpr(), atomicSlot);
 
      Address valueAddr = cgf.getBuilder().createGetMember(
          loc, atomicSlot.getAddress(), "value_addr", 0);
      RValue rvalue = RValue::getAggregate(valueAddr, atomicSlot.isVolatile());
      return emitFinalDestCopy(valueType, rvalue);
    }
    case CK_LValueToRValue:
      // If we're loading from a volatile type, force the destination
      // into existence.
      if (e->getSubExpr()->getType().isVolatileQualified())
        cgf.cgm.errorNYI(e->getSourceRange(),
                         "AggExprEmitter: volatile lvalue-to-rvalue cast");
      [[fallthrough]];
    case CK_NoOp:
    case CK_UserDefinedConversion:
    case CK_ConstructorConversion:
      assert(cgf.getContext().hasSameUnqualifiedType(e->getSubExpr()->getType(),
                                                     e->getType()) &&
             "Implicit cast types must be compatible");
      Visit(e->getSubExpr());
      break;
    case CK_ToUnion: {
      if (dest.isIgnored()) {
        cgf.emitAnyExpr(e->getSubExpr(), AggValueSlot::ignored(),
                        /*ignoreResult=*/true);
        break;
      }
      QualType ty = e->getSubExpr()->getType();
      Address castPtr = dest.getAddress().withElementType(cgf.getBuilder(),
                                                          cgf.convertType(ty));
      emitInitializationToLValue(e->getSubExpr(),
                                 cgf.makeAddrLValue(castPtr, ty));
      break;
    }
    default:
      cgf.cgm.errorNYI(e->getSourceRange(),
                       std::string("AggExprEmitter: VisitCastExpr: ") +
                           e->getCastKindName());
      break;
    }
  }
  void VisitStmt(Stmt *s) {
    cgf.cgm.errorNYI(s->getSourceRange(),
                     std::string("AggExprEmitter::VisitStmt: ") +
                         s->getStmtClassName());
  }
  void VisitParenExpr(ParenExpr *pe) { Visit(pe->getSubExpr()); }
  void VisitGenericSelectionExpr(GenericSelectionExpr *ge) {
    Visit(ge->getResultExpr());
  }
  void VisitCoawaitExpr(CoawaitExpr *e) {
    cgf.cgm.errorNYI(e->getSourceRange(), "AggExprEmitter: VisitCoawaitExpr");
  }
  void VisitCoyieldExpr(CoyieldExpr *e) {
    cgf.cgm.errorNYI(e->getSourceRange(), "AggExprEmitter: VisitCoyieldExpr");
  }
  void VisitUnaryCoawait(UnaryOperator *e) {
    cgf.cgm.errorNYI(e->getSourceRange(), "AggExprEmitter: VisitUnaryCoawait");
  }
  void VisitUnaryExtension(UnaryOperator *e) { Visit(e->getSubExpr()); }
  void VisitSubstNonTypeTemplateParmExpr(SubstNonTypeTemplateParmExpr *e) {
    cgf.cgm.errorNYI(e->getSourceRange(),
                     "AggExprEmitter: VisitSubstNonTypeTemplateParmExpr");
  }
  void VisitConstantExpr(ConstantExpr *e) {
    ensureDest(cgf.getLoc(e->getSourceRange()), e->getType());
 
    if (mlir::Attribute result = ConstantEmitter(cgf).tryEmitConstantExpr(e)) {
      mlir::Value resultVal = cgf.getBuilder().getConstant(
          cgf.getLoc(e->getSourceRange()), mlir::cast<mlir::TypedAttr>(result));
      LValue destLVal = cgf.makeAddrLValue(dest.getAddress(), e->getType());
      cgf.emitStoreThroughLValue(RValue::get(resultVal), destLVal);
      return;
    }
 
    // It isn't clear that it is possible to get to here,  but this branch is
    // present in classic codegen, so we leave it here too.
    return Visit(e->getSubExpr());
  }
  void VisitMemberExpr(MemberExpr *e) { emitAggLoadOfLValue(e); }
  void VisitUnaryDeref(UnaryOperator *e) { emitAggLoadOfLValue(e); }
  void VisitStringLiteral(StringLiteral *e) { emitAggLoadOfLValue(e); }
  void VisitCompoundLiteralExpr(CompoundLiteralExpr *e);
 
  void VisitPredefinedExpr(const PredefinedExpr *e) { emitAggLoadOfLValue(e); }
  void VisitBinaryOperator(const BinaryOperator *e) {
    cgf.cgm.errorNYI(e->getSourceRange(),
                     "AggExprEmitter: VisitBinaryOperator");
  }
  void VisitPointerToDataMemberBinaryOperator(const BinaryOperator *e) {
    cgf.cgm.errorNYI(e->getSourceRange(),
                     "AggExprEmitter: VisitPointerToDataMemberBinaryOperator");
  }
  void VisitBinComma(const BinaryOperator *e) {
    cgf.emitIgnoredExpr(e->getLHS());
    Visit(e->getRHS());
  }
  void VisitBinCmp(const BinaryOperator *e) {
    assert(cgf.getContext().hasSameType(e->getLHS()->getType(),
                                        e->getRHS()->getType()));
    const ComparisonCategoryInfo &cmpInfo =
        cgf.getContext().CompCategories.getInfoForType(e->getType());
    assert(cmpInfo.Record->isTriviallyCopyable() &&
           "cannot copy non-trivially copyable aggregate");
 
    QualType argTy = e->getLHS()->getType();
 
    if (!argTy->isIntegralOrEnumerationType() && !argTy->isRealFloatingType() &&
        !argTy->isNullPtrType() && !argTy->isPointerType() &&
        !argTy->isMemberPointerType() && !argTy->isAnyComplexType())
      cgf.cgm.errorNYI(e->getBeginLoc(), "aggregate three-way comparison");
 
    mlir::Location loc = cgf.getLoc(e->getSourceRange());
    CIRGenBuilderTy &builder = cgf.getBuilder();
 
    if (e->getType()->isAnyComplexType())
      cgf.cgm.errorNYI(e->getBeginLoc(), "VisitBinCmp: complex type");
 
    if (e->getType()->isAggregateType())
      cgf.cgm.errorNYI(e->getBeginLoc(), "VisitBinCmp: aggregate type");
 
    mlir::Value lhs = cgf.emitAnyExpr(e->getLHS()).getValue();
    mlir::Value rhs = cgf.emitAnyExpr(e->getRHS()).getValue();
 
    mlir::Value resultScalar;
    if (argTy->isNullPtrType()) {
      resultScalar =
          builder.getConstInt(loc, cmpInfo.getEqualOrEquiv()->getIntValue());
    } else {
      llvm::APSInt ltRes = cmpInfo.getLess()->getIntValue();
      llvm::APSInt eqRes = cmpInfo.getEqualOrEquiv()->getIntValue();
      llvm::APSInt gtRes = cmpInfo.getGreater()->getIntValue();
      if (!cmpInfo.isPartial()) {
        cir::CmpOrdering ordering = cmpInfo.isStrong()
                                        ? cir::CmpOrdering::Strong
                                        : cir::CmpOrdering::Weak;
        resultScalar = builder.createThreeWayCmpTotalOrdering(
            loc, lhs, rhs, ltRes, eqRes, gtRes, ordering);
      } else {
        // Partial ordering.
        llvm::APSInt unorderedRes = cmpInfo.getUnordered()->getIntValue();
        resultScalar = builder.createThreeWayCmpPartialOrdering(
            loc, lhs, rhs, ltRes, eqRes, gtRes, unorderedRes);
      }
    }
 
    emitComparisonResult(e, loc, cmpInfo, resultScalar);
  }
 
  void VisitTypeTraitExpr(const TypeTraitExpr *e) {
    assert(e->isStoredAsComparisonResult() &&
           "expected a strong_ordering type trait with a stored value");
 
    const ComparisonCategoryInfo &cmpInfo =
        cgf.getContext().CompCategories.getInfoForType(e->getType());
    const auto result =
        ComparisonCategoryResult(e->getAPValue().getInt().getZExtValue());
    mlir::Location loc = cgf.getLoc(e->getSourceRange());
    mlir::Value resultValue = cgf.getBuilder().getConstInt(
        loc, cmpInfo.getValueInfo(result)->getIntValue());
 
    emitComparisonResult(e, loc, cmpInfo, resultValue);
  }
 
  void VisitCXXRewrittenBinaryOperator(CXXRewrittenBinaryOperator *e) {
    Visit(e->getSemanticForm());
  }
  void VisitObjCMessageExpr(ObjCMessageExpr *e) {
    cgf.cgm.errorNYI(e->getSourceRange(),
                     "AggExprEmitter: VisitObjCMessageExpr");
  }
  void VisitObjCIVarRefExpr(ObjCIvarRefExpr *e) {
    cgf.cgm.errorNYI(e->getSourceRange(),
                     "AggExprEmitter: VisitObjCIVarRefExpr");
  }
 
  void VisitDesignatedInitUpdateExpr(DesignatedInitUpdateExpr *e) {
    AggValueSlot dest = ensureSlot(cgf.getLoc(e->getExprLoc()), e->getType());
    LValue destLV = cgf.makeAddrLValue(dest.getAddress(), e->getType());
    emitInitializationToLValue(e->getBase(), destLV);
    VisitInitListExpr(e->getUpdater());
  }
  void VisitAbstractConditionalOperator(const AbstractConditionalOperator *e) {
    mlir::Location loc = cgf.getLoc(e->getSourceRange());
 
    CIRGenFunction::OpaqueValueMapping binding(cgf, e);
    CIRGenFunction::ConditionalEvaluation eval(cgf);
 
    // Save whether the destination's lifetime is externally managed.
    bool isExternallyDestructed = dest.isExternallyDestructed();
    bool destructNonTrivialCStruct =
        !isExternallyDestructed &&
        e->getType().isDestructedType() == QualType::DK_nontrivial_c_struct;
    isExternallyDestructed |= destructNonTrivialCStruct;
 
    // emitIfOnBoolExpr terminates each region; an unconditional yield here
    // would keep alive the dead block a noreturn arm leaves behind.
    cgf.emitIfOnBoolExpr(
        e->getCond(),
        /*thenBuilder=*/
        [&](mlir::OpBuilder &b, mlir::Location loc) {
          eval.beginEvaluation();
          {
            CIRGenFunction::LexicalScope lexScope{cgf, loc,
                                                  b.getInsertionBlock()};
            cgf.curLexScope->setAsTernary();
            dest.setExternallyDestructed(isExternallyDestructed);
            assert(!cir::MissingFeatures::incrementProfileCounter());
            Visit(e->getTrueExpr());
          }
          eval.endEvaluation();
        },
        loc,
        /*elseBuilder=*/
        [&](mlir::OpBuilder &b, mlir::Location loc) {
          eval.beginEvaluation();
          {
            CIRGenFunction::LexicalScope lexScope{cgf, loc,
                                                  b.getInsertionBlock()};
            cgf.curLexScope->setAsTernary();
 
            // If the result of an agg expression is unused, then the emission
            // of the LHS might need to create a destination slot. That's fine
            // with us, and we can safely emit the RHS into the same slot, but
            // we shouldn't claim that it's already being destructed.
            dest.setExternallyDestructed(isExternallyDestructed);
            assert(!cir::MissingFeatures::incrementProfileCounter());
            Visit(e->getFalseExpr());
          }
          eval.endEvaluation();
        },
        loc);
 
    if (destructNonTrivialCStruct)
      cgf.cgm.errorNYI(
          e->getSourceRange(),
          "Abstract conditional aggregate: destructNonTrivialCStruct");
  }
  void VisitChooseExpr(const ChooseExpr *e) { Visit(e->getChosenSubExpr()); }
  void VisitCXXParenListInitExpr(CXXParenListInitExpr *e) {
    visitCXXParenListOrInitListExpr(e, e->getInitExprs(),
                                    e->getInitializedFieldInUnion(),
                                    e->getArrayFiller());
  }
 
  void VisitArrayInitLoopExpr(const ArrayInitLoopExpr *e) {
    CIRGenFunction::OpaqueValueMapping binding(cgf, e->getCommonExpr());
    uint64_t numElements = e->getArraySize().getZExtValue();
 
    if (!numElements)
      return;
 
    const mlir::Location loc = cgf.getLoc(e->getSourceRange());
 
    if (!e->getType()->isConstantArrayType())
      cgf.cgm.errorNYI(e->getSourceRange(),
                       "VisitArrayInitLoopExpr: Non-constant array");
 
    Address dest = ensureSlot(loc, e->getType()).getAddress();
    cir::ArrayType arrayTy = cast<cir::ArrayType>(dest.getElementType());
 
    emitArrayInit(dest, arrayTy, e->getType(),
                  const_cast<ArrayInitLoopExpr *>(e), {}, e->getSubExpr());
  }
 
  void VisitImplicitValueInitExpr(ImplicitValueInitExpr *e) {
    QualType ty = e->getType();
    mlir::Location loc = cgf.getLoc(e->getSourceRange());
    AggValueSlot slot = ensureSlot(loc, ty);
    emitNullInitializationToLValue(loc,
                                   cgf.makeAddrLValue(slot.getAddress(), ty));
  }
  void VisitNoInitExpr(NoInitExpr *e) {
    cgf.cgm.errorNYI(e->getSourceRange(), "AggExprEmitter: VisitNoInitExpr");
  }
  void VisitCXXDefaultArgExpr(CXXDefaultArgExpr *dae) {
    CIRGenFunction::CXXDefaultArgExprScope scope(cgf, dae);
    Visit(dae->getExpr());
  }
  void VisitCXXInheritedCtorInitExpr(const CXXInheritedCtorInitExpr *e) {
    AggValueSlot slot =
        ensureSlot(cgf.getLoc(e->getSourceRange()), e->getType());
    cgf.emitInheritedCXXConstructorCall(e->getConstructor(),
                                        e->constructsVBase(), slot.getAddress(),
                                        e->inheritedFromVBase(), e);
  }
 
  /// Emit the initializer for a std::initializer_list initialized with a
  /// real initializer list.
  void VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *e) {
    ASTContext &ctx = cgf.getContext();
    CIRGenBuilderTy &builder = cgf.getBuilder();
    mlir::Location loc = cgf.getLoc(e->getExprLoc());
 
    LValue array = cgf.emitLValue(e->getSubExpr());
    assert(array.isSimple() && "initializer_list array not a simple lvalue");
    Address arrayPtr = array.getAddress();
 
    const ConstantArrayType *arrayType =
        ctx.getAsConstantArrayType(e->getSubExpr()->getType());
    assert(arrayType && "std::initializer_list constructed from non-array");
 
    auto *record = e->getType()->castAsRecordDecl();
    assert(record->getNumFields() == 2 &&
           "Expected std::initializer_list to only have two fields");
 
    RecordDecl::field_iterator field = record->field_begin();
    assert(field != record->field_end() &&
           ctx.hasSameType(field->getType()->getPointeeType(),
                           arrayType->getElementType()) &&
           "Expected std::initializer_list first field to be const E *");
 
    // Start pointer.
    AggValueSlot dest = ensureSlot(loc, e->getType());
    LValue destLV = cgf.makeAddrLValue(dest.getAddress(), e->getType());
    LValue start =
        cgf.emitLValueForFieldInitialization(destLV, *field, field->getName());
 
    mlir::Value arrayStart = arrayPtr.emitRawPointer();
    cgf.emitStoreThroughLValue(RValue::get(arrayStart), start);
    ++field;
    assert(field != record->field_end() &&
           "Expected std::initializer_list to have two fields");
 
    cir::ConstantOp size = builder.getConstInt(loc, arrayType->getSize());
    LValue endOrLength =
        cgf.emitLValueForFieldInitialization(destLV, *field, field->getName());
    if (ctx.hasSameType(field->getType(), ctx.getSizeType())) {
      // Length.
      cgf.emitStoreThroughLValue(RValue::get(size), endOrLength);
    } else {
      // End pointer.
      assert(field->getType()->isPointerType() &&
             ctx.hasSameType(field->getType()->getPointeeType(),
                             arrayType->getElementType()) &&
             "Expected std::initializer_list second field to be const E *");
      mlir::Value arrayEnd = builder.createPtrStride(loc, arrayStart, size);
      cgf.emitStoreThroughLValue(RValue::get(arrayEnd), endOrLength);
    }
  }
 
  void VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *e) {
    cgf.cgm.errorNYI(e->getSourceRange(),
                     "AggExprEmitter: VisitCXXScalarValueInitExpr");
  }
  void VisitCXXTypeidExpr(CXXTypeidExpr *e) { emitAggLoadOfLValue(e); }
  void VisitMaterializeTemporaryExpr(MaterializeTemporaryExpr *e) {
    Visit(e->getSubExpr());
  }
  void VisitOpaqueValueExpr(OpaqueValueExpr *e) {
    cgf.cgm.errorNYI(e->getSourceRange(),
                     "AggExprEmitter: VisitOpaqueValueExpr");
  }
 
  void VisitPseudoObjectExpr(PseudoObjectExpr *e) {
    cgf.cgm.errorNYI(e->getSourceRange(),
                     "AggExprEmitter: VisitPseudoObjectExpr");
  }
 
  void VisitVAArgExpr(VAArgExpr *e) {
    // emitVAArg returns an aggregate value (not a pointer) at the CIR level.
    // ABI-specific pointer handling will be done later in LoweringPrepare.
    mlir::Value vaArgValue = cgf.emitVAArg(e);
 
    // Create a temporary alloca to hold the aggregate value.
    mlir::Location loc = cgf.getLoc(e->getSourceRange());
    Address tmpAddr = cgf.createMemTemp(e->getType(), loc, "vaarg.tmp");
 
    // Store the va_arg result into the temporary.
    cgf.emitAggregateStore(vaArgValue, tmpAddr);
 
    // Create an LValue from the temporary address.
    LValue tmpLValue = cgf.makeAddrLValue(tmpAddr, e->getType());
 
    // Copy the aggregate value from temporary to destination.
    emitFinalDestCopy(e->getType(), tmpLValue);
  }
 
  void VisitCXXThrowExpr(const CXXThrowExpr *e) { cgf.emitCXXThrowExpr(e); }
  void VisitAtomicExpr(AtomicExpr *e) {
    RValue result = cgf.emitAtomicExpr(e);
    emitFinalDestCopy(e->getType(), result);
  }
};
 
} // namespace
 
static bool isTrivialFiller(Expr *e) {
  if (!e)
    return true;
 
  if (isa<ImplicitValueInitExpr>(e))
    return true;
 
  if (auto *ile = dyn_cast<InitListExpr>(e)) {
    if (ile->getNumInits())
      return false;
    return isTrivialFiller(ile->getArrayFiller());
  }
 
  if (const auto *cons = dyn_cast_or_null<CXXConstructExpr>(e))
    return cons->getConstructor()->isDefaultConstructor() &&
           cons->getConstructor()->isTrivial();
 
  return false;
}
 
/// Given an expression with aggregate type that represents a value lvalue, this
/// method emits the address of the lvalue, then loads the result into DestPtr.
void AggExprEmitter::emitAggLoadOfLValue(const Expr *e) {
  LValue lv = cgf.emitLValue(e);
 
  // If the type of the l-value is atomic, then do an atomic load.
  if (lv.getType()->isAtomicType() || cgf.isLValueSuitableForInlineAtomic(lv)) {
    cgf.emitAtomicLoad(lv, e->getExprLoc(), dest);
    return;
  }
 
  emitFinalDestCopy(e->getType(), lv);
}
 
void AggExprEmitter::VisitCompoundLiteralExpr(CompoundLiteralExpr *e) {
  if (dest.isPotentiallyAliased() && e->getType().isPODType(cgf.getContext())) {
    // For a POD type, just emit a load of the lvalue + a copy, because our
    // compound literal might alias the destination.
    emitAggLoadOfLValue(e);
    return;
  }
 
  AggValueSlot slot = ensureSlot(cgf.getLoc(e->getSourceRange()), e->getType());
 
  // Block-scope compound literals are destroyed at the end of the enclosing
  // scope in C.
  bool destruct =
      !cgf.getLangOpts().CPlusPlus && !slot.isExternallyDestructed();
  if (destruct)
    slot.setExternallyDestructed();
 
  cgf.emitAggExpr(e->getInitializer(), slot);
 
  if (destruct)
    if ([[maybe_unused]] QualType::DestructionKind dtorKind =
            e->getType().isDestructedType())
      cgf.cgm.errorNYI(e->getSourceRange(), "compound literal with destructor");
}
 
void AggExprEmitter::emitArrayInit(Address destPtr, cir::ArrayType arrayTy,
                                   QualType arrayQTy, Expr *e,
                                   ArrayRef<Expr *> args, Expr *arrayFiller) {
  CIRGenBuilderTy &builder = cgf.getBuilder();
  const mlir::Location loc = cgf.getLoc(e->getSourceRange());
 
  const uint64_t numInitElements = args.size();
 
  bool setArrayInitLoopExprScope = isa<ArrayInitLoopExpr>(e);
 
  const QualType elementType =
      cgf.getContext().getAsArrayType(arrayQTy)->getElementType();
 
  const QualType elementPtrType = cgf.getContext().getPointerType(elementType);
 
  const mlir::Type cirElementType = cgf.convertType(elementType);
  const cir::PointerType cirElementPtrType =
      builder.getPointerTo(cirElementType);
 
  auto begin = cir::CastOp::create(builder, loc, cirElementPtrType,
                                   cir::CastKind::array_to_ptrdecay,
                                   destPtr.getPointer());
 
  const CharUnits elementSize =
      cgf.getContext().getTypeSizeInChars(elementType);
  const CharUnits elementAlign =
      destPtr.getAlignment().alignmentOfArrayElement(elementSize);
 
  // Exception safety requires us to destroy all the already-constructed
  // members if an initializer throws. For that, we'll need an EH cleanup.
  QualType::DestructionKind dtorKind = elementType.isDestructedType();
  Address endOfInit = Address::invalid();
  assert(!cir::MissingFeatures::cleanupDeactivationScope());
 
  if (dtorKind && cgf.getLangOpts().Exceptions) {
    endOfInit = cgf.createTempAlloca(cirElementPtrType, cgf.getPointerAlign(),
                                     loc, "arrayinit.endOfInit");
    builder.createStore(loc, begin, endOfInit);
 
    cgf.pushIrregularPartialArrayCleanup(begin, endOfInit, elementType,
                                         elementAlign,
                                         cgf.getDestroyer(dtorKind));
  }
 
  // The 'current element to initialize'.  The invariants on this
  // variable are complicated.  Essentially, after each iteration of
  // the loop, it points to the last initialized element, except
  // that it points to the beginning of the array before any
  // elements have been initialized.
  mlir::Value element = begin;
 
  // Don't build the 'one' before the cycle to avoid
  // emmiting the redundant `cir.const 1` instrs.
  mlir::Value one;
 
  // Emit the explicit initializers.
  for (uint64_t i = 0; i != numInitElements; ++i) {
    // Advance to the next element.
    if (i > 0) {
      one = builder.getConstantInt(loc, cgf.ptrDiffTy, i);
      element = builder.createPtrStride(loc, begin, one);
 
      // Tell the cleanup that it needs to destroy up to this element.
      if (endOfInit.isValid())
        builder.createStore(loc, element, endOfInit);
    }
 
    const Address address = Address(element, cirElementType, elementAlign);
    const LValue elementLV = cgf.makeAddrLValue(address, elementType);
    emitInitializationToLValue(args[i], elementLV);
  }
 
  const uint64_t numArrayElements = arrayTy.getSize();
 
  // Check whether there's a non-trivial array-fill expression.
  const bool hasTrivialFiller = isTrivialFiller(arrayFiller);
 
  // Any remaining elements need to be zero-initialized, possibly
  // using the filler expression.  We can skip this if the we're
  // emitting to zeroed memory.
  if (numInitElements != numArrayElements &&
      !(dest.isZeroed() && hasTrivialFiller &&
        cgf.getTypes().isZeroInitializable(elementType))) {
    // Advance to the start of the rest of the array.
    if (numInitElements) {
      one = builder.getConstantInt(loc, cgf.ptrDiffTy, 1);
      element = cir::PtrStrideOp::create(builder, loc, cirElementPtrType,
                                         element, one);
 
      if (endOfInit.isValid())
        builder.createStore(loc, element, endOfInit);
    }
 
    // Allocate the temporary variable
    // to store the pointer to first unitialized element
    const Address tmpAddr = cgf.createTempAlloca(
        cirElementPtrType, cgf.getPointerAlign(), loc, "arrayinit.temp");
    LValue tmpLV = cgf.makeAddrLValue(tmpAddr, elementPtrType);
    cgf.emitStoreThroughLValue(RValue::get(element), tmpLV);
 
    // Compute the end of array
    cir::ConstantOp numArrayElementsConst = builder.getConstInt(
        loc, mlir::cast<cir::IntType>(cgf.ptrDiffTy), numArrayElements);
    mlir::Value end = cir::PtrStrideOp::create(builder, loc, cirElementPtrType,
                                               begin, numArrayElementsConst);
 
    builder.createDoWhile(
        loc,
        /*condBuilder=*/
        [&](mlir::OpBuilder &b, mlir::Location loc) {
          cir::LoadOp currentElement = builder.createLoad(loc, tmpAddr);
          cir::CmpOp cmp = cir::CmpOp::create(builder, loc, cir::CmpOpKind::ne,
                                              currentElement, end);
          builder.createCondition(cmp);
        },
        /*bodyBuilder=*/
        [&](mlir::OpBuilder &b, mlir::Location loc) {
          cir::LoadOp currentElement = builder.createLoad(loc, tmpAddr);
 
          // Emit the actual filler expression.
          LValue elementLV = cgf.makeAddrLValue(
              Address(currentElement, cirElementType, elementAlign),
              elementType);
 
          mlir::Value idx;
          if (setArrayInitLoopExprScope)
            idx = cir::PtrDiffOp::create(b, loc, cgf.ptrDiffTy, currentElement,
                                         begin);
 
          CIRGenFunction::ArrayInitLoopExprScope loopExprScope(
              cgf, setArrayInitLoopExprScope, idx);
 
          if (arrayFiller)
            emitInitializationToLValue(arrayFiller, elementLV);
          else
            emitNullInitializationToLValue(loc, elementLV);
 
          // Advance pointer and store them to temporary variable
          cir::ConstantOp one = builder.getConstInt(
              loc, mlir::cast<cir::IntType>(cgf.ptrDiffTy), 1);
          auto nextElement = cir::PtrStrideOp::create(
              builder, loc, cirElementPtrType, currentElement, one);
 
          // Tell the EH cleanup that we finished with the last element.
          if (endOfInit.isValid())
            builder.createStore(loc, nextElement, endOfInit);
 
          cgf.emitStoreThroughLValue(RValue::get(nextElement), tmpLV);
 
          builder.createYield(loc);
        });
  }
}
 
/// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
void AggExprEmitter::emitFinalDestCopy(QualType type, RValue src) {
  assert(src.isAggregate() && "value must be aggregate value!");
  LValue srcLV = cgf.makeAddrLValue(src.getAggregateAddress(), type);
  emitFinalDestCopy(type, srcLV, CIRGenFunction::EVK_RValue);
}
 
/// Perform the final copy to destPtr, if desired.
void AggExprEmitter::emitFinalDestCopy(
    QualType type, const LValue &src,
    CIRGenFunction::ExprValueKind srcValueKind) {
  // If dest is ignored, then we're evaluating an aggregate expression
  // in a context that doesn't care about the result.  Note that loads
  // from volatile l-values force the existence of a non-ignored
  // destination.
  if (dest.isIgnored())
    return;
 
  if (srcValueKind == CIRGenFunction::EVK_RValue) {
    if (type.isNonTrivialToPrimitiveDestructiveMove() == QualType::PCK_Struct) {
      cgf.cgm.errorNYI("emitFinalDestCopy: EVK_RValue & PCK_Struct");
    }
  } else {
    if (type.isNonTrivialToPrimitiveCopy() == QualType::PCK_Struct) {
      cgf.cgm.errorNYI("emitFinalDestCopy: !EVK_RValue & PCK_Struct");
    }
  }
 
  assert(!cir::MissingFeatures::aggValueSlotVolatile());
  assert(!cir::MissingFeatures::aggEmitFinalDestCopyRValue());
  assert(!cir::MissingFeatures::aggValueSlotGC());
 
  AggValueSlot srcAgg = AggValueSlot::forLValue(src, AggValueSlot::IsDestructed,
                                                AggValueSlot::IsAliased,
                                                AggValueSlot::MayOverlap);
  emitCopy(type, dest, srcAgg);
}
 
/// Perform a copy from the source into the destination.
///
/// \param type - the type of the aggregate being copied; qualifiers are
///   ignored
void AggExprEmitter::emitCopy(QualType type, const AggValueSlot &dest,
                              const AggValueSlot &src) {
  assert(!cir::MissingFeatures::aggValueSlotGC());
 
  // If the result of the assignment is used, copy the LHS there also.
  // It's volatile if either side is.  Use the minimum alignment of
  // the two sides.
  LValue destLV = cgf.makeAddrLValue(dest.getAddress(), type);
  LValue srcLV = cgf.makeAddrLValue(src.getAddress(), type);
  assert(!cir::MissingFeatures::aggValueSlotVolatile());
  cgf.emitAggregateCopy(destLV, srcLV, type, dest.mayOverlap(),
                        dest.isVolatile() || src.isVolatile());
}
 
void AggExprEmitter::emitInitializationToLValue(Expr *e, LValue lv) {
  const QualType type = lv.getType();
 
  if (isa<ImplicitValueInitExpr, CXXScalarValueInitExpr>(e)) {
    const mlir::Location loc = e->getSourceRange().isValid()
                                   ? cgf.getLoc(e->getSourceRange())
                                   : *cgf.currSrcLoc;
    return emitNullInitializationToLValue(loc, lv);
  }
 
  if (isa<NoInitExpr>(e))
    return;
 
  if (type->isReferenceType()) {
    RValue rv = cgf.emitReferenceBindingToExpr(e);
    return cgf.emitStoreThroughLValue(rv, lv);
  }
 
  switch (cgf.getEvaluationKind(type)) {
  case cir::TEK_Complex:
    cgf.emitComplexExprIntoLValue(e, lv, /*isInit*/ true);
    break;
  case cir::TEK_Aggregate:
    cgf.emitAggExpr(e, AggValueSlot::forLValue(lv, AggValueSlot::IsDestructed,
                                               AggValueSlot::IsNotAliased,
                                               AggValueSlot::MayOverlap,
                                               dest.isZeroed()));
 
    return;
  case cir::TEK_Scalar:
    if (lv.isSimple())
      cgf.emitScalarInit(e, cgf.getLoc(e->getSourceRange()), lv);
    else
      cgf.emitStoreThroughLValue(RValue::get(cgf.emitScalarExpr(e)), lv);
    return;
  }
}
 
void AggExprEmitter::VisitCXXConstructExpr(const CXXConstructExpr *e) {
  AggValueSlot slot = ensureSlot(cgf.getLoc(e->getSourceRange()), e->getType());
  cgf.emitCXXConstructExpr(e, slot);
}
 
void AggExprEmitter::emitNullInitializationToLValue(mlir::Location loc,
                                                    LValue lv) {
  const QualType type = lv.getType();
 
  // If the destination slot is already zeroed out before the aggregate is
  // copied into it, we don't have to emit any zeros here.
  if (dest.isZeroed() && cgf.getTypes().isZeroInitializable(type))
    return;
 
  if (cgf.hasScalarEvaluationKind(type)) {
    // For non-aggregates, we can store the appropriate null constant.
    mlir::Value null = cgf.cgm.emitNullConstant(type, loc);
    if (lv.isSimple()) {
      cgf.emitStoreOfScalar(null, lv, /* isInitialization */ true);
      return;
    }
 
    cgf.emitStoreThroughBitfieldLValue(RValue::get(null), lv);
    return;
  }
 
  // There's a potential optimization opportunity in combining
  // memsets; that would be easy for arrays, but relatively
  // difficult for structures with the current code.
  cgf.emitNullInitialization(loc, lv.getAddress(), lv.getType());
}
 
void AggExprEmitter::emitComparisonResult(const Expr *e, mlir::Location loc,
                                          const ComparisonCategoryInfo &cmpInfo,
                                          mlir::Value resultValue) {
  // Create the return value in the destination slot.
  ensureDest(loc, e->getType());
  LValue destLVal = cgf.makeAddrLValue(dest.getAddress(), e->getType());
 
  // Emit the address of the first (and only) field in the comparison category
  // type, and initialize it from the constant integer value produced above.
  const FieldDecl *resultField = *cmpInfo.Record->field_begin();
  LValue fieldLVal = cgf.emitLValueForFieldInitialization(
      destLVal, resultField, resultField->getName());
  cgf.emitStoreThroughLValue(RValue::get(resultValue), fieldLVal);
}
 
void AggExprEmitter::VisitLambdaExpr(LambdaExpr *e) {
  CIRGenFunction::SourceLocRAIIObject loc{cgf, cgf.getLoc(e->getSourceRange())};
  AggValueSlot slot = ensureSlot(cgf.getLoc(e->getSourceRange()), e->getType());
  LValue slotLV = cgf.makeAddrLValue(slot.getAddress(), e->getType());
 
  // We'll need to enter cleanup scopes in case any of the element
  // initializers throws an exception or contains branch out of the expressions.
  CIRGenFunction::CleanupDeactivationScope deactivationScope(cgf);
 
  for (auto [curField, capture, captureInit] : llvm::zip(
           e->getLambdaClass()->fields(), e->captures(), e->capture_inits())) {
    // Pick a name for the field.
    llvm::StringRef fieldName = curField->getName();
    if (capture.capturesVariable()) {
      assert(!curField->isBitField() && "lambdas don't have bitfield members!");
      ValueDecl *v = capture.getCapturedVar();
      fieldName = v->getName();
      cgf.cgm.lambdaFieldToName[curField] = fieldName;
    } else if (capture.capturesThis()) {
      cgf.cgm.lambdaFieldToName[curField] = "this";
    } else {
      cgf.cgm.errorNYI(e->getSourceRange(), "Unhandled capture kind");
      cgf.cgm.lambdaFieldToName[curField] = "unhandled-capture-kind";
    }
 
    // Emit initialization
    LValue lv =
        cgf.emitLValueForFieldInitialization(slotLV, curField, fieldName);
    if (curField->hasCapturedVLAType())
      cgf.cgm.errorNYI(e->getSourceRange(), "lambda captured VLA type");
 
    emitInitializationToLValue(captureInit, lv);
 
    // Push a destructor if necessary.
    if (QualType::DestructionKind dtorKind =
            curField->getType().isDestructedType()) {
      assert(lv.isSimple());
      cgf.pushDestroyAndDeferDeactivation(NormalAndEHCleanup, lv.getAddress(),
                                          curField->getType(),
                                          cgf.getDestroyer(dtorKind), false);
    }
  }
}
 
void AggExprEmitter::VisitExprWithCleanups(ExprWithCleanups *e) {
  CIRGenFunction::FullExprCleanupScope fullExprScope(cgf, e->getSubExpr());
  Visit(e->getSubExpr());
}
 
void AggExprEmitter::VisitCallExpr(const CallExpr *e) {
  if (e->getCallReturnType(cgf.getContext())->isReferenceType()) {
    cgf.cgm.errorNYI(e->getSourceRange(), "reference return type");
    return;
  }
 
  withReturnValueSlot(
      e, [&](ReturnValueSlot slot) { return cgf.emitCallExpr(e, slot); });
}
 
void AggExprEmitter::withReturnValueSlot(
    const Expr *e, llvm::function_ref<RValue(ReturnValueSlot)> fn) {
  QualType retTy = e->getType();
 
  assert(!cir::MissingFeatures::aggValueSlotDestructedFlag());
  bool requiresDestruction =
      retTy.isDestructedType() == QualType::DK_nontrivial_c_struct;
  if (requiresDestruction)
    cgf.cgm.errorNYI(
        e->getSourceRange(),
        "withReturnValueSlot: return value requiring destruction is NYI");
 
  // If it makes no observable difference, save a memcpy + temporary.
  //
  // We need to always provide our own temporary if destruction is required.
  // Otherwise, fn will emit its own, notice that it's "unused", and end its
  // lifetime before we have the chance to emit a proper destructor call.
  assert(!cir::MissingFeatures::aggValueSlotAlias());
  assert(!cir::MissingFeatures::aggValueSlotGC());
 
  Address retAddr = dest.getAddress();
  assert(!cir::MissingFeatures::emitLifetimeMarkers());
 
  assert(!cir::MissingFeatures::aggValueSlotVolatile());
  assert(!cir::MissingFeatures::aggValueSlotDestructedFlag());
  fn(ReturnValueSlot(retAddr));
}
 
void AggExprEmitter::VisitInitListExpr(InitListExpr *e) {
  if (e->hadArrayRangeDesignator())
    llvm_unreachable("GNU array range designator extension");
 
  if (e->isTransparent())
    return Visit(e->getInit(0));
 
  visitCXXParenListOrInitListExpr(
      e, e->inits(), e->getInitializedFieldInUnion(), e->getArrayFiller());
}
 
void AggExprEmitter::visitCXXParenListOrInitListExpr(
    Expr *e, ArrayRef<Expr *> args, FieldDecl *initializedFieldInUnion,
    Expr *arrayFiller) {
 
  const mlir::Location loc = cgf.getLoc(e->getSourceRange());
  const AggValueSlot dest = ensureSlot(loc, e->getType());
 
  if (e->getType()->isConstantArrayType()) {
    cir::ArrayType arrayTy =
        cast<cir::ArrayType>(dest.getAddress().getElementType());
    emitArrayInit(dest.getAddress(), arrayTy, e->getType(), e, args,
                  arrayFiller);
    return;
  } else if (e->getType()->isVariableArrayType()) {
    cgf.cgm.errorNYI(e->getSourceRange(),
                     "visitCXXParenListOrInitListExpr variable array type");
    return;
  }
 
  if (e->getType()->isArrayType()) {
    cgf.cgm.errorNYI(e->getSourceRange(),
                     "visitCXXParenListOrInitListExpr array type");
    return;
  }
 
  assert(e->getType()->isRecordType() && "Only support structs/unions here!");
 
  // Do struct initialization; this code just sets each individual member
  // to the approprate value.  This makes bitfield support automatic;
  // the disadvantage is that the generated code is more difficult for
  // the optimizer, especially with bitfields.
  unsigned numInitElements = args.size();
  auto *record = e->getType()->castAsRecordDecl();
 
  // We'll need to enter cleanup scopes in case any of the element
  // initializers throws an exception.
  CIRGenFunction::CleanupDeactivationScope deactivateCleanups(cgf);
 
  unsigned curInitIndex = 0;
 
  // Emit initialization of base classes.
  if (auto *cxxrd = dyn_cast<CXXRecordDecl>(record)) {
    assert(numInitElements >= cxxrd->getNumBases() &&
           "missing initializer for base class");
    for (auto &base : cxxrd->bases()) {
      assert(!base.isVirtual() && "should not see vbases here");
      CXXRecordDecl *baseRD = base.getType()->getAsCXXRecordDecl();
      Address address = cgf.getAddressOfDirectBaseInCompleteClass(
          loc, dest.getAddress(), cxxrd, baseRD,
          /*baseIsVirtual=*/false);
      assert(!cir::MissingFeatures::aggValueSlotGC());
      AggValueSlot aggSlot = AggValueSlot::forAddr(
          address, Qualifiers(), AggValueSlot::IsDestructed,
          AggValueSlot::IsNotAliased,
          cgf.getOverlapForBaseInit(cxxrd, baseRD, false));
      cgf.emitAggExpr(args[curInitIndex++], aggSlot);
 
      if (QualType::DestructionKind dtorKind =
              base.getType().isDestructedType())
        cgf.pushDestroyAndDeferDeactivation(dtorKind, address, base.getType());
    }
  }
 
  // Prepare a 'this' for CXXDefaultInitExprs.
  CIRGenFunction::FieldConstructionScope fcScope(cgf, dest.getAddress());
 
  LValue destLV = cgf.makeAddrLValue(dest.getAddress(), e->getType());
 
  if (record->isUnion()) {
    // Only initialize one field of a union. The field itself is
    // specified by the initializer list.
    if (!initializedFieldInUnion) {
      // Empty union; we have nothing to do.
 
      // Make sure that it's really an empty and not a failure of
      // semantic analysis.
      assert(llvm::all_of(record->fields(),
                          [](const FieldDecl *f) {
                            return f->isUnnamedBitField() ||
                                   f->isAnonymousStructOrUnion();
                          }) &&
             "Only unnamed bitfields or anonymous class allowed");
      return;
    }
 
    // FIXME: volatility
    FieldDecl *initedField = initializedFieldInUnion;
 
    LValue fieldLV = cgf.emitLValueForFieldInitialization(
        destLV, initedField, initedField->getName());
 
    if (numInitElements) {
      // Store the initializer into the field
      emitInitializationToLValue(args[0], fieldLV);
    } else {
      // Default-initialize to null.
      emitNullInitializationToLValue(loc, fieldLV);
    }
    return;
  }
 
  // Here we iterate over the fields; this makes it simpler to both
  // default-initialize fields and skip over unnamed fields.
  for (const FieldDecl *field : record->fields()) {
    // We're done once we hit the flexible array member.
    if (field->getType()->isIncompleteArrayType())
      break;
 
    // Always skip anonymous bitfields.
    if (field->isUnnamedBitField())
      continue;
 
    // We're done if we reach the end of the explicit initializers, we
    // have a zeroed object, and the rest of the fields are
    // zero-initializable.
    if (curInitIndex == numInitElements && dest.isZeroed() &&
        cgf.getTypes().isZeroInitializable(e->getType()))
      break;
    LValue lv =
        cgf.emitLValueForFieldInitialization(destLV, field, field->getName());
    // We never generate write-barriers for initialized fields.
    assert(!cir::MissingFeatures::setNonGC());
 
    if (curInitIndex < numInitElements) {
      // Store the initializer into the field.
      CIRGenFunction::SourceLocRAIIObject loc{
          cgf, cgf.getLoc(record->getSourceRange())};
      emitInitializationToLValue(args[curInitIndex++], lv);
    } else {
      // We're out of initializers; default-initialize to null
      emitNullInitializationToLValue(cgf.getLoc(e->getSourceRange()), lv);
    }
 
    // Push a destructor if necessary.
    // FIXME: if we have an array of structures, all explicitly
    // initialized, we can end up pushing a linear number of cleanups.
    if (QualType::DestructionKind dtorKind =
            field->getType().isDestructedType()) {
      assert(lv.isSimple());
      cgf.pushDestroyAndDeferDeactivation(NormalAndEHCleanup, lv.getAddress(),
                                          field->getType(),
                                          cgf.getDestroyer(dtorKind), false);
    }
 
    // From classic codegen, maybe not useful for CIR:
    // If the GEP didn't get used because of a dead zero init or something
    // else, clean it up for -O0 builds and general tidiness.
  }
}
 
// TODO(cir): This could be shared with classic codegen.
AggValueSlot::Overlap_t CIRGenFunction::getOverlapForBaseInit(
    const CXXRecordDecl *rd, const CXXRecordDecl *baseRD, bool isVirtual) {
  // If the most-derived object is a field declared with [[no_unique_address]],
  // the tail padding of any virtual base could be reused for other subobjects
  // of that field's class.
  if (isVirtual)
    return AggValueSlot::MayOverlap;
 
  // If the base class is laid out entirely within the nvsize of the derived
  // class, its tail padding cannot yet be initialized, so we can issue
  // stores at the full width of the base class.
  const ASTRecordLayout &layout = getContext().getASTRecordLayout(rd);
  if (layout.getBaseClassOffset(baseRD) +
          getContext().getASTRecordLayout(baseRD).getSize() <=
      layout.getNonVirtualSize())
    return AggValueSlot::DoesNotOverlap;
 
  // The tail padding may contain values we need to preserve.
  return AggValueSlot::MayOverlap;
}
 
void CIRGenFunction::emitAggExpr(const Expr *e, AggValueSlot slot) {
  AggExprEmitter(*this, slot).Visit(const_cast<Expr *>(e));
}
 
void CIRGenFunction::emitAggregateCopy(LValue dest, LValue src, QualType ty,
                                       AggValueSlot::Overlap_t mayOverlap,
                                       bool isVolatile) {
  // TODO(cir): this function needs improvements, commented code for now since
  // this will be touched again soon.
  assert(!ty->isAnyComplexType() && "Unexpected copy of complex");
 
  Address destPtr = dest.getAddress();
  Address srcPtr = src.getAddress();
 
  if (getLangOpts().CPlusPlus) {
    if (auto *record = ty->getAsCXXRecordDecl()) {
      assert((record->hasTrivialCopyConstructor() ||
              record->hasTrivialCopyAssignment() ||
              record->hasTrivialMoveConstructor() ||
              record->hasTrivialMoveAssignment() ||
              record->hasAttr<TrivialABIAttr>() || record->isUnion()) &&
             "Trying to aggregate-copy a type without a trivial copy/move "
             "constructor or assignment operator");
      // Ignore empty classes in C++.
      if (record->isEmpty())
        return;
    }
  }
 
  assert(!cir::MissingFeatures::cudaSupport());
 
  // Aggregate assignment turns into llvm.memcpy.  This is almost valid per
  // C99 6.5.16.1p3, which states "If the value being stored in an object is
  // read from another object that overlaps in anyway the storage of the first
  // object, then the overlap shall be exact and the two objects shall have
  // qualified or unqualified versions of a compatible type."
  //
  // memcpy is not defined if the source and destination pointers are exactly
  // equal, but other compilers do this optimization, and almost every memcpy
  // implementation handles this case safely.  If there is a libc that does not
  // safely handle this, we can add a target hook.
 
  // Get data size info for this aggregate. Don't copy the tail padding if this
  // might be a potentially-overlapping subobject, since the tail padding might
  // be occupied by a different object. Otherwise, copying it is fine.
  TypeInfoChars typeInfo;
  if (mayOverlap)
    typeInfo = getContext().getTypeInfoDataSizeInChars(ty);
  else
    typeInfo = getContext().getTypeInfoInChars(ty);
 
  assert(!cir::MissingFeatures::aggValueSlotVolatile());
 
  // Don't do any of the memmove_collectable tests if GC isn't set.
  if (cgm.getLangOpts().getGC() != LangOptions::NonGC)
    cgm.errorNYI("emitAggregateCopy: GC");
 
  // If the data size (excluding tail padding) differs from the full type size,
  // use skip_tail_padding to avoid clobbering tail padding that may be occupied
  // by other objects (e.g. fields marked with [[no_unique_address]]).
  CharUnits dataSize = typeInfo.Width;
  bool skipTailPadding =
      mayOverlap && dataSize != getContext().getTypeSizeInChars(ty);
  // NOTE(cir): original codegen would normally convert destPtr and srcPtr to
  // i8* since memcpy operates on bytes. We don't need that in CIR because
  // cir.copy will operate on any CIR pointer that points to a sized type.
  builder.createCopy(destPtr, srcPtr, isVolatile, skipTailPadding);
 
  assert(!cir::MissingFeatures::opTBAA());
}
 
// TODO(cir): This could be shared with classic codegen.
AggValueSlot::Overlap_t
CIRGenFunction::getOverlapForFieldInit(const FieldDecl *fd) {
  if (!fd->hasAttr<NoUniqueAddressAttr>() || !fd->getType()->isRecordType())
    return AggValueSlot::DoesNotOverlap;
 
  // If the field lies entirely within the enclosing class's nvsize, its tail
  // padding cannot overlap any already-initialized object. (The only subobjects
  // with greater addresses that might already be initialized are vbases.)
  const RecordDecl *classRD = fd->getParent();
  const ASTRecordLayout &layout = getContext().getASTRecordLayout(classRD);
  if (layout.getFieldOffset(fd->getFieldIndex()) +
          getContext().getTypeSize(fd->getType()) <=
      (uint64_t)getContext().toBits(layout.getNonVirtualSize()))
    return AggValueSlot::DoesNotOverlap;
 
  // The tail padding may contain values we need to preserve.
  return AggValueSlot::MayOverlap;
}
 
LValue CIRGenFunction::emitAggExprToLValue(const Expr *e) {
  assert(hasAggregateEvaluationKind(e->getType()) && "Invalid argument!");
  Address temp = createMemTemp(e->getType(), getLoc(e->getSourceRange()));
  LValue lv = makeAddrLValue(temp, e->getType());
  emitAggExpr(e, AggValueSlot::forLValue(lv, AggValueSlot::IsNotDestructed,
                                         AggValueSlot::IsNotAliased,
                                         AggValueSlot::DoesNotOverlap));
  return lv;
}