CIRABIRewriteContext.cpp

Go to the documentation of this file.

//===- CIRABIRewriteContext.cpp - CIR ABI rewrite context ----------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
 
#include "CIRABIRewriteContext.h"
#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
#include "mlir/IR/Builders.h"
#include "mlir/IR/Dominance.h"
#include "clang/CIR/Dialect/Builder/CIRBaseBuilder.h"
#include "clang/CIR/Dialect/IR/CIRAttrs.h"
#include "clang/CIR/Dialect/IR/CIRDataLayout.h"
#include "clang/CIR/Dialect/IR/CIRDialect.h"
#include "clang/CIR/Dialect/IR/CIRTypes.h"
#include "clang/CIR/MissingFeatures.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/IR/Attributes.h"
#include "llvm/Support/MathExtras.h"
#include <algorithm>
#include <array>
#include <utility>
 
using namespace cir;
using namespace mlir;
using namespace mlir::abi;
 
// This rewrite context supports the Direct (with or without coercion),
// Extend, Ignore, Indirect-return (sret), Indirect-argument (byval and
// non-byval), and Expand (struct flattening) classifications.
//
// An Indirect argument is byval or not, following its classification's
// byVal flag.  byval is a by-value parameter the ABI passes in memory rather
// than registers, usually for its size.  Non-byval is a by-value parameter
// whose type cannot be copied freely, because it has a non-trivial copy
// constructor, move constructor, or destructor, so the callee works on the
// caller's own object rather than a copy.
//
// At the call site both forward the caller's storage, since byval is copied
// on the caller's side of the call.  byval falls back to a slot of its own
// when the operand's storage cannot be handed on.  At the callee, byval fills
// the CIRGen param slot with a copy of the incoming object, while non-byval
// rewires that slot to the incoming pointer so the body mutates the caller's
// storage in place.
//
// For Expand, the single struct argument is replaced by N scalar arguments
// (one per field).  At the callee, the N field block arguments are stored
// directly into the parameter's own alloca (the CIRGen spill slot).  At the
// call site, the struct operand is decomposed into its fields by reading
// each member from the source alloca (get_member + load) when the operand is
// a load of an alloca, or via cir.extract_member otherwise.
//
// For Direct + canFlatten (where the coerced type is a multi-field struct), the
// coerced struct is similarly passed as N separate arguments, one per field.
// The callee stores the N scalar block args into one slot sized and aligned for
// both the coerced struct and the original argument type, which the body reads.
// The call site reads each field of the coerced struct out of memory holding
// the operand as a separate call argument.
 
namespace {
 
/// Return the coerced RecordType for a Direct classification that should be
/// flattened into individual scalar arguments, or a null type if the
/// classification does not call for flattening.
///
/// Flattening applies when all four conditions hold:
///   1. The classification is Direct with a non-null coercedType.
///   2. canFlatten is set.
///   3. The coercedType is a struct (not a union).
///   4. The struct has more than one field (single-field structs are already
///      scalar; flattening them produces no benefit and classic CodeGen skips
///      them for the same reason).
cir::RecordType getFlattenedCoercedType(const ArgClassification &ac) {
  if (ac.kind != ArgKind::Direct || !ac.coercedType || !ac.canFlatten)
    return {};
  auto recTy = dyn_cast<cir::RecordType>(ac.coercedType);
  if (!recTy || !recTy.isStruct() || recTy.getNumElements() <= 1)
    return {};
  return recTy;
}
 
/// Build the new argument-type list for a function whose ABI classification
/// is \p fc.  Handles Direct (with or without coercion), Extend, Ignore,
/// Indirect (byval and non-byval), and Expand (struct flattening) arguments.
/// The sret return pointer, when present, is prepended by
/// rewriteFunctionDefinition rather than here.
mlir::LogicalResult
buildNewArgTypes(ArrayRef<mlir::Type> oldArgTypes,
                 const FunctionClassification &fc,
                 SmallVectorImpl<mlir::Type> &newArgTypes,
                 function_ref<mlir::InFlightDiagnostic()> emitError) {
  assert(newArgTypes.empty() && "expected an empty output vector");
  newArgTypes.reserve(oldArgTypes.size());
  for (auto [idx, ac] : llvm::enumerate(fc.argInfos)) {
    mlir::Type origTy = oldArgTypes[idx];
    switch (ac.kind) {
    case ArgKind::Direct:
      // Direct with canFlatten and a struct coerced type: push one wire type
      // per field of the coerced struct rather than the struct itself.
      // Single-field coerced structs fall through to the non-flatten path —
      // the struct is already scalar-sized and flattening adds no value.
      if (cir::RecordType flatTy = getFlattenedCoercedType(ac)) {
        llvm::append_range(newArgTypes, flatTy.getMembers());
      } else {
        // Direct with a coerced type: the lowered signature uses the coerced
        // type; the body still expects origTy and insertArgCoercion recovers it
        // from a coercion slot.  Direct without coercion is a pass-through.
        newArgTypes.push_back(ac.coercedType ? ac.coercedType : origTy);
      }
      break;
    case ArgKind::Ignore:
      break;
    case ArgKind::Expand: {
      // Flatten the struct into one wire argument per field.  The
      // reassembly in the callee body and the decomposition at the call
      // site are handled by insertArgCoercion and rewriteCallSite.
      auto recTy = cast<cir::RecordType>(origTy);
      assert(recTy.isStruct() &&
             "Expand classification requires a struct type, not a union");
      assert(!recTy.getMembers().empty() &&
             "Expand classification requires at least one struct field");
      llvm::append_range(newArgTypes, recTy.getMembers());
      break;
    }
    case ArgKind::Extend:
      // Extend keeps the original (narrow) type in the signature; the
      // sign/zero extension is communicated to LLVM via the llvm.signext /
      // llvm.zeroext arg attribute, attached separately below.  Any
      // coercedType the classifier set on the Extend ArgClassification is
      // informational (typically the register-width type the value gets
      // extended to in registers) but does not change the CIR signature.
      newArgTypes.push_back(origTy);
      break;
    case ArgKind::Indirect:
      newArgTypes.push_back(cir::PointerType::get(origTy));
      break;
    }
  }
  return mlir::success();
}
 
/// Compute the new return type for a function whose return classification
/// is \p retInfo.  Direct returns keep (or coerce to) their type, Ignore and
/// Indirect (sret) returns become void, Extend keeps its type; Expand emits
/// an error.
mlir::Type
computeNewReturnType(mlir::Type origRetTy, const ArgClassification &retInfo,
                     mlir::MLIRContext *ctx,
                     function_ref<mlir::InFlightDiagnostic()> emitError) {
  switch (retInfo.kind) {
  case ArgKind::Direct:
    // Direct return with a coerced type uses the coerced type on the wire;
    // the rewriter inserts a coercion before each cir.return.
    return retInfo.coercedType ? retInfo.coercedType : origRetTy;
  case ArgKind::Ignore:
    return cir::VoidType::get(ctx);
  case ArgKind::Expand:
    emitError() << "Expand return is not allowed (classic codegen rejects "
                << "it in EmitFunctionEpilog)";
    return nullptr;
  case ArgKind::Extend:
    // Same convention as Extend args: keep the original return type in the
    // signature; the sign/zero extension is communicated via the
    // llvm.signext / llvm.zeroext res attribute attached separately below.
    return origRetTy;
  case ArgKind::Indirect:
    // sret: the value is returned through a pointer argument that the ABI
    // synthesizes (rewriteFunctionDefinition prepends it to the argument
    // list); it is not part of the source-level signature, so the wire
    // return type becomes void.
    return cir::VoidType::get(ctx);
  }
  llvm_unreachable("all ArgKind cases handled");
}
 
/// Create a typed poison constant to stand in for a value the body of a
/// function (or the result of a call) still references but whose ABI
/// classification is Ignore.  Using poison is honest -- the value is
/// genuinely unused at the ABI boundary -- and avoids a fake alloca+load
/// pattern that would suggest we have a value when we don't.
mlir::Value createIgnoredValue(mlir::OpBuilder &builder, mlir::Location loc,
                               mlir::Type ty) {
  return cir::ConstantOp::create(builder, loc, ty, cir::PoisonAttr::get(ty));
}
 
/// Whether \p coercedTy is wider than the type a value of type \p origTy has
/// in memory, so the coerced value passes bits the value does not define and
/// cannot be noundef.  This is the comparison classic CodeGen makes.  A bool
/// vector's memory type is its storage integer, one bit per element and at
/// least a byte wide, not its power-of-two size.
static bool isCoercionWiderThanMemory(mlir::Type origTy, mlir::Type coercedTy,
                                      const mlir::DataLayout &dl) {
  uint64_t memoryBits = dl.getTypeSizeInBits(origTy).getFixedValue();
  if (auto vecTy = mlir::dyn_cast<cir::VectorType>(origTy);
      vecTy && mlir::isa<cir::BoolType>(vecTy.getElementType()))
    memoryBits = vecTy.getBoolStorageWidth();
  return dl.getTypeSizeInBits(coercedTy).getFixedValue() > memoryBits;
}
 
/// Whether updateArgAttrs has anything to change: an arg that is Ignore (its
/// slot is dropped), Extend (needs llvm.signext / llvm.zeroext), Indirect
/// (needs the pointer attributes), Expand (changes the argument count), or a
/// coerced Direct, which may be flattened or may lose llvm.noundef.
static bool needsArgAttrUpdate(const FunctionClassification &fc) {
  return llvm::any_of(fc.argInfos, [](const ArgClassification &ac) {
    return ac.kind == ArgKind::Ignore || ac.kind == ArgKind::Extend ||
           ac.kind == ArgKind::Indirect || ac.kind == ArgKind::Expand ||
           (ac.kind == ArgKind::Direct && ac.coercedType);
  });
}
 
/// Build an updated arg_attrs ArrayAttr that drops Ignore'd args, adds
/// llvm.signext / llvm.zeroext on Extend args, adds the pointer attributes
/// for Indirect args in place of llvm.nofpclass, drops llvm.noundef from a
/// coerced Direct arg that passes undefined bits, gives each field of a
/// Direct+canFlatten argument the argument's attributes, and gives each field
/// of an Expand argument an empty dict.  Direct and Extend slots keep their
/// other existing attributes.  \p origArgTypes provides the pre-rewrite type
/// for each arg slot.
mlir::ArrayAttr updateArgAttrs(mlir::MLIRContext *ctx, mlir::ModuleOp module,
                               ArrayRef<mlir::Type> origArgTypes,
                               mlir::ArrayAttr existingArgAttrs,
                               const FunctionClassification &fc,
                               const mlir::DataLayout &dl) {
  mlir::Builder builder(ctx);
  SmallVector<mlir::Attribute> newArgAttrs;
  newArgAttrs.reserve(fc.argInfos.size());
  for (auto [oldIdx, ac] : llvm::enumerate(fc.argInfos)) {
    if (ac.kind == ArgKind::Ignore)
      continue;
    mlir::DictionaryAttr existing = builder.getDictionaryAttr({});
    if (existingArgAttrs && oldIdx < existingArgAttrs.size())
      existing = mlir::cast<mlir::DictionaryAttr>(existingArgAttrs[oldIdx]);
    if (ac.kind == ArgKind::Direct && ac.coercedType &&
        isCoercionWiderThanMemory(origArgTypes[oldIdx], ac.coercedType, dl)) {
      mlir::NamedAttrList attrs(existing);
      attrs.erase(mlir::LLVM::LLVMDialect::getNoUndefAttrName());
      existing = attrs.getDictionary(ctx);
    }
    if (cir::RecordType flatTy = getFlattenedCoercedType(ac)) {
      // Direct + canFlatten: as in classic CodeGen, each flattened field
      // carries the attributes of the argument it came from.
      newArgAttrs.append(flatTy.getNumElements(), existing);
    } else if (ac.kind == ArgKind::Expand) {
      // An expanded record has no attributes classic CodeGen would copy to its
      // fields, so each field gets an empty dict.
      auto recTy = cast<cir::RecordType>(origArgTypes[oldIdx]);
      newArgAttrs.append(recTy.getNumElements(), builder.getDictionaryAttr({}));
    } else if (ac.kind == ArgKind::Extend) {
      StringRef attrName = ac.signExtend
                               ? mlir::LLVM::LLVMDialect::getSExtAttrName()
                               : mlir::LLVM::LLVMDialect::getZExtAttrName();
      mlir::NamedAttrList attrs(existing);
      attrs.set(attrName, builder.getUnitAttr());
      newArgAttrs.push_back(attrs.getDictionary(ctx));
    } else if (ac.kind == ArgKind::Indirect) {
      // Indirect lowering hands the callee a pointer.  llvm.align and
      // llvm.noundef describe that pointer, not the bytes it points to, and
      // llvm.nofpclass, which describes the value, does not apply to a
      // pointer.  Without byval the pointer points to the caller's own object.
      // With byval the backend copies the pointee on the caller's side of the
      // call, so the callee gets an object of its own even when the pointer
      // points to the caller's own object.
      mlir::Type pointeeTy = origArgTypes[oldIdx];
      mlir::NamedAttrList attrs(existing);
      attrs.erase(mlir::LLVM::LLVMDialect::getNoFPClassAttrName());
      attrs.set(mlir::LLVM::LLVMDialect::getAlignAttrName(),
                builder.getI64IntegerAttr(ac.indirectAlign.value()));
      attrs.set(mlir::LLVM::LLVMDialect::getNoUndefAttrName(),
                builder.getUnitAttr());
      if (ac.byVal) {
        // llvm.byval(T) records the pre-rewrite arg type because the opaque
        // LLVM pointer cannot carry it.  Classic adds llvm.noalias under
        // -fpass-by-value-is-noalias, which CIR does not plumb through.
        assert(!cir::MissingFeatures::noaliasOnByvalAttr());
        attrs.set(mlir::LLVM::LLVMDialect::getByValAttrName(),
                  mlir::TypeAttr::get(pointeeTy));
      } else {
        // llvm.nofreeobj says the object cannot be freed while the callee
        // runs, which holds because the caller owns it across the call.
        attrs.set(mlir::LLVM::LLVMDialect::getNoFreeObjAttrName(),
                  builder.getUnitAttr());
        // llvm.dead_on_return says the object's lifetime ends in the callee,
        // which holds unless the caller still has a destructor to run on it.
        // A record with no layout entry has no destructor.
        auto recTy = mlir::dyn_cast<cir::RecordType>(pointeeTy);
        cir::RecordLayoutAttr layout =
            recTy ? cir::tryGetRecordLayout(module, recTy.getName())
                  : cir::RecordLayoutAttr();
        if (!layout || layout.getHasTrivialDtor())
          attrs.set(mlir::LLVM::LLVMDialect::getDeadOnReturnAttrName(),
                    builder.getI64IntegerAttr(static_cast<int64_t>(
                        llvm::DeadOnReturnInfo().toIntValue())));
        attrs.set(mlir::LLVM::LLVMDialect::getDereferenceableAttrName(),
                  builder.getI64IntegerAttr(
                      dl.getTypeSize(pointeeTy).getFixedValue()));
      }
      newArgAttrs.push_back(attrs.getDictionary(ctx));
    } else {
      newArgAttrs.push_back(existing);
    }
  }
  return builder.getArrayAttr(newArgAttrs);
}
 
/// Build an updated res_attrs ArrayAttr (single entry, since CIR funcs have
/// at most one result) that adds llvm.signext / llvm.zeroext on an Extend
/// return and drops llvm.noundef from a coerced return that passes undefined
/// bits.  Preserves any other existing res attributes.  Returns
/// \p existingResAttrs unchanged, which may be null, when there is nothing to
/// add or drop.
mlir::ArrayAttr updateResAttrs(mlir::MLIRContext *ctx,
                               mlir::ArrayAttr existingResAttrs,
                               const ArgClassification &retInfo,
                               mlir::Type origRetTy,
                               const mlir::DataLayout &dl) {
  bool needsNoUndefDrop =
      retInfo.kind == ArgKind::Direct && retInfo.coercedType &&
      isCoercionWiderThanMemory(origRetTy, retInfo.coercedType, dl);
  if (retInfo.kind != ArgKind::Extend && !needsNoUndefDrop)
    return existingResAttrs;
 
  mlir::NamedAttrList attrs;
  if (existingResAttrs && !existingResAttrs.empty())
    attrs.append(
        mlir::cast<mlir::DictionaryAttr>(existingResAttrs[0]).getValue());
  if (retInfo.kind == ArgKind::Extend)
    attrs.set(retInfo.signExtend ? "llvm.signext" : "llvm.zeroext",
              mlir::UnitAttr::get(ctx));
  if (needsNoUndefDrop)
    attrs.erase(mlir::LLVM::LLVMDialect::getNoUndefAttrName());
  return mlir::ArrayAttr::get(ctx, {attrs.getDictionary(ctx)});
}
 
/// The number of bytes a coercion memory slot needs to hold a value of type
/// \p ty without truncating it. For most types this is the ordinary storage
/// size. For a _BitInt it is deliberately the value's own literal byte
/// footprint (ceil(width/8)) rather than the wider, ABI-alignment-padded
/// footprint a _BitInt gets as a record member (see
/// cir::IntType::getStorageTypeWidth): this coercion is about how many bytes
/// the *value* needs to round-trip, not how a record would lay it out, and
/// those are genuinely different questions for a _BitInt (e.g. _BitInt(33)
/// only needs 5 bytes here, even though it occupies 8 padded bytes as a
/// record member).
static uint64_t coercionByteSize(mlir::Type ty, const mlir::DataLayout &dl) {
  if (auto intTy = mlir::dyn_cast<cir::IntType>(ty))
    return llvm::divideCeil(intTy.getWidth(), 8);
  return dl.getTypeSize(ty);
}
 
/// A \p wantTy-typed view of the storage \p base points at.  When \p wantTy is
/// \p base's own pointee type, \p base is returned unchanged and \p offset must
/// be 0.  Any other type is reached by a bitcast, preceded by a byte stride
/// of \p offset bytes when \p offset is nonzero.  The view keeps \p base's
/// address space.  Any operations created are added to \p createdOps.
static mlir::Value
emitViewAtOffset(mlir::OpBuilder &builder, mlir::Location loc,
                 mlir::Type wantTy, mlir::Value base,
                 SmallPtrSetImpl<mlir::Operation *> &createdOps,
                 unsigned offset) {
  auto basePtrTy = mlir::cast<cir::PointerType>(base.getType());
  if (wantTy == basePtrTy.getPointee()) {
    assert(offset == 0 && "a view of the storage's own type has no offset");
    return base;
  }
  mlir::ptr::MemorySpaceAttrInterface addrSpace = basePtrTy.getAddrSpace();
  mlir::Value view = base;
  if (offset != 0) {
    auto u8Ty = cir::IntType::get(builder.getContext(), 8, /*isSigned=*/false);
    auto u8PtrTy = cir::PointerType::get(u8Ty, addrSpace);
    auto u8Base = cir::CastOp::create(builder, loc, u8PtrTy,
                                      cir::CastKind::bitcast, base);
    createdOps.insert(u8Base);
    auto strideTy =
        cir::IntType::get(builder.getContext(), 64, /*isSigned=*/true);
    auto strideVal = cir::ConstantOp::create(
        builder, loc, cir::IntAttr::get(strideTy, offset));
    createdOps.insert(strideVal);
    view = cir::PtrStrideOp::create(builder, loc, u8PtrTy, u8Base, strideVal);
    createdOps.insert(view.getDefiningOp());
  }
  auto typedView = cir::CastOp::create(builder, loc,
                                       cir::PointerType::get(wantTy, addrSpace),
                                       cir::CastKind::bitcast, view);
  createdOps.insert(typedView);
  return typedView;
}
 
/// A pointer to \p pointee in the target's alloca address space, which every
/// temporary this pass allocates has to live in.
static cir::PointerType getAllocaPtrTy(mlir::Type pointee,
                                       const mlir::DataLayout &dl) {
  mlir::ptr::MemorySpaceAttrInterface addrSpace;
  if (auto asAttr = mlir::dyn_cast_if_present<mlir::IntegerAttr>(
          dl.getAllocaMemorySpace()))
    addrSpace = cir::TargetAddressSpaceAttr::get(pointee.getContext(),
                                                 asAttr.getUInt());
  return cir::PointerType::get(pointee, addrSpace);
}
 
/// A coercion slot for \p srcTy and \p dstTy, which must differ.  It is typed
/// as \p srcTy unless coercionByteSize says \p dstTy is larger, aligned for
/// both, and placed at the start of \p slotBlock.  \p offset, where the
/// coerced side sits within the slot, is only checked by assertions.  The
/// alloca is added to \p createdOps.
static cir::AllocaOp createCoercionSlot(
    mlir::OpBuilder &builder, mlir::Location loc, mlir::Type srcTy,
    mlir::Type dstTy, mlir::Block *slotBlock, const mlir::DataLayout &dl,
    SmallPtrSetImpl<mlir::Operation *> &createdOps, unsigned offset) {
  assert(srcTy != dstTy && "callers must pre-check that the types differ");
 
  uint64_t srcAlign = dl.getTypeABIAlignment(srcTy);
  uint64_t dstAlign = dl.getTypeABIAlignment(dstTy);
  uint64_t allocaAlign = std::max(srcAlign, dstAlign);
  mlir::Type slotTy = coercionByteSize(srcTy, dl) >= coercionByteSize(dstTy, dl)
                          ? srcTy
                          : dstTy;
 
  // Sizes are compared two ways on purpose: relative size in
  // coercionByteSize terms, which is how slotTy was picked, and capacity in
  // getTypeSize terms, which is what the alloca is given.
  [[maybe_unused]] mlir::Type coercedTy = ((slotTy == srcTy) ? dstTy : srcTy);
  assert((offset == 0 ||
          coercionByteSize(coercedTy, dl) < coercionByteSize(slotTy, dl)) &&
         "a direct offset must land on the coerced side, the smaller one");
  assert((offset == 0 ||
          offset + dl.getTypeSize(coercedTy) <= dl.getTypeSize(slotTy)) &&
         "coerce slot too small for offset access");
  assert((offset == 0 || offset % dl.getTypeABIAlignment(coercedTy) == 0) &&
         "a direct offset must be aligned for the coerced access");
 
  mlir::OpBuilder::InsertionGuard guard(builder);
  builder.setInsertionPointToStart(slotBlock);
  auto alloca = cir::AllocaOp::create(builder, loc, getAllocaPtrTy(slotTy, dl),
                                      builder.getStringAttr("coerce"),
                                      builder.getI64IntegerAttr(allocaAlign));
  createdOps.insert(alloca);
  return alloca;
}
 
/// Coerce \p src into a temporary memory slot typed for \p dstTy at the
/// current builder insertion point, and return the destination-typed pointer
/// to that slot without loading the value back out.  This is the shared
/// memory half of emitCoercion: callers that want the whole coerced value use
/// emitCoercion (below); callers that want to read individual members of a
/// coerced struct (the call-site struct flattening) take the returned pointer
/// and emit their own cir.get_member + cir.load per field.  Lowers uniformly
/// for scalar, vector, and record types.
///
/// The slot is sized to the larger of the two types so that neither the store
/// nor a later load ever runs past it: the coerced ABI type can be larger
/// than the original (e.g. a 12-byte aggregate passed as `{i64, i64}`), so
/// accessing the destination through a source-sized slot would over-read.
/// Alignment is max(srcAlign, dstAlign) to satisfy both accesses.  The slot
/// is written through a source-typed view and returned as a destination-typed
/// view.
///
/// The temporary alloca is placed at the start of \p slotBlock, which must
/// dominate every use of the coerced value and must be a block that ends up
/// inside the enclosing function's entry block after any later outlining.
///
/// \p offset is where the coerced value sits within the larger of the two
/// types, non-zero when the ABI passes a value in a register read from
/// partway into its storage.  The coerced type must be the smaller of the
/// two, so one that can exceed the value it coerces, such as a multi-field
/// register tuple, must not be given an offset.
///
/// Any operations the helper creates are appended to \p createdOps so the
/// caller can leave them out when it later rewires the source value's uses,
/// which would otherwise clobber the store's value operand.
mlir::Value emitCoercionToMemory(mlir::OpBuilder &builder, mlir::Location loc,
                                 mlir::Type dstTy, mlir::Value src,
                                 mlir::Block *slotBlock,
                                 const mlir::DataLayout &dl,
                                 SmallPtrSetImpl<mlir::Operation *> &createdOps,
                                 unsigned offset) {
  mlir::Type srcTy = src.getType();
  cir::AllocaOp alloca = createCoercionSlot(builder, loc, srcTy, dstTy,
                                            slotBlock, dl, createdOps, offset);
  mlir::Type slotTy = alloca.getAllocaType();
 
  // Store through a source-typed view of the slot.
  mlir::Value srcSlot = emitViewAtOffset(
      builder, loc, srcTy, alloca, createdOps, srcTy == slotTy ? 0 : offset);
  auto store = cir::StoreOp::create(builder, loc, src, srcSlot);
  createdOps.insert(store);
 
  // Return a destination-typed view of the slot.
  return emitViewAtOffset(builder, loc, dstTy, alloca, createdOps,
                          dstTy == slotTy ? 0 : offset);
}
 
/// Cast \p src to \p dstTy when both are pointers to the same pointee, so only
/// the address space changes.  Returns null when a real coercion is needed.
static mlir::Value
emitAddrSpaceCoercion(mlir::OpBuilder &builder, mlir::Location loc,
                      mlir::Type dstTy, mlir::Value src,
                      SmallPtrSetImpl<mlir::Operation *> &createdOps,
                      unsigned offset) {
  auto srcPtrTy = dyn_cast<cir::PointerType>(src.getType());
  auto dstPtrTy = dyn_cast<cir::PointerType>(dstTy);
  if (offset || !srcPtrTy || !dstPtrTy ||
      srcPtrTy.getPointee() != dstPtrTy.getPointee())
    return {};
  assert(srcPtrTy.getAddrSpace() != dstPtrTy.getAddrSpace() &&
         "pointers to the same pointee must differ in address space");
  auto cast = cir::CastOp::create(builder, loc, dstTy,
                                  cir::CastKind::address_space, src);
  createdOps.insert(cast);
  return cast;
}
 
/// Coerce \p src to type \p dstTy by going through memory and load the whole
/// coerced value back out.  Builds on emitCoercionToMemory, adding the final
/// load of the destination-typed view.  A pointer that only changes address
/// space is cast instead.
mlir::Value emitCoercion(mlir::OpBuilder &builder, mlir::Location loc,
                         mlir::Type dstTy, mlir::Value src,
                         mlir::Block *slotBlock, const mlir::DataLayout &dl,
                         SmallPtrSetImpl<mlir::Operation *> &createdOps,
                         unsigned offset) {
  if (mlir::Value cast =
          emitAddrSpaceCoercion(builder, loc, dstTy, src, createdOps, offset))
    return cast;
 
  mlir::Value dstSlot = emitCoercionToMemory(builder, loc, dstTy, src,
                                             slotBlock, dl, createdOps, offset);
  auto load = cir::LoadOp::create(builder, loc, dstSlot);
  createdOps.insert(load);
  return load;
}
 
/// Convenience overload for callers that don't need the createdOps set
/// (e.g. call-site coercion where we don't replaceAllUsesExcept).
mlir::Value emitCoercion(mlir::OpBuilder &builder, mlir::Location loc,
                         mlir::Type dstTy, mlir::Value src,
                         mlir::Block *slotBlock, const mlir::DataLayout &dl,
                         unsigned offset) {
  SmallPtrSet<mlir::Operation *, 4> ignored;
  return emitCoercion(builder, loc, dstTy, src, slotBlock, dl, ignored, offset);
}
 
/// The block a coercion slot's alloca belongs at the start of.
///
/// Normally the enclosing function's entry block, where HoistAllocas expects
/// allocas to be.  A body carrying a call is not always inside a function
/// when this pass runs, though, because LoweringPrepare runs after it: a
/// namespace-scope `T g = makeT();` is still in its cir.global ctor region,
/// and an OpenACC recipe's init and destroy bodies are in regions the module
/// owns.  Those take the outermost region below the module, which dominates
/// the whole body and travels with it when the body is outlined.
mlir::Block *coercionSlotBlock(mlir::Operation *op) {
  if (auto funcOp = op->getParentOfType<mlir::FunctionOpInterface>())
    return &funcOp->getRegion(0).front();
  mlir::Region *region = op->getParentRegion();
  while (mlir::Region *outer = region->getParentRegion()) {
    if (mlir::isa<mlir::ModuleOp>(outer->getParentOp()))
      break;
    region = outer;
  }
  assert(!region->empty() && "coercion slot needs a block to hold the alloca");
  return &region->front();
}
 
/// \p val's defining load, if it is simple, meaning neither volatile nor
/// atomic.  Null otherwise: a non-simple load's access has to survive as
/// written, and a call result or any other first-class value has no defining
/// load at all.
static cir::LoadOp maybeGetSimpleLoad(mlir::Value val) {
  cir::LoadOp load = val.getDefiningOp<cir::LoadOp>();
  if (!load || load.getIsVolatile() || load.getMemOrder())
    return {};
  return load;
}
 
/// \p val's defining load, if it loads a record and is not atomic.  Null
/// otherwise.  The load may be volatile, and then its bytes may only be read
/// through a volatile copy, never in place.
static cir::LoadOp maybeGetCopyableRecordLoad(mlir::Value val) {
  cir::LoadOp load = val.getDefiningOp<cir::LoadOp>();
  if (!load || load.getMemOrder() ||
      !mlir::isa<cir::RecordType>(load.getType()))
    return {};
  return load;
}
 
/// Emits an error and returns failure if \p val, an operand of \p user, is an
/// atomic load of a record.  The record's bytes are read through a copy or in
/// place, neither of which keeps the load atomic.
static mlir::LogicalResult rejectAtomicRecordLoad(mlir::Operation *user,
                                                  mlir::Value val) {
  auto load = val.getDefiningOp<cir::LoadOp>();
  if (!load || !load.getMemOrder() ||
      !mlir::isa<cir::RecordType>(load.getType()))
    return mlir::success();
  return user->emitOpError()
         << "atomic load of a record operand not yet implemented in "
            "CallConvLowering";
}
 
/// Whether the storage the record load \p load reads is an alloca that holds
/// \p coercedTy at \p offset.  With no offset, the coerced type's alloc size
/// has to fit in the record's.  At an offset, the coerced value has to end
/// within the record's alloc size.
static bool canReadInPlace(cir::LoadOp load, mlir::Type coercedTy,
                           const cir::CIRDataLayout &dl, unsigned offset) {
  if (!cir::getUnderlyingAlloca(load.getAddr()))
    return false;
  uint64_t readEnd =
      offset == 0 ? dl.getTypeAllocSize(coercedTy).getFixedValue()
                  : offset + dl.getTypeStoreSize(coercedTy).getFixedValue();
  return readEnd <= dl.getTypeAllocSize(load.getType()).getFixedValue();
}
 
/// Read \p coercedTy at \p offset straight out of the storage the record load
/// \p load reads, which must be an alloca.  The read goes just before \p load,
/// so it reads the bytes \p load reads, and takes the alloca's alignment at
/// \p offset.
static mlir::Value emitInPlaceCoercedLoad(mlir::OpBuilder &builder,
                                          mlir::Location loc, cir::LoadOp load,
                                          mlir::Type coercedTy,
                                          unsigned offset) {
  cir::AllocaOp src = cir::getUnderlyingAlloca(load.getAddr());
  assert(src && "the record load must read an alloca");
  mlir::OpBuilder::InsertionGuard guard(builder);
  builder.setInsertionPoint(load);
  SmallPtrSet<mlir::Operation *, 4> ignored;
  mlir::Value view = emitViewAtOffset(builder, loc, coercedTy, load.getAddr(),
                                      ignored, offset);
  auto coerced = cir::LoadOp::create(builder, loc, view);
  coerced.setAlignment(llvm::MinAlign(src.getAlignment(), offset));
  return coerced;
}
 
/// Copy the bytes of the record \p load reads into a new coercion slot for
/// \p coercedTy at \p offset, placed at the start of \p slotBlock, and return
/// the slot.  The slot is at least as aligned as the storage copied from.  The
/// copy goes just before \p load, so it reads the bytes \p load reads, and is
/// volatile if \p load is.
static cir::AllocaOp
emitCopiedCoercionSlot(mlir::OpBuilder &builder, mlir::Location loc,
                       cir::LoadOp load, mlir::Type coercedTy,
                       mlir::Block *slotBlock, const mlir::DataLayout &dl,
                       unsigned offset) {
  auto recTy = cast<cir::RecordType>(load.getType());
  uint64_t srcAlign;
  if (cir::AllocaOp src = cir::getUnderlyingAlloca(load.getAddr()))
    srcAlign = src.getAlignment();
  else
    srcAlign = load.getAlignment().value_or(dl.getTypeABIAlignment(recTy));
  SmallPtrSet<mlir::Operation *, 4> ignored;
  cir::AllocaOp slot = createCoercionSlot(builder, loc, recTy, coercedTy,
                                          slotBlock, dl, ignored, offset);
  slot.setAlignment(std::max(slot.getAlignment(), srcAlign));
  assert((offset == 0 || slot.getAllocaType() == recTy) &&
         "a direct offset must leave the record as the slot's type");
  mlir::OpBuilder::InsertionGuard guard(builder);
  builder.setInsertionPoint(load);
  mlir::Value recView =
      emitViewAtOffset(builder, loc, recTy, slot, ignored, /*offset=*/0);
  auto copy =
      cir::CopyOp::create(builder, loc, recView, load.getAddr(),
                          builder.getI64IntegerAttr(slot.getAlignment()),
                          builder.getI64IntegerAttr(srcAlign));
  copy.setIsVolatile(load.getIsVolatile());
  return slot;
}
 
/// Copy the bytes of the record \p load reads into a coercion slot, as
/// emitCopiedCoercionSlot does, then read \p coercedTy out of the slot at
/// \p offset.  The read takes the slot's alignment at \p offset.
static mlir::Value emitCopiedCoercedLoad(mlir::OpBuilder &builder,
                                         mlir::Location loc, cir::LoadOp load,
                                         mlir::Type coercedTy,
                                         mlir::Block *slotBlock,
                                         const mlir::DataLayout &dl,
                                         unsigned offset) {
  cir::AllocaOp slot = emitCopiedCoercionSlot(builder, loc, load, coercedTy,
                                              slotBlock, dl, offset);
  mlir::OpBuilder::InsertionGuard guard(builder);
  builder.setInsertionPoint(load);
  SmallPtrSet<mlir::Operation *, 4> ignored;
  mlir::Value coercedView =
      emitViewAtOffset(builder, loc, coercedTy, slot, ignored, offset);
  auto coerced = cir::LoadOp::create(builder, loc, coercedView);
  coerced.setAlignment(llvm::MinAlign(slot.getAlignment(), offset));
  return coerced;
}
 
/// Read \p coercedTy at \p offset out of the bytes the record load \p load
/// reads: straight out of its storage if \p load is not volatile and
/// canReadInPlace accepts it, and otherwise out of a coercion slot at the start
/// of \p slotBlock that those bytes are copied into.  emitInPlaceCoercedLoad
/// does not mark its read volatile, so a volatile load is read through the
/// volatile copy instead.  Either way no new load of the record type is
/// emitted.
static mlir::Value emitCoercedRecordLoad(
    mlir::OpBuilder &builder, mlir::Location loc, cir::LoadOp load,
    mlir::Type coercedTy, mlir::Block *slotBlock, const mlir::DataLayout &dl,
    const cir::CIRDataLayout &cirDataLayout, unsigned offset) {
  if (!load.getIsVolatile() &&
      canReadInPlace(load, coercedTy, cirDataLayout, offset))
    return emitInPlaceCoercedLoad(builder, loc, load, coercedTy, offset);
  return emitCopiedCoercedLoad(builder, loc, load, coercedTy, slotBlock, dl,
                               offset);
}
 
/// Whether a cir.copy of \p ty into \p dst that stands in for a store of it
/// skips tail padding.  Unless \p dst is an alloca, which holds a complete
/// object, it can be a potentially-overlapping subobject, such as a
/// [[no_unique_address]] member or a base, whose tail padding holds another
/// object.  Classic CodeGen writes only the data size into a destination it is
/// handed (EmitCall) and into one that may overlap (EmitAggregateCopy).
/// Copying the whole object into an alloca is equivalent, since nothing else
/// lives in its tail padding.  A struct has tail padding when its last members
/// are pad members, which CIRGen emits for padding another object may reuse and
/// for some other structs, such as an over-aligned one.
static bool shouldSkipTailPadding(mlir::Value dst, mlir::Type ty) {
  auto structTy = mlir::dyn_cast<cir::StructType>(ty);
  if (!structTy || cir::getUnderlyingAlloca(dst))
    return false;
  llvm::ArrayRef<mlir::Type> members = structTy.getMembers();
  llvm::ArrayRef<cir::RecordMemberKind> kinds = structTy.getMemberKinds();
  for (auto [member, kind] : llvm::reverse(llvm::zip_equal(members, kinds)))
    if (cir::memberOwnsBytes(member))
      return kind == cir::RecordMemberKind::Pad;
  return false;
}
 
/// Rewire the uses of the record value \p val, whose bytes sit at \p addr,
/// which is aligned to \p addrAlign, so that each one finds the record in
/// memory.  \p val may already have been retyped to the type it is passed as, a
/// coerced type or a pointer, but its uses still expect the record.
///
/// A store of \p val becomes a cir.copy from \p addr at the store, volatile if
/// the store is and skipping tail padding as shouldSkipTailPadding says.  A
/// store of the record value writes only what its lowered type stores, so
/// another union member's data past those bytes would be lost.  A consumer may
/// be rewritten before \p val's defining op, and has then already stored \p val
/// into a slot of its own.  That store becomes a copy here too.
///
/// Every other use is pointed at one cir.load of \p addr created at the
/// builder's insertion point.  Only a call argument, a return and a read of a
/// member that is not itself a record may use it, since those either read
/// \p addr when they are rewritten or read a field.  Uses by an operation in
/// \p exceptions are left alone.  Fails, after reporting it, on an atomic store
/// of \p val or a use by anything else, before anything is rewired.
static mlir::LogicalResult
rewireRecordUses(mlir::OpBuilder &builder, mlir::Location loc, mlir::Value val,
                 mlir::Value addr, uint64_t addrAlign,
                 const mlir::DataLayout &dl,
                 const SmallPtrSetImpl<mlir::Operation *> &exceptions) {
  mlir::Type recTy = mlir::cast<cir::PointerType>(addr.getType()).getPointee();
  assert(mlir::isa<cir::RecordType>(recTy) && "only a record is rewired");
  SmallVector<cir::StoreOp> stores;
  SmallVector<mlir::OpOperand *> otherUses;
  for (mlir::OpOperand &use : val.getUses()) {
    mlir::Operation *user = use.getOwner();
    if (exceptions.contains(user))
      continue;
    if (auto store = mlir::dyn_cast<cir::StoreOp>(user)) {
      if (store.getMemOrder())
        return store.emitOpError()
               << "atomic store of a record argument or result not yet "
                  "implemented in CallConvLowering";
      stores.push_back(store);
      continue;
    }
    auto member = mlir::dyn_cast<cir::ExtractMemberOp>(user);
    bool isFieldRead = member && !mlir::isa<cir::RecordType>(member.getType());
    if (!isFieldRead &&
        !mlir::isa<cir::CIRCallOpInterface, cir::ReturnOp>(user))
      return user->emitOpError()
             << "use of a record argument or result other than by a store, "
                "call, return or member read not yet implemented in "
                "CallConvLowering";
    otherUses.push_back(&use);
  }
 
  mlir::IntegerAttr srcAlign = builder.getI64IntegerAttr(addrAlign);
  for (cir::StoreOp store : stores) {
    mlir::OpBuilder::InsertionGuard guard(builder);
    builder.setInsertionPoint(store);
    uint64_t dstAlign =
        store.getAlignment().value_or(dl.getTypeABIAlignment(recTy));
    auto copy =
        cir::CopyOp::create(builder, store.getLoc(), store.getAddr(), addr,
                            builder.getI64IntegerAttr(dstAlign), srcAlign);
    copy.setIsVolatile(store.getIsVolatile());
    copy.setSkipTailPadding(shouldSkipTailPadding(store.getAddr(), recTy));
    store->erase();
  }
 
  if (otherUses.empty())
    return mlir::success();
  auto load = cir::LoadOp::create(builder, loc, addr);
  load.setAlignment(addrAlign);
  for (mlir::OpOperand *use : otherUses)
    use->set(load);
  return mlir::success();
}
 
/// Point the uses of \p val outside \p exceptions at the value of \p val's
/// original type that \p view, a view of the start of a coercion slot, holds.
/// For a record, rewireRecordUses points the uses at the slot.  Any other type
/// is loaded out of it at the builder's insertion point.
static mlir::LogicalResult
replaceUsesFromSlot(mlir::OpBuilder &builder, mlir::Location loc,
                    mlir::Value val, mlir::Value view,
                    const mlir::DataLayout &dl,
                    const SmallPtrSetImpl<mlir::Operation *> &exceptions) {
  if (!mlir::isa<cir::RecordType>(
          mlir::cast<cir::PointerType>(view.getType()).getPointee())) {
    auto load = cir::LoadOp::create(builder, loc, view);
    val.replaceAllUsesExcept(load, exceptions);
    return mlir::success();
  }
  cir::AllocaOp slot = cir::getUnderlyingAlloca(view);
  assert(slot && "the view must start at the coercion slot");
  return rewireRecordUses(builder, loc, val, view, slot.getAlignment(), dl,
                          exceptions);
}
 
/// Coerce \p src, which is \p val retyped or the value replacing it, back to
/// \p origTy through a coercion slot at the start of \p slotBlock, and point
/// the uses of \p val outside \p createdOps at the result with
/// replaceUsesFromSlot.  A direct \p offset lands on the coerced side, so the
/// view of \p origTy starts at the slot.  A pointer that only changes address
/// space is cast instead.  The coercion's operations are added to
/// \p createdOps.
static mlir::LogicalResult
emitCoercionAndReplaceUses(mlir::OpBuilder &builder, mlir::Location loc,
                           mlir::Type origTy, mlir::Value src, mlir::Value val,
                           mlir::Block *slotBlock, const mlir::DataLayout &dl,
                           SmallPtrSetImpl<mlir::Operation *> &createdOps,
                           unsigned offset) {
  if (mlir::Value cast = emitAddrSpaceCoercion(builder, loc, origTy, src,
                                               createdOps, offset)) {
    val.replaceAllUsesExcept(cast, createdOps);
    return mlir::success();
  }
  mlir::Value view = emitCoercionToMemory(builder, loc, origTy, src, slotBlock,
                                          dl, createdOps, offset);
  return replaceUsesFromSlot(builder, loc, val, view, dl, createdOps);
}
 
/// Coerce the value each cir.return returns to the new (coerced) return type.
///
/// When the returned value is a non-atomic load of a record, the coerced value
/// is read by emitCoercedRecordLoad, and the record load is erased once nothing
/// else uses it.  Any other value is stored into a coercion slot and read back.
/// A record with no storage behind it, such as a constant, defines no bytes
/// beyond its fields, so the store writes every byte it defines.  A call
/// result's store is turned into a copy when its call is rewritten later.
/// Fails, after reporting it, on an atomic load of a record.
mlir::LogicalResult
insertReturnCoercion(mlir::FunctionOpInterface funcOp, mlir::Type origRetTy,
                     mlir::Type coercedRetTy, mlir::OpBuilder &builder,
                     const mlir::DataLayout &dl,
                     const cir::CIRDataLayout &cirDataLayout, unsigned offset) {
  SmallVector<cir::ReturnOp> returns;
  funcOp.walk([&](cir::ReturnOp r) { returns.push_back(r); });
  for (cir::ReturnOp r : returns) {
    if (r.getInput().empty())
      continue;
    mlir::Value origVal = r.getInput()[0];
    if (origVal.getType() == coercedRetTy)
      continue;
 
    if (failed(rejectAtomicRecordLoad(r, origVal)))
      return mlir::failure();
    if (cir::LoadOp recLoad = maybeGetCopyableRecordLoad(origVal)) {
      mlir::Value coerced = emitCoercedRecordLoad(
          builder, r.getLoc(), recLoad, coercedRetTy,
          &funcOp->getRegion(0).front(), dl, cirDataLayout, offset);
      r->setOperand(0, coerced);
      if (recLoad->use_empty())
        recLoad->erase();
      continue;
    }
 
    builder.setInsertionPoint(r);
    mlir::Value coerced =
        emitCoercion(builder, r.getLoc(), coercedRetTy, origVal,
                     &funcOp->getRegion(0).front(), dl, offset);
    r->setOperand(0, coerced);
  }
  return mlir::success();
}
 
/// \p recordVal's defining load, if it is simple and its address resolves to
/// an alloca.  Null otherwise.
static cir::LoadOp getWholeRecordLoad(mlir::Value recordVal) {
  cir::LoadOp load = maybeGetSimpleLoad(recordVal);
  if (!load || !cir::getUnderlyingAlloca(load.getAddr()))
    return {};
  return load;
}
 
/// Returns \p load if the address it read from can be passed in place of the
/// loaded value for an indirect argument that needs \p minAlign alignment,
/// and null otherwise, including when \p load is null.  A byval argument
/// also needs storageUnwrittenBetween.  On null the caller reports a
/// non-byval argument or fills a byval slot of its own.
///
/// The address has to be a pointer to the loaded type in the default address
/// space, since that is what the rewritten parameter is, and a cir.load pins
/// the pointee type without pinning the address space.  The storage it names
/// has to be a slot allocated here, reached through storage-preserving casts,
/// already stating at least \p minAlign.  The enclosing function's own
/// non-byval parameter qualifies as well, since its slot stands until
/// finalizeParameterSlots and prepareNonByvalParameters has restated it with
/// the alignment that parameter promises rather than the one CIRGen chose
/// for a local copy.  A byval parameter's slot keeps CIRGen's alignment, so
/// it qualifies only where that already covers \p minAlign.
///
/// Raising a slot's alignment here would not hold for one that stands in
/// for a parameter, since finalizeParameterSlots replaces it with the
/// incoming pointer, which promises only what the caller gave it.
static cir::LoadOp forwardableIndirectLoad(cir::LoadOp load,
                                           uint64_t minAlign) {
  if (!load || load.getAddr().getType() !=
                   cir::PointerType::get(load.getResult().getType()))
    return {};
  cir::AllocaOp slot = cir::getUnderlyingAlloca(load.getAddr());
  return slot && slot.getAlignment() >= minAlign ? load : cir::LoadOp();
}
 
/// True when \p aliasAnalysis shows that nothing between \p load and \p call
/// can write the storage \p load read, so a byval argument may name that
/// storage rather than a copy of it.  byval takes the callee's copy at the
/// call, while the operand carries the value as of the load, so a write in
/// between would reach the callee that the operand does not hold.
/// prepareNonByvalParameters reads a parameter at its spill, which leaves the
/// body's own stores in between.
///
/// A load in another block is not forwardable here, since the paths from it
/// to the call are not walked.
static bool storageUnwrittenBetween(cir::LoadOp load, mlir::Operation *call,
                                    mlir::AliasAnalysis &aliasAnalysis) {
  if (load->getBlock() != call->getBlock())
    return false;
  for (mlir::Operation *op = load->getNextNode(); op != call;
       op = op->getNextNode()) {
    assert(op && "the load must precede the call in the block they share");
    if (op->getNumRegions() != 0 ||
        aliasAnalysis.getModRef(op, load.getAddr()).isMod())
      return false;
  }
  return true;
}
 
/// The alignment a copy may claim for the storage \p load read, which is the
/// one \p load states, or the underlying slot's when it states none.  A load
/// with neither names storage this pass cannot reason about, so the copy is
/// left to assume nothing.
static mlir::IntegerAttr loadSourceAlignment(cir::LoadOp load,
                                             mlir::OpBuilder &builder) {
  if (mlir::IntegerAttr stated = load.getAlignmentAttr())
    return stated;
  if (cir::AllocaOp slot = cir::getUnderlyingAlloca(load.getAddr()))
    return builder.getI64IntegerAttr(slot.getAlignment());
  return builder.getI64IntegerAttr(1);
}
 
/// Decompose a struct value into one scalar call argument per field of \p
/// recTy, appending the field values to \p newArgs.  When \p structVal is a
/// simple load from an alloca, read each field with cir.get_member +
/// cir.load from the address the load used, emitted at the original load's
/// position so they observe the same memory state, and record the now-dead
/// whole-struct load in \p deadRecordLoads for later erasure.  Otherwise (a
/// call result, compound literal, or a volatile or atomic load) extract each
/// field from the value with cir.extract_member.  Loading the members from
/// memory rather than extracting from a whole-struct value keeps the result in
/// a form SROA can promote (it does not reason about extractvalue).  Shared by
/// the Expand and Direct+canFlatten argument paths.
static void emitStructFieldArgs(mlir::OpBuilder &builder, mlir::Location loc,
                                mlir::Value structVal, cir::RecordType recTy,
                                SmallVectorImpl<mlir::Value> &newArgs,
                                SmallVectorImpl<cir::LoadOp> &deadRecordLoads) {
  cir::LoadOp srcLoad = getWholeRecordLoad(structVal);
 
  if (srcLoad) {
    mlir::OpBuilder::InsertionGuard guard(builder);
    builder.setInsertionPoint(srcLoad);
    cir::PointerType baseTy = srcLoad.getAddr().getType();
    for (auto [f, fieldTy] : llvm::enumerate(recTy.getMembers())) {
      mlir::Type fieldPtrTy =
          cir::PointerType::get(fieldTy, baseTy.getAddrSpace());
      mlir::Value fieldPtr = cir::GetMemberOp::create(
          builder, loc, fieldPtrTy, srcLoad.getAddr(), /*name=*/"",
          /*index=*/f);
      newArgs.push_back(cir::LoadOp::create(builder, loc, fieldPtr));
    }
    deadRecordLoads.push_back(srcLoad);
  } else {
    for (unsigned f = 0; f < recTy.getNumElements(); ++f)
      newArgs.push_back(
          cir::ExtractMemberOp::create(builder, loc, structVal, f));
  }
}
 
/// Erase the loads that a rewritten call left unused.  The old call must
/// already be erased, since until then it still counts as a user.  One load can
/// feed two operands of the same call, as in f(s, s), so \p loads can hold the
/// same load twice.  A load that another op still reads is left alone.
static void eraseDeadRecordLoads(ArrayRef<cir::LoadOp> loads) {
  llvm::SmallSetVector<mlir::Operation *, 4> uniqueLoads(llvm::from_range,
                                                         loads);
  for (mlir::Operation *load : uniqueLoads)
    if (load->use_empty())
      load->erase();
}
 
/// The store that spills indirect parameter \p blockArg, if the parameter's
/// only use is that store.  Null otherwise.
static cir::StoreOp maybeFindParamSpill(mlir::BlockArgument blockArg) {
  if (!blockArg.hasOneUse())
    return {};
  auto store = dyn_cast<cir::StoreOp>(*blockArg.user_begin());
  if (!store || store.getValue() != blockArg)
    return {};
  return store;
}
 
/// The store that spills non-byval indirect parameter \p blockArg, and the
/// slot it spills into.  prepareNonByvalParameters has already established
/// that the spill is the block argument's only use and that it stores into
/// an alloca, which is what the assertions below rest on.  Both results are
/// null when the parameter has no spill, which is so only when nothing uses
/// it at all.
static std::pair<cir::StoreOp, cir::AllocaOp>
findParamSpill(mlir::BlockArgument blockArg) {
  if (blockArg.use_empty())
    return {};
  cir::StoreOp store = maybeFindParamSpill(blockArg);
  assert(store && "non-byval arg's only use must be its CIRGen param spill");
  return {store, cast<cir::AllocaOp>(store.getAddr().getDefiningOp())};
}
 
/// For each Direct arg with a coerced type, change the block argument's type to
/// the coerced type and insert a coercion at function entry that maps it back
/// to the original type for body uses, which find a record in the coercion slot
/// through rewireRecordUses.  For each Indirect byval arg, change the block
/// argument's type to a pointer, then replace a record's single param spill
/// with a copy from that pointer.  Without such a spill, a record's uses find
/// it at the pointer through rewireRecordUses, and a value of any other type is
/// loaded from the pointer at entry.  For each Indirect non-byval arg, change
/// the block argument to a pointer and queue the param-slot alloca to be
/// replaced by it (no entry load or byte-copy) so the body operates on the
/// caller's storage in place.  For each Expand arg, replace the single struct
/// block argument with N scalar block arguments (one per field) and store each
/// field directly into the parameter's own alloca (the CIRGen spill slot),
/// erasing the original whole-struct store.
///
/// \p hasSRetArg is true when the function has an sret return (a hidden return
/// pointer is prepended as block argument 0).  Expand arguments expand the
/// block argument count, so a running index tracks the current block argument
/// position rather than computing the classification index + \p hasSRetArg
/// directly.  Fails, after reporting it, when rewireRecordUses does.
mlir::LogicalResult
insertArgCoercion(mlir::FunctionOpInterface funcOp,
                  const FunctionClassification &fc, mlir::OpBuilder &builder,
                  const mlir::DataLayout &dl, bool hasSRetArg,
                  SmallVectorImpl<std::pair<cir::AllocaOp, mlir::BlockArgument>>
                      &pendingParamSlots) {
  mlir::Region &body = funcOp->getRegion(0);
  if (body.empty())
    return mlir::success();
  mlir::Block &entry = body.front();
 
  // Running block argument index.  Each non-Expand classification occupies
  // one block argument slot; each Expand classification occupies N slots
  // (one per struct field), so the running index must be incremented by N
  // rather than 1 after processing an Expand arg.
  unsigned blockArgIdx = hasSRetArg ? 1 : 0;
 
  for (const ArgClassification &ac : fc.argInfos) {
    assert(blockArgIdx < entry.getNumArguments() &&
           "classification count must not exceed entry block arguments");
 
    if (ac.kind == ArgKind::Expand) {
      // The block arg at blockArgIdx currently has the original struct type.
      // Replace it with N scalar args (one per field) and store each field
      // directly into the parameter's own alloca.
      mlir::BlockArgument origArg = entry.getArgument(blockArgIdx);
      auto recTy = cast<cir::RecordType>(origArg.getType());
      assert(recTy.isStruct() &&
             "Expand classification requires a struct type, not a union");
      unsigned numFields = recTy.getNumElements();
      assert(numFields > 0 &&
             "Expand classification requires at least one struct field");
      mlir::Location loc = funcOp.getLoc();
 
      // CIRGen spills every by-value struct parameter into its local alloca
      // with a single store before any other use, so the struct block arg's
      // only use is that spill.  Capture it and the destination alloca so the
      // expanded fields can be stored straight into that alloca, preserving
      // the alloca's variable name and `init` flag and avoiding a
      // reassemble-then-reload roundtrip.  DCE may have run earlier and
      // removed the spill (leaving the block arg unused); tolerate that by
      // only flattening the signature and emitting no field stores.
      cir::StoreOp paramStore;
      cir::AllocaOp destAlloca;
      if (!origArg.use_empty()) {
        assert(origArg.hasOneUse() &&
               "Expand arg must have exactly one use (the CIRGen param spill)");
        paramStore = cast<cir::StoreOp>(*origArg.user_begin());
        assert(paramStore.getValue() == origArg &&
               "Expand arg's use must be the value operand of its store");
        destAlloca = cast<cir::AllocaOp>(paramStore.getAddr().getDefiningOp());
      }
 
      // Erase the original whole-struct spill before retyping the block
      // argument, so the store is never left feeding a type-mismatched value.
      // The field stores take its place, just before the following operation
      // (the spill always precedes the entry block's terminator).
      mlir::Operation *fieldStoreInsertPt = nullptr;
      if (paramStore) {
        fieldStoreInsertPt = paramStore->getNextNode();
        assert(fieldStoreInsertPt &&
               "param spill must be followed by a block terminator");
        paramStore->erase();
      }
 
      // Split the single struct block arg into N scalar field block args (slot
      // 0 reuses the original; slots 1..N-1 are inserted after it).  The
      // reshape needs no insertion point.  The field stores are gated on the
      // same destAlloca condition: when the spill survived we set the insert
      // point to its old slot (which sits after the CIRGen allocas) and store
      // each field there; when DCE removed the spill the parameter is dead, so
      // we only reshape the signature and emit no stores.
      if (destAlloca)
        builder.setInsertionPoint(fieldStoreInsertPt);
      for (auto [f, fieldTy] : llvm::enumerate(recTy.getMembers())) {
        if (f == 0)
          origArg.setType(fieldTy);
        else
          entry.insertArgument(blockArgIdx + f, fieldTy, loc);
        if (!destAlloca)
          continue;
        mlir::Type fieldPtrTy = cir::PointerType::get(
            fieldTy,
            mlir::cast<cir::PointerType>(destAlloca.getType()).getAddrSpace());
        auto fieldPtr = cir::GetMemberOp::create(builder, loc, fieldPtrTy,
                                                 destAlloca, /*name=*/"",
                                                 /*index=*/f);
        cir::StoreOp::create(builder, loc, entry.getArgument(blockArgIdx + f),
                             fieldPtr);
      }
 
      blockArgIdx += numFields;
      continue;
    }
 
    mlir::BlockArgument blockArg = entry.getArgument(blockArgIdx);
 
    if (cir::RecordType flatTy = getFlattenedCoercedType(ac)) {
      // Direct + canFlatten: the coerced type is a struct whose fields are
      // passed as separate arguments.  As in the Expand path, the single block
      // arg becomes N scalar block args, and each is stored straight into a
      // slot sized and aligned for both the coerced struct and the original
      // argument type, which the body then reads.
      unsigned numFields = flatTy.getNumElements();
      assert(numFields >= 2 && "getFlattenedCoercedType guarantees >1 fields");
      Type origTy = blockArg.getType();
      Location loc = funcOp.getLoc();
 
      // Change slot 0 to field 0's type; insert slots 1..N-1 after it.
      blockArg.setType(flatTy.getElementType(0));
      for (unsigned f = 1; f < numFields; ++f)
        entry.insertArgument(blockArgIdx + f, flatTy.getElementType(f), loc);
 
      // setInsertionPointToStart: see comment in the Expand arm above.
      builder.setInsertionPointToStart(&entry);
      assert(!ac.directOffset &&
             "the coerced struct starts at slot offset 0 here, so a flattened "
             "coercion cannot honor a direct offset");
      SmallPtrSet<Operation *, 8> flattenOps;
      cir::AllocaOp slot;
      if (origTy == flatTy) {
        slot = cir::AllocaOp::create(
            builder, loc, getAllocaPtrTy(flatTy, dl),
            builder.getStringAttr("coerce"),
            builder.getI64IntegerAttr(dl.getTypeABIAlignment(flatTy)));
        flattenOps.insert(slot);
      } else {
        slot = createCoercionSlot(builder, loc, flatTy, origTy, &entry, dl,
                                  flattenOps, /*offset=*/0);
      }
      Value flatView = emitViewAtOffset(builder, loc, flatTy, slot, flattenOps,
                                        /*offset=*/0);
      auto flatPtrTy = cast<cir::PointerType>(flatView.getType());
      for (auto [f, fieldTy] : llvm::enumerate(flatTy.getMembers())) {
        Type fieldPtrTy =
            cir::PointerType::get(fieldTy, flatPtrTy.getAddrSpace());
        auto fieldPtr = cir::GetMemberOp::create(builder, loc, fieldPtrTy,
                                                 flatView, /*name=*/"",
                                                 /*index=*/f);
        flattenOps.insert(fieldPtr);
        auto storeOp = cir::StoreOp::create(
            builder, loc, entry.getArgument(blockArgIdx + f), fieldPtr);
        flattenOps.insert(storeOp);
      }
 
      // Point the original body uses of the struct block arg, now field 0, at
      // the original-type view of the slot.
      Value origView = emitViewAtOffset(builder, loc, origTy, slot, flattenOps,
                                        /*offset=*/0);
      if (failed(replaceUsesFromSlot(builder, loc, blockArg, origView, dl,
                                     flattenOps)))
        return mlir::failure();
 
      blockArgIdx += numFields;
      continue;
    }
 
    if (ac.kind == ArgKind::Direct && ac.coercedType) {
      mlir::Type oldArgTy = blockArg.getType();
      mlir::Type newArgTy = ac.coercedType;
      if (oldArgTy == newArgTy) {
        ++blockArgIdx;
        continue;
      }
      blockArg.setType(newArgTy);
 
      // The coercion's own operations are left out of the rewiring, since they
      // read blockArg as the coerced type.
      builder.setInsertionPointToStart(&entry);
      SmallPtrSet<mlir::Operation *, 4> coercionOps;
      if (failed(emitCoercionAndReplaceUses(builder, funcOp.getLoc(), oldArgTy,
                                            blockArg, blockArg, &entry, dl,
                                            coercionOps, ac.directOffset)))
        return mlir::failure();
    } else if (ac.kind == ArgKind::Indirect) {
      // byval and non-byval both lower to !cir.ptr<T>, and which it is shows
      // up only in the attrs updateArgAttrs applies.  Body lowering differs:
      // the body addresses a byval parameter through the CIRGen slot it was
      // spilled into, so that slot has to be filled from the incoming
      // pointer, while a non-byval parameter's slot is replaced by the
      // pointer so the body works on the caller's storage in place.
      auto ptrTy = cir::PointerType::get(blockArg.getType());
 
      if (!ac.byVal) {
        // Without byval, drop the spill store and let the slot's uses read the
        // incoming pointer, so the body operates on the caller's storage in
        // place.  A byte-copy would be wrong for non-trivially-copyable
        // aggregates (e.g. libstdc++ SSO std::string, where it would leave
        // `_M_p` aliasing the source's `_M_local_buf`).
        auto [paramStore, destAlloca] = findParamSpill(blockArg);
 
        if (paramStore)
          paramStore->erase();
 
        // Update the block argument to point to its original type.
        blockArg.setType(ptrTy);
 
        // Pointing the slot's uses at the incoming pointer waits until every
        // call site has been rewritten.  A call that hands this parameter
        // straight on recognizes it by the slot its operand was loaded from,
        // and collapsing the slot here would leave that call reading a block
        // argument with no defining operation to inspect.  A dead spill DCE
        // already removed leaves nothing to collapse.
        if (destAlloca)
          pendingParamSlots.emplace_back(destAlloca, blockArg);
      } else {
        // A record's spill slot is filled by a byte copy of the incoming
        // object rather than by a store of the loaded value: its padding
        // bytes can hold another union member's live data, which a value
        // store would leave unwritten.  A _BitInt keeps its load and spill
        // store, which are its conversion between value and in-memory form.
        cir::StoreOp paramStore;
        if (mlir::isa<cir::RecordType>(blockArg.getType()))
          paramStore = maybeFindParamSpill(blockArg);
 
        // Erasing the spill before the block argument is retyped keeps the
        // store from being left with a value operand of the wrong type.
        mlir::Value destAddr;
        mlir::IntegerAttr destAlign;
        mlir::Operation *spillPos = nullptr;
        bool spillIsVolatile = false;
        if (paramStore) {
          if (paramStore.getMemOrder())
            return paramStore.emitOpError()
                   << "atomic store of a record argument or result not yet "
                      "implemented in CallConvLowering";
          spillIsVolatile = paramStore.getIsVolatile();
          destAddr = paramStore.getAddr();
          cir::AllocaOp destAlloca = cir::getUnderlyingAlloca(destAddr);
          destAlign = paramStore.getAlignmentAttr();
          if (!destAlign && destAlloca)
            destAlign = destAlloca.getAlignmentAttr();
          spillPos = paramStore->getNextNode();
          assert(spillPos && "param spill must be followed by a block "
                             "terminator");
          paramStore->erase();
        }
 
        blockArg.setType(ptrTy);
 
        if (destAddr) {
          mlir::OpBuilder::InsertionGuard guard(builder);
          builder.setInsertionPoint(spillPos);
          auto copy = cir::CopyOp::create(
              builder, funcOp.getLoc(), destAddr, blockArg, destAlign,
              builder.getI64IntegerAttr(ac.indirectAlign.value()));
          copy.setIsVolatile(spillIsVolatile);
          copy.setSkipTailPadding(
              shouldSkipTailPadding(destAddr, ptrTy.getPointee()));
        } else if (!blockArg.use_empty()) {
          builder.setInsertionPointToStart(&entry);
          if (mlir::isa<cir::RecordType>(ptrTy.getPointee())) {
            SmallPtrSet<mlir::Operation *, 1> noExceptions;
            if (failed(rewireRecordUses(builder, funcOp.getLoc(), blockArg,
                                        blockArg, ac.indirectAlign.value(), dl,
                                        noExceptions)))
              return mlir::failure();
          } else {
            auto loadOp =
                cir::LoadOp::create(builder, funcOp.getLoc(), blockArg);
            SmallPtrSet<mlir::Operation *, 1> loadOps = {loadOp};
            blockArg.replaceAllUsesExcept(loadOp.getResult(), loadOps);
          }
        }
      }
    }
    // Ignore, Extend, and Direct-without-coerce need no block-level changes.
 
    ++blockArgIdx;
  }
  return mlir::success();
}
 
/// Rewrite each cir.return so the return value flows through the sret
/// pointer (the prepended first block argument) and the function returns
/// void.
///
/// CIRGen emits a local `__retval` alloca and emits `cir.return %loaded`
/// where `%loaded = cir.load __retval`.  The naive lowering -- store the
/// loaded SSA value through the sret pointer -- byte-copies the record,
/// which is wrong for non-trivially-copyable types: e.g. libstdc++'s SSO
/// `std::string` has a `_M_p` pointer that aliases the source's internal
/// `_M_local_buf`, so a byte-copy leaves the destination pointing at the
/// source's (now-dying) stack storage and the destination's destructor
/// later `free()`s a stack pointer.
///
/// Instead, route construction directly into the sret slot: find the
/// `__retval` alloca, replace its uses with the sret pointer, and drop the
/// trailing `cir.load __retval` so the rewritten return has no operand.
/// The CIRGen-emitted constructor / store-into-`__retval` then targets the
/// sret slot uniformly, matching classic CodeGen's "construct directly into
/// `%agg.result`" pattern.
///
/// CIRGen emits one `%v = cir.load %__retval` / `cir.return %v` pair per
/// return statement, and every such load reads the single `__retval`
/// alloca (CIR does not merge returns into a shared epilogue block).  The
/// alloca is therefore rewired to the sret pointer once; each cir.return is
/// then collapsed to a bare return and its now-dead load erased.  This
/// `cir.return (cir.load <alloca>)` shape is an invariant guaranteed by
/// CIRGen, so it is asserted via `cast<>` rather than guarded with a
/// fallback.
void insertSRetStores(mlir::FunctionOpInterface funcOp, mlir::Type origRetTy,
                      mlir::OpBuilder &builder) {
  mlir::Value sretPtr = funcOp.getArguments()[0];
 
  SmallVector<cir::ReturnOp> returnOps;
  funcOp->walk([&](cir::ReturnOp retOp) { returnOps.push_back(retOp); });
 
  cir::AllocaOp retAlloca = nullptr;
  for (cir::ReturnOp retOp : returnOps) {
    // Every cir.return in an sret function must carry the loaded return
    // value -- a bare return would mean the sret slot was never written.
    assert(!retOp.getInput().empty() &&
           "cir.return in sret function must have an operand");
 
    cir::LoadOp retLoad =
        mlir::cast<cir::LoadOp>(retOp.getInput()[0].getDefiningOp());
 
    // Rewire the shared `__retval` alloca to the sret pointer once.
    // replaceAllUsesWith updates every load of the alloca (including those
    // feeding the other cir.return ops) to read from sretPtr instead, so
    // all returns are covered by this single rewiring.  Only then is the
    // now-unused alloca safe to erase.
    if (!retAlloca) {
      retAlloca = mlir::cast<cir::AllocaOp>(retLoad.getAddr().getDefiningOp());
      retAlloca.getResult().replaceAllUsesWith(sretPtr);
      retAlloca->erase();
    }
 
    // The sret slot now holds the return value directly; replace the
    // value-carrying return with a void return (no operand).
    builder.setInsertionPoint(retOp);
    cir::ReturnOp::create(builder, retOp.getLoc());
    retOp->erase();
    if (retLoad.use_empty())
      retLoad->erase();
  }
}
 
/// Build the attribute dictionary for the sret slot (slot 0 of an
/// sret-returning function or call).  Matches classic CodeGen's
/// `sret(T) align A [noalias] writable dead_on_unwind`.  noalias is only
/// valid on the callee's parameter, not at the call site, so it is gated by
/// \p withNoalias.  Key order is irrelevant: DictionaryAttr sorts by name.
SmallVector<mlir::NamedAttribute> buildSretSlotAttrs(mlir::OpBuilder &builder,
                                                     mlir::Type retTy,
                                                     uint64_t align,
                                                     bool withNoalias) {
  SmallVector<mlir::NamedAttribute> attrs;
  // The sret type must be carried explicitly: LLVM's sret attribute requires
  // it, and once the CIR `!cir.ptr<retTy>` lowers to an opaque LLVM `ptr` the
  // pointee type can no longer be recovered from the pointer.
  attrs.push_back(
      builder.getNamedAttr("llvm.sret", mlir::TypeAttr::get(retTy)));
  attrs.push_back(
      builder.getNamedAttr("llvm.align", builder.getI64IntegerAttr(align)));
  if (withNoalias)
    attrs.push_back(
        builder.getNamedAttr("llvm.noalias", builder.getUnitAttr()));
  attrs.push_back(builder.getNamedAttr("llvm.writable", builder.getUnitAttr()));
  attrs.push_back(
      builder.getNamedAttr("llvm.dead_on_unwind", builder.getUnitAttr()));
  return attrs;
}
 
/// Prepend the sret slot's attrs at position 0 of newCall's arg_attrs.
/// Called after the call has been rewritten with the sret pointer at
/// operand 0, so the operand count now includes the sret slot.  \p argAttrs
/// must already be shaped for the rewritten argument list (Extend slots
/// carry signext/zeroext, Ignore slots dropped); it is shifted to slots
/// 1..N behind the sret slot.
void applySretSlotAttrs(cir::CallOp newCall, mlir::ArrayAttr argAttrs,
                        mlir::Type retTy, uint64_t align,
                        mlir::OpBuilder &builder) {
  mlir::MLIRContext *ctx = newCall->getContext();
  SmallVector<mlir::NamedAttribute> sretAttrs =
      buildSretSlotAttrs(builder, retTy, align, /*withNoalias=*/false);
 
  SmallVector<mlir::Attribute> newArgAttrs;
  newArgAttrs.reserve(newCall.getArgOperands().size());
  newArgAttrs.push_back(mlir::DictionaryAttr::get(ctx, sretAttrs));
  if (argAttrs)
    llvm::append_range(newArgAttrs, argAttrs);
  assert(newArgAttrs.size() <= newCall.getArgOperands().size() &&
         "arg_attrs wider than the rewritten call's operand list");
  newArgAttrs.resize(newCall.getArgOperands().size(),
                     mlir::DictionaryAttr::get(ctx));
  newCall->setAttr("arg_attrs", mlir::ArrayAttr::get(ctx, newArgAttrs));
}
 
/// Copy the call attributes from \p source to the rebuilt \p target. The
/// callee is already established by CallOp::create and may have been rewritten,
/// so it is not copied. Existing discardable attributes on \p target take
/// precedence over attributes from \p source.
static void copyCallAttributes(cir::CallOp source, cir::CallOp target) {
  source->getName().walkInherentAttrs(source, [&](llvm::StringRef name,
                                                  mlir::Attribute &attr) {
    if (name != cir::CIRDialect::getCalleeAttrName())
      target->setInherentAttr(mlir::StringAttr::get(source->getContext(), name),
                              attr);
  });
  for (mlir::NamedAttribute attr : source->getDiscardableAttrs())
    if (!target->hasDiscardableAttr(attr.getName()))
      target->setDiscardableAttr(attr.getName(), attr.getValue());
}
 
/// For an indirect call, prepend the callee function pointer as operand 0 so
/// CallOp::create rebuilds it as an indirect call, bitcasting it to a function
/// pointer that takes \p args, less the last \p numEllipsisArgs, and returns
/// \p retTy.  Those last arguments were passed through the callee's ellipsis
/// and stay out of the function type, as in classic CodeGen's
/// GetFunctionType: LLVM treats operands past a call's function type as
/// variadic, and some backends pass variadic arguments differently.  No-op
/// for direct calls.
static void prependIndirectCallee(cir::CallOp call,
                                  SmallVectorImpl<mlir::Value> &args,
                                  mlir::Type retTy, unsigned numEllipsisArgs,
                                  mlir::OpBuilder &builder) {
  if (!call.isIndirect())
    return;
  assert(numEllipsisArgs <= args.size() &&
         "more ellipsis arguments than arguments");
  mlir::Value calleePtr = call.getIndirectCall();
  SmallVector<mlir::Type> paramTypes;
  paramTypes.reserve(args.size() - numEllipsisArgs);
  llvm::transform(ArrayRef(args).drop_back(numEllipsisArgs),
                  std::back_inserter(paramTypes),
                  [](mlir::Value v) { return v.getType(); });
  // Lowering builds an indirect call's LLVM function type from the callee
  // pointer's pointee and takes the call's result from that type, so the
  // pointee's return type has to track the rewrite: an sret return would
  // leave a result the call no longer produces, and a coerced return one of
  // the wrong type.  The ellipsis has to survive for the same reason: the
  // rebuilt pointee is what makes the lowered call variadic, and only a
  // variadic call gets the vector-register count that the x86_64 SysV ABI
  // passes in AL and that the callee's va_arg reads back.
  cir::FuncType calleeFnTy = getIndirectCalleeType(call);
  assert((calleeFnTy.isVarArg() || numEllipsisArgs == 0) &&
         "only a variadic callee takes arguments through an ellipsis");
  auto newPtrTy = cir::PointerType::get(
      cir::FuncType::get(paramTypes, retTy, calleeFnTy.isVarArg()));
  if (calleePtr.getType() != newPtrTy)
    calleePtr = cir::CastOp::create(builder, call.getLoc(), newPtrTy,
                                    cir::CastKind::bitcast, calleePtr);
  args.insert(args.begin(), calleePtr);
}
 
/// Rewrite an indirect-return (sret) call site: prepend a return-slot pointer
/// as operand 0, make the call return void, and either reuse a dominating
/// single-use store destination as the slot (so construction flows directly
/// into it) or allocate a fresh slot and point the result's uses at it, a
/// record's through rewireRecordUses and any other type's through a load of the
/// slot after the call.  \p newArgs is the already-shaped (Ignore-dropped,
/// coercion-applied) non-sret argument list, the last \p numEllipsisArgs of
/// which were passed through the callee's ellipsis.  The caller guarantees the
/// call has a result and an indirect-return classification.  Fails, after
/// reporting it, when rewireRecordUses does.
mlir::LogicalResult rewriteIndirectReturnCall(
    cir::CallOp call, const FunctionClassification &fc,
    ArrayRef<mlir::Value> newArgs, unsigned numEllipsisArgs,
    mlir::Type origRetTy, ArrayRef<mlir::Type> origCallArgTypes,
    mlir::OpBuilder &builder, const mlir::DataLayout &dl) {
  mlir::MLIRContext *ctx = call->getContext();
  auto ptrTy = cir::PointerType::get(origRetTy);
  builder.setInsertionPoint(call);
  uint64_t sretAlign = fc.returnInfo.indirectAlign.value();
 
  // CIRGen emits `cir.store %callResult, %dest` when the call's result is
  // bound to a local (e.g. `T s = make();`).  Allocating a fresh sret slot
  // and copying into %dest would byte-copy the record, which is wrong for
  // non-trivially-copyable types (the libstdc++ SSO `_M_p` pointer
  // survives a byte-copy but ends up pointing at the dying temp's local
  // buffer, so the destination's destructor later `free()`s a stack
  // pointer).  When the result has a single store-into-%dest use, use
  // %dest as the sret slot directly so construction flows into it,
  // matching classic CodeGen's "pass %s as sret" pattern.  %dest must
  // dominate the call so the rewritten call (which takes it as operand 0)
  // does not use a value before its definition.
  mlir::Value sretSlot = nullptr;
  cir::StoreOp reuseStore = nullptr;
  if (call.getResult().hasOneUse()) {
    mlir::Operation *user = *call.getResult().getUsers().begin();
    if (auto store = mlir::dyn_cast<cir::StoreOp>(user))
      if (store.getValue() == call.getResult() &&
          store.getAddr().getType() == ptrTy &&
          mlir::DominanceInfo().properlyDominates(store.getAddr(), call)) {
        sretSlot = store.getAddr();
        reuseStore = store;
      }
  }
  if (!sretSlot) {
    auto alloca = cir::AllocaOp::create(
        builder, call.getLoc(), ptrTy,
        /*name=*/builder.getStringAttr("sret"),
        /*alignment=*/builder.getI64IntegerAttr(sretAlign));
    sretSlot = alloca;
  }
 
  SmallVector<mlir::Value> sretArgs;
  sretArgs.push_back(sretSlot);
  sretArgs.append(newArgs.begin(), newArgs.end());
 
  // numEllipsisArgs counts from the end, so the sret slot prepended here stays
  // in the retyped signature, as it does in classic CodeGen.
  mlir::Type sretVoidTy = cir::VoidType::get(ctx);
  prependIndirectCallee(call, sretArgs, sretVoidTy, numEllipsisArgs, builder);
  auto newCall = cir::CallOp::create(
      builder, call.getLoc(), call.getCalleeAttr(), sretVoidTy, sretArgs);
  copyCallAttributes(call, newCall);
  newCall->removeAttr("res_attrs");
 
  // Shape the per-argument attrs exactly as the non-sret path does, through
  // updateArgAttrs, before prepending the sret slot, so sret composes with
  // every argument classification.
  mlir::ArrayAttr argAttrs = call->getAttrOfType<mlir::ArrayAttr>("arg_attrs");
  if (needsArgAttrUpdate(fc))
    argAttrs = updateArgAttrs(ctx, call->getParentOfType<mlir::ModuleOp>(),
                              origCallArgTypes, argAttrs, fc, dl);
  applySretSlotAttrs(newCall, argAttrs, origRetTy, sretAlign, builder);
 
  if (reuseStore) {
    // The callee now constructs directly into the destination slot, so the
    // original store-from-result is redundant; dropping it avoids a
    // byte-copy of the record.
    reuseStore->erase();
  } else if (mlir::isa<cir::RecordType>(origRetTy)) {
    builder.setInsertionPointAfter(newCall);
    SmallPtrSet<mlir::Operation *, 1> noExceptions;
    if (failed(rewireRecordUses(builder, call.getLoc(), call.getResult(),
                                sretSlot, sretAlign, dl, noExceptions)))
      return mlir::failure();
  } else {
    builder.setInsertionPointAfter(newCall);
    auto load = cir::LoadOp::create(builder, call.getLoc(), origRetTy, sretSlot,
                                    /*isDeref=*/mlir::UnitAttr(),
                                    /*isVolatile=*/mlir::UnitAttr(),
                                    /*is_nontemporal=*/mlir::UnitAttr(),
                                    /*alignment=*/mlir::IntegerAttr(),
                                    /*sync_scope=*/cir::SyncScopeKindAttr(),
                                    /*mem_order=*/cir::MemOrderAttr(),
                                    /*invariant=*/mlir::UnitAttr());
    call.getResult().replaceAllUsesWith(load);
  }
  call->erase();
  return mlir::success();
}
 
/// Whether \p ty, a type the classifier named for one register of a
/// coercion, is carried in a vector register.
bool isSSERegisterClass(mlir::Type ty) {
  return mlir::isa<cir::VectorType, cir::FPTypeInterface>(ty);
}
 
} // namespace
 
cir::FuncType cir::getIndirectCalleeType(cir::CIRCallOpInterface call) {
  assert(call.isIndirect() && "expected an indirect call");
  return cast<cir::FuncType>(
      cast<cir::PointerType>(call.getIndirectCall().getType()).getPointee());
}
 
/// Bring \p funcOp's non-byval indirect parameter \p argNo into the shape the
/// rest of the rewrite assumes.  \p claimedSlots carries the slots \p funcOp's
/// earlier non-byval indirect parameters took.  The shape itself, and the
/// cases reported rather than repaired, are documented on
/// CIRABIRewriteContext::prepareNonByvalParameters in the header.
static mlir::LogicalResult prepareOneNonByvalParameter(
    cir::FuncOp funcOp, unsigned argNo, const ArgClassification &ac,
    mlir::BlockArgument blockArg, mlir::DominanceInfo &dom,
    SmallPtrSetImpl<mlir::Operation *> &claimedSlots) {
  // The spill is the store that writes the parameter itself.  Any other use
  // consumes the record value.  At -O1 and above such a use comes from
  // cir-simplify: CIRGen marks a const-qualified parameter's slot const, so
  // the load of it folds to the stored parameter.
  cir::StoreOp spill;
  cir::StoreOp extraSpill;
  mlir::Operation *otherUse = nullptr;
  SmallVector<mlir::OpOperand *> callArgs;
  for (mlir::OpOperand &use : blockArg.getUses()) {
    auto store = dyn_cast<cir::StoreOp>(use.getOwner());
    if (store && store.getValue() == blockArg) {
      // Which of two stores is the spill decides which slot stands in for the
      // parameter, and the use list is in no particular order, so there is
      // nothing to prefer between them.
      if (spill)
        extraSpill = store;
      else
        spill = store;
      continue;
    }
    // Only an argument is served by a load of the slot.  Any other consumer
    // belongs to a rewrite that reads the parameter its own way: a returned
    // record, for one, is rewritten through the sret slot, which assumes the
    // returned load names the return slot and not this one.
    auto call = dyn_cast<cir::CIRCallOpInterface>(use.getOwner());
    if (call && llvm::is_contained(call.getArgOperands(), blockArg))
      callArgs.push_back(&use);
    else if (!otherUse)
      otherUse = use.getOwner();
  }
 
  if (extraSpill)
    return extraSpill->emitOpError()
           << "non-byval parameter " << argNo
           << " spilled more than once is not yet implemented in "
              "CallConvLowering";
 
  if (otherUse)
    return otherUse->emitOpError()
           << "non-byval parameter " << argNo
           << " consumed other than as a call argument is not yet implemented "
              "in CallConvLowering";
 
  if (!spill) {
    // Every other kind of use was reported above, so the parameter has no
    // uses at all and needs neither a slot nor a read.
    if (callArgs.empty())
      return mlir::success();
 
    // Without a spill the parameter becomes the incoming pointer directly,
    // which is the storage an argument taken from it has to name.  Giving it
    // the spill it lacks lets the checks and the read below apply unchanged,
    // and neither survives the pass: insertArgCoercion erases the store and
    // finalizeParameterSlots replaces the slot.
    mlir::OpBuilder builder(funcOp.getContext());
    builder.setInsertionPointToStart(blockArg.getOwner());
    auto synthesized = cir::AllocaOp::create(
        builder, funcOp.getLoc(), cir::PointerType::get(blockArg.getType()),
        builder.getStringAttr("nonbyval.param"),
        builder.getI64IntegerAttr(ac.indirectAlign.value()));
    spill =
        cir::StoreOp::create(builder, funcOp.getLoc(), blockArg, synthesized);
  }
 
  // The incoming pointer replaces the slot itself, in the default address
  // space.  Retargeting a cast of it would leave the allocation's other views
  // reading storage nothing writes.  Storage in another address space, or
  // storage that is not a local alloca at all, is not something the incoming
  // pointer can be substituted for.
  cir::AllocaOp slot = spill.getAddr().getDefiningOp<cir::AllocaOp>();
  if (!slot ||
      spill.getAddr().getType() != cir::PointerType::get(blockArg.getType()))
    return spill->emitOpError()
           << "non-byval parameter " << argNo
           << " spilled to storage that cannot take the incoming pointer is "
              "not yet implemented in CallConvLowering";
 
  // One slot stands in for one non-byval parameter, since the incoming
  // pointer replaces it.  Two of them spilled to the same slot are each
  // spilled once, so the check above cannot see the collision and only
  // comparing the slots can.
  if (!claimedSlots.insert(slot).second)
    return spill->emitOpError()
           << "non-byval parameter " << argNo
           << " sharing its spill slot with another parameter is not yet "
              "implemented in CallConvLowering";
 
  // CIRGen picked the slot's alignment for a local copy of the record, but the
  // slot is about to stand in for the parameter, and a call forwarding it may
  // only promise what the incoming pointer does.
  slot.setAlignment(ac.indirectAlign.value());
 
  if (callArgs.empty())
    return mlir::success();
 
  // A reader the spill does not dominate could read the slot before anything
  // wrote it, and a load placed at the spill would not dominate it either.
  for (mlir::OpOperand *callArg : callArgs)
    if (!dom.properlyDominates(spill, callArg->getOwner()))
      return callArg->getOwner()->emitOpError()
             << "non-byval parameter " << argNo
             << " read before its spill is not yet implemented in "
                "CallConvLowering";
 
  // Read the record back out of the slot instead, so every reader sees the
  // shape CIRGen emits without cir-simplify: a load naming the storage the
  // argument would have to name anyway.  Reading at the spill, not at the
  // reader, keeps the value the parameter's own whatever the body later
  // stores into the slot, and the load promises only the alignment the
  // classification gives the incoming pointer.
  //
  // A call that forwards the argument consumes the load once it recognizes
  // the slot.  One that wants a copy keeps it, reading the incoming pointer
  // once finalizeParameterSlots replaces the slot.
  mlir::OpBuilder builder(spill);
  builder.setInsertionPointAfter(spill);
  auto reload = cir::LoadOp::create(builder, spill.getLoc(), slot);
  reload.setAlignment(ac.indirectAlign.value());
  for (mlir::OpOperand *callArg : callArgs)
    callArg->set(reload.getResult());
  return mlir::success();
}
 
mlir::LogicalResult CIRABIRewriteContext::prepareNonByvalParameters(
    cir::FuncOp funcOp, const FunctionClassification &fc) {
  if (!funcOp.isDefinition())
    return mlir::success();
  mlir::Region &body = funcOp->getRegion(0);
  if (body.empty())
    return mlir::success();
  mlir::Block &entry = body.front();
  mlir::DominanceInfo dom;
  SmallPtrSet<mlir::Operation *, 4> claimedSlots;
 
  // No signature has been rewritten yet, so no sret pointer has been prepended
  // and no Expand argument has been split into its fields.  Every
  // classification therefore still maps to the entry block argument at its own
  // index.
  for (auto [idx, ac] : llvm::enumerate(fc.argInfos)) {
    if (ac.kind != ArgKind::Indirect || ac.byVal)
      continue;
    assert(idx < entry.getNumArguments() &&
           "classification count must not exceed entry block arguments");
    if (failed(prepareOneNonByvalParameter(
            funcOp, idx, ac, entry.getArgument(idx), dom, claimedSlots)))
      return mlir::failure();
  }
  return mlir::success();
}
 
void CIRABIRewriteContext::finalizeParameterSlots() {
  for (auto [slot, incoming] : pendingParamSlots) {
    slot.getResult().replaceAllUsesWith(incoming);
    slot->erase();
  }
  pendingParamSlots.clear();
}
 
mlir::LogicalResult CIRABIRewriteContext::rewriteFunctionDefinition(
    mlir::FunctionOpInterface funcOpInterface, const FunctionClassification &fc,
    mlir::OpBuilder &builder) {
  // The pass driver (CallConvLoweringPass) only ever hands us cir.func ops.
  // Cast once at the top so the rest of the function reads in CIR's own
  // vocabulary, and so we can dispatch to the CIRGlobalValueInterface for
  // isDefinition() (FunctionOpInterface alone does not inherit from
  // CIRGlobalValueInterface).
  cir::FuncOp funcOp = mlir::cast<cir::FuncOp>(funcOpInterface);
 
  if (!fc.needsRewrite())
    return mlir::success();
 
  ArrayRef<mlir::Type> oldArgTypes = funcOp.getArgumentTypes();
  ArrayRef<mlir::Type> oldResultTypes = funcOp.getResultTypes();
  mlir::MLIRContext *ctx = funcOp->getContext();
 
  // CIR follows LLVM IR's single-result rule: a function returns either
  // zero or one value.  Document the invariant so a future multi-result
  // change forces us to revisit the return-handling below.
  assert(oldResultTypes.size() <= 1 &&
         "CIR functions return zero or one value");
 
  SmallVector<mlir::Type> newArgTypes;
  if (mlir::failed(buildNewArgTypes(oldArgTypes, fc, newArgTypes,
                                    [&]() { return funcOp.emitOpError(); })))
    return mlir::failure();
 
  mlir::Type voidTy = cir::VoidType::get(ctx);
  mlir::Type origRetTy = oldResultTypes.empty() ? voidTy : oldResultTypes[0];
  mlir::Type newRetTy = computeNewReturnType(
      origRetTy, fc.returnInfo, ctx, [&]() { return funcOp.emitOpError(); });
  if (!newRetTy)
    return mlir::failure();
  SmallVector<mlir::Type> newResultTypes = {newRetTy};
 
  // sret return: the value is returned through a pointer the ABI inserts as
  // argument 0.  This pointer is not part of the function's source-level
  // signature -- it is synthesized here -- and the wire return type was
  // already set to void by computeNewReturnType.  Every classification index
  // therefore maps to a block argument shifted by one in the body handling
  // below.
  bool hasSRet =
      fc.returnInfo.kind == ArgKind::Indirect && !oldResultTypes.empty();
  if (hasSRet)
    newArgTypes.insert(newArgTypes.begin(), cir::PointerType::get(origRetTy));
 
  if (funcOp.isDefinition()) {
    mlir::Region &body = funcOp->getRegion(0);
    if (!body.empty()) {
      // Prepend the sret pointer block argument and route every cir.return
      // through it before any index-based argument handling below (which
      // then accounts for the +1 offset).
      if (hasSRet) {
        body.front().insertArgument(0u, cir::PointerType::get(origRetTy),
                                    funcOp.getLoc());
        insertSRetStores(funcOp, origRetTy, builder);
      }
 
      // Reshape the entry block for the classifications that change what a
      // parameter looks like to the body: a Direct arg with a coerced type
      // gets a memory roundtrip back to its original type, an Indirect arg
      // becomes a pointer, and an Expand arg becomes one block argument per
      // field.  Done before the Ignore-drop below so the entry block
      // argument indices used here still refer to the original positions.
      if (failed(insertArgCoercion(funcOp, fc, builder, dl, hasSRet,
                                   pendingParamSlots)))
        return mlir::failure();
 
      // Direct return with coerced type: insert a coercion at every
      // cir.return so the returned value matches the (coerced) return
      // type in the new function signature set below.
      if (fc.returnInfo.kind == ArgKind::Direct && fc.returnInfo.coercedType &&
          !oldResultTypes.empty() && fc.returnInfo.coercedType != origRetTy &&
          failed(insertReturnCoercion(
              funcOp, origRetTy, fc.returnInfo.coercedType, builder, dl,
              cirDataLayout, fc.returnInfo.directOffset)))
        return mlir::failure();
 
      mlir::Block &entry = body.front();
 
      // Drop each Ignored argument's block argument, replacing any remaining
      // body uses with a poison constant (an Ignore arg is not passed at the
      // ABI level, so any use is vacuous; poison says exactly that).  Walk
      // forward with a running block-argument index that mirrors
      // insertArgCoercion: an Expand arg or a Direct+canFlatten arg occupies N
      // slots, every other kept kind one.  On erase, do not advance the index
      // -- the next block argument shifts into the vacated slot.
      unsigned blockArgIdx = hasSRet ? 1 : 0;
      for (auto [i, ac] : llvm::enumerate(fc.argInfos)) {
        if (blockArgIdx >= entry.getNumArguments())
          break;
        if (ac.kind == ArgKind::Ignore) {
          mlir::BlockArgument arg = entry.getArgument(blockArgIdx);
          if (!arg.use_empty()) {
            builder.setInsertionPointToStart(&entry);
            mlir::Value poison =
                createIgnoredValue(builder, funcOp.getLoc(), arg.getType());
            arg.replaceAllUsesWith(poison);
          }
          entry.eraseArgument(blockArgIdx);
          continue;
        }
        if (cir::RecordType flatTy = getFlattenedCoercedType(ac))
          blockArgIdx += flatTy.getNumElements();
        else if (ac.kind == ArgKind::Expand)
          blockArgIdx += cast<cir::RecordType>(oldArgTypes[i]).getNumElements();
        else
          ++blockArgIdx;
      }
    }
 
    // When the return is classified Ignore but the original function had
    // a non-void return type, every cir.return becomes a naked return.
    // This relies on the invariant that computeNewReturnType has set
    // newRetTy = void for Ignore above, and that the function type is
    // updated below to match.  Asserting this keeps the dependency
    // explicit.
    if (fc.returnInfo.kind == ArgKind::Ignore && !oldResultTypes.empty()) {
      assert(mlir::isa<cir::VoidType>(newRetTy) &&
             "Ignore-return path requires the new return type to be void");
      SmallVector<cir::ReturnOp> returns;
      funcOp.walk([&](cir::ReturnOp r) { returns.push_back(r); });
      for (cir::ReturnOp r : returns) {
        if (r.getNumOperands() == 0)
          continue;
        builder.setInsertionPoint(r);
        cir::ReturnOp::create(builder, r.getLoc());
        r.erase();
      }
    }
  }
 
  mlir::Type newFnTy = funcOp.cloneTypeWith(newArgTypes, newResultTypes);
  funcOp.setFunctionTypeAttr(mlir::TypeAttr::get(newFnTy));
 
  // Rebuild arg_attrs when the function has an sret slot (slot 0 needs the
  // sret attribute set) or any arg needs updateArgAttrs.
  if (hasSRet || needsArgAttrUpdate(fc)) {
    auto existing = funcOp->getAttrOfType<mlir::ArrayAttr>("arg_attrs");
    mlir::ArrayAttr updated =
        updateArgAttrs(ctx, module, oldArgTypes, existing, fc, dl);
    if (hasSRet) {
      // Prepend the sret slot's attribute dict (slot 0); the per-argument
      // dicts shift to slots 1..N.  noalias is valid only on the callee's
      // parameter, so it is added only for definitions.
      SmallVector<mlir::NamedAttribute> sretAttrs = buildSretSlotAttrs(
          builder, origRetTy, fc.returnInfo.indirectAlign.value(),
          /*withNoalias=*/funcOp.isDefinition());
      SmallVector<mlir::Attribute> withSret;
      withSret.push_back(mlir::DictionaryAttr::get(ctx, sretAttrs));
      llvm::append_range(withSret, updated);
      funcOp->setAttr("arg_attrs", mlir::ArrayAttr::get(ctx, withSret));
    } else {
      funcOp->setAttr("arg_attrs", updated);
    }
  }
 
  if (mlir::isa<cir::VoidType>(newRetTy)) {
    funcOp->removeAttr("res_attrs");
  } else if (fc.returnInfo.kind == ArgKind::Extend ||
             fc.returnInfo.kind == ArgKind::Direct) {
    auto existing = funcOp->getAttrOfType<mlir::ArrayAttr>("res_attrs");
    if (mlir::ArrayAttr updated =
            updateResAttrs(ctx, existing, fc.returnInfo, origRetTy, dl))
      funcOp->setAttr("res_attrs", updated);
  }
 
  return mlir::success();
}
 
mlir::LogicalResult
CIRABIRewriteContext::rewriteCallSite(mlir::Operation *callOp,
                                      const FunctionClassification &fc,
                                      mlir::OpBuilder &builder) {
  // The rewrite below pairs the classification with the call's operands one
  // for one, so a call passing arguments through an ellipsis needs a
  // classification built from its own operands rather than from the callee's
  // declared parameters.  Both directions of a mismatch have to be reported
  // before the pass-through early return, or a call whose declared parameters
  // happen to be pass-through is left as written with its surplus operands
  // never classified.
  //
  // A surplus operand means fc was built from the callee's declared
  // parameters alone.  A shortfall means the callee was declared no_proto,
  // which turns off the verifier's argument-count check altogether.
  unsigned numOperands =
      mlir::cast<cir::CIRCallOpInterface>(callOp).getNumArgOperands();
  if (numOperands > fc.argInfos.size())
    return callOp->emitOpError()
           << "variadic arguments not yet implemented in CallConvLowering";
  if (numOperands < fc.argInfos.size())
    return callOp->emitOpError()
           << "call passes fewer arguments than the callee declares, which is "
              "not yet implemented in CallConvLowering";
 
  if (!fc.needsRewrite())
    return mlir::success();
 
  if (mlir::isa<cir::TryCallOp>(callOp))
    return callOp->emitOpError()
           << "TryCallOp not yet implemented in CallConvLowering";
 
  auto call = mlir::cast<cir::CallOp>(callOp);
  mlir::MLIRContext *ctx = callOp->getContext();
  mlir::Block *slotBlock = coercionSlotBlock(call);
 
  builder.setInsertionPoint(call);
 
  SmallVector<mlir::Value> newArgs;
  mlir::ValueRange argOperands = call.getArgOperands();
  newArgs.reserve(argOperands.size());
 
  // Loads that the new call leaves unused: a Direct (coerced or flattened) or
  // Expand record is read out of its load's storage or a copy of it, and an
  // Indirect argument passes or copies from the address the load read from.
  // The old call still uses them, so erase them only after it is gone.
  SmallVector<cir::LoadOp> deadRecordLoads;
 
  // Capture original arg types before building newArgs (an Indirect argument
  // changes the wire argument from T to !cir.ptr<T>, so we save the
  // pre-rewrite types here for use in updateArgAttrs).
  SmallVector<mlir::Type> origCallArgTypes;
  llvm::append_range(origCallArgTypes, argOperands.getTypes());
 
  // Operands past an indirect callee's declared parameters were passed
  // through its ellipsis.  Note where they begin in newArgs, so
  // prependIndirectCallee can leave them out of the retyped callee type.  A
  // direct call has no callee pointer to retype, so it records no boundary.
  unsigned numDeclared = fc.argInfos.size();
  if (call.isIndirect()) {
    numDeclared = getIndirectCalleeType(call).getNumInputs();
    assert(numDeclared <= fc.argInfos.size() &&
           "the classification covers at least the declared parameters");
  }
  // Recorded before the Ignore skip below, so an ignored first ellipsis
  // argument still marks where the ellipsis arguments begin in newArgs.
  std::optional<unsigned> ellipsisStart;
  for (auto [idx, ac] : llvm::enumerate(fc.argInfos)) {
    if (idx == numDeclared)
      ellipsisStart = newArgs.size();
    if (ac.kind == ArgKind::Ignore)
      continue;
    mlir::Value arg = argOperands[idx];
    if (cir::RecordType flatTy = getFlattenedCoercedType(ac)) {
      // Direct + canFlatten: pass one scalar call argument per field of the
      // ABI-coerced struct.  A non-atomic record load is read in place or
      // through a copy of its storage, as emitCoercedRecordLoad decides, and
      // any other operand is stored into a coercion slot first.  When the types
      // already match, the fields come from the struct value directly (reading
      // from its source alloca when possible).
      if (arg.getType() != flatTy) {
        SmallPtrSet<mlir::Operation *, 4> coercionOps;
        assert(!ac.directOffset &&
               "the coerced struct starts at slot offset 0 here, so a "
               "flattened coercion cannot honor a direct offset");
        if (failed(rejectAtomicRecordLoad(call, arg)))
          return mlir::failure();
        mlir::OpBuilder::InsertionGuard guard(builder);
        mlir::Value coercedPtr;
        uint64_t baseAlign = 0;
        if (cir::LoadOp recLoad = maybeGetCopyableRecordLoad(arg)) {
          mlir::Value base = recLoad.getAddr();
          if (!recLoad.getIsVolatile() &&
              canReadInPlace(recLoad, flatTy, cirDataLayout, /*offset=*/0)) {
            cir::AllocaOp baseAlloca = cir::getUnderlyingAlloca(base);
            assert(baseAlloca && "canReadInPlace requires an alloca");
            baseAlign = baseAlloca.getAlignment();
          } else {
            base = emitCopiedCoercionSlot(builder, call.getLoc(), recLoad,
                                          flatTy, slotBlock, dl, /*offset=*/0);
          }
          builder.setInsertionPoint(recLoad);
          coercedPtr = emitViewAtOffset(builder, call.getLoc(), flatTy, base,
                                        coercionOps, /*offset=*/0);
          deadRecordLoads.push_back(recLoad);
        } else {
          coercedPtr =
              emitCoercionToMemory(builder, call.getLoc(), flatTy, arg,
                                   slotBlock, dl, coercionOps, /*offset=*/0);
        }
        auto coercedPtrTy = mlir::cast<cir::PointerType>(coercedPtr.getType());
        for (auto [f, fieldTy] : llvm::enumerate(flatTy.getMembers())) {
          mlir::Type fieldPtrTy =
              cir::PointerType::get(fieldTy, coercedPtrTy.getAddrSpace());
          auto fieldPtr =
              cir::GetMemberOp::create(builder, call.getLoc(), fieldPtrTy,
                                       coercedPtr, /*name=*/"", /*index=*/f);
          auto field = cir::LoadOp::create(builder, call.getLoc(), fieldTy,
                                           fieldPtr.getResult());
          // Storage read in place may be less aligned than the coerced struct,
          // so each field claims only what its offset in it allows.
          if (baseAlign)
            field.setAlignment(
                llvm::MinAlign(baseAlign, flatTy.getElementOffset(dl, f)));
          newArgs.push_back(field);
        }
      } else {
        emitStructFieldArgs(builder, call.getLoc(), arg, flatTy, newArgs,
                            deadRecordLoads);
      }
    } else if (ac.kind == ArgKind::Expand) {
      // Decompose the struct value into its constituent scalar fields and
      // pass each as a separate argument.
      auto recTy = cast<cir::RecordType>(arg.getType());
      assert(recTy.isStruct() &&
             "Expand classification requires a struct type, not a union");
      emitStructFieldArgs(builder, call.getLoc(), arg, recTy, newArgs,
                          deadRecordLoads);
    } else if (ac.kind == ArgKind::Direct && ac.coercedType &&
               arg.getType() != ac.coercedType) {
      if (failed(rejectAtomicRecordLoad(call, arg)))
        return mlir::failure();
      if (cir::LoadOp recLoad = maybeGetCopyableRecordLoad(arg)) {
        newArgs.push_back(emitCoercedRecordLoad(
            builder, call.getLoc(), recLoad, ac.coercedType, slotBlock, dl,
            cirDataLayout, ac.directOffset));
        deadRecordLoads.push_back(recLoad);
      } else {
        newArgs.push_back(emitCoercion(builder, call.getLoc(), ac.coercedType,
                                       arg, slotBlock, dl, ac.directOffset));
      }
    } else if (ac.kind == ArgKind::Indirect) {
      uint64_t minAlign = ac.indirectAlign.value();
 
      // A record operand that names storage is passed from that storage
      // rather than from the loaded value: a record value carries only the
      // fields of its type, so a store leaves the type's padding bytes
      // unwritten, and for a union those can be another member's live data.
      // A byval _BitInt is passed from its value, since the load and the
      // store below are its conversion between value and in-memory form.
      cir::LoadOp srcLoad;
      if (!ac.byVal || mlir::isa<cir::RecordType>(arg.getType()))
        srcLoad = maybeGetSimpleLoad(arg);
 
      // A non-byval argument must name the caller's storage, so that the
      // object the callee operates on is the one the caller destroys.  That
      // means passing the address the operand was loaded from rather than the
      // loaded value, so a store to that storage after the load is visible to
      // the callee.  byval fills a slot below, this cannot.
      if (!ac.byVal) {
        cir::LoadOp fwdLoad = forwardableIndirectLoad(srcLoad, minAlign);
        if (!fwdLoad)
          return call->emitOpError()
                 << "non-byval indirect argument that does not name the "
                    "caller's storage is not yet implemented in "
                    "CallConvLowering";
        newArgs.push_back(fwdLoad.getAddr());
        deadRecordLoads.push_back(fwdLoad);
        continue;
      }
 
      // byval hands the callee a copy taken at the call, so naming the object's
      // own storage leaves it with the same copy and saves this one.  That
      // holds only while nothing writes that storage between the load and the
      // call, since the operand carries the value as of the load.  A volatile
      // load is not forwarded, since the copy byval takes at the call is not
      // volatile.
      if (cir::LoadOp fwdLoad = forwardableIndirectLoad(srcLoad, minAlign);
          fwdLoad && storageUnwrittenBetween(fwdLoad, call.getOperation(),
                                             aliasAnalysis)) {
        newArgs.push_back(fwdLoad.getAddr());
        deadRecordLoads.push_back(fwdLoad);
        continue;
      }
 
      // Otherwise fill a slot, at the load's position when there is one so
      // the copy reads the memory state the load read.
      if (failed(rejectAtomicRecordLoad(call, arg)))
        return mlir::failure();
      if (!srcLoad)
        srcLoad = maybeGetCopyableRecordLoad(arg);
      auto ptrTy = cir::PointerType::get(arg.getType());
      mlir::IntegerAttr slotAlign = builder.getI64IntegerAttr(minAlign);
      mlir::OpBuilder::InsertionGuard guard(builder);
      if (srcLoad)
        builder.setInsertionPoint(srcLoad);
 
      auto slot =
          cir::AllocaOp::create(builder, call.getLoc(), ptrTy,
                                builder.getStringAttr("byval"), slotAlign);
      if (srcLoad) {
        auto copy = cir::CopyOp::create(builder, call.getLoc(), slot,
                                        srcLoad.getAddr(), slotAlign,
                                        loadSourceAlignment(srcLoad, builder));
        copy.setIsVolatile(srcLoad.getIsVolatile());
        deadRecordLoads.push_back(srcLoad);
      } else {
        // A record with no storage behind it, such as a constant, defines no
        // bytes beyond its fields.  A producer rewritten later turns this store
        // into a copy through rewireRecordUses.
        cir::StoreOp::create(builder, call.getLoc(), arg, slot);
      }
      newArgs.push_back(slot);
    } else {
      newArgs.push_back(arg);
    }
  }
  unsigned numEllipsisArgs =
      ellipsisStart ? newArgs.size() - *ellipsisStart : 0;
 
  bool hasResult = call.getNumResults() > 0;
  mlir::Type origRetTy =
      hasResult ? call.getResult().getType() : cir::VoidType::get(ctx);
 
  // An indirect (sret) return has a different call shape than the coerce /
  // extend / ignore return handling further down (the value is returned
  // through a prepended pointer slot, not as a result), so dispatch to a
  // dedicated helper for it; everything below handles the by-value returns.
  if (fc.returnInfo.kind == ArgKind::Indirect && hasResult) {
    if (failed(rewriteIndirectReturnCall(call, fc, newArgs, numEllipsisArgs,
                                         origRetTy, origCallArgTypes, builder,
                                         dl)))
      return mlir::failure();
    eraseDeadRecordLoads(deadRecordLoads);
    return mlir::success();
  }
 
  mlir::Type callRetTy = origRetTy;
  if (fc.returnInfo.kind == ArgKind::Ignore && hasResult)
    callRetTy = cir::VoidType::get(ctx);
  bool returnNeedsCoercion =
      hasResult && fc.returnInfo.kind == ArgKind::Direct &&
      fc.returnInfo.coercedType && fc.returnInfo.coercedType != origRetTy;
  if (returnNeedsCoercion)
    callRetTy = fc.returnInfo.coercedType;
 
  builder.setInsertionPoint(call);
  prependIndirectCallee(call, newArgs, callRetTy, numEllipsisArgs, builder);
  auto newCall = cir::CallOp::create(builder, call.getLoc(),
                                     call.getCalleeAttr(), callRetTy, newArgs);
  copyCallAttributes(call, newCall);
 
  // Direct return with coercion: the new call returns the coerced type, which
  // is stored into a coercion slot where the old call's uses find the original
  // type.
  if (returnNeedsCoercion) {
    builder.setInsertionPointAfter(newCall);
    SmallPtrSet<mlir::Operation *, 4> coercionOps;
    if (failed(emitCoercionAndReplaceUses(builder, call.getLoc(), origRetTy,
                                          newCall.getResult(), call.getResult(),
                                          slotBlock, dl, coercionOps,
                                          fc.returnInfo.directOffset)))
      return mlir::failure();
  }
 
  if (needsArgAttrUpdate(fc)) {
    auto existing = call->getAttrOfType<mlir::ArrayAttr>("arg_attrs");
    newCall->setAttr("arg_attrs", updateArgAttrs(ctx, module, origCallArgTypes,
                                                 existing, fc, dl));
  }
  // Add llvm.signext / llvm.zeroext to an Extend return, and drop llvm.noundef
  // from a coerced Direct return that passes undefined bits.
  if (fc.returnInfo.kind == ArgKind::Extend ||
      (hasResult && fc.returnInfo.kind == ArgKind::Direct)) {
    auto existing = call->getAttrOfType<mlir::ArrayAttr>("res_attrs");
    if (mlir::ArrayAttr updated =
            updateResAttrs(ctx, existing, fc.returnInfo, origRetTy, dl))
      newCall->setAttr("res_attrs", updated);
  } else if (hasResult && mlir::isa<cir::VoidType>(callRetTy)) {
    newCall->removeAttr("res_attrs");
  }
 
  if (hasResult && fc.returnInfo.kind == ArgKind::Ignore) {
    // The new call returns void, but the original call's result may still
    // have uses.  Substitute a poison constant of the original type so
    // those uses remain well-formed without pretending we have a real
    // value at the ABI boundary.
    if (!call.getResult().use_empty()) {
      builder.setInsertionPointAfter(newCall);
      mlir::Value poison =
          createIgnoredValue(builder, call.getLoc(), origRetTy);
      call.getResult().replaceAllUsesWith(poison);
    }
  } else if (hasResult && !returnNeedsCoercion) {
    // returnNeedsCoercion already wired up the coerced result above.
    call.getResult().replaceAllUsesWith(newCall.getResult());
  }
 
  call->erase();
  eraseDeadRecordLoads(deadRecordLoads);
 
  return mlir::success();
}
 
void CIRABIRewriteContext::rewriteFunctionAddress(cir::GetGlobalOp addrOp,
                                                  cir::FuncOp funcOp,
                                                  mlir::OpBuilder &builder) {
  auto oldPtrTy = mlir::cast<cir::PointerType>(addrOp.getAddr().getType());
  cir::FuncType newFuncTy = funcOp.getFunctionType();
  // An extension rides on an argument attribute and leaves the signature
  // alone, so such a callee still matches the written type.
  if (newFuncTy == oldPtrTy.getPointee())
    return;
 
  // The verifier requires the retype even when nothing reads the address.
  addrOp.getAddr().setType(cir::PointerType::get(newFuncTy));
  if (addrOp.getAddr().use_empty())
    return;
 
  // A later indirect call through the written type is classified and coerced
  // on its own, so casting back leaves it to that rewrite.
  mlir::OpBuilder::InsertionGuard guard(builder);
  builder.setInsertionPointAfter(addrOp);
  auto bitcast = cir::CastOp::create(builder, addrOp.getLoc(), oldPtrTy,
                                     cir::CastKind::bitcast, addrOp.getAddr());
  addrOp.getAddr().replaceAllUsesExcept(bitcast.getResult(), bitcast);
}
 
namespace {
 
/// What one `va_arg` expansion needs from the op it rewrites.  The x86-64
/// cursor fields are reached through `vaFields` by index: 0 gp_offset,
/// 1 fp_offset, 2 overflow_arg_area, 3 reg_save_area.
struct VAArgFetch {
  VAArgFetch(mlir::Location loc, mlir::Value valist,
             llvm::ArrayRef<mlir::Type> vaFields, mlir::Type resultTy,
             const ArgClassification &ac, const mlir::DataLayout &dl,
             mlir::ModuleOp module)
      : loc(loc), valist(valist), vaFields(vaFields), resultTy(resultTy),
        ac(ac), dl(dl), module(module) {
    assert(vaFields.size() == 4 &&
           "the x86-64 va_list is a four-field cursor, checked by the caller");
  }
 
  mlir::Location loc;
  mlir::Value valist;
  llvm::ArrayRef<mlir::Type> vaFields;
  mlir::Type resultTy;
  const ArgClassification &ac;
  const mlir::DataLayout &dl;
  mlir::ModuleOp module;
};
 
/// The cursor fields a register fetch reads, and the predicate saying every
/// class it needs still has room.  An offset is null when the fetch needs no
/// register of that class.
struct RegisterCursor {
  mlir::Value gpOffsetP;
  mlir::Value fpOffsetP;
  mlir::Value gpOffset;
  mlir::Value fpOffset;
  mlir::Value inRegs;
};
 
mlir::LogicalResult reportVAArgNYI(cir::VAArgOp op, llvm::StringRef what) {
  op->emitOpError() << "va_arg of " << what
                    << " not yet implemented in CallConvLowering";
  return mlir::failure();
}
 
/// Sets \p isRegPair when a two-register argument is a coerced pair, one
/// register per member, rather than a single wide scalar, and records in
/// \p pairIsSse which element is SSE class.  Fails on a coercion that is not
/// two eightbytes.
mlir::LogicalResult classifyRegisterPair(cir::VAArgOp op,
                                         const ArgClassification &ac,
                                         std::array<bool, 2> &pairIsSse,
                                         bool &isRegPair) {
  auto pairTy = mlir::dyn_cast<cir::RecordType>(ac.coercedType);
  if (!pairTy)
    return mlir::success();
 
  if (pairTy.getNumElements() != 2)
    return reportVAArgNYI(op, "a register coercion that is not two eightbytes");
 
  assert(!ac.directOffset &&
         "a pair already spans both eightbytes, so it cannot also start "
         "partway into the value");
  for (auto [i, memberTy] : llvm::enumerate(pairTy.getMembers()))
    pairIsSse[i] = isSSERegisterClass(memberTy);
  isRegPair = true;
  return mlir::success();
}
 
/// The alignment an argument is placed at.  A record can require more than
/// the types of its members imply, from an `aligned` attribute on the record
/// or on one of its fields, and neither raises the alignment of any type.
/// Only the record layout knows, so the member-derived value alone can be too
/// small.
uint64_t argumentAreaAlign(mlir::Type ty, mlir::ModuleOp modOp,
                           const mlir::DataLayout &dl) {
  uint64_t align = dl.getTypeABIAlignment(ty);
  if (auto recTy = mlir::dyn_cast<cir::RecordType>(ty))
    if (auto layout = cir::tryGetRecordLayout(modOp, recTy.getName()))
      align = std::max<uint64_t>(align, layout.getRecordAlign());
  return align;
}
 
mlir::Value roundPointerUpToAlignment(CIRBaseBuilderTy &b, mlir::Location loc,
                                      mlir::Value bytePtr, uint64_t align,
                                      const mlir::DataLayout &dl) {
  assert(mlir::cast<cir::PointerType>(bytePtr.getType()).getPointee() ==
             b.getUIntNTy(8) &&
         "the bump strides in bytes, so the pointee must be u8");
  assert(llvm::isPowerOf2_64(align) &&
         "mask rounding needs a power-of-two alignment");
  mlir::Value bumped =
      b.createPtrStride(loc, bytePtr, b.getSignedInt(loc, align - 1, 32));
  std::optional<uint64_t> indexWidth =
      dl.getTypeIndexBitwidth(bytePtr.getType());
  assert(indexWidth && "a pointer in the argument area has an index width");
  mlir::Value mask = b.getSignedInt(loc, -static_cast<int64_t>(align),
                                    static_cast<unsigned>(*indexWidth));
  return cir::PtrMaskOp::create(b, loc, bytePtr.getType(), bumped, mask);
}
 
/// Reads the argument's address out of the overflow area, which also advances
/// the cursor past the argument.
mlir::Value buildOverflowAddrAndAdvance(CIRBaseBuilderTy &b,
                                        const VAArgFetch &f) {
  cir::IntType byteTy = b.getUIntNTy(8);
  mlir::Value overflowP = b.createGetMember(
      f.loc, b.getPointerTo(f.vaFields[2]), f.valist, "overflow_arg_area", 2);
  mlir::Value overflow = b.createLoad(f.loc, overflowP);
  mlir::Value bytePtr = b.createPtrBitcast(overflow, byteTy);
 
  uint64_t tyAlign = argumentAreaAlign(f.resultTy, f.module, f.dl);
  if (tyAlign > 8)
    bytePtr = roundPointerUpToAlignment(b, f.loc, bytePtr, tyAlign, f.dl);
 
  uint64_t stride =
      llvm::alignTo(f.dl.getTypeAllocSize(f.resultTy).getFixedValue(), 8);
  mlir::Value strideVal = b.getSignedInt(f.loc, stride, 32);
  mlir::Value next = b.createPtrStride(f.loc, bytePtr, strideVal);
  b.createStore(f.loc, next, overflowP);
  return bytePtr;
}
 
/// Loads the cursor offsets and builds the predicate that sends the fetch to
/// the register-save area.  Both offsets count from the start of that area, so
/// the integer limit is the six GP registers at 48 and the vector limit is
/// those plus the eight SSE registers at 176.
RegisterCursor buildRegisterGate(CIRBaseBuilderTy &b, const VAArgFetch &f,
                                 unsigned neededInt, unsigned neededSse) {
  RegisterCursor cursor;
  if (neededInt) {
    cursor.gpOffsetP = b.createGetMember(f.loc, b.getPointerTo(f.vaFields[0]),
                                         f.valist, "gp_offset", 0);
    cursor.gpOffset = b.createLoad(f.loc, cursor.gpOffsetP);
    mlir::Value limit =
        b.getConstantInt(f.loc, cursor.gpOffset.getType(), 48 - neededInt * 8);
    cursor.inRegs =
        b.createCompare(f.loc, cir::CmpOpKind::le, cursor.gpOffset, limit);
  }
  if (neededSse) {
    cursor.fpOffsetP = b.createGetMember(f.loc, b.getPointerTo(f.vaFields[1]),
                                         f.valist, "fp_offset", 1);
    cursor.fpOffset = b.createLoad(f.loc, cursor.fpOffsetP);
    mlir::Value limit = b.getConstantInt(f.loc, cursor.fpOffset.getType(),
                                         176 - neededSse * 16);
    mlir::Value fitsInFp =
        b.createCompare(f.loc, cir::CmpOpKind::le, cursor.fpOffset, limit);
    cursor.inRegs = cursor.inRegs
                        ? b.createLogicalAnd(f.loc, cursor.inRegs, fitsInFp)
                        : fitsInFp;
  }
  return cursor;
}
 
/// Copies each half of a non-contiguous pair out of the register-save area
/// into \p regPairTemp, laid out as the coerced pair.
void reassembleRegisterPair(CIRBaseBuilderTy &b, mlir::Location loc,
                            const ArgClassification &ac,
                            const RegisterCursor &cursor,
                            const std::array<bool, 2> &pairIsSse,
                            mlir::Value regSaveArea, mlir::Value regPairTemp) {
  auto pairTy = mlir::cast<cir::RecordType>(ac.coercedType);
  // Both halves of an all-SSE pair sit in 16-byte slots, which classic
  // CodeGen tells the load about.  It leaves a mixed pair to the element's
  // own alignment, so match that rather than claiming the slot there.
  bool bothSse = pairIsSse[0] && pairIsSse[1];
  // Track how many of each class came before, since a slot is reached from
  // its class's own cursor.
  unsigned seenOfClass[2] = {0, 0};
  for (unsigned i = 0; i < 2; ++i) {
    bool isSse = pairIsSse[i];
    mlir::Value base = isSse ? cursor.fpOffset : cursor.gpOffset;
    unsigned regSize = isSse ? 16 : 8;
    unsigned prior = seenOfClass[isSse];
    ++seenOfClass[isSse];
    mlir::Value off = base;
    if (prior) {
      off = b.createAdd(loc, base,
                        b.getConstantInt(loc, base.getType(), prior * regSize));
    }
    mlir::Value src = b.createPtrStride(loc, regSaveArea, off);
    mlir::Type elemTy = pairTy.getElementType(i);
    mlir::Value elemPtr = b.createPtrBitcast(src, elemTy);
    mlir::Value val = bothSse ? b.createAlignedLoad(loc, elemPtr, 16)
                              : b.createLoad(loc, elemPtr);
    b.createStore(
        loc, val,
        b.createGetMember(loc, b.getPointerTo(elemTy), regPairTemp, "", i));
  }
}
 
/// The bytes carried by the registers of this fetch's one class.
uint64_t registerSlotSize(unsigned neededInt, unsigned neededSse) {
  assert(!(neededInt && neededSse) &&
         "a fetch needing both classes is a pair, reassembled elsewhere");
  return neededSse ? neededSse * 16 : neededInt * 8;
}
 
/// Whether the register cannot be read in place, either because it carries
/// less than the whole result or because its slot is under-aligned for it.
bool needsTempCopy(const VAArgFetch &f, unsigned neededInt,
                   unsigned neededSse) {
  uint64_t tySize = f.dl.getTypeSize(f.resultTy).getFixedValue();
  if (f.ac.coercedType &&
      (f.ac.directOffset || registerSlotSize(neededInt, neededSse) < tySize))
    return true;
  // A slot is only as aligned as its class, 8 for a GP register and 16 for an
  // SSE one, so a fetch the ABI places more strictly is copied through a temp.
  uint64_t slotAlign = neededSse ? 16 : 8;
  return argumentAreaAlign(f.resultTy, f.module, f.dl) > slotAlign;
}
 
/// Copies what the registers carry into \p temp, which is the size of the
/// whole result, and returns the address to read the result from.
mlir::Value copyRegisterToTemp(CIRBaseBuilderTy &b, mlir::Location loc,
                               const VAArgFetch &f, mlir::Value regAddr,
                               mlir::Value temp, unsigned neededInt,
                               unsigned neededSse) {
  cir::IntType byteTy = b.getUIntNTy(8);
  uint64_t tySize = f.dl.getTypeSize(f.resultTy).getFixedValue();
 
  if (f.ac.coercedType &&
      (f.ac.directOffset || registerSlotSize(neededInt, neededSse) < tySize)) {
    // The registers carry less than the whole result, either because the
    // eightbytes below directOffset hold no field or because the result is
    // wider than the registers carrying it.  Copy only what they carry, so
    // that reading the temp cannot run on into the neighboring slot.
    mlir::Value val = b.createAlignedLoad(
        loc, b.createPtrBitcast(regAddr, f.ac.coercedType), 8);
    mlir::Value dst = temp;
    if (f.ac.directOffset) {
      mlir::Value tempByte = b.createPtrBitcast(temp, byteTy);
      mlir::Value offset = b.getSignedInt(loc, f.ac.directOffset, 32);
      dst = b.createPtrStride(loc, tempByte, offset);
    }
    b.createStore(loc, val, b.createPtrBitcast(dst, f.ac.coercedType));
    return b.createPtrBitcast(temp, byteTy);
  }
 
  mlir::Value regView = b.createPtrBitcast(regAddr, f.resultTy);
  if (mlir::isa<cir::RecordType>(f.resultTy))
    cir::CopyOp::create(
        b, loc, temp, regView,
        b.getI64IntegerAttr(argumentAreaAlign(f.resultTy, f.module, f.dl)),
        b.getI64IntegerAttr(8));
  else
    b.createStore(loc, b.createAlignedLoad(loc, regView, 8), temp);
  return b.createPtrBitcast(temp, byteTy);
}
 
void advanceRegisterCursors(CIRBaseBuilderTy &b, mlir::Location loc,
                            const RegisterCursor &cursor, unsigned neededInt,
                            unsigned neededSse) {
  if (neededInt) {
    b.createStore(loc,
                  b.createAdd(loc, cursor.gpOffset,
                              b.getConstantInt(loc, cursor.gpOffset.getType(),
                                               neededInt * 8)),
                  cursor.gpOffsetP);
  }
  if (neededSse) {
    b.createStore(loc,
                  b.createAdd(loc, cursor.fpOffset,
                              b.getConstantInt(loc, cursor.fpOffset.getType(),
                                               neededSse * 16)),
                  cursor.fpOffsetP);
  }
}
 
} // namespace
 
mlir::LogicalResult
CIRABIRewriteContext::rewriteVAArg(mlir::Operation *vaArgOp,
                                   const ArgClassification &ac,
                                   mlir::OpBuilder &opBuilder) {
  auto op = mlir::cast<cir::VAArgOp>(vaArgOp);
  CIRBaseBuilderTy builder(opBuilder);
  mlir::Location loc = op.getLoc();
  mlir::Type resultTy = op.getType();
  mlir::Value valist = op.getArgList();
 
  // An ignored type travels in no register and no stack slot, so the fetch
  // reads nothing and must leave the cursor unchanged.  The value holds no
  // bytes, so poison stands in for it.
  if (ac.kind == ArgKind::Ignore) {
    builder.setInsertionPoint(op);
    op.getResult().replaceAllUsesWith(
        createIgnoredValue(builder, loc, resultTy));
    op->erase();
    return mlir::success();
  }
 
  if (ac.kind == ArgKind::Indirect && !ac.byVal)
    return reportVAArgNYI(op, "a non-trivially-copyable type");
 
  // neededInt counts 8-byte integer slots and neededSse counts 16-byte vector
  // slots.  Zero of both means the type travels in memory and is read
  // straight from the overflow area.
  unsigned neededInt = ac.neededIntRegs;
  unsigned neededSse = ac.neededSseRegs;
 
  // Which coerced-pair element (0 = low eightbyte, 1 = high) is SSE rather
  // than INTEGER class.  Only meaningful when isRegPair is set.
  std::array<bool, 2> pairIsSse = {false, false};
  bool isRegPair = false;
  if (ac.kind == ArgKind::Direct && neededInt + neededSse == 2 &&
      ac.coercedType) {
    if (classifyRegisterPair(op, ac, pairIsSse, isRegPair).failed())
      return mlir::failure();
  }
 
  auto vaListRecTy = mlir::dyn_cast<cir::RecordType>(
      mlir::cast<cir::PointerType>(valist.getType()).getPointee());
  if (!vaListRecTy || vaListRecTy.getNumElements() != 4) {
    return reportVAArgNYI(op,
                          "a va_list that is not the four-field gp_offset / "
                          "fp_offset / overflow_arg_area / reg_save_area "
                          "cursor");
  }
 
  const VAArgFetch fetch{loc, valist, vaListRecTy.getMembers(), resultTy, ac,
                         dl,  module};
  cir::IntType byteTy = builder.getUIntNTy(8);
 
  builder.setInsertionPoint(op);
 
  mlir::Value addr;
  if (neededInt == 0 && neededSse == 0) {
    addr = buildOverflowAddrAndAdvance(builder, fetch);
  } else {
    RegisterCursor cursor =
        buildRegisterGate(builder, fetch, neededInt, neededSse);
 
    // A two-eightbyte pair that is purely INTEGER class is contiguous in the
    // register-save area, since GP slots are 8-byte packed, so the address of
    // its low eightbyte is already the address of the whole value.  A pure
    // SSE pair or a mixed pair is not contiguous, since SSE slots are 16-byte
    // spaced and a mixed pair's halves live in disjoint areas, so each half is
    // copied into a temp laid out as the coerced pair.
    bool pairNeedsReassembly = isRegPair && neededSse != 0;
 
    // Both temps are allocated here, since a ternary arm yields their address.
    mlir::Value regPairTemp;
    if (pairNeedsReassembly) {
      // The temp is written one member at a time through the coerced pair, so
      // it has to meet that type's alignment, and read back as the result, so
      // it has to meet the result's alignment too.
      regPairTemp = builder.createAlloca(
          loc, builder.getPointerTo(ac.coercedType), "vaarg.reg",
          clang::CharUnits::fromQuantity(
              std::max(dl.getTypeABIAlignment(ac.coercedType),
                       argumentAreaAlign(resultTy, module, dl))));
    }
 
    mlir::Value regTemp;
    bool copyThroughTemp =
        !pairNeedsReassembly && needsTempCopy(fetch, neededInt, neededSse);
    if (copyThroughTemp) {
      regTemp =
          builder.createAlloca(loc, builder.getPointerTo(resultTy), "vaarg.reg",
                               clang::CharUnits::fromQuantity(
                                   argumentAreaAlign(resultTy, module, dl)));
    }
 
    addr = cir::TernaryOp::create(
               builder, loc, cursor.inRegs,
               /*trueBuilder=*/
               [&](mlir::OpBuilder &ob, mlir::Location l) {
                 CIRBaseBuilderTy b(ob);
                 mlir::Value regSaveArea = b.createLoad(
                     l, b.createGetMember(l, b.getPointerTo(fetch.vaFields[3]),
                                          valist, "reg_save_area", 3));
                 regSaveArea = b.createPtrBitcast(regSaveArea, byteTy);
 
                 mlir::Value regAddr;
                 if (pairNeedsReassembly) {
                   reassembleRegisterPair(b, l, ac, cursor, pairIsSse,
                                          regSaveArea, regPairTemp);
                   regAddr = b.createPtrBitcast(regPairTemp, byteTy);
                 } else {
                   mlir::Value off =
                       neededSse ? cursor.fpOffset : cursor.gpOffset;
                   regAddr = b.createPtrStride(l, regSaveArea, off);
                   if (copyThroughTemp) {
                     regAddr = copyRegisterToTemp(b, l, fetch, regAddr, regTemp,
                                                  neededInt, neededSse);
                   }
                 }
 
                 advanceRegisterCursors(b, l, cursor, neededInt, neededSse);
                 cir::YieldOp::create(b, l, regAddr);
               },
               /*falseBuilder=*/
               [&](mlir::OpBuilder &ob, mlir::Location l) {
                 CIRBaseBuilderTy b(ob);
                 mlir::Value memAddr = buildOverflowAddrAndAdvance(b, fetch);
                 cir::YieldOp::create(b, l, memAddr);
               })
               .getResult();
  }
 
  // Every path above places the result at least this well aligned.
  mlir::Value resultAddr = builder.createPtrBitcast(addr, resultTy);
  uint64_t resultAlign = argumentAreaAlign(resultTy, module, dl);
  if (mlir::isa<cir::RecordType>(resultTy)) {
    SmallPtrSet<mlir::Operation *, 1> noExceptions;
    if (failed(rewireRecordUses(builder, loc, op.getResult(), resultAddr,
                                resultAlign, dl, noExceptions)))
      return mlir::failure();
  } else {
    op.getResult().replaceAllUsesWith(
        builder.createAlignedLoad(loc, resultAddr, resultAlign));
  }
  op->erase();
  return mlir::success();
}