CallConvLoweringPass.cpp

Go to the documentation of this file.

//===- CallConvLoweringPass.cpp - Lower CIR to ABI calling convention ----===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This pass walks every cir.func and cir.call in the module, computes a
// FunctionClassification for it (via either an ABI target or a pre-built
// classification injected as a function attribute), and dispatches to
// CIRABIRewriteContext to perform the actual IR rewriting.
//
// Two driver modes (mutually exclusive):
//
//   target=test
//     Use the MLIR test ABI target (mlir/lib/ABI/Targets/Test/) to classify
//     each function.  Predictable rules that approximate x86_64 SysV.  Real
//     targets (x86_64, AArch64) will be added once the LLVM ABI library
//     ships them.
//
//   classification-attr=<name>
//     Read a DictionaryAttr named <name> from each cir.func and parse it via
//     mlir::abi::test::parseClassificationAttr.  Used by tests to inject any
//     classification (including shapes the test target itself does not
//     produce) without depending on a real ABI target.
//
// The pass requires a `dlti.dl_spec` attribute on the module so the
// classifier can query type sizes and alignments.
//
//===----------------------------------------------------------------------===//
 
#include "PassDetail.h"
#include "TargetLowering/CIRABIRewriteContext.h"
 
#include "mlir/ABI/ABIRewriteContext.h"
#include "mlir/ABI/ABITypeMapper.h"
#include "mlir/ABI/Targets/Test/TestTarget.h"
#include "mlir/Dialect/DLTI/DLTI.h"
#include "mlir/IR/Builders.h"
#include "mlir/IR/BuiltinOps.h"
#include "mlir/IR/SymbolTable.h"
#include "mlir/Interfaces/DataLayoutInterfaces.h"
#include "mlir/Pass/Pass.h"
#include "clang/CIR/Dialect/IR/CIRDialect.h"
#include "clang/CIR/Dialect/Passes.h"
#include "clang/CIR/MissingFeatures.h"
#include "llvm/ABI/FunctionInfo.h"
#include "llvm/ABI/TargetInfo.h"
#include "llvm/ABI/Types.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/TypeSwitch.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/Support/MathExtras.h"
 
#include <algorithm>
#include <array>
 
using namespace mlir;
using namespace mlir::abi;
using namespace cir;
 
namespace mlir {
#define GEN_PASS_DEF_CALLCONVLOWERING
#include "clang/CIR/Dialect/Passes.h.inc"
} // namespace mlir
 
namespace {
 
//===----------------------------------------------------------------------===//
// x86_64 System V classifier bridge
//
// Maps CIR types to llvm::abi::Type, runs the LLVM ABI Lowering Library's SysV
// x86_64 classifier, and converts the result back into the dialect-agnostic
// mlir::abi::FunctionClassification that CIRABIRewriteContext consumes.
// isSupportedType says which CIR types the bridge handles, and a signature
// naming any other fails the pass instead of being misclassified.
//===----------------------------------------------------------------------===//
 
/// Whether a struct's declared argument-passing kind (from the module's
/// record-layout metadata) allows it to be passed in registers.  A record with
/// no layout entry (e.g. an anonymous struct) has no C++ non-trivial reason to
/// be forced to memory, so it defaults to can-pass-in-registers.
static bool recordCanPassInRegs(ModuleOp modOp, cir::RecordType recTy) {
  auto layout = cir::tryGetRecordLayout(modOp, recTy.getName());
  if (!layout)
    return true;
  return layout.getArgPassingKind() == cir::ArgPassingKind::CanPassInRegs;
}
 
/// Whether the classifier could give this type the SSEUP class, looking
/// through arrays and records at the types they hold.
static bool mayReachSseUp(mlir::Type ty, const DataLayout &dl) {
  // The classifier sizes a vector with its width rounded up to a power of two.
  if (isa<cir::VectorType>(ty))
    return llvm::PowerOf2Ceil(dl.getTypeSizeInBits(ty).getFixedValue()) >= 128;
  if (auto fpTy = dyn_cast<cir::FPTypeInterface>(ty))
    return &fpTy.getFloatSemantics() == &llvm::APFloat::IEEEquad();
  if (auto arrTy = dyn_cast<cir::ArrayType>(ty))
    return mayReachSseUp(arrTy.getElementType(), dl);
  if (auto recTy = dyn_cast<cir::RecordType>(ty))
    return recTy.isComplete() &&
           llvm::any_of(recTy.getMembers(),
                        [&](mlir::Type m) { return mayReachSseUp(m, dl); });
  // The rest classify integer, SSE or memory.  A complex is SSE at float and
  // double width and memory above that, so it needs no walk of its own.
  assert((isa<cir::IntType, cir::BoolType, cir::PointerType, cir::VPtrType,
              cir::VoidType, cir::ComplexType, cir::BitFieldType>(ty)) &&
         "unhandled type in the SSEUP walk");
  return false;
}
 
/// Whether the record's size has no SSEUP coerce.  That coerce is a vector as
/// wide as the record, so only 128, 256 and 512 bits have one.  A record past
/// 512 bits classifies memory before any coerce is asked for.
static bool sseUpCoerceSizeUnsupported(uint64_t recordBits,
                                       llvm::ArrayRef<mlir::Type> members,
                                       const DataLayout &dl) {
  if (recordBits == 128 || recordBits == 256 || recordBits == 512)
    return false;
  if (recordBits < 128 || recordBits > 512)
    return false;
  return llvm::any_of(members,
                      [&](mlir::Type m) { return mayReachSseUp(m, dl); });
}
 
/// Whether a member is an empty record, looking through arrays, since an array
/// of empty records supplies no bytes either.
static bool memberIsEmptyRecord(mlir::Type ty) {
  while (auto arrTy = dyn_cast<cir::ArrayType>(ty))
    ty = arrTy.getElementType();
  auto recTy = dyn_cast<cir::RecordType>(ty);
  return recTy && recTy.isEmptyForABI();
}
 
/// A record's declared alignment, which the ABI uses for the byval and sret
/// alignment of an indirect argument.  DataLayout derives alignment from the
/// members, so it cannot see `__attribute__((aligned(N)))`.  The declared value
/// comes from the module's record-layout metadata instead.  CIRGen emits an
/// entry for every record it names, so the computed fallback only serves
/// hand-written CIR.
static llvm::Align recordDeclaredAlign(ModuleOp modOp, cir::RecordType recTy,
                                       const DataLayout &dl) {
  auto layout = cir::tryGetRecordLayout(modOp, recTy.getName());
  if (!layout)
    return llvm::Align(dl.getTypeABIAlignment(recTy));
  return llvm::Align(layout.getRecordAlign());
}
 
/// Whether \p ty, or an aggregate member/element reached by value (never
/// through a pointer), is an incomplete record.  Such a record has no known
/// layout, so no eightbyte classification can be built for it.
static bool hasIncompleteRecordByValue(mlir::Type ty) {
  if (auto recTy = dyn_cast<cir::RecordType>(ty))
    return !recTy.isComplete() ||
           llvm::any_of(recTy.getMembers(), hasIncompleteRecordByValue);
  if (auto arrTy = dyn_cast<cir::ArrayType>(ty))
    return hasIncompleteRecordByValue(arrTy.getElementType());
  return false;
}
 
/// Whether \p ty has the x87 double extended format, wrapped in LongDoubleType
/// or not.
static bool isX87DoubleExtended(mlir::Type ty) {
  auto fpTy = dyn_cast<cir::FPTypeInterface>(ty);
  return fpTy &&
         &fpTy.getFloatSemantics() == &llvm::APFloat::x87DoubleExtended();
}
 
/// Whether \p ty is, or holds by value, an x87 double extended value, alone or
/// as the element of a _Complex or a vector.
static bool holdsX87DoubleExtended(mlir::Type ty) {
  if (isa<cir::FPTypeInterface>(ty))
    return isX87DoubleExtended(ty);
  if (auto complexTy = dyn_cast<cir::ComplexType>(ty))
    return isX87DoubleExtended(complexTy.getElementType());
  if (auto vecTy = dyn_cast<cir::VectorType>(ty))
    return isX87DoubleExtended(vecTy.getElementType());
  if (auto arrTy = dyn_cast<cir::ArrayType>(ty))
    return arrTy.getSize() && holdsX87DoubleExtended(arrTy.getElementType());
  if (auto recTy = dyn_cast<cir::RecordType>(ty))
    return llvm::any_of(recTy.getMembers(), holdsX87DoubleExtended);
  return false;
}
 
/// Returns how many bytes from the start of \p ty a value of it reaches.  A
/// scalar reaches the end of its storage, a record the furthest end of its
/// non-padding members, and an array the end of its last element, so padding
/// at the end of a record does not count.
static uint64_t getDataExtentInBytes(mlir::Type ty, const DataLayout &dl) {
  if (auto recTy = dyn_cast<cir::RecordType>(ty)) {
    uint64_t extent = 0;
    for (auto [idx, memberTy, kind] :
         llvm::enumerate(recTy.getMembers(), recTy.getMemberKinds())) {
      if (kind == cir::RecordMemberKind::Pad)
        continue;
      extent = std::max(extent, recTy.getElementOffset(dl, idx) +
                                    getDataExtentInBytes(memberTy, dl));
    }
    return extent;
  }
  if (auto arrTy = dyn_cast<cir::ArrayType>(ty)) {
    if (!arrTy.getSize())
      return 0;
    mlir::Type elemTy = arrTy.getElementType();
    return (arrTy.getSize() - 1) * dl.getTypeSize(elemTy).getFixedValue() +
           getDataExtentInBytes(elemTy, dl);
  }
  return dl.getTypeSize(ty).getFixedValue();
}
 
/// The CIR types the x86_64 bridge handles.  Scalars: an integer up to 128
/// bits (including `__int128`), a `_BitInt` of any width, pointer, vtable
/// pointer, bool, void, or any floating-point type.  Aggregates: a complete
/// struct or union whose members are all themselves supported, or an array
/// of a supported element type.  Also a `_Complex`, or a fixed-width vector,
/// of a supported element type.  Everything else is reported NYI at the
/// reject() choke point in classifyX86_64Function.
static bool isSupportedType(mlir::Type ty, const DataLayout &dl) {
  // A pointer is only handled in the default address space (null) or an
  // already-lowered target address space.  A LangAddressSpaceAttr must be
  // lowered before this pass, so reject it rather than silently dropping it.
  if (auto ptrTy = dyn_cast<cir::PointerType>(ty))
    return !ptrTy.getAddrSpace() ||
           mlir::isa<cir::TargetAddressSpaceAttr>(ptrTy.getAddrSpace());
  // cir::VPtrType carries no address-space parameter yet, so its target
  // address space cannot be checked here even though it is not always the
  // default one.
  if (isa<cir::VPtrType>(ty)) {
    assert(!cir::MissingFeatures::addressSpace());
    return true;
  }
  if (isa<cir::VoidType, cir::BoolType>(ty))
    return true;
  // Every CIR floating-point type carries the semantics the classifier
  // switches on, so all of them are handled.
  if (isa<cir::FPTypeInterface>(ty))
    return true;
  if (auto intTy = dyn_cast<cir::IntType>(ty)) {
    // Integers up to 64 bits, __int128, and _BitInt of any width are
    // handled: the classifier extends a width below 32, widens 33 through 63
    // to i64, coerces 65 through 127 to a {i64, i64} pair, and passes 32, 64,
    // and 128 in the natural type.  A _BitInt wider than 128 bits classifies
    // Indirect, the same path an oversized aggregate already takes.
    // Non-_BitInt intermediate widths (65..127) do not arise from C and stay
    // rejected.
    if (intTy.getIsBitInt())
      return true;
    return intTy.getWidth() <= 64 || intTy.getWidth() == 128;
  }
  if (auto complexTy = dyn_cast<cir::ComplexType>(ty))
    return isSupportedType(complexTy.getElementType(), dl);
  if (auto vecTy = dyn_cast<cir::VectorType>(ty)) {
    // A scalable vector has no size the SysV eightbyte rules can read, and
    // x86_64 has no calling convention for one.
    if (vecTy.getIsScalable())
      return false;
    // mapCIRType maps a bool as a one-bit integer, so a bool vector is sized
    // one bit per element, as clang packs it.  Vectors of a _BitInt whose
    // width is not a whole number of bytes are not handled yet.
    mlir::Type elemTy = vecTy.getElementType();
    if (auto elemInt = dyn_cast<cir::IntType>(elemTy)) {
      if (elemInt.getWidth() % 8)
        return false;
    } else if (!isa<cir::FPTypeInterface, cir::BoolType>(elemTy)) {
      return false;
    }
    return isSupportedType(elemTy, dl);
  }
  if (auto arrTy = dyn_cast<cir::ArrayType>(ty))
    return isSupportedType(arrTy.getElementType(), dl);
  // A bit-field is classified from the type it was declared with and the bits
  // it holds, and is loaded through its access unit, so both types have to be
  // ones the bridge handles.  A zero-width bit-field has no unit of its own.
  if (auto bfTy = dyn_cast<cir::BitFieldType>(ty)) {
    if (mlir::Type storageTy = bfTy.getStorageType())
      if (!isSupportedType(storageTy, dl))
        return false;
    return llvm::all_of(bfTy.getFields(), [&](cir::BitFieldDeclAttr decl) {
      return isSupportedType(decl.getDeclaredType(), dl);
    });
  }
  if (auto recTy = dyn_cast<cir::RecordType>(ty)) {
    // An incomplete record has no layout to classify.
    if (!recTy.isComplete())
      return false;
    // The members are checked first so that everything below reasons only
    // about types the bridge handles.
    if (!llvm::all_of(recTy.getMembers(),
                      [&](mlir::Type m) { return isSupportedType(m, dl); }))
      return false;
    if (recTy.isUnion()) {
      llvm::ArrayRef<mlir::Type> members = recTy.getMembers();
      uint64_t recordBits = dl.getTypeSizeInBits(recTy).getFixedValue();
      // Refuse a union that could reach SSEUP at a size no coerce exists for.
      // The classifier asserts on those rather than returning a coerce.
      if (sseUpCoerceSizeUnsupported(recordBits, members, dl))
        return false;
      // A member-less union is all padding, which classifies Ignore up to two
      // eightbytes.  Past that SysV says MEMORY regardless of content, and
      // there is no member here to build the Indirect coercion from.
      if (members.empty() && recordBits > 128)
        return false;
      // A union is moved as a value of its storage type, and an x87 value
      // stores fewer bytes than it occupies, so the bytes another member
      // holds past them would be lost.  A double extended member stores its
      // first 10 bytes, so it can share a union with other such members and
      // with members whose data ends within those bytes, provided it is the
      // storage type.  An x87 value held any other way, in a _Complex, a
      // vector, a record or an array, is measured at its full storage, which
      // reaches past them.
      if (members.size() > 1 && llvm::any_of(members, holdsX87DoubleExtended)) {
        uint64_t x87StoreBytes = llvm::APFloat::semanticsSizeInBits(
                                     llvm::APFloat::x87DoubleExtended()) /
                                 8;
        if (!isX87DoubleExtended(
                mlir::cast<cir::UnionType>(recTy).getUnionStorageType(dl)) ||
            !llvm::all_of(members, [&](mlir::Type m) {
              return isX87DoubleExtended(m) ||
                     getDataExtentInBytes(m, dl) <= x87StoreBytes;
            }))
          return false;
      }
      // A `_BitInt` access unit past an eightbyte has byte-array storage in
      // CIR and integer storage in classic CodeGen, so the narrowing walk
      // finds an i8 in the second eightbyte where classic keeps an i64.
      if (llvm::any_of(members, [&](mlir::Type m) {
            auto bfTy = dyn_cast<cir::BitFieldType>(m);
            if (!bfTy || dl.getTypeSizeInBits(bfTy).getFixedValue() <= 64)
              return false;
            return llvm::any_of(bfTy.getFields(), [](cir::BitFieldDeclAttr d) {
              auto intTy = dyn_cast<cir::IntType>(d.getDeclaredType());
              return intTy && intTy.getIsBitInt();
            });
          }))
        return false;
    } else {
      // A struct's `empty` member that occupies bytes is later read as an
      // unnamed bit-field.  An empty-for-ABI record occupies its own padding,
      // and reading that as a unit would hand the classifier storage classic
      // CodeGen ignores.  A zero-sized one is dropped before classification,
      // so a misaligned one never reaches the rule that sends its record to
      // memory.  A union needs neither rule: mapCIRType skips such a variant
      // instead of reading it as a unit, and every variant sits at offset
      // zero.
      for (auto [idx, memberTy, kind] :
           llvm::enumerate(recTy.getMembers(), recTy.getMemberKinds())) {
        if (kind != cir::RecordMemberKind::Empty)
          continue;
        if (dl.getTypeSizeInBits(memberTy).getFixedValue()) {
          if (memberIsEmptyRecord(memberTy))
            return false;
        } else if (recTy.getElementOffset(dl, idx) %
                   dl.getTypeABIAlignment(memberTy)) {
          return false;
        }
      }
    }
    return true;
  }
  return false;
}
 
/// Convert an llvm::abi::Type coercion type back to a scalar CIR type.
static mlir::Type abiTypeToCIR(const llvm::abi::Type *ty, MLIRContext *ctx) {
  if (!ty)
    return nullptr;
  return llvm::TypeSwitch<const llvm::abi::Type *, mlir::Type>(ty)
      .Case(
          [&](const llvm::abi::VoidType *) { return cir::VoidType::get(ctx); })
      .Case([&](const llvm::abi::IntegerType *intTy) -> mlir::Type {
        // mapCIRType maps a bool as a one-bit integer.
        if (intTy->isBool())
          return cir::BoolType::get(ctx);
        return cir::IntType::get(ctx, intTy->getSizeInBits().getFixedValue(),
                                 intTy->isSigned(), intTy->isBitInt());
      })
      .Case([&](const llvm::abi::FloatType *fltTy) {
        return cir::getFloatingPointType(*fltTy->getSemantics(), ctx);
      })
      .Case([&](const llvm::abi::PointerType *) {
        return cir::PointerType::get(cir::VoidType::get(ctx));
      })
      .Case([&](const llvm::abi::VectorType *vecTy) -> mlir::Type {
        mlir::Type elemCIR = abiTypeToCIR(vecTy->getElementType(), ctx);
        if (!elemCIR)
          return nullptr;
        return cir::VectorType::get(elemCIR,
                                    vecTy->getNumElements().getFixedValue());
      })
      .Case([&](const llvm::abi::RecordType *recTy) -> mlir::Type {
        SmallVector<mlir::Type> fieldTypes;
        fieldTypes.reserve(recTy->getFields().size());
        for (const auto &field : recTy->getFields()) {
          mlir::Type fieldCIR = abiTypeToCIR(field.FieldType, ctx);
          if (!fieldCIR)
            return nullptr;
          fieldTypes.push_back(fieldCIR);
        }
        // Coercion types are plain register tuples, not the source record.
        return cir::StructType::get(
            ctx, fieldTypes, /*packed=*/false,
            /*is_class=*/false, cir::RecordType::getAllDataKinds(fieldTypes));
      })
      .Default([](const llvm::abi::Type *) -> mlir::Type { return nullptr; });
}
 
/// Returns \p ty with a LongDoubleType vector element replaced by its
/// underlying format, which is the element abiTypeToCIR builds for the same
/// float.
static mlir::Type unwrapLongDoubleVectorElement(mlir::Type ty) {
  if (auto vecTy = dyn_cast<cir::VectorType>(ty)) {
    if (auto longDoubleTy =
            dyn_cast<cir::LongDoubleType>(vecTy.getElementType()))
      return cir::VectorType::get(longDoubleTy.getUnderlying(), vecTy.getSize(),
                                  vecTy.getIsScalable());
  }
  return ty;
}
 
/// Map a CIR type to an llvm::abi::Type.  classifyX86_64Function pre-filters
/// the signature, so only the scalar and struct/array types handled here can
/// reach this function.
static const llvm::abi::Type *mapCIRType(mlir::Type type,
                                         mlir::abi::ABITypeMapper &typeMapper,
                                         const DataLayout &dl, ModuleOp modOp) {
  llvm::abi::TypeBuilder &tb = typeMapper.getTypeBuilder();
  return llvm::TypeSwitch<mlir::Type, const llvm::abi::Type *>(type)
      .Case([&](cir::IntType intTy) {
        return tb.getIntegerType(intTy.getWidth(),
                                 llvm::Align(dl.getTypeABIAlignment(type)),
                                 intTy.isSigned(), intTy.getIsBitInt());
      })
      .Case([&](cir::PointerType ptrTy) {
        unsigned addrSpace = 0;
        if (auto targetAsAttr =
                dyn_cast_if_present<cir::TargetAddressSpaceAttr>(
                    ptrTy.getAddrSpace()))
          addrSpace = targetAsAttr.getValue();
        return tb.getPointerType(dl.getTypeSizeInBits(type),
                                 llvm::Align(dl.getTypeABIAlignment(type)),
                                 addrSpace);
      })
      .Case([&](cir::VPtrType) {
        // cir::VPtrType carries no address-space parameter yet, so this
        // always maps into the default one until that gap closes.
        assert(!cir::MissingFeatures::addressSpace());
        return tb.getPointerType(dl.getTypeSizeInBits(type),
                                 llvm::Align(dl.getTypeABIAlignment(type)));
      })
      .Case([&](cir::BoolType) {
        // A bool is a one-bit integer to the classifier, as QualTypeMapper
        // maps it, so a bool vector is sized one bit per element.
        return tb.getIntegerType(1, llvm::Align(dl.getTypeABIAlignment(type)),
                                 /*Signed=*/false);
      })
      .Case([&](cir::VoidType) { return tb.getVoidType(); })
      .Case([&](cir::FPTypeInterface fpTy) {
        // LongDoubleType reports its underlying format's semantics, so the
        // classifier sees x87 or IEEE quad rather than the wrapper.
        return tb.getFloatType(fpTy.getFloatSemantics(),
                               llvm::Align(dl.getTypeABIAlignment(type)));
      })
      .Case([&](cir::ComplexType complexTy) {
        return tb.getComplexType(
            mapCIRType(complexTy.getElementType(), typeMapper, dl, modOp),
            llvm::Align(dl.getTypeABIAlignment(type)));
      })
      .Case([&](cir::VectorType vecTy) {
        // isSupportedType rejects a scalable vector, so the element count is
        // always fixed here.
        return tb.getVectorType(
            mapCIRType(vecTy.getElementType(), typeMapper, dl, modOp),
            llvm::ElementCount::getFixed(vecTy.getSize()),
            llvm::Align(dl.getTypeABIAlignment(type)));
      })
      .Case([&](cir::ArrayType arrTy) {
        const llvm::abi::Type *elemAbi =
            mapCIRType(arrTy.getElementType(), typeMapper, dl, modOp);
        return tb.getArrayType(elemAbi, arrTy.getSize(),
                               dl.getTypeSizeInBits(type).getFixedValue());
      })
      .Case([&](cir::RecordType recTy) -> const llvm::abi::Type * {
        llvm::abi::RecordFlags flags = llvm::abi::RecordFlags::None;
        if (recordCanPassInRegs(modOp, recTy))
          flags = flags | llvm::abi::RecordFlags::CanPassInRegisters;
        llvm::TypeSize sizeBits = llvm::TypeSize::getFixed(
            dl.getTypeSizeInBits(type).getFixedValue());
        llvm::Align align = recordDeclaredAlign(modOp, recTy, dl);
 
        // Mapped with no fields, an empty record reaches Ignore on its own.
        // The size still matters: past two eightbytes SysV says memory whatever
        // the content.
        // TODO: AArch64 needs the unadjusted alignment. We'll need to add that
        // to RecordLayoutAttr.
        if (recTy.isEmptyForABI())
          return tb.getRecordType(
              /*Fields=*/{}, sizeBits, align, /*UnadjustedAlign=*/align,
              llvm::abi::StructPacking::Default,
              /*BaseClasses=*/{}, /*VirtualBaseClasses=*/{}, flags);
 
        SmallVector<llvm::abi::FieldInfo> fields;
        fields.reserve(recTy.getMembers().size());
 
        // The classifier reads one entry per source bit-field, so a unit is
        // spread back out into the fields it holds.  x86_64 is little-endian,
        // so the unit's first byte holds the first of them, and the rest
        // follow at the widths ahead of them.
        //
        // A coercion for an eightbyte is built from the type at that
        // eightbyte's first byte, where the unit's first field sits, so that
        // field's entry carries the unit rather than its own declared type.
        // The bits holding user data are a separate question, and the answer
        // is the extent of the type each field was declared with, which the
        // eightbyte rules read through to decide whether a narrower coercion
        // would drop something.  One entry cannot answer both, so a unit
        // narrower than its first field's declaration takes an extra entry for
        // that declaration, holding no bits of its own.  A zero-width
        // bit-field has no unit and so contributes only the declaration.
        //
        // A width of zero is how the classifier recognizes the entries that
        // take no part in assigning eightbyte classes, so the width goes
        // through as declared.
        auto addAccessUnit = [&](cir::BitFieldType bfTy,
                                 uint64_t unitOffsetBits) {
          mlir::Type unitTy = bfTy.getStorageType();
          uint64_t unitBits =
              unitTy ? dl.getTypeSizeInBits(unitTy).getFixedValue() : 0;
          for (auto [idx, decl] : llvm::enumerate(bfTy.getFields())) {
            const uint64_t offsetBits =
                unitOffsetBits + bfTy.getFieldBitOffset(idx);
            const bool holdsUnit = idx == 0 && unitTy;
            if (holdsUnit)
              fields.push_back(llvm::abi::FieldInfo(
                  mapCIRType(unitTy, typeMapper, dl, modOp), offsetBits,
                  /*IsBitField=*/true, decl.getWidth(),
                  /*IsUnnamedBitField=*/decl.getIsUnnamed()));
            mlir::Type declaredTy = decl.getDeclaredType();
            if (!holdsUnit ||
                dl.getTypeSizeInBits(declaredTy).getFixedValue() > unitBits)
              fields.push_back(llvm::abi::FieldInfo(
                  mapCIRType(declaredTy, typeMapper, dl, modOp), offsetBits,
                  /*IsBitField=*/true, holdsUnit ? 0 : decl.getWidth(),
                  /*IsUnnamedBitField=*/holdsUnit || decl.getIsUnnamed()));
          }
        };
 
        // The size passed here spans the tail padding, so an eightbyte covers
        // the whole union rather than just the member the classifier reduces
        // it to.
        if (recTy.isUnion()) {
          // Classify only the variants that supply data, judged by each
          // variant's own storage rather than by its mark.  One supplying
          // none is skipped so it does not appear as a spurious argument.
          // Whether the record is empty at all is settled from the marks
          // before this.  Every variant starts at offset zero, so a
          // bit-field variant needs no offset of its own.
          assert(!llvm::is_contained(recTy.getMemberKinds(),
                                     cir::RecordMemberKind::Pad) &&
                 "a union member cannot be marked pad");
          for (mlir::Type variantTy : recTy.getMembers()) {
            if (auto bfTy = dyn_cast<cir::BitFieldType>(variantTy)) {
              // A variant that is an unnamed bit-field holds data all the
              // same, so it is flagged rather than skipped, as in the struct
              // path below.
              addAccessUnit(bfTy, /*unitOffsetBits=*/0);
              continue;
            }
            if (dl.getTypeSizeInBits(variantTy).getFixedValue() == 0 ||
                memberIsEmptyRecord(variantTy))
              continue;
            fields.push_back(llvm::abi::FieldInfo(
                mapCIRType(variantTy, typeMapper, dl, modOp)));
          }
          return tb.getUnionType(fields, sizeBits, align,
                                 /*UnadjustedAlign=*/align,
                                 llvm::abi::StructPacking::Default, flags);
        }
 
        // Padding is dropped, so the eightbyte rules see its bytes as holding
        // nothing, while the record's full size set above still spans them.  An
        // unnamed bit-field is flagged rather than dropped, since classic
        // CodeGen ignores it when assigning eightbyte classes but counts it
        // when choosing the coerce type, and a member's presence alone cannot
        // say both.
        for (auto [idx, fieldTy, kind] :
             llvm::enumerate(recTy.getMembers(), recTy.getMemberKinds())) {
          if (kind == cir::RecordMemberKind::Pad)
            continue;
          if (auto bfTy = dyn_cast<cir::BitFieldType>(fieldTy)) {
            addAccessUnit(bfTy, recTy.getElementOffset(dl, idx) * 8);
            continue;
          }
          // An `empty` member that is not a bit-field still stands for bytes
          // no field of the source reads, so it is flagged the same way.
          bool isUnnamed = kind == cir::RecordMemberKind::Empty;
          uint64_t widthBits = dl.getTypeSizeInBits(fieldTy).getFixedValue();
          if (isUnnamed && widthBits == 0)
            continue;
          assert((!isUnnamed || !memberIsEmptyRecord(fieldTy)) &&
                 "an empty-for-ABI member must not reach the classifier as an "
                 "unnamed bit-field");
          fields.push_back(llvm::abi::FieldInfo(
              mapCIRType(fieldTy, typeMapper, dl, modOp),
              recTy.getElementOffset(dl, idx) * 8,
              /*IsBitField=*/isUnnamed, isUnnamed ? widthBits : 0,
              /*IsUnnamedBitField=*/isUnnamed));
        }
 
        return tb.getRecordType(
            fields, sizeBits, align, /*UnadjustedAlign=*/align,
            llvm::abi::StructPacking::Default,
            /*BaseClasses=*/{}, /*VirtualBaseClasses=*/{}, flags);
      })
      .Default([](mlir::Type) -> const llvm::abi::Type * {
        llvm_unreachable(
            "mapCIRType: type not pre-filtered by classifyX86_64Function");
      });
}
 
/// Convert an llvm::abi::ArgInfo into the ArgClassification consumed by
/// CIRABIRewriteContext.
///
/// Direct: the value passes in register(s).  A coercion is forwarded in the
/// four cases where the value has to be rebuilt on the wire: an aggregate
/// unpacked into the register(s) holding it, a scalar too wide for one register
/// split into a tuple of them, a scalar the classifier widens to fill its
/// eightbyte, and a value whose live bytes start partway into its storage
/// because a leading eightbyte holds no field (getDirectOffset).  canFlatten
/// follows the classifier's CanBeFlattened, so the rewriter splits a
/// multi-field coerced struct into individual wire arguments unless the
/// classifier asked to keep it intact.  Any other scalar passes in its
/// natural CIR type, which a null coercion denotes.  A coercion this bridge
/// cannot represent yields std::nullopt so the caller reports NYI rather than
/// silently passing the value unchanged.
///
/// Extend: bool or a sub-register integer needs a signext/zeroext attribute.
/// The x86_64 classifier (llvm/lib/ABI/Targets/X86.cpp) only returns Extend
/// for an integer or bool operand, so any other origTy is asserted rather
/// than silently handled.
///
/// Indirect: an aggregate that does not fit in registers, or a scalar too
/// wide to fit (a `_BitInt` over 128 bits), is passed via a pointer (sret
/// for returns, byval for arguments).
///
/// Ignore: a void return, or an empty record dropped from the signature.
static std::optional<ArgClassification>
convertABIArgInfo(const llvm::abi::ArgInfo &info, MLIRContext *ctx,
                  mlir::Type origTy) {
  if (info.isDirect()) {
    unsigned offset = info.getDirectOffset();
    // The classifier names a coerce type even where it matches the natural
    // type, so a non-null coerce does not by itself mean a rewrite is needed.
    const llvm::abi::Type *coerceAbi = info.getCoerceToType();
    bool isAggregate = isa_and_present<cir::RecordType, cir::ArrayType>(origTy);
    // For a _Complex or a vector the classifier's coerce is only sometimes the
    // natural type, so it has to be read rather than assumed.
    bool comparesAgainstCoerce =
        coerceAbi && isa_and_present<cir::ComplexType, cir::VectorType>(origTy);
    bool coerceIsRegisterTuple =
        isa_and_present<llvm::abi::RecordType>(coerceAbi);
    // A tuple coercion cannot be read at an offset, so refuse the pair rather
    // than emit a read of the wrong bytes.
    if (offset && coerceIsRegisterTuple)
      return std::nullopt;
    // Compare widths rather than identity: a coerce no wider than the natural
    // type carries the same value and needs no rewrite.
    auto origInt = dyn_cast_if_present<cir::IntType>(origTy);
    const auto *coerceInt =
        dyn_cast_if_present<llvm::abi::IntegerType>(coerceAbi);
    bool coerceWidensScalar =
        origInt && coerceInt &&
        coerceInt->getSizeInBits().getFixedValue() > origInt.getWidth();
    // Leaving the rest alone also avoids a lossy round trip: abiTypeToCIR
    // drops the LongDoubleType wrapper and a pointer's pointee, so comparing a
    // scalar against its own coerce would report a difference that is not
    // one.  Passing a value through reads it from byte 0, so one living at
    // an offset cannot take this path.
    if (!offset && !isAggregate && !comparesAgainstCoerce &&
        !coerceIsRegisterTuple && !coerceWidensScalar)
      return ArgClassification::getDirect();
    mlir::Type coerced = abiTypeToCIR(coerceAbi, ctx);
    if (!coerced)
      return std::nullopt;
    bool coerceIsNatural = coerced == unwrapLongDoubleVectorElement(origTy);
    // An offset would be lost here, since a coercion to the value's own type
    // is indistinguishable from no coercion at all.
    if (offset && coerceIsNatural)
      return std::nullopt;
    // Coercing a value to the type it already has would add a memory round
    // trip for nothing.
    if (comparesAgainstCoerce && coerceIsNatural)
      return ArgClassification::getDirect();
    ArgClassification classified =
        ArgClassification::getDirect(coerced, offset);
    classified.canFlatten = info.getCanBeFlattened();
    return classified;
  }
  // An extended value is always read from byte 0 of its own storage, so
  // there is no offset to honor here.
  if (info.isExtend()) {
    if (isa_and_present<cir::BoolType>(origTy))
      return ArgClassification::getExtend(nullptr, info.isSignExt());
    assert((!origTy || isa<cir::IntType>(origTy)) &&
           "the x86_64 classifier only returns Extend for integers and bool");
    mlir::Type extendedTy = abiTypeToCIR(info.getCoerceToType(), ctx);
    return ArgClassification::getExtend(extendedTy, info.isSignExt());
  }
  if (info.isIndirect())
    return ArgClassification::getIndirect(info.getIndirectAlign(),
                                          info.getIndirectByVal());
  // AArch64 pure scalable aggregates use CoerceAndExpand. This bridge lowers
  // x86_64 classifications only.
  assert(!info.isCoerceAndExpand() &&
         "CoerceAndExpand is not expected for x86_64");
  assert(info.isIgnore() && "Unexpected classification");
  return ArgClassification::getIgnore();
}
 
/// Where \p fnTy's declared parameters end and its ellipsis arguments begin.
///
/// The only x86_64 rule that reads this boundary sends an unnamed vector wider
/// than 128 bits to memory.
static llvm::abi::RequiredArgs requiredArgs(cir::FuncType fnTy) {
  if (!fnTy.isVarArg())
    return llvm::abi::RequiredArgs::All;
  return llvm::abi::RequiredArgs(fnTy.getNumInputs());
}
 
/// Classify an x86_64 SysV signature (return type + argument types) using the
/// LLVM ABI library.  Shared by the cir.func path, the variadic-call path and
/// the indirect-call path (the latter classifies from the callee function
/// pointer's pointee FuncType).  \p required marks where the declared
/// parameters in \p inputs end.  The classifier treats every argument past that
/// point as passed through an ellipsis.  Returns std::nullopt and emits an NYI
/// error via \p emitError if the signature uses a type the bridge does not
/// handle yet.
static std::optional<FunctionClassification> classifyX86_64Signature(
    mlir::Type retCIR, mlir::TypeRange inputs, llvm::abi::RequiredArgs required,
    MLIRContext *ctx, const DataLayout &dl,
    mlir::abi::ABITypeMapper &typeMapper,
    const llvm::abi::TargetInfo &targetInfo, ModuleOp modOp,
    llvm::function_ref<mlir::InFlightDiagnostic()> emitError) {
  assert(retCIR && "signature return type must be non-null");
  assert((!required.allowsOptionalArgs() ||
          required.getNumRequiredArgs() <= inputs.size()) &&
         "declared parameters cannot outnumber the classified arguments");
  bool voidRet = isa<cir::VoidType>(retCIR);
 
  auto reject = [&](mlir::Type t) -> bool {
    if (isSupportedType(t, dl))
      return false;
    emitError()
        << "x86_64 calling-convention lowering not yet implemented for type "
        << t;
    return true;
  };
  if (!voidRet && reject(retCIR))
    return std::nullopt;
  for (mlir::Type a : inputs)
    if (reject(a))
      return std::nullopt;
 
  const llvm::abi::Type *retAbi =
      voidRet ? typeMapper.getTypeBuilder().getVoidType()
              : mapCIRType(retCIR, typeMapper, dl, modOp);
  SmallVector<const llvm::abi::Type *> argAbi;
  for (mlir::Type a : inputs)
    argAbi.push_back(mapCIRType(a, typeMapper, dl, modOp));
 
  std::unique_ptr<llvm::abi::FunctionInfo> fi = llvm::abi::FunctionInfo::create(
      llvm::CallingConv::C, retAbi, argAbi, required);
  targetInfo.computeInfo(*fi);
 
  // convertABIArgInfo returns nullopt when the classifier picks a coercion this
  // bridge cannot represent.
  auto nyiCoercion = [&](mlir::Type t) {
    emitError() << "x86_64 calling-convention lowering not yet "
                   "implemented for the ABI coercion of type "
                << t;
  };
 
  FunctionClassification fc;
  fc.returnsVoid = voidRet;
  mlir::Type origRet = voidRet ? mlir::Type() : retCIR;
  std::optional<ArgClassification> retAc =
      convertABIArgInfo(fi->getReturnInfo(), ctx, origRet);
  if (!retAc) {
    nyiCoercion(retCIR);
    return std::nullopt;
  }
  fc.returnInfo = *retAc;
  for (unsigned i = 0, e = fi->arg_size(); i < e; ++i) {
    mlir::Type origArg = i < inputs.size() ? inputs[i] : mlir::Type();
    const llvm::abi::ArgInfo &argInfo = fi->getArgInfo(i).Info;
    std::optional<ArgClassification> ac =
        convertABIArgInfo(argInfo, ctx, origArg);
    if (!ac) {
      nyiCoercion(origArg);
      return std::nullopt;
    }
    ac->neededIntRegs = argInfo.getNeededIntRegs();
    ac->neededSseRegs = argInfo.getNeededSseRegs();
    fc.argInfos.push_back(*ac);
  }
  return fc;
}
 
/// The AVX level to classify \p func at: \p base, raised if the function's own
/// recorded feature list enables a wider vector.
static llvm::abi::X86AVXABILevel funcAvxLevel(cir::FuncOp func,
                                              llvm::abi::X86AVXABILevel base) {
  // Only a `target` attribute may raise the level.  A multiversioned function
  // carries a raised feature list too, and must stay at the module's level.
  assert(!cir::MissingFeatures::opFuncMultiVersioning());
 
  auto features = func->getAttrOfType<mlir::StringAttr>("cir.target-features");
  if (!features)
    return base;
  // A '-' entry disables the feature, so match a whole '+' entry rather than
  // searching for the name.  avx512f implies avx, so both entries are present:
  // return on the wider name so one pass suffices.
  bool avx = false;
  for (llvm::StringRef feature : llvm::split(features.getValue(), ',')) {
    if (!feature.consume_front("+"))
      continue;
    if (feature == "avx512f")
      return std::max(base, llvm::abi::X86AVXABILevel::AVX512);
    avx |= feature == "avx";
  }
  return avx ? std::max(base, llvm::abi::X86AVXABILevel::AVX) : base;
}
 
/// Classify a cir.func for x86_64 SysV using the LLVM ABI library.  Returns
/// std::nullopt and emits an NYI error if the signature uses a type the bridge
/// does not handle yet.
static std::optional<FunctionClassification>
classifyX86_64Function(cir::FuncOp func, const DataLayout &dl,
                       mlir::abi::ABITypeMapper &typeMapper,
                       const llvm::abi::TargetInfo &targetInfo,
                       ModuleOp modOp) {
  cir::FuncType fnTy = func.getFunctionType();
  return classifyX86_64Signature(fnTy.getReturnType(), fnTy.getInputs(),
                                 requiredArgs(fnTy), func->getContext(), dl,
                                 typeMapper, targetInfo, modOp,
                                 [&]() { return func.emitOpError(); });
}
 
/// Classify the single type fetched by a `cir.va_arg` as an unnamed argument
/// (RequiredArgs(0)) with a full register budget.  Returns std::nullopt and
/// emits an NYI error if the type is not one this bridge handles.
static std::optional<ArgClassification> classifyX86_64VarArgType(
    mlir::Type ty, MLIRContext *ctx, const DataLayout &dl,
    mlir::abi::ABITypeMapper &typeMapper,
    const llvm::abi::TargetInfo &targetInfo, ModuleOp modOp,
    llvm::function_ref<mlir::InFlightDiagnostic()> emitError) {
  std::optional<FunctionClassification> fc = classifyX86_64Signature(
      cir::VoidType::get(ctx), mlir::TypeRange(ty), llvm::abi::RequiredArgs(0),
      ctx, dl, typeMapper, targetInfo, modOp, emitError);
  if (!fc)
    return std::nullopt;
  return fc->argInfos[0];
}
 
/// Classify a call that passes arguments through an ellipsis.  The callee's
/// own classification covers only its declared parameters, but an ellipsis
/// argument competes for the same argument registers as a declared one, so
/// what the ABI does with it depends on the whole argument list: the same
/// small struct is passed in registers early in the list and in memory once
/// the integer registers are gone.  Classifying from the call's operands
/// rather than the callee's signature is what makes that accounting right.
static std::optional<FunctionClassification> classifyX86_64VariadicCall(
    cir::CIRCallOpInterface call, cir::FuncType calleeTy, const DataLayout &dl,
    mlir::abi::ABITypeMapper &typeMapper,
    const llvm::abi::TargetInfo &targetInfo, ModuleOp modOp) {
  assert(calleeTy.isVarArg() &&
         "only a variadic callee can take more operands than it declares");
  Operation *op = call.getOperation();
  return classifyX86_64Signature(
      calleeTy.getReturnType(), call.getArgOperands().getTypes(),
      requiredArgs(calleeTy), op->getContext(), dl, typeMapper, targetInfo,
      modOp, [&]() { return op->emitOpError(); });
}
 
/// Whether \p fc gives the callee access to memory through a pointer the ABI
/// introduced: an sret slot for an indirect return, or an indirect argument.
static bool hasIndirectSlot(const FunctionClassification &fc) {
  if (fc.returnInfo.kind == ArgKind::Indirect)
    return true;
  return llvm::any_of(fc.argInfos, [](const ArgClassification &ac) {
    return ac.kind == ArgKind::Indirect;
  });
}
 
/// Record on \p op that the memory its pointer arguments address may be read
/// and written.  Absent effects already allow that, since an absent attribute
/// means the effects are unknown, so they are left absent.
template <typename OpTy> static void widenForArgMemory(OpTy op) {
  cir::MemoryEffectsAttr effects = op.getMemoryEffectsAttr();
  if (!effects)
    return;
  op.setMemoryEffectsAttr(cir::MemoryEffectsAttr::get(
      effects.getContext(), effects.getOther(), cir::ModRefInfo::ModRef,
      effects.getInaccessibleMem(), effects.getErrnoMem(),
      effects.getTargetMem0(), effects.getTargetMem1()));
}
 
#ifndef NDEBUG
/// Whether \p callFc classifies a call's leading arguments and its return
/// exactly as \p calleeFc classifies the callee's declared parameters and
/// return.  A function definition and its call sites are rewritten from
/// separate classifications, so the two would silently disagree on the wire
/// format if an ellipsis argument could ever change how a declared parameter
/// is passed.
static bool classifiesSamePrefix(const FunctionClassification &calleeFc,
                                 const FunctionClassification &callFc) {
  if (callFc.argInfos.size() < calleeFc.argInfos.size())
    return false;
  return calleeFc.returnInfo == callFc.returnInfo &&
         std::equal(calleeFc.argInfos.begin(), calleeFc.argInfos.end(),
                    callFc.argInfos.begin());
}
#endif
 
struct CallConvLoweringPass
    : public impl::CallConvLoweringBase<CallConvLoweringPass> {
  using CallConvLoweringBase::CallConvLoweringBase;
 
  CallConvLoweringPass(const CallConvLoweringOptions &options,
                       const llvm::abi::X86ABICompatInfo &x86AbiCompat)
      : CallConvLoweringBase(options), x86AbiCompat(x86AbiCompat) {}
 
  void runOnOperation() override;
 
  /// The x86_64 flags whose value depends on the target and the requested ABI
  /// compatibility version.  Carried outside the pass options because the
  /// struct has no command-line parser, so a cir-opt run gets the library
  /// defaults rather than a target's values.
  llvm::abi::X86ABICompatInfo x86AbiCompat;
};
 
/// Record on \p fc whether \p returnType is CIR's void.  The x86_64 classifier
/// answers this itself, but the other two drivers cannot: the test target is
/// dialect-neutral and has no notion of CIR's void, and the
/// classification-attr schema carries no return type at all.  Both route
/// through here so a classification always reaches needsRewrite paired with
/// the return type it was built from.
static std::optional<FunctionClassification>
withReturnVoidness(std::optional<FunctionClassification> fc,
                   mlir::Type returnType) {
  if (fc)
    fc->returnsVoid = mlir::isa<cir::VoidType>(returnType);
  return fc;
}
 
/// Classify \p func using whichever driver mode is configured.  Returns
/// std::nullopt and emits an error on the function if classification fails
/// (e.g. injection-driver mode but the function is missing the attribute,
/// or the attribute is malformed).
std::optional<FunctionClassification>
classifyFunction(cir::FuncOp func, const DataLayout &dl,
                 cir::CallConvTarget target, StringRef classificationAttrName) {
  ArrayRef<Type> argTypes = func.getFunctionType().getInputs();
  Type returnType = func.getFunctionType().getReturnType();
 
  if (!classificationAttrName.empty()) {
    auto attr = func->getAttrOfType<DictionaryAttr>(classificationAttrName);
    if (!attr) {
      func.emitOpError()
          << "missing classification attribute '" << classificationAttrName
          << "' (CallConvLowering driver mode 'classification-attr')";
      return std::nullopt;
    }
    return withReturnVoidness(mlir::abi::test::parseClassificationAttr(
                                  attr, [&]() { return func.emitOpError(); }),
                              returnType);
  }
 
  // The x86_64 target is handled directly in runOnOperation (it needs a shared
  // ABITypeMapper and TargetInfo), so only the test target reaches here.
  assert(target == cir::CallConvTarget::Test &&
         "classifyFunction only handles the test target");
  return withReturnVoidness(mlir::abi::test::classify(argTypes, returnType, dl),
                            returnType);
}
 
/// Find the cir.func declaration matching a direct cir.call / cir.try_call
/// callee, if any.  Returns nullptr if the callee is indirect or the symbol
/// cannot be resolved.  Takes a SymbolTable instead of a ModuleOp so the
/// symbol lookup is amortized across all the call sites the driver walks
/// (ModuleOp::lookupSymbol is linear per call).
cir::FuncOp lookupCallee(Operation *callOp, SymbolTable &symbolTable) {
  FlatSymbolRefAttr callee;
  if (auto call = dyn_cast<cir::CallOp>(callOp))
    callee = call.getCalleeAttr();
  else if (auto tryCall = dyn_cast<cir::TryCallOp>(callOp))
    callee = tryCall.getCalleeAttr();
  else
    return nullptr;
  if (!callee)
    return nullptr;
  return symbolTable.lookup<cir::FuncOp>(callee.getValue());
}
 
void CallConvLoweringPass::runOnOperation() {
  ModuleOp moduleOp = getOperation();
  MLIRContext *ctx = &getContext();
 
  bool haveTarget = target != cir::CallConvTarget::None;
  bool haveAttr = !classificationAttr.empty();
  if (haveTarget == haveAttr) {
    moduleOp.emitOpError() << "CallConvLowering requires exactly one of "
                              "'target' or 'classification-attr' pass options";
    signalPassFailure();
    return;
  }
 
  if (!moduleOp->hasAttr(DLTIDialect::kDataLayoutAttrName)) {
    moduleOp.emitOpError()
        << "CallConvLowering requires a DataLayout (dlti.dl_spec attribute "
           "on the module)";
    signalPassFailure();
    return;
  }
 
  DataLayout dl(moduleOp);
  CIRABIRewriteContext rewriteCtx(moduleOp, dl);
  // A non-byval indirect parameter's slot outlives the rewrite that retypes
  // the parameter, so that a call forwarding the parameter can still recognize
  // it.  Draining on scope exit collapses those slots whichever way this
  // function returns.
  llvm::scope_exit drainParamSlots(
      [&] { rewriteCtx.finalizeParameterSlots(); });
  SymbolTable symbolTable(moduleOp);
 
  // A per-function target attribute can raise the AVX level, so one classifier
  // per module would misclassify a wide vector in such a function.
  static constexpr unsigned numAvxLevels =
      static_cast<unsigned>(llvm::abi::X86AVXABILevel::Last) + 1;
  bool isX86 = target == cir::CallConvTarget::X86_64;
  std::optional<mlir::abi::ABITypeMapper> x86TypeMapper;
  std::array<std::unique_ptr<llvm::abi::TargetInfo>, numAvxLevels> x86Targets;
  if (isX86)
    x86TypeMapper.emplace(dl);
  auto x86TargetFor =
      [&](llvm::abi::X86AVXABILevel level) -> const llvm::abi::TargetInfo & {
    assert(static_cast<unsigned>(level) < numAvxLevels &&
           "a new X86AVXABILevel must move X86AVXABILevel::Last");
    std::unique_ptr<llvm::abi::TargetInfo> &slot =
        x86Targets[static_cast<unsigned>(level)];
    if (!slot)
      slot = llvm::abi::createX86_64TargetInfo(
          x86TypeMapper->getTypeBuilder(), level,
          /*Has64BitPointers=*/true, x86AbiCompat);
    return *slot;
  };
  llvm::abi::X86AVXABILevel baseAvxLevel = x86AvxAbiLevel.getValue();
  auto avxLevelFor = [&](cir::FuncOp func) -> llvm::abi::X86AVXABILevel {
    if (!allowsX86TargetAttrAvx || !func)
      return baseAvxLevel;
    return funcAvxLevel(func, baseAvxLevel);
  };
 
  // Classify every cir.func up front.  No IR mutation happens here, so
  // later walks can consult any function's classification regardless of
  // visitation order.
  llvm::MapVector<cir::FuncOp, FunctionClassification> classifications;
  bool anyFailed = false;
  moduleOp.walk([&](cir::FuncOp f) {
    // A complete type is required at any call or definition, so only a
    // declaration can carry an incomplete-by-value parameter or return type,
    // and no translation unit can ever call or define it with real argument
    // data.  Leave it unclassified.
    cir::FuncType fnTy = f.getFunctionType();
    if (f.isDeclaration() &&
        (hasIncompleteRecordByValue(fnTy.getReturnType()) ||
         llvm::any_of(fnTy.getInputs(), hasIncompleteRecordByValue)))
      return;
    std::optional<FunctionClassification> fc;
    if (isX86)
      fc = classifyX86_64Function(f, dl, *x86TypeMapper,
                                  x86TargetFor(avxLevelFor(f)), moduleOp);
    else
      fc = classifyFunction(f, dl, target, classificationAttr);
    if (!fc) {
      anyFailed = true;
      return;
    }
    classifications.insert({f, std::move(*fc)});
  });
  if (anyFailed) {
    signalPassFailure();
    return;
  }
 
  // Build a callee-to-callers index.  One module walk collects every direct
  // cir.call / cir.try_call to each cir.func; the loop below rewrites a
  // function and all of its call sites together.  Indirect or unresolved
  // callees are skipped here; rewriteCallSite errors on those at the end.
  //
  // A call that passes arguments through an ellipsis gets its own
  // classification, recorded here while every signature is still in its
  // original form.  The callee's classification covers only its declared
  // parameters and cannot describe those extra arguments.
  llvm::DenseMap<cir::FuncOp, SmallVector<Operation *>> callers;
  // Keyed on the call op collected below, looked up once when that same op is
  // rewritten.  A key must never come from an op created during the rewrite:
  // a recycled address could match an unrelated entry.
  llvm::DenseMap<Operation *, FunctionClassification> variadicCallSites;
  moduleOp.walk([&](Operation *op) {
    auto call = dyn_cast<cir::CIRCallOpInterface>(op);
    if (!call)
      return;
    cir::FuncOp callee = lookupCallee(op, symbolTable);
    if (!callee)
      return;
    callers[callee].push_back(op);
 
    // Only the x86_64 driver classifies per call site.  Under the other
    // drivers the classification comes from a fixed per-function source, so
    // such a call stays short a classification and rewriteCallSite reports it.
    cir::FuncType calleeTy = callee.getFunctionType();
    if (!isX86 || call.getNumArgOperands() <= calleeTy.getNumInputs())
      return;
    // A callee declared without a prototype also takes more operands than it
    // declares, and the verifier allows it.  Those extra arguments are named
    // rather than passed through an ellipsis, so the accounting below does not
    // describe them.
    if (!calleeTy.isVarArg()) {
      op->emitOpError() << "extra arguments to a callee without a prototype "
                           "not yet implemented in CallConvLowering";
      anyFailed = true;
      return;
    }
    // The callee's level, not the caller's: this pass rewrites the definition
    // and its call sites from one classification, so they have to agree.
    // Classic instead arranges every call site from the caller and reports a
    // caller whose level disagrees with its callee in checkFunctionCallABI,
    // which has no equivalent here yet.
    std::optional<FunctionClassification> fc =
        classifyX86_64VariadicCall(call, calleeTy, dl, *x86TypeMapper,
                                   x86TargetFor(avxLevelFor(callee)), moduleOp);
    if (!fc) {
      anyFailed = true;
      return;
    }
    variadicCallSites.insert({op, std::move(*fc)});
  });
  if (anyFailed) {
    signalPassFailure();
    return;
  }
 
  // A cir.get_global holding a function's address carries the signature the
  // source wrote, which the verifier ties to the callee, so it goes stale when
  // that callee is rewritten.  A function address in a global initializer is a
  // GlobalViewAttr instead, whose recorded type the verifier does not tie to
  // the callee and which is dropped at opaque-pointer lowering, so it needs no
  // counterpart.
  llvm::DenseMap<cir::FuncOp, SmallVector<cir::GetGlobalOp>> addressTakers;
  moduleOp.walk([&](cir::GetGlobalOp getGlobal) {
    auto ptrTy = cast<cir::PointerType>(getGlobal.getAddr().getType());
    if (!isa<cir::FuncType>(ptrTy.getPointee()))
      return;
    // A get_global's pointee must equal the named symbol's type, and the
    // GlobalOp verifier rejects a function type there, so this names a
    // cir.func.
    auto callee = cast<cir::FuncOp>(symbolTable.lookup(getGlobal.getName()));
    addressTakers[callee].push_back(getGlobal);
  });
 
  // Restate every non-byval indirect parameter's slot alignment and route any
  // use of such a parameter as a call argument through a load of that slot,
  // before any definition or call site is rewritten.  Doing this up front
  // makes the forwarding decision independent of the order the callee and its
  // caller were declared in.
  for (auto &kv : classifications) {
    if (failed(rewriteCtx.prepareNonByvalParameters(kv.first, kv.second))) {
      signalPassFailure();
      return;
    }
  }
 
  // An sret slot or an indirect argument is a pointer the source never
  // wrote, and an ellipsis can carry one the declared parameters do not
  // describe.  The callee reads and writes that memory whatever const or
  // pure recorded about it, so the recorded effects have to widen.
  for (auto &kv : classifications) {
    cir::FuncOp func = kv.first;
    if (hasIndirectSlot(kv.second) || func.getFunctionType().isVarArg())
      widenForArgMemory(func);
  }
 
  // Rewrite each function together with every direct call to it and every op
  // holding its address.  By the time we move on to function F+1, F's
  // signature and every reference to F have already been brought into
  // alignment, and F+1..FN are still in their original (mutually consistent)
  // form, so the IR is verifier-clean at every outer-iteration boundary.
  //
  // There is still a brief inner window where F's signature has been
  // rewritten but its references have not yet caught up -- we have no way to
  // mutate both sides of a call atomically.  No verifier runs inside the
  // pass, and at pass exit the module is verifier-clean.  Fusing the inner
  // loops here keeps the invalid window per-function rather than module-wide.
  OpBuilder builder(ctx);
  for (auto &kv : classifications) {
    cir::FuncOp func = kv.first;
    const FunctionClassification &fc = kv.second;
    if (failed(rewriteCtx.rewriteFunctionDefinition(func, fc, builder))) {
      signalPassFailure();
      return;
    }
    for (Operation *callOp : callers.lookup(func)) {
      const FunctionClassification *callFc = &fc;
      if (auto it = variadicCallSites.find(callOp);
          it != variadicCallSites.end()) {
        callFc = &it->second;
        assert(classifiesSamePrefix(fc, *callFc) &&
               "a call site's declared parameters must be classified the same "
               "way as the callee's");
      }
      if (hasIndirectSlot(*callFc))
        widenForArgMemory(cast<cir::CIRCallOpInterface>(callOp));
      if (failed(rewriteCtx.rewriteCallSite(callOp, *callFc, builder))) {
        signalPassFailure();
        return;
      }
    }
    for (cir::GetGlobalOp addrOp : addressTakers.lookup(func))
      rewriteCtx.rewriteFunctionAddress(addrOp, func, builder);
  }
 
  // Rewrite indirect call sites.  The callee is opaque, so classify from the
  // function pointer's pointee FuncType and let rewriteCallSite retype the
  // callee pointer to match the coerced signature.  Collect the calls first:
  // when an sret rewrite reuses a single-use store's destination as the return
  // slot it erases that store, which is the operation a live walk has already
  // cached as the next one to visit.
  SmallVector<cir::CIRCallOpInterface> indirectCalls;
  moduleOp.walk([&](cir::CIRCallOpInterface c) {
    if (c.isIndirect())
      indirectCalls.push_back(c);
  });
  for (cir::CIRCallOpInterface c : indirectCalls) {
    // classification-attr mode injects a per-function classification, which
    // cannot describe a callee resolved at run time.  Report it rather than
    // leave the indirect call unrewritten while direct calls are coerced.
    if (!classificationAttr.empty()) {
      c->emitOpError() << "indirect call cannot be classified in the "
                          "'classification-attr' driver mode";
      signalPassFailure();
      return;
    }
    cir::FuncType funcTy = getIndirectCalleeType(c);
    auto classifySignature =
        [&](mlir::TypeRange argTypes) -> std::optional<FunctionClassification> {
      // A callee resolved at run time carries no features of its own, so the
      // level comes from the function containing the call, which is the
      // declaration classic arranges every call site from.
      if (isX86)
        return classifyX86_64Signature(
            funcTy.getReturnType(), argTypes, requiredArgs(funcTy), ctx, dl,
            *x86TypeMapper,
            x86TargetFor(avxLevelFor(c->getParentOfType<cir::FuncOp>())),
            moduleOp, [&]() { return c->emitOpError(); });
      return withReturnVoidness(
          mlir::abi::test::classify(argTypes, funcTy.getReturnType(), dl),
          funcTy.getReturnType());
    };
 
    // An argument passed through an ellipsis has no counterpart in the
    // pointee's parameter list.  On x86_64 such a call is classified from its
    // own operands, as a direct one is.  Under target=test it is classified
    // from the pointee alone, and rewriteCallSite reports the call.
    bool classifyCallSite =
        isX86 && c.getNumArgOperands() > funcTy.getNumInputs();
    std::optional<FunctionClassification> fc =
        classifyCallSite
            ? classifyX86_64VariadicCall(
                  c, funcTy, dl, *x86TypeMapper,
                  x86TargetFor(avxLevelFor(c->getParentOfType<cir::FuncOp>())),
                  moduleOp)
            : classifySignature(funcTy.getInputs());
    if (!fc) {
      signalPassFailure();
      return;
    }
    if (hasIndirectSlot(*fc))
      widenForArgMemory(c);
    if (failed(rewriteCtx.rewriteCallSite(c.getOperation(), *fc, builder))) {
      signalPassFailure();
      return;
    }
  }
 
  if (isX86) {
    // Collect the fetches before rewriting any, because rewriting one erases
    // it, and a live walk holds the op it is about to visit next.
    SmallVector<cir::VAArgOp> vaArgs;
    moduleOp.walk([&](cir::VAArgOp v) { vaArgs.push_back(v); });
    for (cir::VAArgOp v : vaArgs) {
      cir::FuncOp enclosing = v->getParentOfType<cir::FuncOp>();
      std::optional<ArgClassification> ac = classifyX86_64VarArgType(
          v.getType(), ctx, dl, *x86TypeMapper,
          x86TargetFor(avxLevelFor(enclosing)), moduleOp,
          [&]() { return v->emitOpError(); });
      if (!ac) {
        signalPassFailure();
        return;
      }
      if (failed(rewriteCtx.rewriteVAArg(v.getOperation(), *ac, builder))) {
        signalPassFailure();
        return;
      }
    }
  }
}
 
} // namespace
 
std::unique_ptr<Pass> mlir::createCallConvLoweringPass() {
  return std::make_unique<CallConvLoweringPass>();
}
 
std::unique_ptr<Pass> mlir::createCallConvLoweringPass(
    cir::CallConvTarget target, llvm::abi::X86AVXABILevel x86AvxAbiLevel,
    bool allowsX86TargetAttrAvx,
    const llvm::abi::X86ABICompatInfo &x86AbiCompat) {
  CallConvLoweringOptions options;
  options.target = target;
  options.x86AvxAbiLevel = x86AvxAbiLevel;
  options.allowsX86TargetAttrAvx = allowsX86TargetAttrAvx;
  return std::make_unique<CallConvLoweringPass>(options, x86AbiCompat);
}