CallConvLoweringPass.cpp

Go to the documentation of this file.

//===- CallConvLoweringPass.cpp - Lower CIR to ABI calling convention ----===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This pass walks every cir.func and cir.call in the module, computes a
// FunctionClassification for it (via either an ABI target or a pre-built
// classification injected as a function attribute), and dispatches to
// CIRABIRewriteContext to perform the actual IR rewriting.
//
// Two driver modes (mutually exclusive):
//
//   target=test
//     Use the MLIR test ABI target (mlir/lib/ABI/Targets/Test/) to classify
//     each function.  Predictable rules that approximate x86_64 SysV.  Real
//     targets (x86_64, AArch64) will be added once the LLVM ABI library
//     ships them.
//
//   classification-attr=<name>
//     Read a DictionaryAttr named <name> from each cir.func and parse it via
//     mlir::abi::test::parseClassificationAttr.  Used by tests to inject any
//     classification (including shapes the test target itself does not
//     produce) without depending on a real ABI target.
//
// The pass requires a `dlti.dl_spec` attribute on the module so the
// classifier can query type sizes and alignments.
//
//===----------------------------------------------------------------------===//
 
#include "CIRABIClassifier.h"
#include "PassDetail.h"
#include "TargetLowering/CIRABIRewriteContext.h"
 
#include "mlir/ABI/ABIRewriteContext.h"
#include "mlir/ABI/ABITypeMapper.h"
#include "mlir/ABI/Targets/Test/TestTarget.h"
#include "mlir/Dialect/DLTI/DLTI.h"
#include "mlir/IR/Builders.h"
#include "mlir/IR/BuiltinOps.h"
#include "mlir/IR/SymbolTable.h"
#include "mlir/Interfaces/DataLayoutInterfaces.h"
#include "mlir/Pass/Pass.h"
#include "clang/CIR/Dialect/IR/CIRDialect.h"
#include "clang/CIR/Dialect/Passes.h"
#include "clang/CIR/MissingFeatures.h"
#include "llvm/ABI/FunctionInfo.h"
#include "llvm/ABI/TargetInfo.h"
#include "llvm/ABI/Types.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/TypeSwitch.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/Support/MathExtras.h"
 
#include <algorithm>
#include <array>
 
using namespace mlir;
using namespace mlir::abi;
using namespace cir;
 
namespace mlir {
#define GEN_PASS_DEF_CALLCONVLOWERING
#include "clang/CIR/Dialect/Passes.h.inc"
} // namespace mlir
 
//===----------------------------------------------------------------------===//
// x86_64 System V classifier bridge
//
// Maps CIR types to llvm::abi::Type, runs the LLVM ABI Lowering Library's SysV
// x86_64 classifier, and converts the result back into the dialect-agnostic
// mlir::abi::FunctionClassification that CIRABIRewriteContext consumes.
// isSupportedType says which CIR types the bridge handles, and a signature
// naming any other fails the pass instead of being misclassified.
//===----------------------------------------------------------------------===//
 
/// Whether a struct's declared argument-passing kind (from the module's
/// record-layout metadata) allows it to be passed in registers.  A record with
/// no layout entry (e.g. an anonymous struct) has no C++ non-trivial reason to
/// be forced to memory, so it defaults to can-pass-in-registers.
static bool recordCanPassInRegs(ModuleOp modOp, cir::RecordType recTy) {
  auto layout = cir::tryGetRecordLayout(modOp, recTy.getName());
  if (!layout)
    return true;
  return layout.getArgPassingKind() == cir::ArgPassingKind::CanPassInRegs;
}
 
/// Whether the classifier could give this type the SSEUP class, looking
/// through arrays and records at the types they hold.
static bool mayReachSseUp(mlir::Type ty, const DataLayout &dl) {
  // The classifier sizes a vector with its width rounded up to a power of two.
  if (isa<cir::VectorType>(ty))
    return llvm::PowerOf2Ceil(dl.getTypeSizeInBits(ty).getFixedValue()) >= 128;
  if (auto fpTy = dyn_cast<cir::FPTypeInterface>(ty))
    return &fpTy.getFloatSemantics() == &llvm::APFloat::IEEEquad();
  if (auto arrTy = dyn_cast<cir::ArrayType>(ty))
    return mayReachSseUp(arrTy.getElementType(), dl);
  if (auto recTy = dyn_cast<cir::RecordType>(ty))
    return recTy.isComplete() &&
           llvm::any_of(recTy.getMembers(),
                        [&](mlir::Type m) { return mayReachSseUp(m, dl); });
  // The rest classify integer, SSE or memory.  A complex is SSE at float and
  // double width and memory above that, so it needs no walk of its own.
  assert((isa<cir::IntType, cir::BoolType, cir::PointerType, cir::VPtrType,
              cir::VoidType, cir::ComplexType, cir::BitFieldType>(ty)) &&
         "unhandled type in the SSEUP walk");
  return false;
}
 
/// Whether the record's size has no SSEUP coerce.  That coerce is a vector as
/// wide as the record, so only 128, 256 and 512 bits have one.  A record past
/// 512 bits classifies memory before any coerce is asked for.
static bool sseUpCoerceSizeUnsupported(uint64_t recordBits,
                                       llvm::ArrayRef<mlir::Type> members,
                                       const DataLayout &dl) {
  if (recordBits == 128 || recordBits == 256 || recordBits == 512)
    return false;
  if (recordBits < 128 || recordBits > 512)
    return false;
  return llvm::any_of(members,
                      [&](mlir::Type m) { return mayReachSseUp(m, dl); });
}
 
/// Whether a member is an empty record, looking through arrays, since an array
/// of empty records supplies no bytes either.
static bool memberIsEmptyRecord(mlir::Type ty) {
  while (auto arrTy = dyn_cast<cir::ArrayType>(ty))
    ty = arrTy.getElementType();
  auto recTy = dyn_cast<cir::RecordType>(ty);
  return recTy && recTy.isEmptyForABI();
}
 
/// A record's declared alignment, which the ABI uses for the byval and sret
/// alignment of an indirect argument.  DataLayout derives alignment from the
/// members, so it cannot see `__attribute__((aligned(N)))`.  The declared value
/// comes from the module's record-layout metadata instead.  CIRGen emits an
/// entry for every record it names, so the computed fallback only serves
/// hand-written CIR.
static llvm::Align recordDeclaredAlign(ModuleOp modOp, cir::RecordType recTy,
                                       const DataLayout &dl) {
  auto layout = cir::tryGetRecordLayout(modOp, recTy.getName());
  if (!layout)
    return llvm::Align(dl.getTypeABIAlignment(recTy));
  return llvm::Align(layout.getRecordAlign());
}
 
/// Whether \p ty, or an aggregate member/element reached by value (never
/// through a pointer), is an incomplete record.  Such a record has no known
/// layout, so no eightbyte classification can be built for it.
static bool hasIncompleteRecordByValue(mlir::Type ty) {
  if (auto recTy = dyn_cast<cir::RecordType>(ty))
    return !recTy.isComplete() ||
           llvm::any_of(recTy.getMembers(), hasIncompleteRecordByValue);
  if (auto arrTy = dyn_cast<cir::ArrayType>(ty))
    return hasIncompleteRecordByValue(arrTy.getElementType());
  return false;
}
 
/// The CIR types the x86_64 bridge handles.  Scalars: an integer up to 128
/// bits (including `__int128`), a `_BitInt` of any width, pointer, vtable
/// pointer, bool, void, or any floating-point type.  Aggregates: a complete
/// struct or union whose members are all themselves supported, or an array
/// of a supported element type.  Also a `_Complex`, or a fixed-width vector,
/// of a supported element type.  Everything else is reported NYI at the
/// reject() choke point in classifyX86_64Function.
static bool isSupportedType(mlir::Type ty, const DataLayout &dl) {
  // A pointer is only handled in the default address space (null) or an
  // already-lowered target address space.  A LangAddressSpaceAttr must be
  // lowered before this pass, so reject it rather than silently dropping it.
  if (auto ptrTy = dyn_cast<cir::PointerType>(ty))
    return !ptrTy.getAddrSpace() ||
           mlir::isa<cir::TargetAddressSpaceAttr>(ptrTy.getAddrSpace());
  // cir::VPtrType carries no address-space parameter yet, so its target
  // address space cannot be checked here even though it is not always the
  // default one.
  if (isa<cir::VPtrType>(ty)) {
    assert(!cir::MissingFeatures::addressSpace());
    return true;
  }
  if (isa<cir::VoidType, cir::BoolType>(ty))
    return true;
  // Every CIR floating-point type carries the semantics the classifier
  // switches on, so all of them are handled.
  if (isa<cir::FPTypeInterface>(ty))
    return true;
  if (auto intTy = dyn_cast<cir::IntType>(ty)) {
    // Integers up to 64 bits, __int128, and _BitInt of any width are
    // handled: the classifier extends a width below 32, widens 33 through 63
    // to i64, coerces 65 through 127 to a {i64, i64} pair, and passes 32, 64,
    // and 128 in the natural type.  A _BitInt wider than 128 bits classifies
    // Indirect, the same path an oversized aggregate already takes.
    // Non-_BitInt intermediate widths (65..127) do not arise from C and stay
    // rejected.
    if (intTy.getIsBitInt())
      return true;
    return intTy.getWidth() <= 64 || intTy.getWidth() == 128;
  }
  if (auto complexTy = dyn_cast<cir::ComplexType>(ty))
    return isSupportedType(complexTy.getElementType(), dl);
  if (auto vecTy = dyn_cast<cir::VectorType>(ty)) {
    // A scalable vector has no size the SysV eightbyte rules can read, and
    // x86_64 has no calling convention for one.
    if (vecTy.isScalable())
      return false;
    // mapCIRType maps a bool as a one-bit integer, so a bool vector is sized
    // one bit per element, as clang packs it.  Vectors of a _BitInt whose
    // width is not a whole number of bytes are not handled yet.
    mlir::Type elemTy = vecTy.getElementType();
    if (auto elemInt = dyn_cast<cir::IntType>(elemTy)) {
      if (elemInt.getWidth() % 8)
        return false;
    } else if (!isa<cir::FPTypeInterface, cir::BoolType>(elemTy)) {
      return false;
    }
    return isSupportedType(elemTy, dl);
  }
  if (auto arrTy = dyn_cast<cir::ArrayType>(ty))
    return isSupportedType(arrTy.getElementType(), dl);
  // A bit-field is classified from the type it was declared with and the bits
  // it holds, and is loaded through its access unit, so both types have to be
  // ones the bridge handles.  A zero-width bit-field has no unit of its own.
  if (auto bfTy = dyn_cast<cir::BitFieldType>(ty)) {
    if (mlir::Type storageTy = bfTy.getStorageType())
      if (!isSupportedType(storageTy, dl))
        return false;
    return llvm::all_of(bfTy.getFields(), [&](cir::BitFieldDeclAttr decl) {
      return isSupportedType(decl.getDeclaredType(), dl);
    });
  }
  if (auto recTy = dyn_cast<cir::RecordType>(ty)) {
    // An incomplete record has no layout to classify.
    if (!recTy.isComplete())
      return false;
    // The members are checked first so that everything below reasons only
    // about types the bridge handles.
    if (!llvm::all_of(recTy.getMembers(),
                      [&](mlir::Type m) { return isSupportedType(m, dl); }))
      return false;
    if (recTy.isUnion()) {
      llvm::ArrayRef<mlir::Type> members = recTy.getMembers();
      uint64_t recordBits = dl.getTypeSizeInBits(recTy).getFixedValue();
      // Refuse a union that could reach SSEUP at a size no coerce exists for.
      // The classifier asserts on those rather than returning a coerce.
      if (sseUpCoerceSizeUnsupported(recordBits, members, dl))
        return false;
      // A member-less union is all padding, which classifies Ignore up to two
      // eightbytes.  Past that SysV says MEMORY regardless of content, and
      // there is no member here to build the Indirect coercion from.
      if (members.empty() && recordBits > 128)
        return false;
      // A `_BitInt` access unit past an eightbyte has byte-array storage in
      // CIR and integer storage in classic CodeGen, so the narrowing walk
      // finds an i8 in the second eightbyte where classic keeps an i64.
      if (llvm::any_of(members, [&](mlir::Type m) {
            auto bfTy = dyn_cast<cir::BitFieldType>(m);
            if (!bfTy || dl.getTypeSizeInBits(bfTy).getFixedValue() <= 64)
              return false;
            return llvm::any_of(bfTy.getFields(), [](cir::BitFieldDeclAttr d) {
              auto intTy = dyn_cast<cir::IntType>(d.getDeclaredType());
              return intTy && intTy.getIsBitInt();
            });
          }))
        return false;
    } else {
      // A struct's `empty` member that occupies bytes is later read as an
      // unnamed bit-field.  An empty-for-ABI record occupies its own padding,
      // and reading that as a unit would hand the classifier storage classic
      // CodeGen ignores.  A zero-sized one is dropped before classification,
      // so a misaligned one never reaches the rule that sends its record to
      // memory.  A union needs neither rule: mapCIRType skips such a variant
      // instead of reading it as a unit, and every variant sits at offset
      // zero.
      for (auto [idx, memberTy, kind] :
           llvm::enumerate(recTy.getMembers(), recTy.getMemberKinds())) {
        if (kind != cir::RecordMemberKind::Empty)
          continue;
        if (dl.getTypeSizeInBits(memberTy).getFixedValue()) {
          if (memberIsEmptyRecord(memberTy))
            return false;
        } else if (recTy.getElementOffset(dl, idx) %
                   dl.getTypeABIAlignment(memberTy)) {
          return false;
        }
      }
    }
    return true;
  }
  return false;
}
 
/// Convert an llvm::abi::Type coercion type back to a scalar CIR type.
static mlir::Type abiTypeToCIR(const llvm::abi::Type *ty, MLIRContext *ctx) {
  if (!ty)
    return nullptr;
  return llvm::TypeSwitch<const llvm::abi::Type *, mlir::Type>(ty)
      .Case(
          [&](const llvm::abi::VoidType *) { return cir::VoidType::get(ctx); })
      .Case([&](const llvm::abi::IntegerType *intTy) -> mlir::Type {
        // mapCIRType maps a bool as a one-bit integer.
        if (intTy->isBool())
          return cir::BoolType::get(ctx);
        return cir::IntType::get(ctx, intTy->getSizeInBits().getFixedValue(),
                                 intTy->isSigned(), intTy->isBitInt());
      })
      .Case([&](const llvm::abi::FloatType *fltTy) {
        return cir::getFloatingPointType(*fltTy->getSemantics(), ctx);
      })
      .Case([&](const llvm::abi::PointerType *ptrTy) {
        auto voidTy = cir::VoidType::get(ctx);
        if (unsigned as = ptrTy->getAddrSpace())
          return cir::PointerType::get(
              voidTy, cir::TargetAddressSpaceAttr::get(ctx, as));
        return cir::PointerType::get(voidTy);
      })
      .Case([&](const llvm::abi::VectorType *vecTy) -> mlir::Type {
        mlir::Type elemCIR = abiTypeToCIR(vecTy->getElementType(), ctx);
        if (!elemCIR)
          return nullptr;
        return cir::VectorType::getFixed(
            elemCIR, vecTy->getNumElements().getFixedValue());
      })
      .Case([&](const llvm::abi::ArrayType *arrTy) -> mlir::Type {
        mlir::Type elemCIR = abiTypeToCIR(arrTy->getElementType(), ctx);
        if (!elemCIR)
          return nullptr;
        return cir::ArrayType::get(elemCIR, arrTy->getNumElements());
      })
      .Case([&](const llvm::abi::RecordType *recTy) -> mlir::Type {
        SmallVector<mlir::Type> fieldTypes;
        fieldTypes.reserve(recTy->getFields().size());
        for (const auto &field : recTy->getFields()) {
          mlir::Type fieldCIR = abiTypeToCIR(field.FieldType, ctx);
          if (!fieldCIR)
            return nullptr;
          fieldTypes.push_back(fieldCIR);
        }
        // Coercion types are plain register tuples, not the source record.
        return cir::StructType::get(
            ctx, fieldTypes, /*packed=*/false,
            /*is_class=*/false, cir::RecordType::getAllDataKinds(fieldTypes));
      })
      .Default([](const llvm::abi::Type *) -> mlir::Type { return nullptr; });
}
 
/// Returns \p ty with a LongDoubleType vector element replaced by its
/// underlying format, which is the element abiTypeToCIR builds for the same
/// float.
static mlir::Type unwrapLongDoubleVectorElement(mlir::Type ty) {
  if (auto vecTy = dyn_cast<cir::VectorType>(ty)) {
    if (auto longDoubleTy =
            dyn_cast<cir::LongDoubleType>(vecTy.getElementType()))
      return cir::VectorType::get(longDoubleTy.getUnderlying(),
                                  vecTy.getSize());
  }
  return ty;
}
 
/// Map a CIR type to an llvm::abi::Type.  classifySignature pre-filters
/// the signature, so only the scalar and struct/array types handled here can
/// reach this function.
static const llvm::abi::Type *mapCIRType(mlir::Type type,
                                         mlir::abi::ABITypeMapper &typeMapper,
                                         const DataLayout &dl, ModuleOp modOp) {
  llvm::abi::TypeBuilder &tb = typeMapper.getTypeBuilder();
  return llvm::TypeSwitch<mlir::Type, const llvm::abi::Type *>(type)
      .Case([&](cir::IntType intTy) {
        return tb.getIntegerType(intTy.getWidth(),
                                 llvm::Align(dl.getTypeABIAlignment(type)),
                                 intTy.isSigned(), intTy.getIsBitInt());
      })
      .Case([&](cir::PointerType ptrTy) {
        unsigned addrSpace = 0;
        if (auto targetAsAttr =
                dyn_cast_if_present<cir::TargetAddressSpaceAttr>(
                    ptrTy.getAddrSpace()))
          addrSpace = targetAsAttr.getValue();
        // Any address-space attribute is a qualifier.
        bool hasAddressSpaceQualifier = ptrTy.getAddrSpace() != nullptr;
        return tb.getPointerType(dl.getTypeSizeInBits(type),
                                 llvm::Align(dl.getTypeABIAlignment(type)),
                                 addrSpace, hasAddressSpaceQualifier);
      })
      .Case([&](cir::VPtrType) {
        // cir::VPtrType carries no address-space parameter yet, so this
        // always maps into the default one until that gap closes.
        assert(!cir::MissingFeatures::addressSpace());
        return tb.getPointerType(dl.getTypeSizeInBits(type),
                                 llvm::Align(dl.getTypeABIAlignment(type)));
      })
      .Case([&](cir::BoolType) {
        // A bool is a one-bit integer to the classifier, as QualTypeMapper
        // maps it, so a bool vector is sized one bit per element.
        return tb.getIntegerType(1, llvm::Align(dl.getTypeABIAlignment(type)),
                                 /*Signed=*/false);
      })
      .Case([&](cir::VoidType) { return tb.getVoidType(); })
      .Case([&](cir::FPTypeInterface fpTy) {
        // LongDoubleType reports its underlying format's semantics, so the
        // classifier sees x87 or IEEE quad rather than the wrapper.
        return tb.getFloatType(fpTy.getFloatSemantics(),
                               llvm::Align(dl.getTypeABIAlignment(type)));
      })
      .Case([&](cir::ComplexType complexTy) {
        return tb.getComplexType(
            mapCIRType(complexTy.getElementType(), typeMapper, dl, modOp),
            llvm::Align(dl.getTypeABIAlignment(type)));
      })
      .Case([&](cir::VectorType vecTy) {
        // isSupportedType rejects a scalable vector, so the element count is
        // always fixed here.
        return tb.getVectorType(
            mapCIRType(vecTy.getElementType(), typeMapper, dl, modOp),
            vecTy.getSize(), llvm::Align(dl.getTypeABIAlignment(type)));
      })
      .Case([&](cir::ArrayType arrTy) {
        const llvm::abi::Type *elemAbi =
            mapCIRType(arrTy.getElementType(), typeMapper, dl, modOp);
        return tb.getArrayType(elemAbi, arrTy.getSize(),
                               dl.getTypeSizeInBits(type).getFixedValue());
      })
      .Case([&](cir::RecordType recTy) -> const llvm::abi::Type * {
        llvm::abi::RecordFlags flags = llvm::abi::RecordFlags::None;
        if (recordCanPassInRegs(modOp, recTy))
          flags = flags | llvm::abi::RecordFlags::CanPassInRegisters;
        llvm::TypeSize sizeBits = llvm::TypeSize::getFixed(
            dl.getTypeSizeInBits(type).getFixedValue());
        llvm::Align align = recordDeclaredAlign(modOp, recTy, dl);
 
        // Mapped with no fields, an empty record reaches Ignore on its own.
        // The size still matters: past two eightbytes SysV says memory whatever
        // the content.
        // TODO: AArch64 needs the unadjusted alignment. We'll need to add that
        // to RecordLayoutAttr.
        if (recTy.isEmptyForABI())
          return tb.getRecordType(
              /*Fields=*/{}, sizeBits, align, /*UnadjustedAlign=*/align,
              llvm::abi::StructPacking::Default,
              /*BaseClasses=*/{}, /*VirtualBaseClasses=*/{}, flags);
 
        SmallVector<llvm::abi::FieldInfo> fields;
        fields.reserve(recTy.getMembers().size());
 
        // The classifier reads one entry per source bit-field, so a unit is
        // spread back out into the fields it holds.  x86_64 is little-endian,
        // so the unit's first byte holds the first of them, and the rest
        // follow at the widths ahead of them.
        //
        // A coercion for an eightbyte is built from the type at that
        // eightbyte's first byte, where the unit's first field sits, so that
        // field's entry carries the unit rather than its own declared type.
        // The bits holding user data are a separate question, and the answer
        // is the extent of the type each field was declared with, which the
        // eightbyte rules read through to decide whether a narrower coercion
        // would drop something.  One entry cannot answer both, so a unit
        // narrower than its first field's declaration takes an extra entry for
        // that declaration, holding no bits of its own.  A zero-width
        // bit-field has no unit and so contributes only the declaration.
        //
        // A width of zero is how the classifier recognizes the entries that
        // take no part in assigning eightbyte classes, so the width goes
        // through as declared.
        auto addAccessUnit = [&](cir::BitFieldType bfTy,
                                 uint64_t unitOffsetBits) {
          mlir::Type unitTy = bfTy.getStorageType();
          uint64_t unitBits =
              unitTy ? dl.getTypeSizeInBits(unitTy).getFixedValue() : 0;
          for (auto [idx, decl] : llvm::enumerate(bfTy.getFields())) {
            const uint64_t offsetBits =
                unitOffsetBits + bfTy.getFieldBitOffset(idx);
            const bool holdsUnit = idx == 0 && unitTy;
            if (holdsUnit)
              fields.push_back(llvm::abi::FieldInfo(
                  mapCIRType(unitTy, typeMapper, dl, modOp), offsetBits,
                  /*IsBitField=*/true, decl.getWidth(),
                  /*IsUnnamedBitField=*/decl.getIsUnnamed()));
            mlir::Type declaredTy = decl.getDeclaredType();
            if (!holdsUnit ||
                dl.getTypeSizeInBits(declaredTy).getFixedValue() > unitBits)
              fields.push_back(llvm::abi::FieldInfo(
                  mapCIRType(declaredTy, typeMapper, dl, modOp), offsetBits,
                  /*IsBitField=*/true, holdsUnit ? 0 : decl.getWidth(),
                  /*IsUnnamedBitField=*/holdsUnit || decl.getIsUnnamed()));
          }
        };
 
        // The size passed here spans the tail padding, so an eightbyte covers
        // the whole union rather than just the member the classifier reduces
        // it to.
        if (recTy.isUnion()) {
          // Classify only the variants that supply data, judged by each
          // variant's own storage rather than by its mark.  One supplying
          // none is skipped so it does not appear as a spurious argument.
          // Whether the record is empty at all is settled from the marks
          // before this.  Every variant starts at offset zero, so a
          // bit-field variant needs no offset of its own.
          assert(!llvm::is_contained(recTy.getMemberKinds(),
                                     cir::RecordMemberKind::Pad) &&
                 "a union member cannot be marked pad");
          for (mlir::Type variantTy : recTy.getMembers()) {
            if (auto bfTy = dyn_cast<cir::BitFieldType>(variantTy)) {
              // A variant that is an unnamed bit-field holds data all the
              // same, so it is flagged rather than skipped, as in the struct
              // path below.
              addAccessUnit(bfTy, /*unitOffsetBits=*/0);
              continue;
            }
            if (dl.getTypeSizeInBits(variantTy).getFixedValue() == 0 ||
                memberIsEmptyRecord(variantTy))
              continue;
            fields.push_back(llvm::abi::FieldInfo(
                mapCIRType(variantTy, typeMapper, dl, modOp)));
          }
          return tb.getUnionType(fields, sizeBits, align,
                                 /*UnadjustedAlign=*/align,
                                 llvm::abi::StructPacking::Default, flags);
        }
 
        // Padding is dropped, so the eightbyte rules see its bytes as holding
        // nothing, while the record's full size set above still spans them.  An
        // unnamed bit-field is flagged rather than dropped, since classic
        // CodeGen ignores it when assigning eightbyte classes but counts it
        // when choosing the coerce type, and a member's presence alone cannot
        // say both.
        for (auto [idx, fieldTy, kind] :
             llvm::enumerate(recTy.getMembers(), recTy.getMemberKinds())) {
          if (kind == cir::RecordMemberKind::Pad)
            continue;
          if (auto bfTy = dyn_cast<cir::BitFieldType>(fieldTy)) {
            addAccessUnit(bfTy, recTy.getElementOffset(dl, idx) * 8);
            continue;
          }
          // An `empty` member that is not a bit-field still stands for bytes
          // no field of the source reads, so it is flagged the same way.
          bool isUnnamed = kind == cir::RecordMemberKind::Empty;
          uint64_t widthBits = dl.getTypeSizeInBits(fieldTy).getFixedValue();
          if (isUnnamed && widthBits == 0)
            continue;
          assert((!isUnnamed || !memberIsEmptyRecord(fieldTy)) &&
                 "an empty-for-ABI member must not reach the classifier as an "
                 "unnamed bit-field");
          fields.push_back(llvm::abi::FieldInfo(
              mapCIRType(fieldTy, typeMapper, dl, modOp),
              recTy.getElementOffset(dl, idx) * 8,
              /*IsBitField=*/isUnnamed, isUnnamed ? widthBits : 0,
              /*IsUnnamedBitField=*/isUnnamed));
        }
 
        return tb.getRecordType(
            fields, sizeBits, align, /*UnadjustedAlign=*/align,
            llvm::abi::StructPacking::Default,
            /*BaseClasses=*/{}, /*VirtualBaseClasses=*/{}, flags);
      })
      .Default([](mlir::Type) -> const llvm::abi::Type * {
        llvm_unreachable(
            "mapCIRType: type not pre-filtered by classifySignature");
      });
}
 
/// Convert an llvm::abi::ArgInfo into the ArgClassification consumed by
/// CIRABIRewriteContext.
///
/// Direct: the value passes in register(s).  A coercion is forwarded in the
/// cases where the value has to be rebuilt on the wire: an aggregate unpacked
/// into the register(s) holding it, a scalar too wide for one register split
/// into a tuple of them, a scalar the classifier widens to fill its
/// eightbyte, a pointer moved into another address space, and a value whose
/// live bytes start partway into its storage because a leading eightbyte
/// holds no field (getDirectOffset).  canFlatten
/// follows the classifier's CanBeFlattened, so the rewriter splits a
/// multi-field coerced struct into individual wire arguments unless the
/// classifier asked to keep it intact.  Any other scalar passes in its
/// natural CIR type, which a null coercion denotes.  A multi-field struct with
/// a null coercion is its own coerced type, so the rewriter splits it into one
/// wire argument per field as classic codegen does.  A coercion this bridge
/// cannot represent yields std::nullopt so the caller reports NYI rather than
/// silently passing the value unchanged.
///
/// Extend: bool or a sub-register integer needs a signext/zeroext attribute.
/// The x86_64 classifier (llvm/lib/ABI/Targets/X86.cpp) only returns Extend
/// for an integer or bool operand, so any other origTy is asserted rather
/// than silently handled.
///
/// Indirect: an aggregate that does not fit in registers, or a scalar too
/// wide to fit (a `_BitInt` over 128 bits), is passed via a pointer (sret
/// for returns, byval for arguments).
///
/// Ignore: a void return, or an empty record dropped from the signature.
static std::optional<ArgClassification>
convertABIArgInfo(const llvm::abi::ArgInfo &info, MLIRContext *ctx,
                  mlir::Type origTy) {
  if (info.isDirect()) {
    unsigned offset = info.getDirectOffset();
    // The classifier names a coerce type even where it matches the natural
    // type, so a non-null coerce does not by itself mean a rewrite is needed.
    const llvm::abi::Type *coerceAbi = info.getCoerceToType();
    if (!coerceAbi) {
      auto recTy = dyn_cast_if_present<cir::RecordType>(origTy);
      if (!recTy || !recTy.isStruct() || !info.getCanBeFlattened())
        return ArgClassification::getDirect();
      ArgClassification classified =
          ArgClassification::getDirect(recTy, /*offset=*/0);
      classified.canFlatten = true;
      return classified;
    }
    bool isAggregate = isa_and_present<cir::RecordType, cir::ArrayType>(origTy);
    // For a _Complex or a vector the classifier's coerce is only sometimes the
    // natural type, so it has to be read rather than assumed.
    bool comparesAgainstCoerce =
        isa_and_present<cir::ComplexType, cir::VectorType>(origTy);
    bool coerceIsRegisterTuple =
        isa_and_present<llvm::abi::RecordType>(coerceAbi);
    // A tuple coercion cannot be read at an offset, so refuse the pair rather
    // than emit a read of the wrong bytes.
    if (offset && coerceIsRegisterTuple)
      return std::nullopt;
    // A pointer moved to another address space keeps its pointee, which
    // abiTypeToCIR cannot recover.
    auto origPtr = dyn_cast_if_present<cir::PointerType>(origTy);
    const auto *coercePtr =
        dyn_cast_if_present<llvm::abi::PointerType>(coerceAbi);
    if (origPtr && coercePtr && !offset) {
      // Compare numbers, since an explicit address_space(0) keeps a
      // target_address_space(0) attribute that a plain pointer lacks.
      auto origAS = dyn_cast_if_present<cir::TargetAddressSpaceAttr>(
          origPtr.getAddrSpace());
      if (coercePtr->getAddrSpace() != (origAS ? origAS.getValue() : 0)) {
        auto coercedPtr = cast<cir::PointerType>(abiTypeToCIR(coerceAbi, ctx));
        ArgClassification classified = ArgClassification::getDirect(
            cir::PointerType::get(origPtr.getPointee(),
                                  coercedPtr.getAddrSpace()),
            /*offset=*/0);
        classified.canFlatten = info.getCanBeFlattened();
        return classified;
      }
    }
    // Compare widths rather than identity: a coerce no wider than the natural
    // type carries the same value and needs no rewrite.
    auto origInt = dyn_cast_if_present<cir::IntType>(origTy);
    const auto *coerceInt =
        dyn_cast_if_present<llvm::abi::IntegerType>(coerceAbi);
    bool coerceWidensScalar =
        origInt && coerceInt &&
        coerceInt->getSizeInBits().getFixedValue() > origInt.getWidth();
    // Leaving the rest alone also avoids a lossy round trip: abiTypeToCIR
    // drops the LongDoubleType wrapper and a pointer's pointee, so comparing a
    // scalar against its own coerce would report a difference that is not
    // one.  Passing a value through reads it from byte 0, so one living at
    // an offset cannot take this path.
    if (!offset && !isAggregate && !comparesAgainstCoerce &&
        !coerceIsRegisterTuple && !coerceWidensScalar)
      return ArgClassification::getDirect();
    mlir::Type coerced = abiTypeToCIR(coerceAbi, ctx);
    if (!coerced)
      return std::nullopt;
    bool coerceIsNatural = coerced == unwrapLongDoubleVectorElement(origTy);
    // An offset would be lost here, since a coercion to the value's own type
    // is indistinguishable from no coercion at all.
    if (offset && coerceIsNatural)
      return std::nullopt;
    // Coercing a value to the type it already has would add a memory round
    // trip for nothing.
    if (comparesAgainstCoerce && coerceIsNatural)
      return ArgClassification::getDirect();
    ArgClassification classified =
        ArgClassification::getDirect(coerced, offset);
    classified.canFlatten = info.getCanBeFlattened();
    return classified;
  }
  // An extended value is always read from byte 0 of its own storage, so
  // there is no offset to honor here.
  if (info.isExtend()) {
    if (isa_and_present<cir::BoolType>(origTy))
      return ArgClassification::getExtend(nullptr, info.isSignExt());
    assert((!origTy || isa<cir::IntType>(origTy)) &&
           "the x86_64 classifier only returns Extend for integers and bool");
    mlir::Type extendedTy = abiTypeToCIR(info.getCoerceToType(), ctx);
    return ArgClassification::getExtend(extendedTy, info.isSignExt());
  }
  // ArgClassification has no address space for the indirect pointer yet.
  if (info.isIndirectAliased() ||
      (info.isIndirect() && info.getIndirectAddrSpace() != 0))
    return std::nullopt;
  if (info.isIndirect())
    return ArgClassification::getIndirect(info.getIndirectAlign(),
                                          info.getIndirectByVal(),
                                          info.getIndirectAddrSpace());
  // AArch64 pure scalable aggregates use CoerceAndExpand. This bridge lowers
  // x86_64 classifications only.
  assert(!info.isCoerceAndExpand() &&
         "CoerceAndExpand is not expected for x86_64");
  assert(info.isIgnore() && "Unexpected classification");
  return ArgClassification::getIgnore();
}
 
/// Where \p fnTy's declared parameters end and its ellipsis arguments begin.
///
/// The only x86_64 rule that reads this boundary sends an unnamed vector wider
/// than 128 bits to memory.
llvm::abi::RequiredArgs cir::requiredArgs(cir::FuncType fnTy) {
  if (!fnTy.isVarArg())
    return llvm::abi::RequiredArgs::All;
  return llvm::abi::RequiredArgs(fnTy.getNumInputs());
}
 
/// Classify a signature (return type + argument types) using the LLVM ABI
/// library.  Shared by the cir.func path, the variadic-call path and the
/// indirect-call path (the latter classifies from the callee function
/// pointer's pointee FuncType).  \p required marks where the declared
/// parameters in \p inputs end.  The classifier treats every argument past that
/// point as passed through an ellipsis.  \p callConv is the calling convention
/// the signature is classified under.  Returns std::nullopt and emits an NYI
/// error via \p emitError if the signature uses a type the bridge does not
/// handle yet.
std::optional<FunctionClassification> CIRABIClassifierCommon::classifySignature(
    mlir::Type retCIR, mlir::TypeRange inputs, llvm::abi::RequiredArgs required,
    const llvm::abi::TargetInfo &targetInfo, llvm::CallingConv::ID callConv,
    llvm::function_ref<mlir::InFlightDiagnostic()> emitError) {
  assert(retCIR && "signature return type must be non-null");
  assert((!required.allowsOptionalArgs() ||
          required.getNumRequiredArgs() <= inputs.size()) &&
         "declared parameters cannot outnumber the classified arguments");
  MLIRContext *ctx = modOp->getContext();
  bool voidRet = isa<cir::VoidType>(retCIR);
 
  auto reject = [&](mlir::Type t) -> bool {
    if (isSupportedType(t, dl))
      return false;
    emitError() << "calling-convention lowering not yet implemented for type "
                << t;
    return true;
  };
  if (!voidRet && reject(retCIR))
    return std::nullopt;
  for (mlir::Type a : inputs)
    if (reject(a))
      return std::nullopt;
 
  const llvm::abi::Type *retAbi =
      voidRet ? typeMapper.getTypeBuilder().getVoidType()
              : mapCIRType(retCIR, typeMapper, dl, modOp);
  SmallVector<const llvm::abi::Type *> argAbi;
  for (mlir::Type a : inputs)
    argAbi.push_back(mapCIRType(a, typeMapper, dl, modOp));
 
  std::unique_ptr<llvm::abi::FunctionInfo> fi =
      llvm::abi::FunctionInfo::create(callConv, retAbi, argAbi, required);
  targetInfo.computeInfo(*fi);
 
  // convertABIArgInfo returns nullopt when the classifier picks a coercion this
  // bridge cannot represent.
  auto nyiCoercion = [&](mlir::Type t) {
    emitError() << "calling-convention lowering not yet "
                   "implemented for the ABI coercion of type "
                << t;
  };
 
  FunctionClassification fc;
  fc.returnsVoid = voidRet;
  mlir::Type origRet = voidRet ? mlir::Type() : retCIR;
  std::optional<ArgClassification> retAc =
      convertABIArgInfo(fi->getReturnInfo(), ctx, origRet);
  if (!retAc) {
    nyiCoercion(retCIR);
    return std::nullopt;
  }
  fc.returnInfo = *retAc;
  for (unsigned i = 0, e = fi->arg_size(); i < e; ++i) {
    mlir::Type origArg = i < inputs.size() ? inputs[i] : mlir::Type();
    const llvm::abi::ArgInfo &argInfo = fi->getArgInfo(i).Info;
    std::optional<ArgClassification> ac =
        convertABIArgInfo(argInfo, ctx, origArg);
    if (!ac) {
      nyiCoercion(origArg);
      return std::nullopt;
    }
    ac->neededIntRegs = argInfo.getNeededIntRegs();
    ac->neededSseRegs = argInfo.getNeededSseRegs();
    fc.argInfos.push_back(*ac);
  }
  return fc;
}
 
/// The AVX level to classify \p func at: \p base, raised if the function's own
/// recorded feature list enables a wider vector.
llvm::abi::X86AVXABILevel cir::funcAvxLevel(cir::FuncOp func,
                                            llvm::abi::X86AVXABILevel base) {
  // Only a `target` attribute may raise the level.  A multiversioned function
  // carries a raised feature list too, and must stay at the module's level.
  assert(!cir::MissingFeatures::opFuncMultiVersioning());
 
  auto features = func->getAttrOfType<mlir::StringAttr>("cir.target-features");
  if (!features)
    return base;
  // A '-' entry disables the feature, so match a whole '+' entry rather than
  // searching for the name.  avx512f implies avx, so both entries are present:
  // return on the wider name so one pass suffices.
  bool avx = false;
  for (llvm::StringRef feature : llvm::split(features.getValue(), ',')) {
    if (!feature.consume_front("+"))
      continue;
    if (feature == "avx512f")
      return std::max(base, llvm::abi::X86AVXABILevel::AVX512);
    avx |= feature == "avx";
  }
  return avx ? std::max(base, llvm::abi::X86AVXABILevel::AVX) : base;
}
 
/// Classify a cir.func using the LLVM ABI library.  Returns std::nullopt and
/// emits an NYI error if the signature uses a type the bridge does not handle
/// yet.
std::optional<FunctionClassification>
CIRABIClassifierCommon::classifyFunction(cir::FuncOp func) {
  cir::FuncType fnTy = func.getFunctionType();
  return classifySignature(fnTy.getReturnType(), fnTy.getInputs(),
                           requiredArgs(fnTy), getTargetInfo(func),
                           convertCallingConv(func.getCallingConv()),
                           [&]() { return func.emitOpError(); });
}
 
/// Classify the single type fetched by a `cir.va_arg` as an unnamed argument
/// (RequiredArgs(0)) with a full register budget.  Returns std::nullopt and
/// emits an NYI error if the type is not one this bridge handles.
std::optional<ArgClassification> CIRABIClassifierCommon::classifyVarArgType(
    mlir::Type ty, const llvm::abi::TargetInfo &targetInfo,
    llvm::function_ref<mlir::InFlightDiagnostic()> emitError) {
  MLIRContext *ctx = modOp->getContext();
  std::optional<FunctionClassification> fc = classifySignature(
      cir::VoidType::get(ctx), mlir::TypeRange(ty), llvm::abi::RequiredArgs(0),
      targetInfo, llvm::CallingConv::C, emitError);
  if (!fc)
    return std::nullopt;
  return fc->argInfos[0];
}
 
/// Classify a call that passes arguments through an ellipsis.  The callee's
/// own classification covers only its declared parameters, but an ellipsis
/// argument competes for the same argument registers as a declared one, so
/// what the ABI does with it depends on the whole argument list: the same
/// small struct is passed in registers early in the list and in memory once
/// the integer registers are gone.  Classifying from the call's operands
/// rather than the callee's signature is what makes that accounting right.
std::optional<FunctionClassification>
CIRABIClassifierCommon::classifyVariadicCall(
    cir::CIRCallOpInterface call, cir::FuncType calleeTy,
    const llvm::abi::TargetInfo &targetInfo, llvm::CallingConv::ID callConv) {
  assert(calleeTy.isVarArg() &&
         "only a variadic callee can take more operands than it declares");
  Operation *op = call.getOperation();
  return classifySignature(calleeTy.getReturnType(),
                           call.getArgOperands().getTypes(),
                           requiredArgs(calleeTy), targetInfo, callConv,
                           [&]() { return op->emitOpError(); });
}
 
/// Whether \p fc gives the callee access to memory through a pointer the ABI
/// introduced: an sret slot for an indirect return, or an indirect argument.
static bool hasIndirectSlot(const FunctionClassification &fc) {
  if (fc.returnInfo.kind == ArgKind::Indirect)
    return true;
  return llvm::any_of(fc.argInfos, [](const ArgClassification &ac) {
    return ac.kind == ArgKind::Indirect;
  });
}
 
/// Record on \p op that the memory its pointer arguments address may be read
/// and written.  Absent effects already allow that, since an absent attribute
/// means the effects are unknown, so they are left absent.
template <typename OpTy> static void widenForArgMemory(OpTy op) {
  cir::MemoryEffectsAttr effects = op.getMemoryEffectsAttr();
  if (!effects)
    return;
  op.setMemoryEffectsAttr(cir::MemoryEffectsAttr::get(
      effects.getContext(), effects.getOther(), cir::ModRefInfo::ModRef,
      effects.getInaccessibleMem(), effects.getErrnoMem(),
      effects.getTargetMem0(), effects.getTargetMem1()));
}
 
#ifndef NDEBUG
/// Whether \p callFc classifies a call's leading arguments and its return
/// exactly as \p calleeFc classifies the callee's declared parameters and
/// return.  A function definition and its call sites are rewritten from
/// separate classifications, so the two would silently disagree on the wire
/// format if an ellipsis argument could ever change how a declared parameter
/// is passed.
static bool classifiesSamePrefix(const FunctionClassification &calleeFc,
                                 const FunctionClassification &callFc) {
  if (callFc.argInfos.size() < calleeFc.argInfos.size())
    return false;
  return calleeFc.returnInfo == callFc.returnInfo &&
         std::equal(calleeFc.argInfos.begin(), calleeFc.argInfos.end(),
                    callFc.argInfos.begin());
}
#endif
 
namespace {
 
struct CallConvLoweringPass
    : public impl::CallConvLoweringBase<CallConvLoweringPass> {
  using CallConvLoweringBase::CallConvLoweringBase;
 
  CallConvLoweringPass(const CallConvLoweringOptions &options,
                       const llvm::abi::X86ABICompatInfo &x86AbiCompat)
      : CallConvLoweringBase(options), x86AbiCompat(x86AbiCompat) {}
 
  void runOnOperation() override;
 
  /// The x86_64 flags whose value depends on the target and the requested ABI
  /// compatibility version.  Carried outside the pass options because the
  /// struct has no command-line parser, so a cir-opt run gets the library
  /// defaults rather than a target's values.
  llvm::abi::X86ABICompatInfo x86AbiCompat;
};
 
} // namespace
 
/// Record on \p fc whether \p returnType is CIR's void.  The x86_64 classifier
/// answers this itself, but the other two drivers cannot: the test target is
/// dialect-neutral and has no notion of CIR's void, and the
/// classification-attr schema carries no return type at all.  Both route
/// through here so a classification always reaches needsRewrite paired with
/// the return type it was built from.
std::optional<FunctionClassification>
cir::withReturnVoidness(std::optional<FunctionClassification> fc,
                        mlir::Type returnType) {
  if (fc)
    fc->returnsVoid = mlir::isa<cir::VoidType>(returnType);
  return fc;
}
 
/// Classify \p func using whichever driver mode is configured.  Returns
/// std::nullopt and emits an error on the function if classification fails
/// (e.g. injection-driver mode but the function is missing the attribute,
/// or the attribute is malformed).
std::optional<FunctionClassification>
cir::classifyFunction(cir::FuncOp func, const DataLayout &dl,
                      cir::CallConvTarget target,
                      StringRef classificationAttrName) {
  ArrayRef<Type> argTypes = func.getFunctionType().getInputs();
  Type returnType = func.getFunctionType().getReturnType();
 
  if (!classificationAttrName.empty()) {
    auto attr = func->getAttrOfType<DictionaryAttr>(classificationAttrName);
    if (!attr) {
      func.emitOpError()
          << "missing classification attribute '" << classificationAttrName
          << "' (CallConvLowering driver mode 'classification-attr')";
      return std::nullopt;
    }
    return withReturnVoidness(mlir::abi::test::parseClassificationAttr(
                                  attr, [&]() { return func.emitOpError(); }),
                              returnType);
  }
 
  // The x86_64 and AMDGPU targets classify through CIRABIClassifierCommon, so
  // only the test target reaches here.
  assert(target == cir::CallConvTarget::Test &&
         "classifyFunction only handles the test target");
  return withReturnVoidness(mlir::abi::test::classify(argTypes, returnType, dl),
                            returnType);
}
 
/// Build the classifier the pass options select.  Exactly one of \p target
/// and \p classificationAttrName is set, which runOnOperation checks before
/// calling.
static std::unique_ptr<CIRABIClassifier>
createABIClassifier(cir::CallConvTarget target,
                    StringRef classificationAttrName, const DataLayout &dl,
                    ModuleOp modOp, const ABIClassifierOptions &options) {
  if (!classificationAttrName.empty())
    return createInjectedABIClassifier(dl, classificationAttrName);
 
  switch (target) {
  case cir::CallConvTarget::Test:
    return createTestTargetABIClassifier(dl);
  case cir::CallConvTarget::X86_64:
    return createX86_64ABIClassifier(dl, modOp, options);
  case cir::CallConvTarget::AMDGPU:
    return createAMDGPUABIClassifier(dl, modOp);
  case cir::CallConvTarget::None:
    break;
  }
  llvm_unreachable("an unset driver mode is rejected before this point");
}
 
namespace {
 
/// Find the cir.func declaration matching a direct cir.call / cir.try_call
/// callee, if any.  Returns nullptr if the callee is indirect or the symbol
/// cannot be resolved.  Takes a SymbolTable instead of a ModuleOp so the
/// symbol lookup is amortized across all the call sites the driver walks
/// (ModuleOp::lookupSymbol is linear per call).
cir::FuncOp lookupCallee(Operation *callOp, SymbolTable &symbolTable) {
  FlatSymbolRefAttr callee;
  if (auto call = dyn_cast<cir::CallOp>(callOp))
    callee = call.getCalleeAttr();
  else if (auto tryCall = dyn_cast<cir::TryCallOp>(callOp))
    callee = tryCall.getCalleeAttr();
  else
    return nullptr;
  if (!callee)
    return nullptr;
  return symbolTable.lookup<cir::FuncOp>(callee.getValue());
}
 
void CallConvLoweringPass::runOnOperation() {
  ModuleOp moduleOp = getOperation();
  MLIRContext *ctx = &getContext();
 
  bool haveTarget = target != cir::CallConvTarget::None;
  bool haveAttr = !classificationAttr.empty();
  if (haveTarget == haveAttr) {
    moduleOp.emitOpError() << "CallConvLowering requires exactly one of "
                              "'target' or 'classification-attr' pass options";
    signalPassFailure();
    return;
  }
 
  if (!moduleOp->hasAttr(DLTIDialect::kDataLayoutAttrName)) {
    moduleOp.emitOpError()
        << "CallConvLowering requires a DataLayout (dlti.dl_spec attribute "
           "on the module)";
    signalPassFailure();
    return;
  }
 
  DataLayout dl(moduleOp);
  CIRABIRewriteContext rewriteCtx(moduleOp, dl);
  // A non-byval indirect parameter's slot outlives the rewrite that retypes
  // the parameter, so that a call forwarding the parameter can still recognize
  // it.  Draining on scope exit collapses those slots whichever way this
  // function returns.
  llvm::scope_exit drainParamSlots(
      [&] { rewriteCtx.finalizeParameterSlots(); });
  SymbolTable symbolTable(moduleOp);
 
  cir::ABIClassifierOptions classifierOptions;
  classifierOptions.x86AvxAbiLevel = x86AvxAbiLevel;
  classifierOptions.allowsX86TargetAttrAvx = allowsX86TargetAttrAvx;
  classifierOptions.x86AbiCompat = x86AbiCompat;
  std::unique_ptr<cir::CIRABIClassifier> classifier = createABIClassifier(
      target, classificationAttr, dl, moduleOp, classifierOptions);
 
  // Classify every cir.func up front.  No IR mutation happens here, so
  // later walks can consult any function's classification regardless of
  // visitation order.
  llvm::MapVector<cir::FuncOp, FunctionClassification> classifications;
  bool anyFailed = false;
  moduleOp.walk([&](cir::FuncOp f) {
    // A complete type is required at any call or definition, so only a
    // declaration can carry an incomplete-by-value parameter or return type,
    // and no translation unit can ever call or define it with real argument
    // data.  Leave it unclassified.
    cir::FuncType fnTy = f.getFunctionType();
    if (f.isDeclaration() &&
        (hasIncompleteRecordByValue(fnTy.getReturnType()) ||
         llvm::any_of(fnTy.getInputs(), hasIncompleteRecordByValue)))
      return;
    std::optional<FunctionClassification> fc = classifier->classifyFunction(f);
    if (!fc) {
      anyFailed = true;
      return;
    }
    classifications.insert({f, std::move(*fc)});
  });
  if (anyFailed) {
    signalPassFailure();
    return;
  }
 
  // Build a callee-to-callers index.  One module walk collects every direct
  // cir.call / cir.try_call to each cir.func; the loop below rewrites a
  // function and all of its call sites together.  Indirect or unresolved
  // callees are skipped here; rewriteCallSite errors on those at the end.
  //
  // A call that passes arguments through an ellipsis gets its own
  // classification, recorded here while every signature is still in its
  // original form.  The callee's classification covers only its declared
  // parameters and cannot describe those extra arguments.
  llvm::DenseMap<cir::FuncOp, SmallVector<Operation *>> callers;
  // Keyed on the call op collected below, looked up once when that same op is
  // rewritten.  A key must never come from an op created during the rewrite:
  // a recycled address could match an unrelated entry.
  llvm::DenseMap<Operation *, FunctionClassification> variadicCallSites;
  moduleOp.walk([&](Operation *op) {
    auto call = dyn_cast<cir::CIRCallOpInterface>(op);
    if (!call)
      return;
    cir::FuncOp callee = lookupCallee(op, symbolTable);
    if (!callee)
      return;
    callers[callee].push_back(op);
 
    cir::FuncType calleeTy = callee.getFunctionType();
    if (call.getNumArgOperands() <= calleeTy.getNumInputs())
      return;
    // A callee declared without a prototype also takes more operands than it
    // declares, and the verifier allows it.  Those extra arguments are named
    // rather than passed through an ellipsis, so the accounting below does not
    // describe them.
    if (!calleeTy.isVarArg()) {
      op->emitOpError() << "extra arguments to a callee without a prototype "
                           "not yet implemented in CallConvLowering";
      anyFailed = true;
      return;
    }
    std::optional<FunctionClassification> fc =
        classifier->classifyVariadicCall(call, callee);
    if (!fc) {
      anyFailed = true;
      return;
    }
    variadicCallSites.insert({op, std::move(*fc)});
  });
  if (anyFailed) {
    signalPassFailure();
    return;
  }
 
  // A cir.get_global holding a function's address carries the signature the
  // source wrote, which the verifier ties to the callee, so it goes stale when
  // that callee is rewritten.  A function address in a global initializer is a
  // GlobalViewAttr instead, whose recorded type the verifier does not tie to
  // the callee and which is dropped at opaque-pointer lowering, so it needs no
  // counterpart.
  llvm::DenseMap<cir::FuncOp, SmallVector<cir::GetGlobalOp>> addressTakers;
  moduleOp.walk([&](cir::GetGlobalOp getGlobal) {
    auto ptrTy = cast<cir::PointerType>(getGlobal.getAddr().getType());
    if (!isa<cir::FuncType>(ptrTy.getPointee()))
      return;
    // A get_global's pointee must equal the named symbol's type, and the
    // GlobalOp verifier rejects a function type there, so this names a
    // cir.func.
    auto callee = cast<cir::FuncOp>(symbolTable.lookup(getGlobal.getName()));
    addressTakers[callee].push_back(getGlobal);
  });
 
  // Restate every non-byval indirect parameter's slot alignment and route any
  // use of such a parameter as a call argument through a load of that slot,
  // before any definition or call site is rewritten.  Doing this up front
  // makes the forwarding decision independent of the order the callee and its
  // caller were declared in.
  for (auto &kv : classifications) {
    if (failed(rewriteCtx.prepareNonByvalParameters(kv.first, kv.second))) {
      signalPassFailure();
      return;
    }
  }
 
  // An sret slot or an indirect argument is a pointer the source never
  // wrote, and an ellipsis can carry one the declared parameters do not
  // describe.  The callee reads and writes that memory whatever const or
  // pure recorded about it, so the recorded effects have to widen.
  for (auto &kv : classifications) {
    cir::FuncOp func = kv.first;
    if (hasIndirectSlot(kv.second) || func.getFunctionType().isVarArg())
      widenForArgMemory(func);
  }
 
  // Rewrite each function together with every direct call to it and every op
  // holding its address.  By the time we move on to function F+1, F's
  // signature and every reference to F have already been brought into
  // alignment, and F+1..FN are still in their original (mutually consistent)
  // form, so the IR is verifier-clean at every outer-iteration boundary.
  //
  // There is still a brief inner window where F's signature has been
  // rewritten but its references have not yet caught up -- we have no way to
  // mutate both sides of a call atomically.  No verifier runs inside the
  // pass, and at pass exit the module is verifier-clean.  Fusing the inner
  // loops here keeps the invalid window per-function rather than module-wide.
  OpBuilder builder(ctx);
  for (auto &kv : classifications) {
    cir::FuncOp func = kv.first;
    const FunctionClassification &fc = kv.second;
    if (failed(rewriteCtx.rewriteFunctionDefinition(func, fc, builder))) {
      signalPassFailure();
      return;
    }
    for (Operation *callOp : callers.lookup(func)) {
      const FunctionClassification *callFc = &fc;
      if (auto it = variadicCallSites.find(callOp);
          it != variadicCallSites.end()) {
        callFc = &it->second;
        assert(classifiesSamePrefix(fc, *callFc) &&
               "a call site's declared parameters must be classified the same "
               "way as the callee's");
      }
      if (hasIndirectSlot(*callFc))
        widenForArgMemory(cast<cir::CIRCallOpInterface>(callOp));
      if (failed(rewriteCtx.rewriteCallSite(callOp, *callFc, builder))) {
        signalPassFailure();
        return;
      }
    }
    for (cir::GetGlobalOp addrOp : addressTakers.lookup(func))
      rewriteCtx.rewriteFunctionAddress(addrOp, func, builder);
  }
 
  // Rewrite indirect call sites.  The callee is opaque, so classify from the
  // function pointer's pointee FuncType and let rewriteCallSite retype the
  // callee pointer to match the coerced signature.  Collect the calls first:
  // when an sret rewrite reuses a single-use store's destination as the return
  // slot it erases that store, which is the operation a live walk has already
  // cached as the next one to visit.
  SmallVector<cir::CIRCallOpInterface> indirectCalls;
  moduleOp.walk([&](cir::CIRCallOpInterface c) {
    if (c.isIndirect())
      indirectCalls.push_back(c);
  });
  for (cir::CIRCallOpInterface c : indirectCalls) {
    std::optional<FunctionClassification> fc =
        classifier->classifyIndirectCall(c);
    if (!fc) {
      signalPassFailure();
      return;
    }
    if (hasIndirectSlot(*fc))
      widenForArgMemory(c);
    if (failed(rewriteCtx.rewriteCallSite(c.getOperation(), *fc, builder))) {
      signalPassFailure();
      return;
    }
  }
 
  // Collect the fetches before rewriting any, because rewriting one erases
  // it, and a live walk holds the op it is about to visit next.
  SmallVector<cir::VAArgOp> vaArgs;
  moduleOp.walk([&](cir::VAArgOp v) { vaArgs.push_back(v); });
  for (cir::VAArgOp v : vaArgs) {
    std::optional<ArgClassification> ac = classifier->classifyVAArg(v);
    if (!ac) {
      signalPassFailure();
      return;
    }
    if (failed(rewriteCtx.rewriteVAArg(v.getOperation(), *ac, builder))) {
      signalPassFailure();
      return;
    }
  }
}
 
} // namespace
 
std::unique_ptr<Pass> mlir::createCallConvLoweringPass() {
  return std::make_unique<CallConvLoweringPass>();
}
 
std::unique_ptr<Pass> mlir::createCallConvLoweringPass(
    cir::CallConvTarget target, llvm::abi::X86AVXABILevel x86AvxAbiLevel,
    bool allowsX86TargetAttrAvx,
    const llvm::abi::X86ABICompatInfo &x86AbiCompat) {
  CallConvLoweringOptions options;
  options.target = target;
  options.x86AvxAbiLevel = x86AvxAbiLevel;
  options.allowsX86TargetAttrAvx = allowsX86TargetAttrAvx;
  return std::make_unique<CallConvLoweringPass>(options, x86AbiCompat);
}