CIRDialect.cpp

Go to the documentation of this file.

//===- CIRDialect.cpp - MLIR CIR ops implementation -----------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file implements the CIR dialect and its operations.
//
//===----------------------------------------------------------------------===//
 
#include "clang/CIR/Dialect/IR/CIRDialect.h"
 
#include "clang/CIR/Dialect/IR/CIRAttrs.h"
#include "clang/CIR/Dialect/IR/CIROpsEnums.h"
#include "clang/CIR/Dialect/IR/CIRTypes.h"
 
#include "mlir/IR/Attributes.h"
#include "mlir/IR/BuiltinOps.h"
#include "mlir/IR/BuiltinTypes.h"
#include "mlir/IR/DialectImplementation.h"
#include "mlir/IR/PatternMatch.h"
#include "mlir/IR/Value.h"
#include "mlir/Interfaces/ControlFlowInterfaces.h"
#include "mlir/Interfaces/FunctionImplementation.h"
#include "mlir/Support/LLVM.h"
 
#include "clang/CIR/Dialect/IR/CIROpsDialect.cpp.inc"
#include "clang/CIR/Dialect/IR/CIROpsEnums.cpp.inc"
#include "clang/CIR/MissingFeatures.h"
#include "llvm/ADT/SetOperations.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/TypeSwitch.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/LogicalResult.h"
 
#include "CIRDialectBytecode.h"
 
using namespace mlir;
using namespace cir;
 
//===----------------------------------------------------------------------===//
// CIR Dialect
//===----------------------------------------------------------------------===//
namespace {
struct CIROpAsmDialectInterface : public OpAsmDialectInterface {
  using OpAsmDialectInterface::OpAsmDialectInterface;
 
  AliasResult getAlias(Type type, raw_ostream &os) const final {
    if (auto recordType = dyn_cast<cir::RecordType>(type)) {
      StringAttr nameAttr = recordType.getName();
      if (!nameAttr)
        os << "rec_anon_" << recordType.getKindAsStr();
      else
        os << "rec_" << nameAttr.getValue();
      return AliasResult::OverridableAlias;
    }
    if (auto intType = dyn_cast<cir::IntType>(type)) {
      // We only provide alias for standard integer types (i.e. integer types
      // whose width is a power of 2 and at least 8).
      unsigned width = intType.getWidth();
      if (width < 8 || !llvm::isPowerOf2_32(width))
        return AliasResult::NoAlias;
      os << intType.getAlias();
      return AliasResult::OverridableAlias;
    }
    if (auto voidType = dyn_cast<cir::VoidType>(type)) {
      os << voidType.getAlias();
      return AliasResult::OverridableAlias;
    }
 
    return AliasResult::NoAlias;
  }
 
  AliasResult getAlias(Attribute attr, raw_ostream &os) const final {
    if (auto boolAttr = mlir::dyn_cast<cir::BoolAttr>(attr)) {
      os << (boolAttr.getValue() ? "true" : "false");
      return AliasResult::FinalAlias;
    }
    if (auto bitfield = mlir::dyn_cast<cir::BitfieldInfoAttr>(attr)) {
      os << "bfi_" << bitfield.getName().str();
      return AliasResult::FinalAlias;
    }
    if (auto dynCastInfoAttr = mlir::dyn_cast<cir::DynamicCastInfoAttr>(attr)) {
      os << dynCastInfoAttr.getAlias();
      return AliasResult::FinalAlias;
    }
    if (auto cmpThreeWayInfoAttr =
            mlir::dyn_cast<cir::CmpThreeWayInfoAttr>(attr)) {
      os << cmpThreeWayInfoAttr.getAlias();
      return AliasResult::FinalAlias;
    }
    return AliasResult::NoAlias;
  }
};
} // namespace
 
void cir::CIRDialect::initialize() {
  registerTypes();
  registerAttributes();
  addOperations<
#define GET_OP_LIST
#include "clang/CIR/Dialect/IR/CIROps.cpp.inc"
      >();
  addInterfaces<CIROpAsmDialectInterface>();
  detail::addBytecodeInterface(this);
}
 
Operation *cir::CIRDialect::materializeConstant(mlir::OpBuilder &builder,
                                                mlir::Attribute value,
                                                mlir::Type type,
                                                mlir::Location loc) {
  return cir::ConstantOp::create(builder, loc, type,
                                 mlir::cast<mlir::TypedAttr>(value));
}
 
static bool isOpenCLVersionAttrName(StringRef attrName) {
  return attrName == CIRDialect::getOpenCLVersionAttrName() ||
         attrName == CIRDialect::getOpenCLCXXVersionAttrName() ||
         attrName == CIRDialect::getOpenCLSPIRVersionAttrName();
}
 
static LogicalResult verifyOpenCLVersionAttrPlacement(Operation *op,
                                                      NamedAttribute attr) {
  StringRef attrName = attr.getName().getValue();
  if (isa<ModuleOp>(op))
    return success();
 
  return op->emitError() << attrName
                         << " attribute must be attached to a module";
}
 
static bool areOpenCLVersionsCompatible(cir::OpenCLVersionAttr openCLVersion,
                                        cir::OpenCLVersionAttr cxxVersion) {
  return (openCLVersion.getMajor() == 2 && openCLVersion.getMinor() == 0 &&
          cxxVersion.getMajor() == 1 && cxxVersion.getMinor() == 0) ||
         (openCLVersion.getMajor() == 3 && openCLVersion.getMinor() == 0 &&
          cxxVersion.getMajor() == 2021 && cxxVersion.getMinor() == 0);
}
 
static LogicalResult verifyOpenCLCXXVersion(ModuleOp module,
                                            cir::OpenCLVersionAttr cxxVersion) {
  Attribute openCLAttr =
      module->getAttr(CIRDialect::getOpenCLVersionAttrName());
  if (!openCLAttr)
    return module.emitError()
           << "module attribute '" << CIRDialect::getOpenCLCXXVersionAttrName()
           << "' requires the companion attribute '"
           << CIRDialect::getOpenCLVersionAttrName() << "'";
 
  auto openCLVersion = dyn_cast<cir::OpenCLVersionAttr>(openCLAttr);
  if (!openCLVersion)
    return success();
 
  if (!areOpenCLVersionsCompatible(openCLVersion, cxxVersion))
    return module.emitError("incompatible OpenCL and C++ for OpenCL versions");
 
  return success();
}
 
static LogicalResult verifyOpenCLVersionAttr(Operation *op,
                                             NamedAttribute attr) {
  if (failed(verifyOpenCLVersionAttrPlacement(op, attr)))
    return failure();
 
  StringRef attrName = attr.getName().getValue();
  auto version = dyn_cast<cir::OpenCLVersionAttr>(attr.getValue());
  if (!version) {
    return op->emitError() << "expected " << attrName
                           << " to be #cir.cl.version";
  }
 
  if (attrName == CIRDialect::getOpenCLCXXVersionAttrName())
    return verifyOpenCLCXXVersion(cast<ModuleOp>(op), version);
 
  return success();
}
 
LogicalResult cir::CIRDialect::verifyRegionArgAttribute(
    Operation *op, unsigned /*regionIndex*/, unsigned /*argIndex*/,
    NamedAttribute attr) {
  if (!isOpenCLVersionAttrName(attr.getName().getValue()))
    return success();
 
  return verifyOpenCLVersionAttrPlacement(op, attr);
}
 
LogicalResult cir::CIRDialect::verifyRegionResultAttribute(
    Operation *op, unsigned /*regionIndex*/, unsigned /*resultIndex*/,
    NamedAttribute attr) {
  if (!isOpenCLVersionAttrName(attr.getName().getValue()))
    return success();
 
  return verifyOpenCLVersionAttrPlacement(op, attr);
}
 
//===----------------------------------------------------------------------===//
// Dialect attribute verification
//===----------------------------------------------------------------------===//
 
static LogicalResult verifyOffloadKind(mlir::ModuleOp module,
                                       cir::OffloadKind expected) {
  auto attr = module->getAttrOfType<cir::OffloadKindAttr>(
      cir::CIRDialect::getOffloadKindAttrName());
  if (!attr)
    return module.emitOpError()
           << "expects '" << cir::CIRDialect::getOffloadKindAttrName()
           << "' offload kind attribute";
  if (attr.getValue() != expected)
    return module.emitOpError()
           << "expects '" << cir::CIRDialect::getOffloadKindAttrName()
           << "' value '" << cir::stringifyOffloadKind(expected) << "'";
  return success();
}
 
// A module marked with `cir.offload.container` holds the host module followed
// by one or more device modules, each tagged with `cir.offload.kind`. Keeping
// the host module first gives later offload passes a simple convention for
// finding the host side while iterating the remaining device modules.
static LogicalResult verifyOffloadContainer(mlir::Operation *op) {
  auto container = mlir::dyn_cast<mlir::ModuleOp>(op);
  if (!container)
    return op->emitOpError()
           << "expects '" << cir::CIRDialect::getOffloadContainerAttrName()
           << "' attribute to be attached to '"
           << mlir::ModuleOp::getOperationName() << "'";
 
  mlir::Block &body = *container.getBody();
  if (body.empty())
    return container.emitOpError()
           << "expects host module as the first nested op";
 
  auto host = mlir::dyn_cast<mlir::ModuleOp>(body.front());
  if (!host)
    return container.emitOpError()
           << "expects host module as the first nested op";
  if (failed(verifyOffloadKind(host, cir::OffloadKind::Host)))
    return failure();
 
  // At least one device module is required after the host module.
  if (std::next(body.begin()) == body.end())
    return container.emitOpError() << "expects at least one device module";
 
  for (auto &op : llvm::drop_begin(body)) {
    auto module = mlir::dyn_cast<mlir::ModuleOp>(op);
    if (!module)
      return container.emitOpError()
             << "expects only nested builtin.module ops";
    if (failed(verifyOffloadKind(module, cir::OffloadKind::Device)))
      return failure();
  }
  return success();
}
 
LogicalResult
cir::CIRDialect::verifyOperationAttribute(mlir::Operation *op,
                                          mlir::NamedAttribute attr) {
  llvm::StringRef attrName = attr.getName().getValue();
  if (isOpenCLVersionAttrName(attrName))
    return verifyOpenCLVersionAttr(op, attr);
 
  if (attrName == getOffloadContainerAttrName()) {
    if (!mlir::isa<mlir::UnitAttr>(attr.getValue()))
      return op->emitOpError() << "expects '" << getOffloadContainerAttrName()
                               << "' to be a unit attribute";
    return verifyOffloadContainer(op);
  }
 
  // The container verifier owns the structural contract between a container
  // and the modules it holds. All this can add is that the kind attribute
  // never lands on something that is not a module.
  if (attrName == getOffloadKindAttrName() && !mlir::isa<mlir::ModuleOp>(op))
    return op->emitOpError() << "expects '" << getOffloadKindAttrName()
                             << "' attribute to be attached to '"
                             << mlir::ModuleOp::getOperationName() << "'";
 
  if (attrName == getModuleFramePointerAttrName()) {
    if (!mlir::isa<mlir::ModuleOp>(op))
      return op->emitOpError() << "expects '" << getModuleFramePointerAttrName()
                               << "' attribute to be attached to '"
                               << mlir::ModuleOp::getOperationName() << "'";
    if (!mlir::isa<cir::FramePointerKindAttr>(attr.getValue()))
      return op->emitOpError() << "expects '" << getModuleFramePointerAttrName()
                               << "' to be a #cir.frame_pointer attribute";
    return success();
  }
 
  // LoweringPrepare uses this attribute directly as the fatbin global's
  // initializer, so it must be a valid #cir.const_array payload for its type.
  if (attrName == getCUDADeviceBinaryAttrName()) {
    auto bytes = mlir::dyn_cast<mlir::StringAttr>(attr.getValue());
    auto arrayTy =
        bytes ? mlir::dyn_cast<cir::ArrayType>(bytes.getType()) : nullptr;
    if (!arrayTy || arrayTy.getSize() != bytes.size())
      return op->emitOpError()
             << "expects '" << getCUDADeviceBinaryAttrName()
             << "' to be a string typed as an array of its length";
    return cir::ConstArrayAttr::verify([&] { return op->emitOpError(); },
                                       arrayTy, bytes,
                                       /*trailingZerosNum=*/0);
  }
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// Helpers
//===----------------------------------------------------------------------===//
 
// Parses one of the keywords provided in the list `keywords` and returns the
// position of the parsed keyword in the list. If none of the keywords from the
// list is parsed, returns -1.
static int parseOptionalKeywordAlternative(AsmParser &parser,
                                           ArrayRef<llvm::StringRef> keywords) {
  for (auto en : llvm::enumerate(keywords)) {
    if (succeeded(parser.parseOptionalKeyword(en.value())))
      return en.index();
  }
  return -1;
}
 
namespace {
template <typename Ty> struct EnumTraits {};
 
#define REGISTER_ENUM_TYPE(Ty)                                                 \
  template <> struct EnumTraits<cir::Ty> {                                     \
    static llvm::StringRef stringify(cir::Ty value) {                          \
      return stringify##Ty(value);                                             \
    }                                                                          \
    static unsigned getMaxEnumVal() { return cir::getMaxEnumValFor##Ty(); }    \
  }
 
REGISTER_ENUM_TYPE(GlobalLinkageKind);
REGISTER_ENUM_TYPE(VisibilityKind);
REGISTER_ENUM_TYPE(CallingConv);
} // namespace
 
/// Parse an enum from the keyword, or default to the provided default value.
/// The return type is the enum type by default, unless overriden with the
/// second template argument.
template <typename EnumTy, typename RetTy = EnumTy>
static RetTy parseOptionalCIRKeyword(AsmParser &parser, EnumTy defaultValue) {
  llvm::SmallVector<llvm::StringRef, 10> names;
  for (unsigned i = 0, e = EnumTraits<EnumTy>::getMaxEnumVal(); i <= e; ++i)
    names.push_back(EnumTraits<EnumTy>::stringify(static_cast<EnumTy>(i)));
 
  int index = parseOptionalKeywordAlternative(parser, names);
  if (index == -1)
    return static_cast<RetTy>(defaultValue);
  return static_cast<RetTy>(index);
}
 
/// Parse an enum from the keyword, return failure if the keyword is not found.
template <typename EnumTy, typename RetTy = EnumTy>
static ParseResult parseCIRKeyword(AsmParser &parser, RetTy &result) {
  llvm::SmallVector<llvm::StringRef, 10> names;
  for (unsigned i = 0, e = EnumTraits<EnumTy>::getMaxEnumVal(); i <= e; ++i)
    names.push_back(EnumTraits<EnumTy>::stringify(static_cast<EnumTy>(i)));
 
  int index = parseOptionalKeywordAlternative(parser, names);
  if (index == -1)
    return failure();
  result = static_cast<RetTy>(index);
  return success();
}
 
// Check if a region's termination omission is valid and, if so, creates and
// inserts the omitted terminator into the region.
static LogicalResult ensureRegionTerm(OpAsmParser &parser, Region &region,
                                      SMLoc errLoc) {
  Location eLoc = parser.getEncodedSourceLoc(parser.getCurrentLocation());
  OpBuilder builder(parser.getBuilder().getContext());
 
  // Insert empty block in case the region is empty to ensure the terminator
  // will be inserted
  if (region.empty())
    builder.createBlock(&region);
 
  Block &block = region.back();
  // Region is properly terminated: nothing to do.
  if (!block.empty() && block.back().hasTrait<OpTrait::IsTerminator>())
    return success();
 
  // Check for invalid terminator omissions.
  if (!region.hasOneBlock())
    return parser.emitError(errLoc,
                            "multi-block region must not omit terminator");
 
  // Terminator was omitted correctly: recreate it.
  builder.setInsertionPointToEnd(&block);
  cir::YieldOp::create(builder, eLoc);
  return success();
}
 
// True if the region's terminator should be omitted.
static bool omitRegionTerm(mlir::Region &r) {
  const auto singleNonEmptyBlock = r.hasOneBlock() && !r.back().empty();
  const auto yieldsNothing = [&r]() {
    auto y = dyn_cast<cir::YieldOp>(r.back().getTerminator());
    return y && y.getArgs().empty();
  };
  return singleNonEmptyBlock && yieldsNothing();
}
 
// Verifies that the given operand is produced by an operation of type
// ExpectedProducerOp.
template <typename ExpectedProducerOp>
static LogicalResult verifyProducedBy(Operation *op, Value operand,
                                      StringRef operandName) {
  Operation *producer = operand.getDefiningOp();
  if (!producer || !isa<ExpectedProducerOp>(producer))
    return op->emitOpError()
           << "operand '" << operandName << "' must be produced by '"
           << ExpectedProducerOp::getOperationName() << "'";
  return success();
}
 
//===----------------------------------------------------------------------===//
// InlineKindAttr (FIXME: remove once FuncOp uses assembly format)
//===----------------------------------------------------------------------===//
 
ParseResult parseInlineKindAttr(OpAsmParser &parser,
                                cir::InlineKindAttr &inlineKindAttr) {
  // Static list of possible inline kind keywords
  static constexpr llvm::StringRef keywords[] = {"no_inline", "always_inline",
                                                 "inline_hint"};
 
  // Parse the inline kind keyword (optional)
  llvm::StringRef keyword;
  if (parser.parseOptionalKeyword(&keyword, keywords).failed()) {
    // Not an inline kind keyword, leave inlineKindAttr empty
    return success();
  }
 
  // Parse the enum value from the keyword
  auto inlineKindResult = ::cir::symbolizeEnum<::cir::InlineKind>(keyword);
  if (!inlineKindResult) {
    return parser.emitError(parser.getCurrentLocation(), "expected one of [")
           << llvm::join(llvm::ArrayRef(keywords), ", ")
           << "] for inlineKind, got: " << keyword;
  }
 
  inlineKindAttr =
      ::cir::InlineKindAttr::get(parser.getContext(), *inlineKindResult);
  return success();
}
 
void printInlineKindAttr(OpAsmPrinter &p, cir::InlineKindAttr inlineKindAttr) {
  if (inlineKindAttr) {
    p << " " << stringifyInlineKind(inlineKindAttr.getValue());
  }
}
 
//===----------------------------------------------------------------------===//
// CIR Custom Parsers/Printers
//===----------------------------------------------------------------------===//
 
static mlir::ParseResult parseOmittedTerminatorRegion(mlir::OpAsmParser &parser,
                                                      mlir::Region &region) {
  auto regionLoc = parser.getCurrentLocation();
  if (parser.parseRegion(region))
    return failure();
  if (ensureRegionTerm(parser, region, regionLoc).failed())
    return failure();
  return success();
}
 
static void printOmittedTerminatorRegion(mlir::OpAsmPrinter &printer,
                                         cir::ScopeOp &op,
                                         mlir::Region &region) {
  printer.printRegion(region,
                      /*printEntryBlockArgs=*/false,
                      /*printBlockTerminators=*/!omitRegionTerm(region));
}
 
mlir::OptionalParseResult
parseGlobalMemorySpace(mlir::AsmParser &p,
                       mlir::ptr::MemorySpaceAttrInterface &attr);
 
void printGlobalMemorySpace(mlir::AsmPrinter &printer, cir::GlobalOp op,
                            mlir::ptr::MemorySpaceAttrInterface attr);
 
//===----------------------------------------------------------------------===//
// AllocaOp
//===----------------------------------------------------------------------===//
 
void cir::AllocaOp::build(mlir::OpBuilder &odsBuilder,
                          mlir::OperationState &odsState, mlir::Type addr,
                          llvm::StringRef name, mlir::IntegerAttr alignment) {
  odsState.addAttribute(getNameAttrName(odsState.name),
                        odsBuilder.getStringAttr(name));
  if (alignment) {
    odsState.addAttribute(getAlignmentAttrName(odsState.name), alignment);
  }
  odsState.addTypes(addr);
}
 
cir::AllocaOp cir::getUnderlyingAlloca(mlir::Value addr) {
  mlir::Value ptr = addr;
  while (cir::CastOp castOp = ptr.getDefiningOp<cir::CastOp>()) {
    if (!castOp.isAllocaPreservingCast())
      break;
    ptr = castOp.getSrc();
  }
  return ptr.getDefiningOp<cir::AllocaOp>();
}
 
//===----------------------------------------------------------------------===//
// ArrayCtor & ArrayDtor
//===----------------------------------------------------------------------===//
 
template <typename Op> static LogicalResult verifyArrayCtorDtor(Op op) {
  auto ptrTy = mlir::cast<cir::PointerType>(op.getAddr().getType());
  mlir::Type pointeeTy = ptrTy.getPointee();
 
  mlir::Block &body = op.getBody().front();
  if (body.getNumArguments() != 1)
    return op.emitOpError("body must have exactly one block argument");
 
  auto expectedEltPtrTy =
      mlir::dyn_cast<cir::PointerType>(body.getArgument(0).getType());
  if (!expectedEltPtrTy)
    return op.emitOpError("block argument must be a !cir.ptr type");
 
  if (op.getNumElements()) {
    auto recTy = mlir::dyn_cast<cir::RecordType>(pointeeTy);
    if (!recTy)
      return op.emitOpError(
          "when 'num_elements' is present, 'addr' must be a pointer to a "
          "!cir.struct or !cir.union type");
 
    if (expectedEltPtrTy != ptrTy)
      return op.emitOpError("when 'num_elements' is present, 'addr' type must "
                            "match the block argument type");
  } else {
    auto arrayTy = mlir::dyn_cast<cir::ArrayType>(pointeeTy);
    if (!arrayTy)
      return op.emitOpError(
          "when 'num_elements' is absent, 'addr' must be a pointer to a "
          "!cir.array type");
 
    mlir::Type innerEltTy = arrayTy.getElementType();
    while (auto nested = mlir::dyn_cast<cir::ArrayType>(innerEltTy))
      innerEltTy = nested.getElementType();
 
    auto recTy = mlir::dyn_cast<cir::RecordType>(innerEltTy);
    if (!recTy)
      return op.emitOpError("the block argument type must be a pointer to a "
                            "!cir.struct or !cir.union type");
 
    if (expectedEltPtrTy.getPointee() != innerEltTy)
      return op.emitOpError(
          "block argument pointee type must match the innermost array "
          "element type");
  }
 
  return success();
}
 
LogicalResult cir::ArrayCtor::verify() {
  if (failed(verifyArrayCtorDtor(*this)))
    return failure();
 
  mlir::Region &partialDtor = getPartialDtor();
  if (!partialDtor.empty()) {
    mlir::Block &dtorBlock = partialDtor.front();
    if (dtorBlock.getNumArguments() != 1)
      return emitOpError("partial_dtor must have exactly one block argument");
 
    auto bodyArgTy = getBody().front().getArgument(0).getType();
    if (dtorBlock.getArgument(0).getType() != bodyArgTy)
      return emitOpError("partial_dtor block argument type must match "
                         "the body block argument type");
  }
  return success();
}
LogicalResult cir::ArrayDtor::verify() { return verifyArrayCtorDtor(*this); }
 
//===----------------------------------------------------------------------===//
// DeleteArrayOp
//===----------------------------------------------------------------------===//
 
LogicalResult cir::DeleteArrayOp::verify() {
  if (getDtorMayThrow() && !getElementDtorAttr())
    return emitOpError(
        "'dtor_may_throw' requires an 'element_dtor' to be present");
  return success();
}
 
//===----------------------------------------------------------------------===//
// AssumeOp
//===----------------------------------------------------------------------===//
 
static void printAssumeBundle(OpAsmPrinter &p, cir::AssumeOp op,
                              cir::AssumeBundleKindAttr kindAttr,
                              OperandRange bundleArgs,
                              TypeRange bundleArgTypes) {
  cir::AssumeBundleKind kind = kindAttr.getValue();
  if (kind == cir::AssumeBundleKind::None)
    return;
 
  p << " " << cir::stringifyAssumeBundleKind(kind);
  if (bundleArgs.empty())
    return;
 
  p << "(";
  p.printOperands(bundleArgs);
  p << " : ";
  llvm::interleaveComma(bundleArgTypes, p);
  p << ")";
}
 
static ParseResult parseAssumeBundle(
    OpAsmParser &p, cir::AssumeBundleKindAttr &bundleKindAttr,
    llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand, 4> &bundleArgs,
    llvm::SmallVector<mlir::Type, 1> &bundleArgTypes) {
  StringRef keyword;
  auto loc = p.getCurrentLocation();
  if (failed(p.parseOptionalKeyword(&keyword))) {
    bundleKindAttr = cir::AssumeBundleKindAttr::get(
        p.getContext(), cir::AssumeBundleKind::None);
    return success();
  }
 
  std::optional<cir::AssumeBundleKind> parsedKind =
      cir::symbolizeAssumeBundleKind(keyword);
  if (!parsedKind)
    return p.emitError(loc, "unknown assume bundle kind '") << keyword << "'";
 
  bundleKindAttr = cir::AssumeBundleKindAttr::get(p.getContext(), *parsedKind);
 
  if (p.parseOptionalLParen())
    return success();
 
  if (p.parseOperandList(bundleArgs) || p.parseColon() ||
      p.parseTypeList(bundleArgTypes) || p.parseRParen())
    return failure();
 
  return success();
}
 
LogicalResult cir::AssumeOp::verify() {
  cir::AssumeBundleKind kind = getBundleKind();
  size_t numArgs = getBundleArgs().size();
 
  if (kind == cir::AssumeBundleKind::None) {
    if (numArgs != 0)
      return emitOpError("unexpected bundle operands for kind 'none'");
    return success();
  }
 
  if (numArgs == 0)
    return emitOpError("expected bundle operands for kind '")
           << cir::stringifyAssumeBundleKind(kind) << "'";
 
  switch (kind) {
  case cir::AssumeBundleKind::Align:
    if (numArgs != 2 && numArgs != 3)
      return emitOpError("align bundle expects 2 or 3 operands");
    break;
  case cir::AssumeBundleKind::SeparateStorage:
    if (numArgs != 2)
      return emitOpError("separate_storage bundle expects 2 operands");
    break;
  case cir::AssumeBundleKind::Dereferenceable:
    if (numArgs != 2)
      return emitOpError("dereferenceable bundle expects 2 operands");
    break;
  default:
    break;
  }
  return success();
}
 
//===----------------------------------------------------------------------===//
// LocalInitOp
//===----------------------------------------------------------------------===//
 
LogicalResult
cir::LocalInitOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
  cir::GlobalOp global = getReferencedGlobal(symbolTable);
  if (!global)
    return emitOpError("'")
           << getGlobalName() << "' does not reference a valid cir.global";
 
  if (getTls() && !global.getTlsModel())
    return emitOpError("access to global not marked thread local");
 
  if (!global.getDynamicInitGuard().has_value())
    return emitOpError("static_local attribute mismatch");
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// RegisterExitDtorOp
//===----------------------------------------------------------------------===//
 
/// Returns true if \p op is nested in the ctor region of a cir.global or
/// cir.local_init without crossing a cir.func or a cir.register_exit_dtor.
static bool isInVarCtorRegion(mlir::Operation *op) {
  for (mlir::Operation *parent = op->getParentOp(); parent;
       parent = parent->getParentOp()) {
    mlir::Region *ctorRegion = nullptr;
    if (auto global = mlir::dyn_cast<cir::GlobalOp>(parent))
      ctorRegion = &global.getCtorRegion();
    else if (auto localInit = mlir::dyn_cast<cir::LocalInitOp>(parent))
      ctorRegion = &localInit.getCtorRegion();
    else if (mlir::isa<cir::FuncOp, cir::RegisterExitDtorOp>(parent))
      return false;
    if (ctorRegion)
      return ctorRegion->isAncestor(op->getParentRegion());
  }
  return false;
}
 
LogicalResult cir::RegisterExitDtorOp::verify() {
  mlir::Block &body = getBody().front();
  if (body.without_terminator().empty())
    return emitOpError("body must destroy the object");
  if (body.getTerminator()->getNumOperands())
    return emitOpError("body must not yield a value");
  if (!isInVarCtorRegion(*this))
    return emitOpError("must be in the ctor region of a cir.global or "
                       "cir.local_init");
  return success();
}
 
LogicalResult
cir::RegisterExitDtorOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
  auto global = symbolTable.lookupNearestSymbolFrom<cir::GlobalOp>(
      *this, getObjectAttr());
  if (!global)
    return emitOpError("'")
           << getObject() << "' does not reference a valid cir.global";
  if (!global.getCtorRegion().empty() || !global.getDtorRegion().empty())
    return emitOpError("object must not have ctor or dtor regions");
  return success();
}
 
//===----------------------------------------------------------------------===//
// ConditionOp
//===----------------------------------------------------------------------===//
 
//===----------------------------------
// BranchOpTerminatorInterface Methods
//===----------------------------------
 
void cir::ConditionOp::getSuccessorRegions(
    ArrayRef<Attribute> operands, SmallVectorImpl<RegionSuccessor> &regions) {
  // TODO(cir): The condition value may be folded to a constant, narrowing
  // down its list of possible successors.
 
  // Parent is a loop: condition may branch to the body, or on the false edge
  // to the per-iteration cleanup region if present, otherwise to the parent op.
  if (auto loopOp = dyn_cast<LoopOpInterface>(getOperation()->getParentOp())) {
    regions.emplace_back(&loopOp.getBody());
    if (mlir::Region *cleanup = loopOp.maybeGetCleanup())
      regions.emplace_back(cleanup);
    else
      regions.emplace_back(getOperation());
    return;
  }
 
  // Parent is an await: condition may branch to resume or suspend regions.
  auto await = cast<AwaitOp>(getOperation()->getParentOp());
  regions.emplace_back(&await.getResume());
  regions.emplace_back(&await.getSuspend());
}
 
MutableOperandRange
cir::ConditionOp::getMutableSuccessorOperands(RegionSuccessor point) {
  // No values are yielded to the successor region.
  return MutableOperandRange(getOperation(), 0, 0);
}
 
MutableOperandRange
cir::ResumeOp::getMutableSuccessorOperands(RegionSuccessor point) {
  // The eh_token operand is not forwarded to the parent region.
  return MutableOperandRange(getOperation(), 0, 0);
}
 
MutableOperandRange
cir::CoroSuspendPoint::getMutableSuccessorOperands(RegionSuccessor point) {
  // The condition operand is not forwarded to successor regions.
  return MutableOperandRange(getOperation(), 0, 0);
}
 
LogicalResult cir::ConditionOp::verify() {
  if (!isa<LoopOpInterface, AwaitOp>(getOperation()->getParentOp()))
    return emitOpError("condition must be within a conditional region");
  return success();
}
 
//===----------------------------------------------------------------------===//
// WhileOp & ForOp
//===----------------------------------------------------------------------===//
 
template <typename LoopOpTy>
static LogicalResult verifyLoopCleanup(LoopOpTy op) {
  std::optional<cir::CleanupKind> cleanupKind = op.getCleanupKind();
 
  // The cleanup kind attribute must be present exactly when a (non-empty)
  // cleanup region is present.
  if (cleanupKind.has_value() == op.getCleanup().empty())
    return op.emitOpError("cleanup kind must be present if and only if the "
                          "cleanup region is non-empty");
 
  // A loop's per-iteration cleanup runs on every normal exit edge (loop exit,
  // end of iteration, break/continue), so an EH-only cleanup is meaningless.
  // Only 'normal' (exceptions disabled) and 'all' (normal + EH unwind) apply.
  if (cleanupKind == cir::CleanupKind::EH)
    return op.emitOpError("loop cleanup kind must be 'normal' or 'all', "
                          "not 'eh'");
 
  return success();
}
 
LogicalResult cir::WhileOp::verify() { return verifyLoopCleanup(*this); }
 
LogicalResult cir::ForOp::verify() { return verifyLoopCleanup(*this); }
 
//===----------------------------------------------------------------------===//
// ConstantOp
//===----------------------------------------------------------------------===//
 
static LogicalResult checkConstantTypes(mlir::Operation *op, mlir::Type opType,
                                        mlir::Attribute attrType) {
  if (isa<cir::ConstPtrAttr>(attrType)) {
    if (!mlir::isa<cir::PointerType>(opType))
      return op->emitOpError(
          "pointer constant initializing a non-pointer type");
    return success();
  }
 
  if (isa<cir::DataMemberAttr, cir::DataMemberOffsetAttr, cir::MethodAttr>(
          attrType)) {
    // More detailed type verifications are already done in
    // DataMemberAttr::verify or MethodAttr::verify. Don't need to repeat here.
    return success();
  }
 
  if (isa<cir::ZeroAttr>(attrType)) {
    if (isa<cir::RecordType, cir::ArrayType, cir::MatrixType, cir::VectorType,
            cir::ComplexType>(opType))
      return success();
    return op->emitOpError(
        "zero expects struct, array, vector, or complex type");
  }
 
  if (mlir::isa<cir::UndefAttr>(attrType)) {
    if (!mlir::isa<cir::VoidType>(opType))
      return success();
    return op->emitOpError("undef expects non-void type");
  }
 
  if (mlir::isa<cir::BoolAttr>(attrType)) {
    if (!mlir::isa<cir::BoolType>(opType))
      return op->emitOpError("result type (")
             << opType << ") must be '!cir.bool' for '" << attrType << "'";
    return success();
  }
 
  if (mlir::isa<cir::IntAttr, cir::FPAttr>(attrType)) {
    auto at = cast<TypedAttr>(attrType);
    if (at.getType() != opType) {
      return op->emitOpError("result type (")
             << opType << ") does not match value type (" << at.getType()
             << ")";
    }
    return success();
  }
 
  if (mlir::isa<cir::BlockAddrDiffAttr, cir::BlockAddrInfoAttr,
                cir::ConstArrayAttr, cir::ConstVectorAttr,
                cir::ConstComplexAttr, cir::ConstRecordAttr,
                cir::GlobalOffsetAttr, cir::GlobalViewAttr, cir::PoisonAttr,
                cir::TypeInfoAttr, cir::VTableAttr>(attrType))
    return success();
 
  assert(isa<TypedAttr>(attrType) && "What else could we be looking at here?");
  return op->emitOpError("global with type ")
         << cast<TypedAttr>(attrType).getType() << " not yet supported";
}
 
LogicalResult cir::ConstantOp::verify() {
  // ODS already generates checks to make sure the result type is valid. We just
  // need to additionally check that the value's attribute type is consistent
  // with the result type.
  return checkConstantTypes(getOperation(), getType(), getValue());
}
 
OpFoldResult cir::ConstantOp::fold(FoldAdaptor /*adaptor*/) {
  return getValue();
}
 
//===----------------------------------------------------------------------===//
// CastOp
//===----------------------------------------------------------------------===//
 
static bool isFloatingPointCastKind(cir::CastKind kind) {
  switch (kind) {
  case cir::CastKind::floating:
  case cir::CastKind::int_to_float:
  case cir::CastKind::float_to_int:
  case cir::CastKind::float_to_bool:
  case cir::CastKind::bool_to_float:
  case cir::CastKind::float_to_complex:
  case cir::CastKind::float_complex_to_real:
  case cir::CastKind::float_complex_to_bool:
  case cir::CastKind::float_complex:
  case cir::CastKind::float_complex_to_int_complex:
  case cir::CastKind::int_complex_to_float_complex:
    return true;
  default:
    return false;
  }
}
 
LogicalResult cir::CastOp::verify() {
  mlir::Type resType = getType();
  mlir::Type srcType = getSrc().getType();
  cir::CastKind kind = getKind();
 
  if (getFenvAttr() && !isFloatingPointCastKind(kind))
    return emitOpError()
           << "'fenv' is only valid for floating-point cast kinds";
 
  // Verify address space casts for pointer types. given that
  // casts for within a different address space are illegal.
  auto srcPtrTy = mlir::dyn_cast<cir::PointerType>(srcType);
  auto resPtrTy = mlir::dyn_cast<cir::PointerType>(resType);
  if (srcPtrTy && resPtrTy && (kind != cir::CastKind::address_space))
    if (srcPtrTy.getAddrSpace() != resPtrTy.getAddrSpace()) {
      return emitOpError() << "result type address space does not match the "
                              "address space of the operand";
    }
 
  auto srcVTy = mlir::dyn_cast<cir::VectorType>(srcType);
  auto resVTy = mlir::dyn_cast<cir::VectorType>(resType);
  if (srcVTy && resVTy) {
    if ((kind == cir::CastKind::int_to_float ||
         kind == cir::CastKind::float_to_int) &&
        srcVTy.getSize() != resVTy.getSize()) {
      return emitOpError()
             << "vector float-to-int and int-to-float casts require "
                "source and destination vectors to have the same number of "
                "elements";
    }
    // Use the element type of the vector to verify the cast kind. (Except for
    // bitcast, see below.)
    srcType = srcVTy.getElementType();
    resType = resVTy.getElementType();
  }
 
  switch (kind) {
  case cir::CastKind::int_to_bool: {
    if (!mlir::isa<cir::BoolType>(resType))
      return emitOpError() << "requires !cir.bool type for result";
    if (!mlir::isa<cir::IntType>(srcType))
      return emitOpError() << "requires !cir.int type for source";
    return success();
  }
  case cir::CastKind::ptr_to_bool: {
    if (!mlir::isa<cir::BoolType>(resType))
      return emitOpError() << "requires !cir.bool type for result";
    if (!mlir::isa<cir::PointerType>(srcType))
      return emitOpError() << "requires !cir.ptr type for source";
    return success();
  }
  case cir::CastKind::integral: {
    if (!mlir::isa<cir::IntType>(resType))
      return emitOpError() << "requires !cir.int type for result";
    if (!mlir::isa<cir::IntType>(srcType))
      return emitOpError() << "requires !cir.int type for source";
    return success();
  }
  case cir::CastKind::array_to_ptrdecay: {
    const auto arrayPtrTy = mlir::dyn_cast<cir::PointerType>(srcType);
    const auto flatPtrTy = mlir::dyn_cast<cir::PointerType>(resType);
    if (!arrayPtrTy || !flatPtrTy)
      return emitOpError() << "requires !cir.ptr type for source and result";
 
    // TODO(CIR): Make sure the AddrSpace of both types are equals
    return success();
  }
  case cir::CastKind::bitcast: {
    // Handle the pointer types first.
    auto srcPtrTy = mlir::dyn_cast<cir::PointerType>(srcType);
    auto resPtrTy = mlir::dyn_cast<cir::PointerType>(resType);
 
    if (srcPtrTy && resPtrTy) {
      return success();
    }
 
    return success();
  }
  case cir::CastKind::floating: {
    if (!mlir::isa<cir::FPTypeInterface>(srcType) ||
        !mlir::isa<cir::FPTypeInterface>(resType))
      return emitOpError() << "requires !cir.float type for source and result";
    return success();
  }
  case cir::CastKind::float_to_int: {
    if (!mlir::isa<cir::FPTypeInterface>(srcType))
      return emitOpError() << "requires !cir.float type for source";
    if (!mlir::dyn_cast<cir::IntType>(resType))
      return emitOpError() << "requires !cir.int type for result";
    return success();
  }
  case cir::CastKind::int_to_ptr: {
    if (!mlir::dyn_cast<cir::IntType>(srcType))
      return emitOpError() << "requires !cir.int type for source";
    if (!mlir::dyn_cast<cir::PointerType>(resType))
      return emitOpError() << "requires !cir.ptr type for result";
    return success();
  }
  case cir::CastKind::ptr_to_int: {
    if (!mlir::dyn_cast<cir::PointerType>(srcType))
      return emitOpError() << "requires !cir.ptr type for source";
    if (!mlir::dyn_cast<cir::IntType>(resType))
      return emitOpError() << "requires !cir.int type for result";
    return success();
  }
  case cir::CastKind::float_to_bool: {
    if (!mlir::isa<cir::FPTypeInterface>(srcType))
      return emitOpError() << "requires !cir.float type for source";
    if (!mlir::isa<cir::BoolType>(resType))
      return emitOpError() << "requires !cir.bool type for result";
    return success();
  }
  case cir::CastKind::bool_to_int: {
    if (!mlir::isa<cir::BoolType>(srcType))
      return emitOpError() << "requires !cir.bool type for source";
    if (!mlir::isa<cir::IntType>(resType))
      return emitOpError() << "requires !cir.int type for result";
    return success();
  }
  case cir::CastKind::int_to_float: {
    if (!mlir::isa<cir::IntType>(srcType))
      return emitOpError() << "requires !cir.int type for source";
    if (!mlir::isa<cir::FPTypeInterface>(resType))
      return emitOpError() << "requires !cir.float type for result";
    return success();
  }
  case cir::CastKind::bool_to_float: {
    if (!mlir::isa<cir::BoolType>(srcType))
      return emitOpError() << "requires !cir.bool type for source";
    if (!mlir::isa<cir::FPTypeInterface>(resType))
      return emitOpError() << "requires !cir.float type for result";
    return success();
  }
  case cir::CastKind::address_space: {
    auto srcPtrTy = mlir::dyn_cast<cir::PointerType>(srcType);
    auto resPtrTy = mlir::dyn_cast<cir::PointerType>(resType);
    if (!srcPtrTy || !resPtrTy)
      return emitOpError() << "requires !cir.ptr type for source and result";
    if (srcPtrTy.getPointee() != resPtrTy.getPointee())
      return emitOpError() << "requires two types differ in addrspace only";
    return success();
  }
  case cir::CastKind::float_to_complex: {
    if (!mlir::isa<cir::FPTypeInterface>(srcType))
      return emitOpError() << "requires !cir.float type for source";
    auto resComplexTy = mlir::dyn_cast<cir::ComplexType>(resType);
    if (!resComplexTy)
      return emitOpError() << "requires !cir.complex type for result";
    if (srcType != resComplexTy.getElementType())
      return emitOpError() << "requires source type match result element type";
    return success();
  }
  case cir::CastKind::int_to_complex: {
    if (!mlir::isa<cir::IntType>(srcType))
      return emitOpError() << "requires !cir.int type for source";
    auto resComplexTy = mlir::dyn_cast<cir::ComplexType>(resType);
    if (!resComplexTy)
      return emitOpError() << "requires !cir.complex type for result";
    if (srcType != resComplexTy.getElementType())
      return emitOpError() << "requires source type match result element type";
    return success();
  }
  case cir::CastKind::float_complex_to_real: {
    auto srcComplexTy = mlir::dyn_cast<cir::ComplexType>(srcType);
    if (!srcComplexTy)
      return emitOpError() << "requires !cir.complex type for source";
    if (!mlir::isa<cir::FPTypeInterface>(resType))
      return emitOpError() << "requires !cir.float type for result";
    if (srcComplexTy.getElementType() != resType)
      return emitOpError() << "requires source element type match result type";
    return success();
  }
  case cir::CastKind::int_complex_to_real: {
    auto srcComplexTy = mlir::dyn_cast<cir::ComplexType>(srcType);
    if (!srcComplexTy)
      return emitOpError() << "requires !cir.complex type for source";
    if (!mlir::isa<cir::IntType>(resType))
      return emitOpError() << "requires !cir.int type for result";
    if (srcComplexTy.getElementType() != resType)
      return emitOpError() << "requires source element type match result type";
    return success();
  }
  case cir::CastKind::float_complex_to_bool: {
    auto srcComplexTy = mlir::dyn_cast<cir::ComplexType>(srcType);
    if (!srcComplexTy || !srcComplexTy.isFloatingPointComplex())
      return emitOpError()
             << "requires floating point !cir.complex type for source";
    if (!mlir::isa<cir::BoolType>(resType))
      return emitOpError() << "requires !cir.bool type for result";
    return success();
  }
  case cir::CastKind::int_complex_to_bool: {
    auto srcComplexTy = mlir::dyn_cast<cir::ComplexType>(srcType);
    if (!srcComplexTy || !srcComplexTy.isIntegerComplex())
      return emitOpError()
             << "requires floating point !cir.complex type for source";
    if (!mlir::isa<cir::BoolType>(resType))
      return emitOpError() << "requires !cir.bool type for result";
    return success();
  }
  case cir::CastKind::float_complex: {
    auto srcComplexTy = mlir::dyn_cast<cir::ComplexType>(srcType);
    if (!srcComplexTy || !srcComplexTy.isFloatingPointComplex())
      return emitOpError()
             << "requires floating point !cir.complex type for source";
    auto resComplexTy = mlir::dyn_cast<cir::ComplexType>(resType);
    if (!resComplexTy || !resComplexTy.isFloatingPointComplex())
      return emitOpError()
             << "requires floating point !cir.complex type for result";
    return success();
  }
  case cir::CastKind::float_complex_to_int_complex: {
    auto srcComplexTy = mlir::dyn_cast<cir::ComplexType>(srcType);
    if (!srcComplexTy || !srcComplexTy.isFloatingPointComplex())
      return emitOpError()
             << "requires floating point !cir.complex type for source";
    auto resComplexTy = mlir::dyn_cast<cir::ComplexType>(resType);
    if (!resComplexTy || !resComplexTy.isIntegerComplex())
      return emitOpError() << "requires integer !cir.complex type for result";
    return success();
  }
  case cir::CastKind::int_complex: {
    auto srcComplexTy = mlir::dyn_cast<cir::ComplexType>(srcType);
    if (!srcComplexTy || !srcComplexTy.isIntegerComplex())
      return emitOpError() << "requires integer !cir.complex type for source";
    auto resComplexTy = mlir::dyn_cast<cir::ComplexType>(resType);
    if (!resComplexTy || !resComplexTy.isIntegerComplex())
      return emitOpError() << "requires integer !cir.complex type for result";
    return success();
  }
  case cir::CastKind::int_complex_to_float_complex: {
    auto srcComplexTy = mlir::dyn_cast<cir::ComplexType>(srcType);
    if (!srcComplexTy || !srcComplexTy.isIntegerComplex())
      return emitOpError() << "requires integer !cir.complex type for source";
    auto resComplexTy = mlir::dyn_cast<cir::ComplexType>(resType);
    if (!resComplexTy || !resComplexTy.isFloatingPointComplex())
      return emitOpError()
             << "requires floating point !cir.complex type for result";
    return success();
  }
  case cir::CastKind::member_ptr_to_bool: {
    if (!mlir::isa<cir::DataMemberType, cir::MethodType>(srcType))
      return emitOpError()
             << "requires !cir.data_member or !cir.method type for source";
    if (!mlir::isa<cir::BoolType>(resType))
      return emitOpError() << "requires !cir.bool type for result";
    return success();
  }
  }
  llvm_unreachable("Unknown CastOp kind?");
}
 
static bool isIntOrBoolCast(cir::CastOp op) {
  auto kind = op.getKind();
  return kind == cir::CastKind::bool_to_int ||
         kind == cir::CastKind::int_to_bool || kind == cir::CastKind::integral;
}
 
static bool isCirFunctionPointerType(mlir::Type ty) {
  const auto ptrTy = mlir::dyn_cast<cir::PointerType>(ty);
  return ptrTy && mlir::isa<cir::FuncType>(ptrTy.getPointee());
}
 
static Value tryFoldCastChain(cir::CastOp op) {
  cir::CastOp head = op, tail = op;
 
  while (op) {
    if (!isIntOrBoolCast(op))
      break;
    head = op;
    op = head.getSrc().getDefiningOp<cir::CastOp>();
  }
 
  if (head != tail) {
    // if bool_to_int -> ...  -> int_to_bool: take the bool
    // as we had it was before all casts
    if (head.getKind() == cir::CastKind::bool_to_int &&
        tail.getKind() == cir::CastKind::int_to_bool)
      return head.getSrc();
 
    // if int_to_bool -> ...  -> int_to_bool: take the result
    // of the first one, as no other casts (and ext casts as well)
    // don't change the first result
    if (head.getKind() == cir::CastKind::int_to_bool &&
        tail.getKind() == cir::CastKind::int_to_bool)
      return head.getResult();
 
    return {};
  }
 
  // Bitcast round-trip on function pointers: T0 -> T1 -> T0 (e.g. no-proto
  // redeclaration vs. actual prototype). Restrict to function pointers so
  // other pointer bitcast chains are unchanged.
  if (tail.getKind() == cir::CastKind::bitcast) {
    auto *inner = tail.getSrc().getDefiningOp();
    if (inner && isCirFunctionPointerType(tail.getType())) {
      auto innerCast = mlir::dyn_cast<cir::CastOp>(inner);
      if (innerCast && innerCast.getKind() == cir::CastKind::bitcast &&
          innerCast.getSrc().getType() == tail.getType() &&
          innerCast.getType() == tail.getSrc().getType()) {
        return innerCast.getSrc();
      }
    }
  }
 
  return {};
}
 
OpFoldResult cir::CastOp::fold(FoldAdaptor adaptor) {
  if (mlir::isa_and_present<cir::PoisonAttr>(adaptor.getSrc())) {
    // Propagate poison value
    return cir::PoisonAttr::get(getContext(), getType());
  }
 
  // Propagate Undef value
  if (mlir::isa_and_present<cir::UndefAttr>(adaptor.getSrc()))
    return cir::UndefAttr::get(getType());
 
  if (getSrc().getType() == getType()) {
    switch (getKind()) {
    case cir::CastKind::integral: {
      llvm::SmallVector<mlir::OpFoldResult, 1> foldResults;
      auto foldOrder = getSrc().getDefiningOp()->fold(foldResults);
      if (foldOrder.succeeded() && mlir::isa<mlir::Attribute>(foldResults[0]))
        return mlir::cast<mlir::Attribute>(foldResults[0]);
      return {};
    }
    case cir::CastKind::bitcast:
    case cir::CastKind::address_space:
    case cir::CastKind::float_complex:
    case cir::CastKind::int_complex: {
      return getSrc();
    }
    default:
      return {};
    }
  }
 
  // Handle cases where a chain of casts cancel out.
  Value result = tryFoldCastChain(*this);
  if (result)
    return result;
 
  // Handle simple constant casts.
  if (auto srcConst = getSrc().getDefiningOp<cir::ConstantOp>()) {
    switch (getKind()) {
    case cir::CastKind::integral: {
      mlir::Type srcTy = getSrc().getType();
      // Don't try to fold vector casts for now.
      assert(mlir::isa<cir::VectorType>(srcTy) ==
             mlir::isa<cir::VectorType>(getType()));
      if (mlir::isa<cir::VectorType>(srcTy))
        break;
 
      auto srcIntTy = mlir::cast<cir::IntType>(srcTy);
      auto dstIntTy = mlir::cast<cir::IntType>(getType());
      auto constIntAttr = srcConst.getValueAttr<cir::IntAttr>();
      if (!constIntAttr)
        return {};
 
      APInt srcValue = constIntAttr.getValue();
      APInt newVal = srcIntTy.isSigned()
                         ? srcValue.sextOrTrunc(dstIntTy.getWidth())
                         : srcValue.zextOrTrunc(dstIntTy.getWidth());
      return cir::IntAttr::get(dstIntTy, newVal);
    }
    default:
      break;
    }
  }
  return {};
}
 
//===----------------------------------------------------------------------===//
// BuiltinIntCastOp
//===----------------------------------------------------------------------===//
 
LogicalResult cir::BuiltinIntCastOp::verify() {
  mlir::Type srcType = getSrc().getType();
  mlir::Type resType = getType();
 
  auto srcCirInt = mlir::dyn_cast<cir::IntType>(srcType);
  auto resCirInt = mlir::dyn_cast<cir::IntType>(resType);
 
  // One side must be a CIR integer the other must be a builtin
  // integer or index type.
  if (static_cast<bool>(srcCirInt) == static_cast<bool>(resCirInt))
    return emitOpError()
           << "requires exactly one '!cir.int' operand or result; the other "
              "must be a builtin integer or 'index' type";
 
  mlir::Type builtinType = srcCirInt ? resType : srcType;
  if (!mlir::isa<mlir::IntegerType, mlir::IndexType>(builtinType))
    return emitOpError() << "requires a builtin integer or 'index' type on the "
                            "non-CIR side";
 
  // The cast preserves bit width. 'index' has no fixed width, so only check
  // when the builtin side is a fixed-width integer.
  if (auto builtinInt = mlir::dyn_cast<mlir::IntegerType>(builtinType)) {
    cir::IntType cirInt = srcCirInt ? srcCirInt : resCirInt;
    if (cirInt.getWidth() != builtinInt.getWidth())
      return emitOpError()
             << "requires the CIR and builtin integer types to have the same "
                "width; use 'cir.cast' for width conversions";
  }
 
  return success();
}
 
OpFoldResult cir::BuiltinIntCastOp::fold(FoldAdaptor adaptor) {
  // Fold: builtin_int_cast(builtin_int_cast(x)) -> x
  // Inner source type must match the cast's result type.
  if (auto inner = getSrc().getDefiningOp<cir::BuiltinIntCastOp>())
    if (inner.getSrc().getType() == getType())
      return inner.getSrc();
  return {};
}
 
//===----------------------------------------------------------------------===//
// CallOp
//===----------------------------------------------------------------------===//
 
mlir::OperandRange cir::CallOp::getArgOperands() {
  if (isIndirect())
    return getArgs().drop_front(1);
  return getArgs();
}
 
mlir::MutableOperandRange cir::CallOp::getArgOperandsMutable() {
  mlir::MutableOperandRange args = getArgsMutable();
  if (isIndirect())
    return args.slice(1, args.size() - 1);
  return args;
}
 
mlir::Value cir::CallOp::getIndirectCall() {
  assert(isIndirect());
  return getOperand(0);
}
 
/// Return the operand at index 'i'.
Value cir::CallOp::getArgOperand(unsigned i) {
  if (isIndirect())
    ++i;
  return getOperand(i);
}
 
/// Return the number of operands.
unsigned cir::CallOp::getNumArgOperands() {
  if (isIndirect())
    return this->getOperation()->getNumOperands() - 1;
  return this->getOperation()->getNumOperands();
}
 
static mlir::ParseResult
parseTryCallDestinations(mlir::OpAsmParser &parser,
                         mlir::OperationState &result) {
  mlir::Block *normalDestSuccessor;
  if (parser.parseSuccessor(normalDestSuccessor))
    return mlir::failure();
 
  if (parser.parseComma())
    return mlir::failure();
 
  mlir::Block *unwindDestSuccessor;
  if (parser.parseSuccessor(unwindDestSuccessor))
    return mlir::failure();
 
  result.addSuccessors(normalDestSuccessor);
  result.addSuccessors(unwindDestSuccessor);
  return mlir::success();
}
 
/// Reject an attribute in an explicit attribute dictionary whose value is not
/// the attribute class expected for its name.  Where an op declares the
/// attribute it is stored as a property, so a value of the wrong class would
/// be dropped without a diagnostic.
static ParseResult checkEffectAttrKinds(mlir::OpAsmParser &parser,
                                        llvm::SMLoc loc,
                                        const mlir::NamedAttrList &attrs) {
  if (mlir::Attribute effects =
          attrs.get(CIRDialect::getMemoryEffectsAttrName()))
    if (!mlir::isa<cir::MemoryEffectsAttr>(effects))
      return parser.emitError(loc, "attribute '")
             << CIRDialect::getMemoryEffectsAttrName()
             << "' must be a #cir.memory_effects attribute";
 
  if (mlir::Attribute uwtable = attrs.get(CIRDialect::getUwtableAttrName()))
    if (!mlir::isa<cir::UnwindTableKindAttr>(uwtable))
      return parser.emitError(loc, "attribute '")
             << CIRDialect::getUwtableAttrName()
             << "' must be a #cir.uwtable attribute";
 
  if (mlir::Attribute framePointer =
          attrs.get(CIRDialect::getFramePointerAttrName()))
    if (!mlir::isa<cir::FramePointerKindAttr>(framePointer))
      return parser.emitError(loc, "attribute '")
             << CIRDialect::getFramePointerAttrName()
             << "' must be a #cir.frame_pointer attribute";
 
  for (llvm::StringRef name :
       {CIRDialect::getNoUnwindAttrName(), CIRDialect::getWillReturnAttrName(),
        CIRDialect::getMustProgressAttrName()})
    if (mlir::Attribute flag = attrs.get(name))
      if (!mlir::isa<mlir::UnitAttr>(flag))
        return parser.emitError(loc, "attribute '")
               << name << "' must be a unit attribute";
 
  return mlir::success();
}
 
static mlir::ParseResult parseCallCommon(mlir::OpAsmParser &parser,
                                         mlir::OperationState &result,
                                         bool hasDestinationBlocks = false) {
  llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand, 4> ops;
  llvm::SMLoc opsLoc;
  mlir::FlatSymbolRefAttr calleeAttr;
 
  // If we cannot parse a string callee, it means this is an indirect call.
  if (!parser
           .parseOptionalAttribute(calleeAttr, CIRDialect::getCalleeAttrName(),
                                   result.attributes)
           .has_value()) {
    OpAsmParser::UnresolvedOperand indirectVal;
    // Do not resolve right now, since we need to figure out the type
    if (parser.parseOperand(indirectVal).failed())
      return failure();
    ops.push_back(indirectVal);
  }
 
  if (parser.parseLParen())
    return mlir::failure();
 
  opsLoc = parser.getCurrentLocation();
  if (parser.parseOperandList(ops))
    return mlir::failure();
  if (parser.parseRParen())
    return mlir::failure();
 
  if (hasDestinationBlocks &&
      parseTryCallDestinations(parser, result).failed()) {
    return ::mlir::failure();
  }
 
  if (parser.parseOptionalKeyword("cc").succeeded()) {
    cir::CallingConv callingConv;
    if (parser.parseLParen().failed())
      return failure();
    if (parseCIRKeyword<cir::CallingConv>(parser, callingConv).failed())
      return parser.emitError(parser.getCurrentLocation(),
                              "unknown calling convention");
    if (parser.parseRParen().failed())
      return failure();
    result.addAttribute(
        CIRDialect::getCallingConvAttrName(),
        cir::CallingConvAttr::get(parser.getContext(), callingConv));
  }
 
  if (parser.parseOptionalKeyword("musttail").succeeded())
    result.addAttribute(CIRDialect::getMustTailAttrName(),
                        mlir::UnitAttr::get(parser.getContext()));
 
  if (parser.parseOptionalKeyword("nothrow").succeeded())
    result.addAttribute(CIRDialect::getNoThrowAttrName(),
                        mlir::UnitAttr::get(parser.getContext()));
 
  if (parser.parseOptionalKeyword("nounwind").succeeded())
    result.addAttribute(CIRDialect::getNoUnwindAttrName(),
                        mlir::UnitAttr::get(parser.getContext()));
 
  if (parser.parseOptionalKeyword("willreturn").succeeded())
    result.addAttribute(CIRDialect::getWillReturnAttrName(),
                        mlir::UnitAttr::get(parser.getContext()));
 
  llvm::SMLoc attrsLoc = parser.getCurrentLocation();
  if (parser.parseOptionalAttrDict(result.attributes))
    return ::mlir::failure();
 
  if (checkEffectAttrKinds(parser, attrsLoc, result.attributes).failed())
    return ::mlir::failure();
 
  if (parser.parseColon())
    return ::mlir::failure();
 
  SmallVector<Type> argTypes;
  SmallVector<DictionaryAttr> argAttrs;
  SmallVector<Type> resultTypes;
  SmallVector<DictionaryAttr> resultAttrs;
  if (call_interface_impl::parseFunctionSignature(parser, argTypes, argAttrs,
                                                  resultTypes, resultAttrs))
    return mlir::failure();
 
  if (resultTypes.size() > 1 || resultAttrs.size() > 1)
    return parser.emitError(
        parser.getCurrentLocation(),
        "functions with multiple return types are not supported");
 
  result.addTypes(resultTypes);
 
  if (parser.resolveOperands(ops, argTypes, opsLoc, result.operands))
    return mlir::failure();
 
  if (!resultAttrs.empty() && resultAttrs[0])
    result.addAttribute(
        CIRDialect::getResAttrsAttrName(),
        mlir::ArrayAttr::get(parser.getContext(), {resultAttrs[0]}));
 
  // ArrayAttr requires a vector of 'Attribute', so we have to do the conversion
  // here into a separate collection.
  llvm::SmallVector<Attribute> convertedArgAttrs;
  bool argAttrsEmpty = true;
 
  llvm::transform(argAttrs, std::back_inserter(convertedArgAttrs),
                  [&](DictionaryAttr da) -> mlir::Attribute {
                    if (da)
                      argAttrsEmpty = false;
                    return da;
                  });
 
  if (!argAttrsEmpty) {
    llvm::ArrayRef argAttrsRef = convertedArgAttrs;
    if (!calleeAttr) {
      // Fixup for indirect calls, which get an extra entry in the 'args' for
      // the indirect type, which doesn't get attributes.
      argAttrsRef = argAttrsRef.drop_front();
    }
    result.addAttribute(CIRDialect::getArgAttrsAttrName(),
                        mlir::ArrayAttr::get(parser.getContext(), argAttrsRef));
  }
 
  return mlir::success();
}
 
static void printCallCommon(mlir::Operation *op,
                            mlir::FlatSymbolRefAttr calleeSym,
                            mlir::Value indirectCallee,
                            mlir::OpAsmPrinter &printer, bool isNothrow,
                            ArrayAttr argAttrs, ArrayAttr resAttrs,
                            mlir::Block *normalDest = nullptr,
                            mlir::Block *unwindDest = nullptr) {
  printer << ' ';
 
  auto callLikeOp = mlir::cast<cir::CIRCallOpInterface>(op);
  auto ops = callLikeOp.getArgOperands();
 
  if (calleeSym) {
    // Direct calls
    printer.printAttributeWithoutType(calleeSym);
  } else {
    // Indirect calls
    assert(indirectCallee);
    printer << indirectCallee;
  }
 
  printer << "(" << ops << ")";
 
  if (normalDest) {
    assert(unwindDest && "expected two successors");
    auto tryCall = cast<cir::TryCallOp>(op);
    printer << ' ' << tryCall.getNormalDest();
    printer << ",";
    printer << ' ';
    printer << tryCall.getUnwindDest();
  }
 
  cir::CallingConv callingConv = callLikeOp.getCallingConv();
  if (callingConv != cir::CallingConv::C)
    printer << " cc(" << stringifyCallingConv(callingConv) << ")";
 
  if (op->hasAttr(CIRDialect::getMustTailAttrName()))
    printer << " musttail";
 
  if (isNothrow)
    printer << " nothrow";
 
  if (op->hasAttr(CIRDialect::getNoUnwindAttrName()))
    printer << " nounwind";
 
  if (op->hasAttr(CIRDialect::getWillReturnAttrName()))
    printer << " willreturn";
 
  llvm::StringRef elidedAttrs[] = {
      CIRDialect::getCalleeAttrName(),
      CIRDialect::getCallingConvAttrName(),
      CIRDialect::getMustTailAttrName(),
      CIRDialect::getNoThrowAttrName(),
      CIRDialect::getNoUnwindAttrName(),
      CIRDialect::getWillReturnAttrName(),
      CIRDialect::getOperandSegmentSizesAttrName(),
      "res_attrs",
      "arg_attrs",
  };
  // TODO: Split inherent and discardable attribute printing instead of
  // materializing a single dictionary that mixes the two storage classes.
  llvm::SmallVector<mlir::NamedAttribute> attrs;
  for (mlir::NamedAttribute attr : op->getDiscardableAttrs())
    if (!llvm::is_contained(elidedAttrs, attr.getName()))
      attrs.push_back(attr);
  op->getName().walkInherentAttrs(op, [&](llvm::StringRef name,
                                          mlir::Attribute &attr) {
    if (!llvm::is_contained(elidedAttrs, name))
      attrs.emplace_back(mlir::StringAttr::get(op->getContext(), name), attr);
  });
  llvm::sort(attrs, [](mlir::NamedAttribute lhs, mlir::NamedAttribute rhs) {
    return lhs.getName().strref() < rhs.getName().strref();
  });
  printer.printOptionalAttrDict(attrs);
  printer << " : ";
  if (calleeSym || !argAttrs) {
    call_interface_impl::printFunctionSignature(
        printer, op->getOperands().getTypes(), argAttrs,
        /*isVariadic=*/false, op->getResultTypes(), resAttrs);
  } else {
    // indirect function calls use an 'arg' type for the type of its indirect
    // argument.  However, we don't store a similar attribute collection.  In
    // order to make `printFunctionSignature` have the attributes line up, we
    // have to make a 'shimmed' copy of the attributes that have a blank set of
    // attributes for the indirect argument.
    llvm::SmallVector<Attribute> shimmedArgAttrs;
    shimmedArgAttrs.push_back(mlir::DictionaryAttr::get(op->getContext(), {}));
    shimmedArgAttrs.append(argAttrs.begin(), argAttrs.end());
    call_interface_impl::printFunctionSignature(
        printer, op->getOperands().getTypes(),
        mlir::ArrayAttr::get(op->getContext(), shimmedArgAttrs),
        /*isVariadic=*/false, op->getResultTypes(), resAttrs);
  }
}
 
mlir::ParseResult cir::CallOp::parse(mlir::OpAsmParser &parser,
                                     mlir::OperationState &result) {
  return parseCallCommon(parser, result);
}
 
void cir::CallOp::print(mlir::OpAsmPrinter &p) {
  mlir::Value indirectCallee = isIndirect() ? getIndirectCall() : nullptr;
  printCallCommon(*this, getCalleeAttr(), indirectCallee, p, getNothrow(),
                  getArgAttrsAttr(), getResAttrsAttr());
}
 
static LogicalResult
verifyCallCommInSymbolUses(mlir::Operation *op,
                           SymbolTableCollection &symbolTable) {
  auto callIf = cast<cir::CIRCallOpInterface>(op);
 
  // An indirect call is checked against the function type its callee pointer
  // points to, and a direct call against its callee's declaration.
  cir::FuncType fnType;
  bool hasPrototype = true;
  auto fnAttr =
      op->getAttrOfType<FlatSymbolRefAttr>(CIRDialect::getCalleeAttrName());
  if (!fnAttr) {
    if (op->getNumOperands() == 0)
      return op->emitOpError("indirect call requires a callee operand");
    mlir::Type calleeTy = callIf.getIndirectCall().getType();
    if (auto ptrTy = mlir::dyn_cast<cir::PointerType>(calleeTy))
      fnType = mlir::dyn_cast<cir::FuncType>(ptrTy.getPointee());
    if (!fnType)
      return op->emitOpError()
             << "indirect callee must be a pointer to a function, but has type "
             << calleeTy;
  } else {
    auto fn = symbolTable.lookupNearestSymbolFrom<cir::FuncOp>(op, fnAttr);
    if (!fn)
      return op->emitOpError() << "'" << fnAttr.getValue()
                               << "' does not reference a valid function";
    fnType = fn.getFunctionType();
    hasPrototype = !fn.getNoProto();
  }
 
  // Verify that the operand and result types match the callee.  Operands are
  // not checked for a direct call to a function declared no_proto.
  if (hasPrototype) {
    unsigned numCallOperands = callIf.getNumArgOperands();
    unsigned numFnOpOperands = fnType.getNumInputs();
 
    if (!fnType.isVarArg() && numCallOperands != numFnOpOperands)
      return op->emitOpError("incorrect number of operands for callee");
    if (fnType.isVarArg() && numCallOperands < numFnOpOperands)
      return op->emitOpError("too few operands for callee");
 
    for (unsigned i = 0, e = numFnOpOperands; i != e; ++i)
      if (callIf.getArgOperand(i).getType() != fnType.getInput(i))
        return op->emitOpError("operand type mismatch: expected operand type ")
               << fnType.getInput(i) << ", but provided "
               << callIf.getArgOperand(i).getType() << " for operand number "
               << i;
  }
 
  assert(!cir::MissingFeatures::opCallCallConv());
 
  // Void function must not return any results.
  if (fnType.hasVoidReturn() && op->getNumResults() != 0)
    return op->emitOpError("callee returns void but call has results");
 
  // Non-void function calls must return exactly one result.
  if (!fnType.hasVoidReturn() && op->getNumResults() != 1)
    return op->emitOpError("incorrect number of results for callee");
 
  // Parent function and return value types must match.
  if (!fnType.hasVoidReturn() &&
      op->getResultTypes().front() != fnType.getReturnType()) {
    return op->emitOpError("result type mismatch: expected ")
           << fnType.getReturnType() << ", but provided "
           << op->getResult(0).getType();
  }
 
  return mlir::success();
}
 
LogicalResult
cir::CallOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
  return verifyCallCommInSymbolUses(*this, symbolTable);
}
 
//===----------------------------------------------------------------------===//
// TryCallOp
//===----------------------------------------------------------------------===//
 
mlir::OperandRange cir::TryCallOp::getArgOperands() {
  if (isIndirect())
    return getArgs().drop_front(1);
  return getArgs();
}
 
mlir::MutableOperandRange cir::TryCallOp::getArgOperandsMutable() {
  mlir::MutableOperandRange args = getArgsMutable();
  if (isIndirect())
    return args.slice(1, args.size() - 1);
  return args;
}
 
mlir::Value cir::TryCallOp::getIndirectCall() {
  assert(isIndirect());
  return getOperand(0);
}
 
/// Return the operand at index 'i'.
Value cir::TryCallOp::getArgOperand(unsigned i) {
  if (isIndirect())
    ++i;
  return getOperand(i);
}
 
/// Return the number of operands.
unsigned cir::TryCallOp::getNumArgOperands() {
  if (isIndirect())
    return this->getOperation()->getNumOperands() - 1;
  return this->getOperation()->getNumOperands();
}
 
LogicalResult
cir::TryCallOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
  return verifyCallCommInSymbolUses(*this, symbolTable);
}
 
mlir::ParseResult cir::TryCallOp::parse(mlir::OpAsmParser &parser,
                                        mlir::OperationState &result) {
  return parseCallCommon(parser, result, /*hasDestinationBlocks=*/true);
}
 
void cir::TryCallOp::print(::mlir::OpAsmPrinter &p) {
  mlir::Value indirectCallee = isIndirect() ? getIndirectCall() : nullptr;
  printCallCommon(*this, getCalleeAttr(), indirectCallee, p, getNothrow(),
                  getArgAttrsAttr(), getResAttrsAttr(), getNormalDest(),
                  getUnwindDest());
}
 
//===----------------------------------------------------------------------===//
// ReturnOp
//===----------------------------------------------------------------------===//
 
static mlir::LogicalResult checkReturnAndFunction(cir::ReturnOp op,
                                                  cir::FuncOp function) {
  // ReturnOps currently only have a single optional operand.
  if (op.getNumOperands() > 1)
    return op.emitOpError() << "expects at most 1 return operand";
 
  // Ensure returned type matches the function signature.
  auto expectedTy = function.getFunctionType().getReturnType();
  auto actualTy =
      (op.getNumOperands() == 0 ? cir::VoidType::get(op.getContext())
                                : op.getOperand(0).getType());
  if (actualTy != expectedTy)
    return op.emitOpError() << "returns " << actualTy
                            << " but enclosing function returns " << expectedTy;
 
  return mlir::success();
}
 
mlir::LogicalResult cir::ReturnOp::verify() {
  // Returns can be present in multiple different scopes, get the
  // wrapping function and start from there.
  auto *fnOp = getOperation()->getParentOp();
  while (!isa<cir::FuncOp>(fnOp))
    fnOp = fnOp->getParentOp();
 
  // Make sure return types match function return type.
  if (checkReturnAndFunction(*this, cast<cir::FuncOp>(fnOp)).failed())
    return failure();
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// IfOp
//===----------------------------------------------------------------------===//
 
ParseResult cir::IfOp::parse(OpAsmParser &parser, OperationState &result) {
  // create the regions for 'then'.
  result.regions.reserve(2);
  Region *thenRegion = result.addRegion();
  Region *elseRegion = result.addRegion();
 
  mlir::Builder &builder = parser.getBuilder();
  OpAsmParser::UnresolvedOperand cond;
  Type boolType = cir::BoolType::get(builder.getContext());
 
  if (parser.parseOperand(cond) ||
      parser.resolveOperand(cond, boolType, result.operands))
    return failure();
 
  // Parse 'then' region.
  mlir::SMLoc parseThenLoc = parser.getCurrentLocation();
  if (parser.parseRegion(*thenRegion, /*arguments=*/{}, /*argTypes=*/{}))
    return failure();
 
  if (ensureRegionTerm(parser, *thenRegion, parseThenLoc).failed())
    return failure();
 
  // If we find an 'else' keyword, parse the 'else' region.
  if (!parser.parseOptionalKeyword("else")) {
    mlir::SMLoc parseElseLoc = parser.getCurrentLocation();
    if (parser.parseRegion(*elseRegion, /*arguments=*/{}, /*argTypes=*/{}))
      return failure();
    if (ensureRegionTerm(parser, *elseRegion, parseElseLoc).failed())
      return failure();
  }
 
  // Parse the optional attribute list.
  if (parser.parseOptionalAttrDict(result.attributes))
    return failure();
  return success();
}
 
void cir::IfOp::print(OpAsmPrinter &p) {
  p << " " << getCondition() << " ";
  mlir::Region &thenRegion = this->getThenRegion();
  p.printRegion(thenRegion,
                /*printEntryBlockArgs=*/false,
                /*printBlockTerminators=*/!omitRegionTerm(thenRegion));
 
  // Print the 'else' regions if it exists and has a block.
  mlir::Region &elseRegion = this->getElseRegion();
  if (!elseRegion.empty()) {
    p << " else ";
    p.printRegion(elseRegion,
                  /*printEntryBlockArgs=*/false,
                  /*printBlockTerminators=*/!omitRegionTerm(elseRegion));
  }
 
  p.printOptionalAttrDict(
      getOperation()->getDiscardableAttrDictionary().getValue());
}
 
/// Default callback for IfOp builders.
void cir::buildTerminatedBody(OpBuilder &builder, Location loc) {
  // add cir.yield to end of the block
  cir::YieldOp::create(builder, loc);
}
 
/// Given the region at `index`, or the parent operation if `index` is None,
/// return the successor regions. These are the regions that may be selected
/// during the flow of control. `operands` is a set of optional attributes that
/// correspond to a constant value for each operand, or null if that operand is
/// not a constant.
void cir::IfOp::getSuccessorRegions(mlir::RegionBranchPoint point,
                                    SmallVectorImpl<RegionSuccessor> &regions) {
  // The `then` and the `else` region branch back to the parent operation.
  if (!point.isParent()) {
    regions.emplace_back(getOperation());
    return;
  }
 
  // Don't consider the else region if it is empty.
  Region *elseRegion = &this->getElseRegion();
  if (elseRegion->empty())
    elseRegion = nullptr;
 
  // If the condition isn't constant, both regions may be executed.
  regions.push_back(RegionSuccessor(&getThenRegion()));
  if (elseRegion)
    regions.push_back(RegionSuccessor(elseRegion));
  else
    regions.emplace_back(getOperation());
}
 
void cir::IfOp::build(OpBuilder &builder, OperationState &result, Value cond,
                      bool withElseRegion, BuilderCallbackRef thenBuilder,
                      BuilderCallbackRef elseBuilder) {
  assert(thenBuilder && "the builder callback for 'then' must be present");
  result.addOperands(cond);
 
  OpBuilder::InsertionGuard guard(builder);
  Region *thenRegion = result.addRegion();
  builder.createBlock(thenRegion);
  thenBuilder(builder, result.location);
 
  Region *elseRegion = result.addRegion();
  if (!withElseRegion)
    return;
 
  builder.createBlock(elseRegion);
  elseBuilder(builder, result.location);
}
 
//===----------------------------------------------------------------------===//
// ScopeOp
//===----------------------------------------------------------------------===//
 
void cir::ScopeOp::build(
    OpBuilder &builder, OperationState &result,
    function_ref<void(OpBuilder &, Type &, Location)> scopeBuilder) {
  assert(scopeBuilder && "the builder callback for 'then' must be present");
 
  OpBuilder::InsertionGuard guard(builder);
  Region *scopeRegion = result.addRegion();
  builder.createBlock(scopeRegion);
  assert(!cir::MissingFeatures::opScopeCleanupRegion());
 
  mlir::Type yieldTy;
  scopeBuilder(builder, yieldTy, result.location);
 
  if (yieldTy)
    result.addTypes(TypeRange{yieldTy});
}
 
void cir::ScopeOp::build(
    OpBuilder &builder, OperationState &result,
    function_ref<void(OpBuilder &, Location)> scopeBuilder) {
  assert(scopeBuilder && "the builder callback for 'then' must be present");
  OpBuilder::InsertionGuard guard(builder);
  Region *scopeRegion = result.addRegion();
  builder.createBlock(scopeRegion);
  assert(!cir::MissingFeatures::opScopeCleanupRegion());
  scopeBuilder(builder, result.location);
}
 
LogicalResult cir::ScopeOp::verify() {
  if (getRegion().empty()) {
    return emitOpError() << "cir.scope must not be empty since it should "
                            "include at least an implicit cir.yield ";
  }
 
  mlir::Block &lastBlock = getRegion().back();
  if (lastBlock.empty() || !lastBlock.mightHaveTerminator() ||
      !lastBlock.getTerminator()->hasTrait<OpTrait::IsTerminator>())
    return emitOpError() << "last block of cir.scope must be terminated";
  return success();
}
 
LogicalResult cir::ScopeOp::fold(FoldAdaptor /*adaptor*/,
                                 SmallVectorImpl<OpFoldResult> &results) {
  // Only fold "trivial" scopes: a single block containing only a `cir.yield`.
  if (!getRegion().hasOneBlock())
    return failure();
  Block &block = getRegion().front();
  if (block.getOperations().size() != 1)
    return failure();
 
  auto yield = dyn_cast<cir::YieldOp>(block.front());
  if (!yield)
    return failure();
 
  // Only fold when the scope produces a value.
  if (getNumResults() != 1 || yield.getNumOperands() != 1)
    return failure();
 
  results.push_back(yield.getOperand(0));
  return success();
}
 
//===----------------------------------------------------------------------===//
// CleanupScopeOp
//===----------------------------------------------------------------------===//
 
static bool isRedundantBeforeReturn(mlir::Region &cleanupRegion) {
  for (mlir::Block &block : cleanupRegion) {
    for (mlir::Operation &op : block) {
      if (isa<cir::YieldOp, cir::LifetimeEndOp, cir::StackRestoreOp>(op))
        continue;
      // The stack restore reloads the pointer saved before the VLA.
      auto loadOp = dyn_cast<cir::LoadOp>(op);
      if (loadOp && loadOp.getResult().hasOneUse() &&
          isa<cir::StackRestoreOp>(*loadOp.getResult().getUsers().begin()))
        continue;
      return false;
    }
  }
  return true;
}
 
LogicalResult cir::CleanupScopeOp::verify() {
  // If the cleanup contains a musttail call, it must be a cleanup that can be
  // skipped on return (such as a lifetime end or a stack restore). Other
  // cleanups must never contain musttail calls.
  cir::CallOp mustTailCall;
  getBodyRegion().walk([&](cir::CallOp callOp) {
    if (!callOp.getMusttail())
      return WalkResult::advance();
    mustTailCall = callOp;
    return WalkResult::interrupt();
  });
  if (!mustTailCall)
    return success();
 
  if (isRedundantBeforeReturn(getCleanupRegion()))
    return success();
 
  InFlightDiagnostic diag =
      emitOpError("cleanup is not redundant before a return, so it cannot be "
                  "skipped by a musttail call");
  diag.attachNote(mustTailCall.getLoc()) << "musttail call is here";
  return diag;
}
 
void cir::CleanupScopeOp::getSuccessorRegions(
    mlir::RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> &regions) {
  // Execution always starts in the body region.
  if (point.isParent()) {
    regions.emplace_back(&getBodyRegion());
    return;
  }
 
  mlir::Operation *term = point.getTerminatorPredecessorOrNull();
  assert(term && "expected a terminator predecessor");
 
  if (term->getParentRegion() == &getBodyRegion()) {
    // cir.resume unwinds out of the body. Every other terminator, such as
    // cir.yield or cir.co_return, is a normal exit.
    cir::CleanupKindAttr kind = getCleanupKindAttr();
    if (isa<cir::ResumeOp>(term) ? kind.isEH() : kind.isNormal()) {
      regions.emplace_back(&getCleanupRegion());
      return;
    }
  }
 
  // Exiting the cleanup region, or a body exit that skips it, leaves the
  // operation. An unwind out of the middle of the body is not a
  // RegionBranchPoint, and neither are cir.return, cir.break, cir.continue or
  // cir.goto, so a cleanup region reached only that way has no predecessor
  // here.
  regions.emplace_back(getOperation());
}
 
LogicalResult cir::CleanupScopeOp::canonicalize(CleanupScopeOp op,
                                                PatternRewriter &rewriter) {
  auto isRegionTrivial = [](Region &region) {
    assert(!region.empty() && "CleanupScopeOp regions must not be empty");
    if (!region.hasOneBlock())
      return false;
    Block &block = llvm::getSingleElement(region);
    return llvm::hasSingleElement(block) &&
           isa<cir::YieldOp>(llvm::getSingleElement(block));
  };
 
  Region &body = op.getBodyRegion();
  Region &cleanup = op.getCleanupRegion();
 
  // An EH-only cleanup scope with an empty body can never trigger its cleanup
  // region — there are no operations in the body that could throw. Erase it.
  if (op.getCleanupKind() == CleanupKind::EH && isRegionTrivial(body)) {
    rewriter.eraseOp(op);
    return success();
  }
 
  // A cleanup scope with a trivial cleanup region has no cleanup to perform.
  // Inline the body into the parent block and erase the scope.
  if (!isRegionTrivial(cleanup) || !body.hasOneBlock())
    return failure();
 
  Block &bodyBlock = body.front();
  if (!isa<cir::YieldOp>(bodyBlock.getTerminator()))
    return failure();
 
  Operation *yield = bodyBlock.getTerminator();
  rewriter.inlineBlockBefore(&bodyBlock, op);
  rewriter.eraseOp(yield);
  rewriter.eraseOp(op);
  return success();
}
 
void cir::CleanupScopeOp::build(
    OpBuilder &builder, OperationState &result, CleanupKind cleanupKind,
    function_ref<void(OpBuilder &, Location)> bodyBuilder,
    function_ref<void(OpBuilder &, Location)> cleanupBuilder) {
  result.addAttribute(getCleanupKindAttrName(result.name),
                      CleanupKindAttr::get(builder.getContext(), cleanupKind));
 
  OpBuilder::InsertionGuard guard(builder);
 
  // Build body region.
  Region *bodyRegion = result.addRegion();
  builder.createBlock(bodyRegion);
  if (bodyBuilder)
    bodyBuilder(builder, result.location);
 
  // Build cleanup region.
  Region *cleanupRegion = result.addRegion();
  builder.createBlock(cleanupRegion);
  if (cleanupBuilder)
    cleanupBuilder(builder, result.location);
}
 
//===----------------------------------------------------------------------===//
// BrOp
//===----------------------------------------------------------------------===//
 
/// Merges blocks connected by a unique unconditional branch.
///
///   ^bb0:              ^bb0:
///     ...                ...
///     cir.br ^bb1  =>    ...
///   ^bb1:                cir.return
///     ...
///     cir.return
LogicalResult cir::BrOp::canonicalize(BrOp op, PatternRewriter &rewriter) {
  Block *src = op->getBlock();
  Block *dst = op.getDest();
 
  // Do not fold self-loops.
  if (src == dst)
    return failure();
 
  // Only merge when this is the unique edge between the blocks.
  if (src->getNumSuccessors() != 1 || dst->getSinglePredecessor() != src)
    return failure();
 
  // Don't merge blocks that start with LabelOp or IndirectBrOp.
  // This is to avoid merging blocks that have an indirect predecessor.
  if (isa<cir::LabelOp, cir::IndirectBrOp>(dst->front()))
    return failure();
 
  // An operand that is an argument of the destination itself (possible in an
  // unreachable cycle) would be replaced by itself, leaving its other uses
  // dangling once the destination is erased.
  if (llvm::any_of(op.getDestOperands(), [&](Value operand) {
        auto arg = dyn_cast<BlockArgument>(operand);
        return arg && arg.getOwner() == dst;
      }))
    return failure();
 
  // Copy the operands out: erasing the branch frees its operand storage.
  SmallVector<Value> operands(op.getDestOperands());
  rewriter.eraseOp(op);
  rewriter.mergeBlocks(dst, src, operands);
  return success();
}
 
mlir::SuccessorOperands cir::BrOp::getSuccessorOperands(unsigned index) {
  assert(index == 0 && "invalid successor index");
  return mlir::SuccessorOperands(getDestOperandsMutable());
}
 
Block *cir::BrOp::getSuccessorForOperands(ArrayRef<Attribute>) {
  return getDest();
}
 
//===----------------------------------------------------------------------===//
// IndirectBrCondOp
//===----------------------------------------------------------------------===//
 
mlir::SuccessorOperands
cir::IndirectBrOp::getSuccessorOperands(unsigned index) {
  assert(index < getNumSuccessors() && "invalid successor index");
  return mlir::SuccessorOperands(getSuccOperandsMutable()[index]);
}
 
ParseResult parseIndirectBrOpSucessors(
    OpAsmParser &parser, Type &flagType,
    SmallVectorImpl<Block *> &succOperandBlocks,
    SmallVectorImpl<SmallVector<OpAsmParser::UnresolvedOperand>> &succOperands,
    SmallVectorImpl<SmallVector<Type>> &succOperandsTypes) {
  if (failed(parser.parseCommaSeparatedList(
          OpAsmParser::Delimiter::Square,
          [&]() {
            Block *destination = nullptr;
            SmallVector<OpAsmParser::UnresolvedOperand> operands;
            SmallVector<Type> operandTypes;
 
            if (parser.parseSuccessor(destination).failed())
              return failure();
 
            if (succeeded(parser.parseOptionalLParen())) {
              if (failed(parser.parseOperandList(
                      operands, OpAsmParser::Delimiter::None)) ||
                  failed(parser.parseColonTypeList(operandTypes)) ||
                  failed(parser.parseRParen()))
                return failure();
            }
            succOperandBlocks.push_back(destination);
            succOperands.emplace_back(operands);
            succOperandsTypes.emplace_back(operandTypes);
            return success();
          },
          "successor blocks")))
    return failure();
  return success();
}
 
void printIndirectBrOpSucessors(OpAsmPrinter &p, cir::IndirectBrOp op,
                                Type flagType, SuccessorRange succs,
                                OperandRangeRange succOperands,
                                const TypeRangeRange &succOperandsTypes) {
  p << "[";
  llvm::interleave(
      llvm::zip(succs, succOperands),
      [&](auto i) {
        p.printNewline();
        p.printSuccessorAndUseList(std::get<0>(i), std::get<1>(i));
      },
      [&] { p << ','; });
  if (!succOperands.empty())
    p.printNewline();
  p << "]";
}
 
//===----------------------------------------------------------------------===//
// BrCondOp
//===----------------------------------------------------------------------===//
 
mlir::SuccessorOperands cir::BrCondOp::getSuccessorOperands(unsigned index) {
  assert(index < getNumSuccessors() && "invalid successor index");
  return SuccessorOperands(index == 0 ? getDestOperandsTrueMutable()
                                      : getDestOperandsFalseMutable());
}
 
Block *cir::BrCondOp::getSuccessorForOperands(ArrayRef<Attribute> operands) {
  if (IntegerAttr condAttr = dyn_cast_if_present<IntegerAttr>(operands.front()))
    return condAttr.getValue().isOne() ? getDestTrue() : getDestFalse();
  return nullptr;
}
 
//===----------------------------------------------------------------------===//
// CaseOp
//===----------------------------------------------------------------------===//
 
void cir::CaseOp::build(OpBuilder &builder, OperationState &result,
                        ArrayAttr value, CaseOpKind kind,
                        OpBuilder::InsertPoint &insertPoint) {
  OpBuilder::InsertionGuard guardSwitch(builder);
  result.addAttribute("value", value);
  result.getOrAddProperties<Properties>().kind =
      cir::CaseOpKindAttr::get(builder.getContext(), kind);
  Region *caseRegion = result.addRegion();
  builder.createBlock(caseRegion);
 
  insertPoint = builder.saveInsertionPoint();
}
 
//===----------------------------------------------------------------------===//
// SwitchOp
//===----------------------------------------------------------------------===//
 
void cir::SwitchOp::build(OpBuilder &builder, OperationState &result,
                          Value cond, BuilderOpStateCallbackRef switchBuilder) {
  assert(switchBuilder && "the builder callback for regions must be present");
  OpBuilder::InsertionGuard guardSwitch(builder);
  Region *switchRegion = result.addRegion();
  builder.createBlock(switchRegion);
  result.addOperands({cond});
  switchBuilder(builder, result.location, result);
}
 
void cir::SwitchOp::collectCases(llvm::SmallVectorImpl<CaseOp> &cases) {
  walk<mlir::WalkOrder::PreOrder>([&](mlir::Operation *op) {
    // Don't walk in nested switch op.
    if (isa<cir::SwitchOp>(op) && op != *this)
      return WalkResult::skip();
 
    if (auto caseOp = dyn_cast<cir::CaseOp>(op))
      cases.push_back(caseOp);
 
    return WalkResult::advance();
  });
}
 
bool cir::SwitchOp::isSimpleForm(llvm::SmallVectorImpl<CaseOp> &cases) {
  collectCases(cases);
 
  if (getBody().empty())
    return false;
 
  if (!isa<YieldOp>(getBody().front().back()))
    return false;
 
  if (!llvm::all_of(getBody().front(),
                    [](Operation &op) { return isa<CaseOp, YieldOp>(op); }))
    return false;
 
  return llvm::all_of(cases, [this](CaseOp op) {
    return op->getParentOfType<SwitchOp>() == *this;
  });
}
 
//===----------------------------------------------------------------------===//
// SwitchFlatOp
//===----------------------------------------------------------------------===//
 
void cir::SwitchFlatOp::build(OpBuilder &builder, OperationState &result,
                              Value value, Block *defaultDestination,
                              ValueRange defaultOperands,
                              ArrayRef<APInt> caseValues,
                              BlockRange caseDestinations,
                              ArrayRef<ValueRange> caseOperands) {
 
  std::vector<mlir::Attribute> caseValuesAttrs;
  for (const APInt &val : caseValues)
    caseValuesAttrs.push_back(cir::IntAttr::get(value.getType(), val));
  mlir::ArrayAttr attrs = ArrayAttr::get(builder.getContext(), caseValuesAttrs);
 
  build(builder, result, value, defaultOperands, caseOperands, attrs,
        defaultDestination, caseDestinations);
}
 
SuccessorOperands cir::SwitchFlatOp::getSuccessorOperands(unsigned index) {
  assert(index < getNumSuccessors() && "invalid successor index");
  if (index == 0)
    return SuccessorOperands(getDefaultOperandsMutable());
  return SuccessorOperands(getCaseOperandsMutable()[index - 1]);
}
 
Block *
cir::SwitchFlatOp::getSuccessorForOperands(ArrayRef<Attribute> operands) {
  auto cond = dyn_cast_if_present<cir::IntAttr>(operands.front());
  if (!cond)
    return nullptr;
  for (auto [value, dest] : llvm::zip(getCaseValues(), getCaseDestinations())) {
    const APInt &caseValue = cast<cir::IntAttr>(value).getValue();
    if (caseValue == cond.getValue())
      return dest;
  }
  return getDefaultDestination();
}
 
/// <cases> ::= `[` (case (`,` case )* )? `]`
/// <case>  ::= integer `:` bb-id (`(` ssa-use-and-type-list `)`)?
static ParseResult parseSwitchFlatOpCases(
    OpAsmParser &parser, Type flagType, mlir::ArrayAttr &caseValues,
    SmallVectorImpl<Block *> &caseDestinations,
    SmallVectorImpl<llvm::SmallVector<OpAsmParser::UnresolvedOperand>>
        &caseOperands,
    SmallVectorImpl<llvm::SmallVector<Type>> &caseOperandTypes) {
  if (failed(parser.parseLSquare()))
    return failure();
  if (succeeded(parser.parseOptionalRSquare()))
    return success();
  llvm::SmallVector<mlir::Attribute> values;
 
  auto parseCase = [&]() {
    int64_t value = 0;
    if (failed(parser.parseInteger(value)))
      return failure();
 
    values.push_back(cir::IntAttr::get(flagType, value));
 
    Block *destination;
    llvm::SmallVector<OpAsmParser::UnresolvedOperand> operands;
    llvm::SmallVector<Type> operandTypes;
    if (parser.parseColon() || parser.parseSuccessor(destination))
      return failure();
    if (!parser.parseOptionalLParen()) {
      if (parser.parseOperandList(operands, OpAsmParser::Delimiter::None,
                                  /*allowResultNumber=*/false) ||
          parser.parseColonTypeList(operandTypes) || parser.parseRParen())
        return failure();
    }
    caseDestinations.push_back(destination);
    caseOperands.emplace_back(operands);
    caseOperandTypes.emplace_back(operandTypes);
    return success();
  };
  if (failed(parser.parseCommaSeparatedList(parseCase)))
    return failure();
 
  caseValues = ArrayAttr::get(flagType.getContext(), values);
 
  return parser.parseRSquare();
}
 
static void printSwitchFlatOpCases(OpAsmPrinter &p, cir::SwitchFlatOp op,
                                   Type flagType, mlir::ArrayAttr caseValues,
                                   SuccessorRange caseDestinations,
                                   OperandRangeRange caseOperands,
                                   const TypeRangeRange &caseOperandTypes) {
  p << '[';
  p.printNewline();
  if (!caseValues) {
    p << ']';
    return;
  }
 
  size_t index = 0;
  llvm::interleave(
      llvm::zip(caseValues, caseDestinations),
      [&](auto i) {
        p << "  ";
        mlir::Attribute a = std::get<0>(i);
        p << mlir::cast<cir::IntAttr>(a).getValue();
        p << ": ";
        p.printSuccessorAndUseList(std::get<1>(i), caseOperands[index++]);
      },
      [&] {
        p << ',';
        p.printNewline();
      });
  p.printNewline();
  p << ']';
}
 
//===----------------------------------------------------------------------===//
// GlobalOp
//===----------------------------------------------------------------------===//
 
static ParseResult parseConstantValue(OpAsmParser &parser,
                                      mlir::Attribute &valueAttr) {
  NamedAttrList attr;
  return parser.parseAttribute(valueAttr, "value", attr);
}
 
static void printConstant(OpAsmPrinter &p, Attribute value) {
  p.printAttribute(value);
}
 
mlir::LogicalResult cir::GlobalOp::verify() {
  // A function is not an object, so it cannot be the type of a global.  A
  // global that holds a function's address carries a pointer type instead.
  if (mlir::isa<cir::FuncType>(getSymType()))
    return emitOpError("global type cannot be a function type");
 
  // Verify that the initial value, if present, is either a unit attribute or
  // an attribute CIR supports.
  if (getInitialValue().has_value()) {
    if (checkConstantTypes(getOperation(), getSymType(), *getInitialValue())
            .failed())
      return failure();
  }
 
  // The initial value can't be recomputed from the type once the ctor is
  // lowered (e.g. a null member pointer becomes -1 after CXXABILowering).
  if (!getCtorRegion().empty() && !getInitialValue().has_value())
    return emitOpError(
        "cannot have a constructor region without an initial value");
 
  std::optional<DynamicInitInfoAttr> info = getDynamicInitInfo();
 
  if ((getDynamicInitGuard().has_value()) && (!info || info->getLocal()) &&
      (!getCtorRegion().empty() || !getDtorRegion().empty()))
    return emitOpError(
        "Cannot have a static-local global-op with a constructor or "
        "destructor, they require in-function initialization via LocalInitOp");
 
  // CIRGen emits 'dynamic_init_guard' and 'dynamic_init_info' together and
  // they are only meaningful together: the guard drives lowering, which reads
  // the info. Require both or neither so malformed .cir can carry neither a
  // guard without info nor a dangling info nothing will read.
  if (getDynamicInitGuard().has_value() != getDynamicInitInfo().has_value())
    return emitOpError("'dynamic_init_guard' and 'dynamic_init_info' must be "
                       "present together");
 
  if (getTlsRefs()) {
    // 'Unordered' TLS globals (variable template instantiations) legitimately
    // carry both: they can't use the shared __tls_init guard, so CIRGen also
    // gives them their own dynamic-init guard, mangled the same as
    // 'tls_refs's own guard name. 'Ordered' TLS globals share __tls_init and
    // have no per-variable guard name, so a dynamic-init guard there would be
    // a mismatch.
    if (getDynamicInitGuard().has_value() && !getTlsRefs()->getGuardName())
      return emitOpError("cannot have a dynamic-init guard combined with "
                         "ordered tls references");
    if (!getTlsModel())
      return emitOpError("'tls_refs' only valid for tls");
  }
 
  if (getAliasee().has_value()) {
    if (getInitialValue().has_value() || !getCtorRegion().empty() ||
        !getDtorRegion().empty())
      return emitOpError("global alias shall not have an initializer or "
                         "constructor/destructor regions");
  }
 
  // TODO(CIR): Many other checks for properties that haven't been upstreamed
  // yet.
 
  return success();
}
 
void cir::GlobalOp::build(
    OpBuilder &odsBuilder, OperationState &odsState, llvm::StringRef sym_name,
    mlir::Type sym_type, bool isConstant,
    mlir::ptr::MemorySpaceAttrInterface addrSpace,
    cir::GlobalLinkageKind linkage,
    function_ref<void(OpBuilder &, Location)> ctorBuilder,
    function_ref<void(OpBuilder &, Location)> dtorBuilder) {
  odsState.addAttribute(getSymNameAttrName(odsState.name),
                        odsBuilder.getStringAttr(sym_name));
  odsState.addAttribute(getSymTypeAttrName(odsState.name),
                        mlir::TypeAttr::get(sym_type));
  auto &properties = odsState.getOrAddProperties<cir::GlobalOp::Properties>();
  properties.setConstant(isConstant);
 
  addrSpace = normalizeDefaultAddressSpace(addrSpace);
  if (addrSpace)
    odsState.addAttribute(getAddrSpaceAttrName(odsState.name), addrSpace);
 
  cir::GlobalLinkageKindAttr linkageAttr =
      cir::GlobalLinkageKindAttr::get(odsBuilder.getContext(), linkage);
  odsState.addAttribute(getLinkageAttrName(odsState.name), linkageAttr);
 
  Region *ctorRegion = odsState.addRegion();
  if (ctorBuilder) {
    odsBuilder.createBlock(ctorRegion);
    ctorBuilder(odsBuilder, odsState.location);
  }
 
  Region *dtorRegion = odsState.addRegion();
  if (dtorBuilder) {
    odsBuilder.createBlock(dtorRegion);
    dtorBuilder(odsBuilder, odsState.location);
  }
}
 
/// Given the region at `index`, or the parent operation if `index` is None,
/// return the successor regions. These are the regions that may be selected
/// during the flow of control. `operands` is a set of optional attributes that
/// correspond to a constant value for each operand, or null if that operand is
/// not a constant.
void cir::GlobalOp::getSuccessorRegions(
    mlir::RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> &regions) {
  // The `ctor` and `dtor` regions always branch back to the parent operation.
  if (!point.isParent()) {
    regions.emplace_back(getOperation());
    return;
  }
 
  // Don't consider the ctor region if it is empty.
  Region *ctorRegion = &this->getCtorRegion();
  if (ctorRegion->empty())
    ctorRegion = nullptr;
 
  // Don't consider the dtor region if it is empty.
  Region *dtorRegion = &this->getDtorRegion();
  if (dtorRegion->empty())
    dtorRegion = nullptr;
 
  // If the condition isn't constant, both regions may be executed.
  if (ctorRegion)
    regions.push_back(RegionSuccessor(ctorRegion));
  if (dtorRegion)
    regions.push_back(RegionSuccessor(dtorRegion));
}
 
static void printComdatName(OpAsmPrinter &p, StringAttr comdat) {
  if (!comdat)
    return;
  p << "comdat";
  if (!comdat.getValue().empty())
    p << "(\"" << comdat.getValue() << "\")";
}
 
static void printComdatName(OpAsmPrinter &p, cir::GlobalOp op,
                            StringAttr comdat) {
  printComdatName(p, comdat);
}
 
static ParseResult parseComdatName(OpAsmParser &parser,
                                   StringAttr &comdatAttr) {
  if (parser.parseOptionalKeyword("comdat").failed())
    return success();
  std::string comdatKey;
  if (succeeded(parser.parseOptionalLParen())) {
    if (parser.parseString(&comdatKey).failed() ||
        parser.parseRParen().failed())
      return failure();
  }
  comdatAttr = parser.getBuilder().getStringAttr(comdatKey);
  return success();
}
 
static void printGlobalOpTypeAndInitialValue(OpAsmPrinter &p, cir::GlobalOp op,
                                             TypeAttr type, Attribute initAttr,
                                             mlir::Region &ctorRegion,
                                             mlir::Region &dtorRegion) {
  auto printType = [&]() { p << ": " << type; };
  // Aliases are definitions but they have no initial value or ctor/dtor; the
  // assembly prints them like declarations (`: type`).
  if (op.isDeclaration() || op.getAliasee()) {
    printType();
    return;
  }
 
  p << "= ";
  if (initAttr)
    printConstant(p, initAttr);
 
  if (!ctorRegion.empty()) {
    p << " ctor ";
    p.printRegion(ctorRegion,
                  /*printEntryBlockArgs=*/false,
                  /*printBlockTerminators=*/false);
  }
 
  if (!dtorRegion.empty()) {
    p << " dtor ";
    p.printRegion(dtorRegion,
                  /*printEntryBlockArgs=*/false,
                  /*printBlockTerminators=*/false);
  }
}
 
static ParseResult parseGlobalOpTypeAndInitialValue(OpAsmParser &parser,
                                                    TypeAttr &typeAttr,
                                                    Attribute &initialValueAttr,
                                                    mlir::Region &ctorRegion,
                                                    mlir::Region &dtorRegion) {
  mlir::Type opTy;
  if (parser.parseOptionalEqual().failed()) {
    // Absence of equal means a declaration, so we need to parse the type.
    //  cir.global @a : !cir.int<s, 32>
    if (parser.parseColonType(opTy))
      return failure();
  } else {
    // Parse constant with initializer, examples:
    //  cir.global @y = 3.400000e+00 : f32
    //  cir.global @rgb = #cir.const_array<[...] : !cir.array<i8 x 3>>
    if (parseConstantValue(parser, initialValueAttr).failed())
      return failure();
 
    assert(mlir::isa<mlir::TypedAttr>(initialValueAttr) &&
           "Non-typed attrs shouldn't appear here.");
    opTy = mlir::cast<mlir::TypedAttr>(initialValueAttr).getType();
 
    // Parse constructor, example:
    //  cir.global @rgb = #cir.zero : type ctor { ... }
    if (!parser.parseOptionalKeyword("ctor")) {
      auto parseLoc = parser.getCurrentLocation();
      if (parser.parseRegion(ctorRegion, /*arguments=*/{}, /*argTypes=*/{}))
        return failure();
      if (ensureRegionTerm(parser, ctorRegion, parseLoc).failed())
        return failure();
    }
 
    // Parse destructor, example:
    //   dtor { ... }
    if (!parser.parseOptionalKeyword("dtor")) {
      auto parseLoc = parser.getCurrentLocation();
      if (parser.parseRegion(dtorRegion, /*arguments=*/{}, /*argTypes=*/{}))
        return failure();
      if (ensureRegionTerm(parser, dtorRegion, parseLoc).failed())
        return failure();
    }
  }
 
  typeAttr = TypeAttr::get(opTy);
  return success();
}
 
//===----------------------------------------------------------------------===//
// GetGlobalOp
//===----------------------------------------------------------------------===//
 
LogicalResult
cir::GetGlobalOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
  // Verify that the result type underlying pointer type matches the type of
  // the referenced cir.global or cir.func op.
  mlir::Operation *op =
      symbolTable.lookupNearestSymbolFrom(*this, getNameAttr());
  if (op == nullptr || !(isa<GlobalOp>(op) || isa<FuncOp>(op)))
    return emitOpError("'")
           << getName()
           << "' does not reference a valid cir.global or cir.func";
 
  mlir::Type symTy;
  mlir::ptr::MemorySpaceAttrInterface symAddrSpaceAttr{};
  if (auto g = dyn_cast<GlobalOp>(op)) {
    symTy = g.getSymType();
    symAddrSpaceAttr = g.getAddrSpaceAttr();
    // Verify that for thread local global access, the global needs to
    // be marked with tls bits.
    if (getTls() && !g.getTlsModel())
      return emitOpError("access to global not marked thread local");
 
    // Verify that the static_local attribute on GetGlobalOp matches the
    // dynamic_init_guard attribute on GlobalOp. GetGlobalOp uses a UnitAttr,
    // GlobalOp uses DynamicInitGuardAttr. Both should be present, or neither.
    bool getGlobalIsStaticLocal = getStaticLocal();
    bool globalIsStaticLocal =
        g.getDynamicInitGuard().has_value() &&
        (!g.getDynamicInitInfo() || g.getDynamicInitInfo()->getLocal());
 
    if (getGlobalIsStaticLocal != globalIsStaticLocal &&
        !getOperation()->getParentOfType<cir::GlobalOp>())
      return emitOpError("static_local attribute mismatch");
  } else if (auto f = dyn_cast<FuncOp>(op)) {
    symTy = f.getFunctionType();
  } else {
    llvm_unreachable("Unexpected operation for GetGlobalOp");
  }
 
  auto resultType = dyn_cast<PointerType>(getAddr().getType());
  if (!resultType || symTy != resultType.getPointee())
    return emitOpError("result type pointee type '")
           << resultType.getPointee() << "' does not match type " << symTy
           << " of the global @" << getName();
 
  if (symAddrSpaceAttr != resultType.getAddrSpace()) {
    return emitOpError()
           << "result type address space does not match the address "
              "space of the global @"
           << getName();
  }
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// VTableAddrPointOp
//===----------------------------------------------------------------------===//
 
LogicalResult
cir::VTableAddrPointOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
  StringRef name = getName();
 
  // Verify that the result type underlying pointer type matches the type of
  // the referenced cir.global.
  auto op =
      symbolTable.lookupNearestSymbolFrom<cir::GlobalOp>(*this, getNameAttr());
  if (!op)
    return emitOpError("'")
           << name << "' does not reference a valid cir.global";
  std::optional<mlir::Attribute> init = op.getInitialValue();
  if (!init)
    return success();
  if (!isa<cir::VTableAttr>(*init))
    return emitOpError("Expected #cir.vtable in initializer for global '")
           << name << "'";
  return success();
}
 
//===----------------------------------------------------------------------===//
// VTTAddrPointOp
//===----------------------------------------------------------------------===//
 
LogicalResult
cir::VTTAddrPointOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
  // VTT ptr is not coming from a symbol.
  if (!getName())
    return success();
  StringRef name = *getName();
 
  // Verify that the result type underlying pointer type matches the type of
  // the referenced cir.global op.
  auto op =
      symbolTable.lookupNearestSymbolFrom<cir::GlobalOp>(*this, getNameAttr());
  if (!op)
    return emitOpError("'")
           << name << "' does not reference a valid cir.global";
  std::optional<mlir::Attribute> init = op.getInitialValue();
  if (!init)
    return success();
  if (!isa<cir::ConstArrayAttr>(*init))
    return emitOpError(
               "Expected constant array in initializer for global VTT '")
           << name << "'";
  return success();
}
 
LogicalResult cir::VTTAddrPointOp::verify() {
  // The operation uses either a symbol or a value to operate, but not both
  if (getName() && getSymAddr())
    return emitOpError("should use either a symbol or value, but not both");
 
  // If not a symbol, stick with the concrete type used for getSymAddr.
  if (getSymAddr())
    return success();
 
  mlir::Type resultType = getAddr().getType();
  mlir::Type resTy = cir::PointerType::get(
      cir::PointerType::get(cir::VoidType::get(getContext())));
 
  if (resultType != resTy)
    return emitOpError("result type must be ")
           << resTy << ", but provided result type is " << resultType;
  return success();
}
 
//===----------------------------------------------------------------------===//
// FuncOp
//===----------------------------------------------------------------------===//
 
/// Returns the name used for the linkage attribute. This *must* correspond to
/// the name of the attribute in ODS.
static llvm::StringRef getLinkageAttrNameString() { return "linkage"; }
 
/// Declared cir.func attributes with no dedicated syntax, which print in and
/// parse from the attribute dictionary instead.
static bool isFuncAttrInDict(llvm::StringRef name) {
  return name == CIRDialect::getMemoryEffectsAttrName() ||
         name == CIRDialect::getUwtableAttrName() ||
         name == CIRDialect::getFramePointerAttrName();
}
 
void cir::FuncOp::build(OpBuilder &builder, OperationState &result,
                        StringRef name, FuncType type,
                        GlobalLinkageKind linkage, CallingConv callingConv) {
  result.addRegion();
  result.addAttribute(getSymNameAttrName(result.name),
                      builder.getStringAttr(name));
  result.addAttribute(getFunctionTypeAttrName(result.name),
                      TypeAttr::get(type));
  result.addAttribute(
      getLinkageAttrNameString(),
      GlobalLinkageKindAttr::get(builder.getContext(), linkage));
  result.addAttribute(getCallingConvAttrName(result.name),
                      CallingConvAttr::get(builder.getContext(), callingConv));
}
 
//===----------------------------------------------------------------------===//
// AnnotationAttr
//===----------------------------------------------------------------------===//
 
LogicalResult
cir::AnnotationAttr::verify(function_ref<InFlightDiagnostic()> emitError,
                            mlir::StringAttr name, mlir::ArrayAttr args) {
  if (!args)
    return success();
  for (mlir::Attribute arg : args) {
    if (!isa<mlir::StringAttr, mlir::IntegerAttr>(arg))
      return emitError() << "annotation args must be StringAttr or IntegerAttr,"
                         << " got " << arg;
  }
  return success();
}
 
ParseResult cir::FuncOp::parse(OpAsmParser &parser, OperationState &state) {
  llvm::SMLoc loc = parser.getCurrentLocation();
  mlir::Builder &builder = parser.getBuilder();
 
  mlir::StringAttr builtinNameAttr = getBuiltinAttrName(state.name);
  mlir::StringAttr coroutineNameAttr = getCoroutineAttrName(state.name);
  mlir::StringAttr inlineKindNameAttr = getInlineKindAttrName(state.name);
  mlir::StringAttr lambdaNameAttr = getLambdaAttrName(state.name);
  mlir::StringAttr noProtoNameAttr = getNoProtoAttrName(state.name);
  mlir::StringAttr comdatNameAttr = getComdatAttrName(state.name);
  mlir::StringAttr alignmentNameAttr = getAlignmentAttrName(state.name);
  mlir::StringAttr preferredAlignmentNameAttr =
      getPreferredAlignmentAttrName(state.name);
  mlir::StringAttr visNameAttr = getSymVisibilityAttrName(state.name);
  mlir::StringAttr dsoLocalNameAttr = getDsoLocalAttrName(state.name);
  mlir::StringAttr funcInfoNameAttr = getFuncInfoAttrName(state.name);
 
  if (::mlir::succeeded(parser.parseOptionalKeyword(builtinNameAttr.strref())))
    state.addAttribute(builtinNameAttr, parser.getBuilder().getUnitAttr());
  if (::mlir::succeeded(
          parser.parseOptionalKeyword(coroutineNameAttr.strref())))
    state.addAttribute(coroutineNameAttr, parser.getBuilder().getUnitAttr());
 
  // Parse optional inline kind attribute
  cir::InlineKindAttr inlineKindAttr;
  if (failed(parseInlineKindAttr(parser, inlineKindAttr)))
    return failure();
  if (inlineKindAttr)
    state.addAttribute(inlineKindNameAttr, inlineKindAttr);
 
  if (::mlir::succeeded(parser.parseOptionalKeyword(lambdaNameAttr.strref())))
    state.addAttribute(lambdaNameAttr, parser.getBuilder().getUnitAttr());
  if (parser.parseOptionalKeyword(noProtoNameAttr).succeeded())
    state.addAttribute(noProtoNameAttr, parser.getBuilder().getUnitAttr());
 
  {
    StringAttr comdatAttr;
    if (parseComdatName(parser, comdatAttr).failed())
      return failure();
    if (comdatAttr)
      state.addAttribute(comdatNameAttr, comdatAttr);
  }
 
  auto parseAlignmentBody = [&](int64_t &value) {
    if (parser.parseLParen().failed() || parser.parseInteger(value).failed() ||
        parser.parseRParen().failed())
      return failure();
 
    if (value <= 0)
      return static_cast<LogicalResult>(parser.emitError(
          loc, "function alignment must be a positive integer"));
 
    return success();
  };
 
  if (parser.parseOptionalKeyword(alignmentNameAttr).succeeded()) {
    int64_t value;
    if (parseAlignmentBody(value).failed())
      return failure();
    state.addAttribute(alignmentNameAttr, builder.getI64IntegerAttr(value));
  }
 
  if (parser.parseOptionalKeyword(preferredAlignmentNameAttr).succeeded()) {
    int64_t value;
    if (parseAlignmentBody(value).failed())
      return failure();
    state.addAttribute(preferredAlignmentNameAttr,
                       builder.getI64IntegerAttr(value));
  }
 
  // Default to external linkage if no keyword is provided.
  state.addAttribute(getLinkageAttrNameString(),
                     GlobalLinkageKindAttr::get(
                         parser.getContext(),
                         parseOptionalCIRKeyword<GlobalLinkageKind>(
                             parser, GlobalLinkageKind::ExternalLinkage)));
 
  ::llvm::StringRef visAttrStr;
  if (parser.parseOptionalKeyword(&visAttrStr, {"private", "public", "nested"})
          .succeeded()) {
    state.addAttribute(visNameAttr,
                       parser.getBuilder().getStringAttr(visAttrStr));
  }
 
  state.getOrAddProperties<cir::FuncOp::Properties>().global_visibility =
      parseOptionalCIRKeyword(parser, cir::VisibilityKind::Default);
 
  if (parser.parseOptionalKeyword(dsoLocalNameAttr).succeeded())
    state.addAttribute(dsoLocalNameAttr, parser.getBuilder().getUnitAttr());
 
  StringAttr nameAttr;
  if (parser.parseSymbolName(nameAttr, getSymNameAttrName(state.name),
                             state.attributes))
    return failure();
  llvm::SmallVector<OpAsmParser::Argument, 8> arguments;
  llvm::SmallVector<mlir::Type> resultTypes;
  llvm::SmallVector<DictionaryAttr> resultAttrs;
  bool isVariadic = false;
  if (function_interface_impl::parseFunctionSignatureWithArguments(
          parser, /*allowVariadic=*/true, arguments, isVariadic, resultTypes,
          resultAttrs))
    return failure();
  llvm::SmallVector<mlir::Type> argTypes;
  llvm::SmallVector<mlir::Attribute> argAttrs;
  bool argAttrsEmpty = true;
  for (OpAsmParser::Argument &arg : arguments) {
    argTypes.push_back(arg.type);
    // Add the 'empty' attribute anyway to make sure the arity matches, but we
    // only want to 'set' the attribute at the top level if there is SOME data
    // along the way.
    argAttrs.push_back(arg.attrs);
    if (arg.attrs)
      argAttrsEmpty = false;
  }
 
  // These should be in sync anyway, but test both of them anyway.
  if (resultTypes.size() > 1 || resultAttrs.size() > 1)
    return parser.emitError(
        loc, "functions with multiple return types are not supported");
 
  mlir::Type returnType =
      (resultTypes.empty() ? cir::VoidType::get(builder.getContext())
                           : resultTypes.front());
 
  cir::FuncType fnType =
      cir::FuncType::getChecked([&]() { return parser.emitError(loc); },
                                argTypes, returnType, isVariadic);
  if (!fnType)
    return failure();
 
  state.addAttribute(getFunctionTypeAttrName(state.name),
                     TypeAttr::get(fnType));
 
  if (!resultAttrs.empty() && resultAttrs[0])
    state.addAttribute(
        getResAttrsAttrName(state.name),
        mlir::ArrayAttr::get(parser.getContext(), {resultAttrs[0]}));
 
  if (!argAttrsEmpty)
    state.addAttribute(getArgAttrsAttrName(state.name),
                       mlir::ArrayAttr::get(parser.getContext(), argAttrs));
 
  bool hasAlias = false;
  mlir::StringAttr aliaseeNameAttr = getAliaseeAttrName(state.name);
  if (parser.parseOptionalKeyword("alias").succeeded()) {
    if (parser.parseLParen().failed())
      return failure();
    mlir::StringAttr aliaseeAttr;
    if (parser.parseOptionalSymbolName(aliaseeAttr).failed())
      return failure();
    state.addAttribute(aliaseeNameAttr, FlatSymbolRefAttr::get(aliaseeAttr));
    if (parser.parseRParen().failed())
      return failure();
    hasAlias = true;
  }
 
  mlir::StringAttr personalityNameAttr = getPersonalityAttrName(state.name);
  if (parser.parseOptionalKeyword("personality").succeeded()) {
    if (parser.parseLParen().failed())
      return failure();
    mlir::StringAttr personalityAttr;
    if (parser.parseOptionalSymbolName(personalityAttr).failed())
      return failure();
    state.addAttribute(personalityNameAttr,
                       FlatSymbolRefAttr::get(personalityAttr));
    if (parser.parseRParen().failed())
      return failure();
  }
 
  // Default to C calling convention if no keyword is provided.
  mlir::StringAttr callConvNameAttr = getCallingConvAttrName(state.name);
  cir::CallingConv callConv = cir::CallingConv::C;
  if (parser.parseOptionalKeyword("cc").succeeded()) {
    if (parser.parseLParen().failed())
      return failure();
    if (parseCIRKeyword<cir::CallingConv>(parser, callConv).failed())
      return parser.emitError(loc) << "unknown calling convention";
    if (parser.parseRParen().failed())
      return failure();
  }
  state.addAttribute(callConvNameAttr,
                     cir::CallingConvAttr::get(parser.getContext(), callConv));
 
  auto parseGlobalDtorCtor =
      [&](StringRef keyword,
          llvm::function_ref<void(std::optional<int> prio)> createAttr)
      -> mlir::LogicalResult {
    if (mlir::succeeded(parser.parseOptionalKeyword(keyword))) {
      std::optional<int> priority;
      if (mlir::succeeded(parser.parseOptionalLParen())) {
        auto parsedPriority = mlir::FieldParser<int>::parse(parser);
        if (mlir::failed(parsedPriority))
          return parser.emitError(parser.getCurrentLocation(),
                                  "failed to parse 'priority', of type 'int'");
        priority = parsedPriority.value_or(int());
        // Parse literal ')'
        if (parser.parseRParen())
          return failure();
      }
      createAttr(priority);
    }
    return success();
  };
 
  // Parse the func_info attribute
  if (parser.parseOptionalKeyword("func_info").succeeded()) {
    if (parser.parseLess().failed())
      return failure();
 
    llvm::SMLoc attrLoc = parser.getCurrentLocation();
    mlir::Attribute attr;
    if (parser.parseAttribute(attr).failed())
      return failure();
    if (!mlir::isa<cir::CXXCtorAttr, cir::CXXDtorAttr, cir::CXXAssignAttr,
                   cir::FuncIdentityAttr>(attr))
      return parser.emitError(attrLoc,
                              "expected a function info attribute, got ")
             << attr;
    state.addAttribute(funcInfoNameAttr, attr);
 
    if (parser.parseGreater().failed())
      return failure();
  }
 
  if (parseGlobalDtorCtor("global_ctor", [&](std::optional<int> priority) {
        mlir::IntegerAttr globalCtorPriorityAttr =
            builder.getI32IntegerAttr(priority.value_or(65535));
        state.addAttribute(getGlobalCtorPriorityAttrName(state.name),
                           globalCtorPriorityAttr);
      }).failed())
    return failure();
 
  if (parseGlobalDtorCtor("global_dtor", [&](std::optional<int> priority) {
        mlir::IntegerAttr globalDtorPriorityAttr =
            builder.getI32IntegerAttr(priority.value_or(65535));
        state.addAttribute(getGlobalDtorPriorityAttrName(state.name),
                           globalDtorPriorityAttr);
      }).failed())
    return failure();
 
  // Parse optional annotations attribute (an ArrayAttr of AnnotationAttr).
  mlir::StringAttr annotationsNameAttr = getAnnotationsAttrName(state.name);
  mlir::ArrayAttr annotationsAttr;
  if (parser.parseOptionalAttribute(annotationsAttr).has_value() &&
      annotationsAttr)
    state.addAttribute(annotationsNameAttr, annotationsAttr);
 
  // Parse the rest of the attributes.
  NamedAttrList parsedAttrs;
  llvm::SMLoc attrsLoc = parser.getCurrentLocation();
  if (parser.parseOptionalAttrDictWithKeyword(parsedAttrs))
    return failure();
 
  // Every other declared attribute has dedicated syntax above.
  for (StringRef disallowed : cir::FuncOp::getAttributeNames()) {
    if (isFuncAttrInDict(disallowed))
      continue;
    if (parsedAttrs.get(disallowed))
      return parser.emitError(loc, "attribute '")
             << disallowed
             << "' should not be specified in the explicit attribute list";
  }
 
  if (checkEffectAttrKinds(parser, attrsLoc, parsedAttrs).failed())
    return failure();
 
  state.attributes.append(parsedAttrs);
 
  // Parse the optional function body.
  auto *body = state.addRegion();
  OptionalParseResult parseResult = parser.parseOptionalRegion(
      *body, arguments, /*enableNameShadowing=*/false);
  if (parseResult.has_value()) {
    if (hasAlias)
      return parser.emitError(loc, "function alias shall not have a body");
    if (failed(*parseResult))
      return failure();
    // Function body was parsed, make sure its not empty.
    if (body->empty())
      return parser.emitError(loc, "expected non-empty function body");
  }
 
  return success();
}
 
// This function corresponds to `llvm::GlobalValue::isDeclaration` and should
// have a similar implementation. We don't currently ifuncs or materializable
// functions, but those should be handled here as they are implemented.
bool cir::FuncOp::isDeclaration() {
  assert(!cir::MissingFeatures::supportIFuncAttr());
 
  std::optional<StringRef> aliasee = getAliasee();
  if (!aliasee)
    return getFunctionBody().empty();
 
  // Aliases are always definitions.
  return false;
}
 
bool cir::FuncOp::isCXXSpecialMemberFunction() {
  // The func_info union can grow forms that are not special members, so the
  // check names the concrete forms rather than testing for presence.
  mlir::Attribute attr = getFuncInfoAttr();
  return attr && mlir::isa<CXXCtorAttr, CXXDtorAttr, CXXAssignAttr>(attr);
}
 
bool cir::FuncOp::isCxxConstructor() {
  auto attr = getFuncInfoAttr();
  return attr && dyn_cast<CXXCtorAttr>(attr);
}
 
bool cir::FuncOp::isCxxDestructor() {
  auto attr = getFuncInfoAttr();
  return attr && dyn_cast<CXXDtorAttr>(attr);
}
 
bool cir::FuncOp::isCxxSpecialAssignment() {
  auto attr = getFuncInfoAttr();
  return attr && dyn_cast<CXXAssignAttr>(attr);
}
 
std::optional<CtorKind> cir::FuncOp::getCxxConstructorKind() {
  mlir::Attribute attr = getFuncInfoAttr();
  if (attr) {
    if (auto ctor = dyn_cast<CXXCtorAttr>(attr))
      return ctor.getCtorKind();
  }
  return std::nullopt;
}
 
std::optional<AssignKind> cir::FuncOp::getCxxSpecialAssignKind() {
  mlir::Attribute attr = getFuncInfoAttr();
  if (attr) {
    if (auto assign = dyn_cast<CXXAssignAttr>(attr))
      return assign.getAssignKind();
  }
  return std::nullopt;
}
 
bool cir::FuncOp::isCxxTrivialMemberFunction() {
  mlir::Attribute attr = getFuncInfoAttr();
  if (attr) {
    if (auto ctor = dyn_cast<CXXCtorAttr>(attr))
      return ctor.getIsTrivial();
    if (auto dtor = dyn_cast<CXXDtorAttr>(attr))
      return dtor.getIsTrivial();
    if (auto assign = dyn_cast<CXXAssignAttr>(attr))
      return assign.getIsTrivial();
  }
  return false;
}
 
mlir::Region *cir::FuncOp::getCallableRegion() {
  // Declarations and aliases have no body to analyze or inline. Returning the
  // empty region would make interprocedural analyses (e.g. SCCP) treat calls
  // to them as having no returns, leaving their results uninitialized.
  if (getBody().empty())
    return nullptr;
  return &getBody();
}
 
void cir::FuncOp::print(OpAsmPrinter &p) {
  if (getBuiltin())
    p << " builtin";
 
  if (getCoroutine())
    p << " coroutine";
 
  printInlineKindAttr(p, getInlineKindAttr());
 
  if (getLambda())
    p << " lambda";
 
  if (getNoProto())
    p << " no_proto";
 
  if (getComdatAttr()) {
    p << ' ';
    printComdatName(p, getComdatAttr());
  }
 
  if (getAlignment())
    p << " alignment(" << *getAlignment() << ')';
 
  if (getPreferredAlignment())
    p << " preferred_alignment(" << *getPreferredAlignment() << ')';
 
  if (getLinkage() != GlobalLinkageKind::ExternalLinkage)
    p << ' ' << stringifyGlobalLinkageKind(getLinkage());
 
  mlir::SymbolTable::Visibility vis = getVisibility();
  if (vis != mlir::SymbolTable::Visibility::Public)
    p << ' ' << vis;
 
  if (getGlobalVisibility() != cir::VisibilityKind::Default)
    p << ' ' << stringifyVisibilityKind(getGlobalVisibility());
 
  if (getDsoLocal())
    p << " dso_local";
 
  p << ' ';
  p.printSymbolName(getSymName());
  cir::FuncType fnType = getFunctionType();
  function_interface_impl::printFunctionSignature(
      p, *this, fnType.getInputs(), fnType.isVarArg(), fnType.getReturnTypes());
 
  if (std::optional<StringRef> aliaseeName = getAliasee()) {
    p << " alias(";
    p.printSymbolName(*aliaseeName);
    p << ")";
  }
 
  if (getCallingConv() != cir::CallingConv::C) {
    p << " cc(";
    p << stringifyCallingConv(getCallingConv());
    p << ")";
  }
 
  if (std::optional<StringRef> personalityName = getPersonality()) {
    p << " personality(";
    p.printSymbolName(*personalityName);
    p << ")";
  }
 
  if (mlir::Attribute funcInfo = getFuncInfoAttr()) {
    p << " func_info<";
    p.printAttribute(funcInfo);
    p << '>';
  }
 
  if (auto globalCtorPriority = getGlobalCtorPriority()) {
    p << " global_ctor";
    if (globalCtorPriority.value() != 65535)
      p << "(" << globalCtorPriority.value() << ")";
  }
 
  if (auto globalDtorPriority = getGlobalDtorPriority()) {
    p << " global_dtor";
    if (globalDtorPriority.value() != 65535)
      p << "(" << globalDtorPriority.value() << ")";
  }
 
  if (mlir::ArrayAttr annotations = getAnnotationsAttr()) {
    p << ' ';
    p.printAttribute(annotations);
  }
 
  // Every other declared attribute is printed by the syntax above.
  llvm::SmallVector<llvm::StringRef> elidedAttrs;
  for (llvm::StringRef name : cir::FuncOp::getAttributeNames())
    if (!isFuncAttrInDict(name))
      elidedAttrs.push_back(name);
  function_interface_impl::printFunctionAttributes(p, *this, elidedAttrs);
 
  // Print the body if this is not an external function.
  Region &body = getOperation()->getRegion(0);
  if (!body.empty()) {
    p << ' ';
    p.printRegion(body, /*printEntryBlockArgs=*/false,
                  /*printBlockTerminators=*/true);
  }
}
 
mlir::LogicalResult cir::FuncOp::verify() {
 
  if (!isDeclaration() && getCoroutine()) {
    int coroutineOpCount = 0;
    this->walk([&](Operation *op) {
      if (isa<CoroutineOp>(op)) {
        coroutineOpCount++;
        if (coroutineOpCount > 1) {
          return mlir::WalkResult::interrupt();
        }
      }
      return mlir::WalkResult::advance();
    });
    if (coroutineOpCount != 1)
      return emitOpError()
             << "coroutine function must have exactly one cir.coroutine op";
  }
 
  llvm::SmallSet<llvm::StringRef, 16> labels;
  llvm::SmallSet<llvm::StringRef, 16> gotos;
  llvm::SmallSet<llvm::StringRef, 16> blockAddresses;
  bool invalidBlockAddress = false;
  getOperation()->walk([&](mlir::Operation *op) {
    if (auto lab = dyn_cast<cir::LabelOp>(op)) {
      labels.insert(lab.getLabel());
    } else if (auto goTo = dyn_cast<cir::GotoOp>(op)) {
      gotos.insert(goTo.getLabel());
    } else if (auto blkAdd = dyn_cast<cir::BlockAddressOp>(op)) {
      if (blkAdd.getBlockAddrInfoAttr().getFunc().getAttr() != getSymName()) {
        // Stop the walk early, no need to continue
        invalidBlockAddress = true;
        return mlir::WalkResult::interrupt();
      }
      blockAddresses.insert(blkAdd.getBlockAddrInfoAttr().getLabel());
    }
    return mlir::WalkResult::advance();
  });
 
  if (invalidBlockAddress)
    return emitOpError() << "blockaddress references a different function";
 
  llvm::SmallSet<llvm::StringRef, 16> mismatched;
  if (!labels.empty() || !gotos.empty()) {
    mismatched = llvm::set_difference(gotos, labels);
 
    if (!mismatched.empty())
      return emitOpError() << "goto/label mismatch";
  }
 
  mismatched.clear();
 
  if (!labels.empty() || !blockAddresses.empty()) {
    mismatched = llvm::set_difference(blockAddresses, labels);
 
    if (!mismatched.empty())
      return emitOpError()
             << "expects an existing label target in the referenced function";
  }
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// AddOp / SubOp
//===----------------------------------------------------------------------===//
 
// The integer-only type constraint on these ops makes the nsw/nuw/sat flag
// type checks unnecessary. Only the mutual-exclusivity between nsw/nuw and
// sat needs to be verified.
 
LogicalResult cir::AddOp::verify() {
  if (getSaturated() && (getNoSignedWrap() || getNoUnsignedWrap()))
    return emitOpError()
           << "the nsw/nuw flags and the saturated flag are mutually exclusive";
  return mlir::success();
}
 
LogicalResult cir::SubOp::verify() {
  if (getSaturated() && (getNoSignedWrap() || getNoUnsignedWrap()))
    return emitOpError()
           << "the nsw/nuw flags and the saturated flag are mutually exclusive";
  return mlir::success();
}
 
//===----------------------------------------------------------------------===//
// TernaryOp
//===----------------------------------------------------------------------===//
 
void cir::TernaryOp::build(
    OpBuilder &builder, OperationState &result, Value cond,
    function_ref<void(OpBuilder &, Location)> trueBuilder,
    function_ref<void(OpBuilder &, Location)> falseBuilder) {
  result.addOperands(cond);
  OpBuilder::InsertionGuard guard(builder);
  Region *trueRegion = result.addRegion();
  builder.createBlock(trueRegion);
  trueBuilder(builder, result.location);
  Region *falseRegion = result.addRegion();
  builder.createBlock(falseRegion);
  falseBuilder(builder, result.location);
 
  // Get result type from whichever branch has a yield (the other may have
  // unreachable from a throw expression)
  cir::YieldOp yield;
  if (trueRegion->back().mightHaveTerminator())
    yield = dyn_cast_or_null<cir::YieldOp>(trueRegion->back().getTerminator());
  if (!yield && falseRegion->back().mightHaveTerminator())
    yield = dyn_cast_or_null<cir::YieldOp>(falseRegion->back().getTerminator());
 
  assert((!yield || yield.getNumOperands() <= 1) &&
         "expected zero or one result type");
  if (yield && yield.getNumOperands() == 1)
    result.addTypes(TypeRange{yield.getOperandTypes().front()});
}
 
//===----------------------------------------------------------------------===//
// SelectOp
//===----------------------------------------------------------------------===//
 
OpFoldResult cir::SelectOp::fold(FoldAdaptor adaptor) {
  if (auto condition =
          mlir::dyn_cast_if_present<cir::BoolAttr>(adaptor.getCondition()))
    return condition.getValue() ? getTrueValue() : getFalseValue();
 
  // cir.select if %0 then x else x -> x
  mlir::Attribute trueValue = adaptor.getTrueValue();
  mlir::Attribute falseValue = adaptor.getFalseValue();
  if (trueValue == falseValue)
    return trueValue;
  if (getTrueValue() == getFalseValue())
    return getTrueValue();
 
  return {};
}
 
LogicalResult cir::SelectOp::verify() {
  // AllTypesMatch already guarantees trueVal and falseVal have matching types.
  auto condTy = dyn_cast<cir::VectorType>(getCondition().getType());
 
  // If condition is not a vector, no further checks are needed.
  if (!condTy)
    return success();
 
  // When condition is a vector, both other operands must also be vectors.
  if (!isa<cir::VectorType>(getTrueValue().getType()) ||
      !isa<cir::VectorType>(getFalseValue().getType())) {
    return emitOpError()
           << "expected both true and false operands to be vector types "
              "when the condition is a vector boolean type";
  }
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// ShiftOp
//===----------------------------------------------------------------------===//
LogicalResult cir::ShiftOp::verify() {
  mlir::Operation *op = getOperation();
  auto op0VecTy = mlir::dyn_cast<cir::VectorType>(op->getOperand(0).getType());
  auto op1VecTy = mlir::dyn_cast<cir::VectorType>(op->getOperand(1).getType());
  if (!op0VecTy ^ !op1VecTy)
    return emitOpError() << "input types cannot be one vector and one scalar";
 
  if (op0VecTy) {
    if (op0VecTy.getSize() != op1VecTy.getSize())
      return emitOpError() << "input vector types must have the same size";
 
    auto opResultTy = mlir::dyn_cast<cir::VectorType>(getType());
    if (!opResultTy)
      return emitOpError() << "the type of the result must be a vector "
                           << "if it is vector shift";
 
    auto op0VecEleTy = mlir::cast<cir::IntType>(op0VecTy.getElementType());
    auto op1VecEleTy = mlir::cast<cir::IntType>(op1VecTy.getElementType());
    if (op0VecEleTy.getWidth() != op1VecEleTy.getWidth())
      return emitOpError()
             << "vector operands do not have the same elements sizes";
 
    auto resVecEleTy = mlir::cast<cir::IntType>(opResultTy.getElementType());
    if (op0VecEleTy.getWidth() != resVecEleTy.getWidth())
      return emitOpError() << "vector operands and result type do not have the "
                              "same elements sizes";
  }
 
  return mlir::success();
}
 
//===----------------------------------------------------------------------===//
// LabelOp Definitions
//===----------------------------------------------------------------------===//
 
LogicalResult cir::LabelOp::verify() {
  mlir::Operation *op = getOperation();
  mlir::Block *blk = op->getBlock();
  if (&blk->front() != op)
    return emitError() << "must be the first operation in a block";
 
  return mlir::success();
}
 
//===----------------------------------------------------------------------===//
// IncOp
//===----------------------------------------------------------------------===//
 
OpFoldResult cir::IncOp::fold(FoldAdaptor adaptor) {
  if (mlir::isa_and_present<cir::PoisonAttr>(adaptor.getInput()))
    return adaptor.getInput();
  return {};
}
 
//===----------------------------------------------------------------------===//
// DecOp
//===----------------------------------------------------------------------===//
 
OpFoldResult cir::DecOp::fold(FoldAdaptor adaptor) {
  if (mlir::isa_and_present<cir::PoisonAttr>(adaptor.getInput()))
    return adaptor.getInput();
  return {};
}
 
//===----------------------------------------------------------------------===//
// MinusOp
//===----------------------------------------------------------------------===//
 
OpFoldResult cir::MinusOp::fold(FoldAdaptor adaptor) {
  if (mlir::isa_and_present<cir::PoisonAttr>(adaptor.getInput()))
    return adaptor.getInput();
 
  // Fold with constant inputs. Floating-point negation is handled by
  // cir::FNegOp.
  if (auto intAttr =
          mlir::dyn_cast_if_present<cir::IntAttr>(adaptor.getInput())) {
    APInt val = intAttr.getValue();
    val.negate();
    return cir::IntAttr::get(getType(), val);
  }
 
  return {};
}
 
//===----------------------------------------------------------------------===//
// FNegOp
//===----------------------------------------------------------------------===//
 
OpFoldResult cir::FNegOp::fold(FoldAdaptor adaptor) {
  if (mlir::isa_and_present<cir::PoisonAttr>(adaptor.getInput()))
    return adaptor.getInput();
 
  // Fold with constant inputs.
  if (auto fpAttr =
          mlir::dyn_cast_if_present<cir::FPAttr>(adaptor.getInput())) {
    APFloat val = fpAttr.getValue();
    val.changeSign();
    return cir::FPAttr::get(getType(), val);
  }
 
  return {};
}
 
//===----------------------------------------------------------------------===//
// NotOp
//===----------------------------------------------------------------------===//
 
OpFoldResult cir::NotOp::fold(FoldAdaptor adaptor) {
  if (mlir::isa_and_present<cir::PoisonAttr>(adaptor.getInput()))
    return adaptor.getInput();
 
  // not(not(x)) -> x is handled by the Involution trait.
 
  // Fold with constant inputs.
  if (mlir::Attribute attr = adaptor.getInput()) {
    if (auto intAttr = mlir::dyn_cast<cir::IntAttr>(attr)) {
      APInt val = intAttr.getValue();
      val.flipAllBits();
      return cir::IntAttr::get(getType(), val);
    }
    if (auto boolAttr = mlir::dyn_cast<cir::BoolAttr>(attr))
      return cir::BoolAttr::get(getContext(), !boolAttr.getValue());
  }
 
  return {};
}
 
//===----------------------------------------------------------------------===//
// BaseClassAddrOp & DerivedClassAddrOp
//===----------------------------------------------------------------------===//
 
LogicalResult cir::BaseClassAddrOp::verify() {
  if (getSrcAddr().getType().getAddrSpace() != getType().getAddrSpace())
    return emitOpError() << "address space mismatch";
  return mlir::success();
}
 
LogicalResult cir::DerivedClassAddrOp::verify() {
  if (getSrcAddr().getType().getAddrSpace() != getType().getAddrSpace())
    return emitOpError() << "address space mismatch";
  return mlir::success();
}
 
//===----------------------------------------------------------------------===//
// BaseDataMemberOp & DerivedDataMemberOp
//===----------------------------------------------------------------------===//
 
static LogicalResult verifyMemberPtrCast(Operation *op, mlir::Value src,
                                         mlir::Type resultTy) {
  // Let the operand type be T1 C1::*, let the result type be T2 C2::*.
  // Verify that T1 and T2 are the same type.
  mlir::Type inputMemberTy;
  mlir::Type resultMemberTy;
  if (mlir::isa<cir::DataMemberType>(src.getType())) {
    inputMemberTy =
        mlir::cast<cir::DataMemberType>(src.getType()).getMemberTy();
    resultMemberTy = mlir::cast<cir::DataMemberType>(resultTy).getMemberTy();
  }
  assert(!cir::MissingFeatures::memberFuncPtrCast());
  if (inputMemberTy != resultMemberTy)
    return op->emitOpError()
           << "member types of the operand and the result do not match";
 
  return mlir::success();
}
 
LogicalResult cir::BaseDataMemberOp::verify() {
  return verifyMemberPtrCast(getOperation(), getSrc(), getType());
}
 
LogicalResult cir::DerivedDataMemberOp::verify() {
  return verifyMemberPtrCast(getOperation(), getSrc(), getType());
}
 
//===----------------------------------------------------------------------===//
// BaseMethodOp & DerivedMethodOp
//===----------------------------------------------------------------------===//
 
LogicalResult cir::BaseMethodOp::verify() {
  return verifyMemberPtrCast(getOperation(), getSrc(), getType());
}
 
LogicalResult cir::DerivedMethodOp::verify() {
  return verifyMemberPtrCast(getOperation(), getSrc(), getType());
}
 
//===----------------------------------------------------------------------===//
// AwaitOp
//===----------------------------------------------------------------------===//
 
void cir::AwaitOp::build(OpBuilder &builder, OperationState &result,
                         cir::AwaitKind kind, BuilderCallbackRef readyBuilder,
                         BuilderCallbackRef suspendBuilder,
                         BuilderCallbackRef resumeBuilder) {
  result.addAttribute(getKindAttrName(result.name),
                      cir::AwaitKindAttr::get(builder.getContext(), kind));
  {
    OpBuilder::InsertionGuard guard(builder);
    Region *readyRegion = result.addRegion();
    builder.createBlock(readyRegion);
    readyBuilder(builder, result.location);
  }
 
  {
    OpBuilder::InsertionGuard guard(builder);
    Region *suspendRegion = result.addRegion();
    builder.createBlock(suspendRegion);
    suspendBuilder(builder, result.location);
  }
 
  {
    OpBuilder::InsertionGuard guard(builder);
    Region *resumeRegion = result.addRegion();
    builder.createBlock(resumeRegion);
    resumeBuilder(builder, result.location);
  }
}
 
void cir::AwaitOp::getSuccessorRegions(
    mlir::RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> &regions) {
  assert(point.isParent() || point.getTerminatorPredecessorOrNull());
 
  // Execution always starts in the ready region.
  if (point.isParent()) {
    regions.emplace_back(&getReady());
    return;
  }
 
  mlir::Region *parentRegion =
      point.getTerminatorPredecessorOrNull()->getParentRegion();
 
  // Branching from ready: the cir.condition terminating it selects between
  // suspending and resuming. Keep in sync with
  // ConditionOp::getSuccessorRegions.
  //
  // TODO: retrieve information from the promise and only push the
  // necessary ones. Example: `std::suspend_never` on initial or final
  // await's might allow suspend region to be skipped.
  if (&getReady() == parentRegion) {
    regions.emplace_back(&getResume());
    regions.emplace_back(&getSuspend());
    return;
  }
 
  // Branching from suspend: continue in resume once the coroutine is resumed
  // (or immediately if suspension is vetoed), or exit to the parent operation
  // if it stays suspended or is destroyed.
  if (&getSuspend() == parentRegion) {
    regions.emplace_back(&getResume());
    regions.emplace_back(getOperation());
    return;
  }
 
  // Branching from resume: exit to the parent operation.
  regions.emplace_back(getOperation());
}
 
LogicalResult cir::AwaitOp::verify() {
  if (!isa<ConditionOp>(this->getReady().back().getTerminator()))
    return emitOpError("ready region must end with cir.condition");
  if (this->getSuspend().empty())
    return emitOpError("suspend region must not be empty");
  if (!isa<CoroSuspendPoint>(this->getSuspend().back().getTerminator()))
    return emitOpError("suspend region must end with cir.coro.suspend_point");
  return success();
}
 
//===----------------------------------------------------------------------===//
// CoReturnOp
//===----------------------------------------------------------------------===//
 
LogicalResult cir::CoReturnOp::verify() {
  mlir::Operation *coRet = getOperation();
  auto coroutine = coRet->getParentOfType<CoroutineOp>();
  if (!coroutine.getBody().isAncestor(getOperation()->getParentRegion()))
    return emitOpError("must be inside the cir.coroutine body region");
  return success();
}
 
//===----------------------------------------------------------------------===//
// CoroutineOp
//===----------------------------------------------------------------------===//
 
void cir::CoroutineOp::build(OpBuilder &builder, OperationState &result,
                             BuilderCallbackRef initialSuspendBuilder,
                             BuilderCallbackRef bodyBuilder,
                             BuilderCallbackRef finalSuspendBuilder,
                             BuilderCallbackRef destroyBuilder,
                             BuilderCallbackRef exitBuilder) {
  {
    OpBuilder::InsertionGuard guard(builder);
    Region *initialRegion = result.addRegion();
    builder.createBlock(initialRegion);
    initialSuspendBuilder(builder, result.location);
  }
 
  {
    OpBuilder::InsertionGuard guard(builder);
    Region *bodyRegion = result.addRegion();
    builder.createBlock(bodyRegion);
    bodyBuilder(builder, result.location);
  }
 
  {
    OpBuilder::InsertionGuard guard(builder);
    Region *finalRegion = result.addRegion();
    builder.createBlock(finalRegion);
    finalSuspendBuilder(builder, result.location);
  }
 
  {
    OpBuilder::InsertionGuard guard(builder);
    Region *destroyRegion = result.addRegion();
    builder.createBlock(destroyRegion);
    destroyBuilder(builder, result.location);
  }
 
  {
    OpBuilder::InsertionGuard guard(builder);
    Region *exitRegion = result.addRegion();
    builder.createBlock(exitRegion);
    exitBuilder(builder, result.location);
  }
}
 
void cir::CoroutineOp::getSuccessorRegions(
    mlir::RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> &regions) {
  // The parent op always enters through initial_suspend get_return_object()
  // (if present) and the init await always run first.
  if (point.isParent()) {
    regions.emplace_back(&getInitialSuspend());
    return;
  }
 
  mlir::Region *parent =
      point.getTerminatorPredecessorOrNull()->getParentRegion();
 
  // initial_suspend either falls into body (resumed, or never actually
  // suspended because await_ready() was true), exits directly, or reaches
  // destroy when the coroutine is destroyed while suspended at its initial
  // suspend point.
  if (parent == &getInitialSuspend()) {
    regions.emplace_back(&getBody());
    regions.emplace_back(&getExit());
    regions.emplace_back(&getDestroy());
    return;
  }
 
  // body can fall through into final_suspend (co_return), exit directly
  // from any suspend_point inside it that just suspends, or reach destroy,
  // either from a suspend_point being destroyed, or from an exception
  // escaping body's own catch-all.
  if (parent == &getBody()) {
    regions.emplace_back(&getFinalSuspend());
    regions.emplace_back(&getExit());    // any suspend_point inside body
    regions.emplace_back(&getDestroy()); // destroy() on any suspend inside body
    return;
  }
 
  // final_suspend's only live edge in valid programs is destroy. Resuming
  // past the final suspend is UB. The ready-immediately edge to exit is
  // kept for structural symmetry with the other two suspend regions even
  // though it's effectively dead.
  if (parent == &getFinalSuspend()) {
    regions.emplace_back(
        &getDestroy());               // the only real exit from final_suspend
    regions.emplace_back(&getExit()); // ready==true edge, rarely taken
    return;
  }
 
  // destroy has two possible outcomes depending on why it was entered:
  // ordinary destroy dispatch falls through to exit (normal return); an
  // exception that reached destroy needs to keep propagating instead, i.e.
  // leave the whole op rather than go through exit's cir.return.
  if (parent == &getDestroy()) {
    regions.emplace_back(&getExit());
    regions.push_back(RegionSuccessor(getOperation())); // unwind out of the op
    return;
  }
  // exit always terminates the op there's no region it loops back into.
  if (parent == &getExit()) {
    regions.emplace_back(getOperation());
    return;
  }
}
 
mlir::ValueRange
cir::CoroutineOp::getSuccessorInputs(RegionSuccessor successor) {
  return ValueRange();
}
 
LogicalResult cir::CoroutineOp::verify() {
  mlir::Region &initialSuspend = getInitialSuspend();
  mlir::Region &body = getBody();
  mlir::Region &finalSuspend = getFinalSuspend();
  mlir::Region &destroy = getDestroy();
  mlir::Region &exit = getExit();
 
  // initial_suspend must contain exactly one 'init'-kind cir.await an
  // optional prelude of ordinary statements (get_return_object() and
  // similar) is allowed before it, but no *other* await may appear.
  int initAwaitCount = 0;
  bool hasInvalidAwait = false;
 
  initialSuspend.walk([&](Operation *op) {
    auto awaitOp = mlir::dyn_cast<AwaitOp>(op);
    if (!awaitOp)
      return mlir::WalkResult::advance();
 
    if (awaitOp.getKind() == cir::AwaitKind::Init)
      ++initAwaitCount;
    else
      hasInvalidAwait = true;
 
    if (initAwaitCount > 1 || hasInvalidAwait)
      return mlir::WalkResult::interrupt();
 
    return mlir::WalkResult::advance();
  });
 
  if (hasInvalidAwait)
    return emitOpError("'initial_suspend' must not contain any cir.await other "
                       "than the 'init' one");
  if (initAwaitCount != 1)
    return emitOpError(
        "must have exactly one 'init' cir.await in 'initial_suspend'");
 
  // final_suspend must contain only one or zero 'final'-kind cir.await
  int finalAwaitCount = 0;
  hasInvalidAwait = false;
 
  finalSuspend.walk([&](Operation *op) {
    auto awaitOp = mlir::dyn_cast<AwaitOp>(op);
    if (!awaitOp)
      return mlir::WalkResult::advance();
 
    if (awaitOp.getKind() == cir::AwaitKind::Final)
      ++finalAwaitCount;
    else
      hasInvalidAwait = true;
 
    if (finalAwaitCount > 1 || hasInvalidAwait)
      return mlir::WalkResult::interrupt();
 
    return mlir::WalkResult::advance();
  });
 
  if (hasInvalidAwait)
    return emitOpError("'final_suspend' must not contain any cir.await other "
                       "than the 'final' one");
  if (finalAwaitCount > 1)
    return emitOpError(
        "must have one or zero 'final' cir.await in 'final_suspend'");
 
  // Each region must end with the terminator its role requires:
  // initial_suspend/final_suspend/destroy yield back into cir.coroutine's
  // own control flow, body ends in a cir.co_return (or a plain yield if
  // some path never reaches one), and exit actually returns from the
  // function.
  if (initialSuspend.empty() || initialSuspend.back().empty())
    return emitOpError("initial_suspend region must not be empty");
  if (!isa<YieldOp>(initialSuspend.back().back()))
    return emitOpError("'initial_suspend' must end with cir.yield");
 
  if (body.empty() || body.back().empty())
    return emitOpError("'body' region must not be empty");
  if (!isa<YieldOp, CoReturnOp>(body.back().back()))
    return emitOpError("'body' must end with cir.yield or cir.co_return");
 
  if (finalSuspend.empty() || finalSuspend.back().empty())
    return emitOpError("'final_suspend' region must not be empty");
  if (!isa<YieldOp>(finalSuspend.back().back()))
    return emitOpError("'final_suspend' must end with cir.yield");
 
  if (destroy.empty() || destroy.back().empty())
    return emitOpError("'destroy' region must not be empty");
  if (!isa<YieldOp>(destroy.back().back()))
    return emitOpError("'destroy' must end with cir.yield");
 
  if (exit.empty() || exit.back().empty())
    return emitOpError("'exit' region must not be empty");
  if (!isa<ReturnOp>(exit.back().back()))
    return emitOpError("'exit' must end with cir.return");
 
  return mlir::success();
}
 
//===----------------------------------------------------------------------===//
// CopyOp Definitions
//===----------------------------------------------------------------------===//
 
// Prints the pointer type(s) for a `cir.copy`. `src` and `dst` share the same
// pointee type but may differ in address space; a single type is printed when
// they match, otherwise both are printed (`src` first).
static void printCopyTypes(mlir::OpAsmPrinter &printer, mlir::Operation *,
                           mlir::Type srcType, mlir::Type dstType) {
  printer.printType(srcType);
  if (srcType != dstType) {
    printer << ", ";
    printer.printType(dstType);
  }
}
 
static mlir::ParseResult parseCopyTypes(mlir::OpAsmParser &parser,
                                        mlir::Type &srcType,
                                        mlir::Type &dstType) {
  if (parser.parseType(srcType))
    return mlir::failure();
  if (parser.parseOptionalComma().succeeded()) {
    if (parser.parseType(dstType))
      return mlir::failure();
  } else {
    dstType = srcType;
  }
  return mlir::success();
}
 
LogicalResult cir::CopyOp::verify() {
  // The pointee types of `src` and `dst` are guaranteed to match by the
  // SameOperandsPointeeType trait; they may still differ in address space.
 
  // A data layout is required for us to know the number of bytes to be copied.
  if (!getType().getPointee().hasTrait<DataLayoutTypeInterface::Trait>())
    return emitError() << "missing data layout for pointee type";
 
  if (getSkipTailPadding() &&
      !mlir::isa<cir::RecordType>(getType().getPointee()))
    return emitError()
           << "skip_tail_padding is only valid for record pointee types";
 
  return mlir::success();
}
 
//===----------------------------------------------------------------------===//
// PtrMaskOp Definitions
//===----------------------------------------------------------------------===//
 
LogicalResult cir::PtrMaskOp::verify() {
  mlir::DataLayout layout = mlir::DataLayout::closest(*this);
  std::optional<uint64_t> indexWidth =
      layout.getTypeIndexBitwidth(getPtr().getType());
  if (!indexWidth)
    return emitOpError() << "pointer has no index width";
 
  uint64_t maskWidth = getMask().getType().getWidth();
  if (maskWidth != *indexWidth)
    return emitOpError() << "mask width " << maskWidth
                         << " must equal the pointer index width "
                         << *indexWidth;
 
  return mlir::success();
}
 
//===----------------------------------------------------------------------===//
// GetRuntimeMemberOp Definitions
//===----------------------------------------------------------------------===//
 
LogicalResult cir::GetRuntimeMemberOp::verify() {
  cir::DataMemberType memberPtrTy = getMember().getType();
 
  if (getAddr().getType().getPointee() != memberPtrTy.getClassTy())
    return emitError() << "record type does not match the member pointer type";
  if (getType().getPointee() != memberPtrTy.getMemberTy())
    return emitError() << "result type does not match the member pointer type";
  return mlir::success();
}
 
//===----------------------------------------------------------------------===//
// GetMethodOp Definitions
//===----------------------------------------------------------------------===//
 
LogicalResult cir::GetMethodOp::verify() {
  cir::MethodType methodTy = getMethod().getType();
 
  // Assume objectTy is !cir.ptr<!T>
  cir::PointerType objectPtrTy = getObject().getType();
  mlir::Type objectTy = objectPtrTy.getPointee();
 
  if (methodTy.getClassTy() != objectTy)
    return emitError() << "method class type and object type do not match";
 
  // Assume methodFuncTy is !cir.func<!Ret (!Args)>
  auto calleeTy = mlir::cast<cir::FuncType>(getCallee().getType().getPointee());
  cir::FuncType methodFuncTy = methodTy.getMemberFuncTy();
 
  // We verify at here that calleeTy is !cir.func<!Ret (!cir.ptr<!void>, !Args)>
  // Note that the first parameter type of the callee is !cir.ptr<!void> instead
  // of !cir.ptr<!T> because the "this" pointer may be adjusted before calling
  // the callee.
 
  if (methodFuncTy.getReturnType() != calleeTy.getReturnType())
    return emitError()
           << "method return type and callee return type do not match";
 
  llvm::ArrayRef<mlir::Type> calleeArgsTy = calleeTy.getInputs();
  llvm::ArrayRef<mlir::Type> methodFuncArgsTy = methodFuncTy.getInputs();
 
  if (calleeArgsTy.empty())
    return emitError() << "callee parameter list lacks receiver object ptr";
 
  auto calleeThisArgPtrTy = mlir::dyn_cast<cir::PointerType>(calleeArgsTy[0]);
  if (!calleeThisArgPtrTy ||
      !mlir::isa<cir::VoidType>(calleeThisArgPtrTy.getPointee())) {
    return emitError()
           << "the first parameter of callee must be a void pointer";
  }
 
  if (calleeArgsTy.size() != methodFuncArgsTy.size())
    return emitError() << "callee and method parameter counts do not match";
 
  if (calleeArgsTy.size() > 1 &&
      calleeArgsTy.slice(1) != methodFuncArgsTy.slice(1))
    return emitError()
           << "callee parameters and method parameters do not match";
 
  return mlir::success();
}
 
//===----------------------------------------------------------------------===//
// GetMemberOp Definitions
//===----------------------------------------------------------------------===//
 
static mlir::Type memberPointeeType(cir::RecordType recordTy, unsigned idx) {
  return cir::memberStorageType(recordTy.getMembers()[idx]);
}
 
LogicalResult cir::GetMemberOp::verify() {
  const auto recordTy = dyn_cast<RecordType>(getAddrTy().getPointee());
  if (!recordTy)
    return emitError() << "expected pointer to a record type";
 
  if (recordTy.getMembers().size() <= getIndex())
    return emitError() << "member index out of bounds";
 
  mlir::Type pointeeTy = memberPointeeType(recordTy, getIndex());
  if (!pointeeTy)
    return emitError() << "member owns no storage to point at";
 
  if (pointeeTy != getType().getPointee())
    return emitError() << "member type mismatch";
 
  if (getAddrTy().getAddrSpace() != getType().getAddrSpace())
    return emitError() << "address space mismatch";
 
  return mlir::success();
}
 
//===----------------------------------------------------------------------===//
// ExtractMemberOp Definitions
//===----------------------------------------------------------------------===//
 
LogicalResult cir::ExtractMemberOp::verify() {
  if (mlir::isa<cir::UnionType>(getRecord().getType()))
    return emitError()
           << "cir.extract_member currently does not support unions";
  auto structTy = mlir::cast<cir::StructType>(getRecord().getType());
  if (structTy.getMembers().size() <= getIndex())
    return emitError() << "member index out of bounds";
  mlir::Type memberTy = structTy.getMembers()[getIndex()];
  if (mlir::isa<cir::BitFieldType>(memberTy))
    return emitError() << "cir.extract_member does not support bit-fields";
  if (memberTy != getType())
    return emitError() << "member type mismatch";
  return mlir::success();
}
 
//===----------------------------------------------------------------------===//
// InsertMemberOp Definitions
//===----------------------------------------------------------------------===//
 
LogicalResult cir::InsertMemberOp::verify() {
  if (mlir::isa<cir::UnionType>(getRecord().getType()))
    return emitError() << "cir.insert_member currently does not support unions";
  auto structTy = mlir::cast<cir::StructType>(getRecord().getType());
  if (structTy.getMembers().size() <= getIndex())
    return emitError() << "member index out of bounds";
  mlir::Type memberTy = structTy.getMembers()[getIndex()];
  if (mlir::isa<cir::BitFieldType>(memberTy))
    return emitError() << "cir.insert_member does not support bit-fields";
  if (memberTy != getValue().getType())
    return emitError() << "member type mismatch";
  // The op trait already checks that the types of $result and $record match.
  return mlir::success();
}
 
//===----------------------------------------------------------------------===//
// VecCreateOp
//===----------------------------------------------------------------------===//
 
OpFoldResult cir::VecCreateOp::fold(FoldAdaptor adaptor) {
  if (llvm::any_of(getElements(), [](mlir::Value value) {
        return !value.getDefiningOp<cir::ConstantOp>();
      }))
    return {};
 
  return cir::ConstVectorAttr::get(
      getType(), mlir::ArrayAttr::get(getContext(), adaptor.getElements()));
}
 
LogicalResult cir::VecCreateOp::verify() {
  // Verify that the number of arguments matches the number of elements in the
  // vector, and that the type of all the arguments matches the type of the
  // elements in the vector.
  const cir::VectorType vecTy = getType();
  if (getElements().size() != vecTy.getSize()) {
    return emitOpError() << "operand count of " << getElements().size()
                         << " doesn't match vector type " << vecTy
                         << " element count of " << vecTy.getSize();
  }
 
  const mlir::Type elementType = vecTy.getElementType();
  for (const mlir::Value element : getElements()) {
    if (element.getType() != elementType) {
      return emitOpError() << "operand type " << element.getType()
                           << " doesn't match vector element type "
                           << elementType;
    }
  }
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// VecExtractOp
//===----------------------------------------------------------------------===//
 
OpFoldResult cir::VecExtractOp::fold(FoldAdaptor adaptor) {
  const auto vectorAttr =
      llvm::dyn_cast_if_present<cir::ConstVectorAttr>(adaptor.getVec());
  if (!vectorAttr)
    return {};
 
  const auto indexAttr =
      llvm::dyn_cast_if_present<cir::IntAttr>(adaptor.getIndex());
  if (!indexAttr)
    return {};
 
  const mlir::ArrayAttr elements = vectorAttr.getElts();
  const uint64_t index = indexAttr.getUInt();
  if (index >= elements.size())
    return {};
 
  return elements[index];
}
 
//===----------------------------------------------------------------------===//
// CmpOp
//===----------------------------------------------------------------------===//
 
LogicalResult cir::CmpOp::verify() {
  if (getFenvAttr() && !cir::isAnyFloatingPointType(getLhs().getType()))
    return emitOpError()
           << "'fenv' is only valid for floating-point comparisons";
  return success();
}
 
//===----------------------------------------------------------------------===//
// VecCmpOp
//===----------------------------------------------------------------------===//
 
LogicalResult cir::VecCmpOp::verify() {
  if (getFenvAttr() && !cir::isFPOrVectorOfFPType(getLhs().getType()))
    return emitOpError()
           << "'fenv' is only valid for floating-point comparisons";
  return success();
}
 
OpFoldResult cir::VecCmpOp::fold(FoldAdaptor adaptor) {
  // Do not fold when fenv is present.
  // TODO(cir): This is overly conservative. We could fold comparisons as long
  //            as we can prove that the operation wouldn't raise exceptions or
  //            when the fenv attribute does not require strict exception
  //            semantics.
  if (getFenvAttr())
    return {};
 
  auto lhsVecAttr =
      mlir::dyn_cast_if_present<cir::ConstVectorAttr>(adaptor.getLhs());
  auto rhsVecAttr =
      mlir::dyn_cast_if_present<cir::ConstVectorAttr>(adaptor.getRhs());
  if (!lhsVecAttr || !rhsVecAttr)
    return {};
 
  mlir::Type inputElemTy =
      mlir::cast<cir::VectorType>(lhsVecAttr.getType()).getElementType();
  if (!isAnyIntegerOrFloatingPointType(inputElemTy))
    return {};
 
  cir::CmpOpKind opKind = adaptor.getKind();
  mlir::ArrayAttr lhsVecElhs = lhsVecAttr.getElts();
  mlir::ArrayAttr rhsVecElhs = rhsVecAttr.getElts();
  uint64_t vecSize = lhsVecElhs.size();
 
  SmallVector<mlir::Attribute, 16> elements(vecSize);
  bool isIntAttr = vecSize && mlir::isa<cir::IntAttr>(lhsVecElhs[0]);
  bool isUnsignedInt =
      isIntAttr && mlir::cast<cir::IntType>(inputElemTy).isUnsigned();
  for (uint64_t i = 0; i < vecSize; i++) {
    mlir::Attribute lhsAttr = lhsVecElhs[i];
    mlir::Attribute rhsAttr = rhsVecElhs[i];
    bool cmpResult = false;
    switch (opKind) {
    case cir::CmpOpKind::lt: {
      if (isIntAttr) {
        if (isUnsignedInt)
          cmpResult = mlir::cast<cir::IntAttr>(lhsAttr).getUInt() <
                      mlir::cast<cir::IntAttr>(rhsAttr).getUInt();
        else
          cmpResult = mlir::cast<cir::IntAttr>(lhsAttr).getSInt() <
                      mlir::cast<cir::IntAttr>(rhsAttr).getSInt();
      } else {
        cmpResult = mlir::cast<cir::FPAttr>(lhsAttr).getValue() <
                    mlir::cast<cir::FPAttr>(rhsAttr).getValue();
      }
      break;
    }
    case cir::CmpOpKind::le: {
      if (isIntAttr) {
        if (isUnsignedInt)
          cmpResult = mlir::cast<cir::IntAttr>(lhsAttr).getUInt() <=
                      mlir::cast<cir::IntAttr>(rhsAttr).getUInt();
        else
          cmpResult = mlir::cast<cir::IntAttr>(lhsAttr).getSInt() <=
                      mlir::cast<cir::IntAttr>(rhsAttr).getSInt();
      } else {
        cmpResult = mlir::cast<cir::FPAttr>(lhsAttr).getValue() <=
                    mlir::cast<cir::FPAttr>(rhsAttr).getValue();
      }
      break;
    }
    case cir::CmpOpKind::gt: {
      if (isIntAttr) {
        if (isUnsignedInt)
          cmpResult = mlir::cast<cir::IntAttr>(lhsAttr).getUInt() >
                      mlir::cast<cir::IntAttr>(rhsAttr).getUInt();
        else
          cmpResult = mlir::cast<cir::IntAttr>(lhsAttr).getSInt() >
                      mlir::cast<cir::IntAttr>(rhsAttr).getSInt();
      } else {
        cmpResult = mlir::cast<cir::FPAttr>(lhsAttr).getValue() >
                    mlir::cast<cir::FPAttr>(rhsAttr).getValue();
      }
      break;
    }
    case cir::CmpOpKind::ge: {
      if (isIntAttr) {
        if (isUnsignedInt)
          cmpResult = mlir::cast<cir::IntAttr>(lhsAttr).getUInt() >=
                      mlir::cast<cir::IntAttr>(rhsAttr).getUInt();
        else
          cmpResult = mlir::cast<cir::IntAttr>(lhsAttr).getSInt() >=
                      mlir::cast<cir::IntAttr>(rhsAttr).getSInt();
      } else {
        cmpResult = mlir::cast<cir::FPAttr>(lhsAttr).getValue() >=
                    mlir::cast<cir::FPAttr>(rhsAttr).getValue();
      }
      break;
    }
    case cir::CmpOpKind::eq: {
      if (isIntAttr) {
        cmpResult = mlir::cast<cir::IntAttr>(lhsAttr).getSInt() ==
                    mlir::cast<cir::IntAttr>(rhsAttr).getSInt();
      } else {
        cmpResult = mlir::cast<cir::FPAttr>(lhsAttr).getValue() ==
                    mlir::cast<cir::FPAttr>(rhsAttr).getValue();
      }
      break;
    }
    case cir::CmpOpKind::ne: {
      if (isIntAttr) {
        cmpResult = mlir::cast<cir::IntAttr>(lhsAttr).getSInt() !=
                    mlir::cast<cir::IntAttr>(rhsAttr).getSInt();
      } else {
        cmpResult = mlir::cast<cir::FPAttr>(lhsAttr).getValue() !=
                    mlir::cast<cir::FPAttr>(rhsAttr).getValue();
      }
      break;
    }
    case cir::CmpOpKind::one: {
      llvm::APFloat::cmpResult cr =
          mlir::cast<cir::FPAttr>(lhsAttr).getValue().compare(
              mlir::cast<cir::FPAttr>(rhsAttr).getValue());
      cmpResult =
          cr != llvm::APFloat::cmpUnordered && cr != llvm::APFloat::cmpEqual;
      break;
    }
    case cir::CmpOpKind::uno: {
      cmpResult = mlir::cast<cir::FPAttr>(lhsAttr).getValue().compare(
                      mlir::cast<cir::FPAttr>(rhsAttr).getValue()) ==
                  llvm::APFloat::cmpUnordered;
      break;
    }
    }
 
    // A true result is all bits set (-1 in two's complement), and a false
    // result is all bits clear. For a 1-bit element type these are the same
    // bit pattern as 1 and 0, respectively.
    elements[i] =
        cir::IntAttr::get(getType().getElementType(), cmpResult ? -1LL : 0LL);
  }
 
  return cir::ConstVectorAttr::get(
      getType(), mlir::ArrayAttr::get(getContext(), elements));
}
 
//===----------------------------------------------------------------------===//
// VecShuffleOp
//===----------------------------------------------------------------------===//
 
OpFoldResult cir::VecShuffleOp::fold(FoldAdaptor adaptor) {
  auto vec1Attr =
      mlir::dyn_cast_if_present<cir::ConstVectorAttr>(adaptor.getVec1());
  auto vec2Attr =
      mlir::dyn_cast_if_present<cir::ConstVectorAttr>(adaptor.getVec2());
  if (!vec1Attr || !vec2Attr)
    return {};
 
  mlir::Type vec1ElemTy =
      mlir::cast<cir::VectorType>(vec1Attr.getType()).getElementType();
 
  mlir::ArrayAttr vec1Elts = vec1Attr.getElts();
  mlir::ArrayAttr vec2Elts = vec2Attr.getElts();
  mlir::ArrayAttr indicesElts = adaptor.getIndices();
 
  SmallVector<mlir::Attribute, 16> elements;
  elements.reserve(indicesElts.size());
 
  uint64_t vec1Size = vec1Elts.size();
  for (const auto &idxAttr : indicesElts.getAsRange<cir::IntAttr>()) {
    if (idxAttr.getSInt() == -1) {
      elements.push_back(cir::UndefAttr::get(vec1ElemTy));
      continue;
    }
 
    uint64_t idxValue = idxAttr.getUInt();
    elements.push_back(idxValue < vec1Size ? vec1Elts[idxValue]
                                           : vec2Elts[idxValue - vec1Size]);
  }
 
  return cir::ConstVectorAttr::get(
      getType(), mlir::ArrayAttr::get(getContext(), elements));
}
 
LogicalResult cir::VecShuffleOp::verify() {
  // The number of elements in the indices array must match the number of
  // elements in the result type.
  if (getIndices().size() != getResult().getType().getSize()) {
    return emitOpError() << ": the number of elements in " << getIndices()
                         << " and " << getResult().getType() << " don't match";
  }
 
  // The element types of the two input vectors and of the result type must
  // match.
  if (getVec1().getType().getElementType() !=
      getResult().getType().getElementType()) {
    return emitOpError() << ": element types of " << getVec1().getType()
                         << " and " << getResult().getType() << " don't match";
  }
 
  const uint64_t maxValidIndex =
      getVec1().getType().getSize() + getVec2().getType().getSize() - 1;
  if (llvm::any_of(
          getIndices().getAsRange<cir::IntAttr>(), [&](cir::IntAttr idxAttr) {
            return idxAttr.getSInt() != -1 && idxAttr.getUInt() > maxValidIndex;
          })) {
    return emitOpError() << ": index for __builtin_shufflevector must be "
                            "less than the total number of vector elements";
  }
  return success();
}
 
//===----------------------------------------------------------------------===//
// VecShuffleDynamicOp
//===----------------------------------------------------------------------===//
 
OpFoldResult cir::VecShuffleDynamicOp::fold(FoldAdaptor adaptor) {
  mlir::Attribute vec = adaptor.getVec();
  mlir::Attribute indices = adaptor.getIndices();
  if (mlir::isa_and_nonnull<cir::ConstVectorAttr>(vec) &&
      mlir::isa_and_nonnull<cir::ConstVectorAttr>(indices)) {
    auto vecAttr = mlir::cast<cir::ConstVectorAttr>(vec);
    auto indicesAttr = mlir::cast<cir::ConstVectorAttr>(indices);
 
    mlir::ArrayAttr vecElts = vecAttr.getElts();
    mlir::ArrayAttr indicesElts = indicesAttr.getElts();
 
    const uint64_t numElements = vecElts.size();
 
    SmallVector<mlir::Attribute, 16> elements;
    elements.reserve(numElements);
 
    const uint64_t maskBits = llvm::NextPowerOf2(numElements - 1) - 1;
    for (const auto &idxAttr : indicesElts.getAsRange<cir::IntAttr>()) {
      uint64_t idxValue = idxAttr.getUInt();
      uint64_t newIdx = idxValue & maskBits;
      elements.push_back(vecElts[newIdx]);
    }
 
    return cir::ConstVectorAttr::get(
        getType(), mlir::ArrayAttr::get(getContext(), elements));
  }
 
  return {};
}
 
LogicalResult cir::VecShuffleDynamicOp::verify() {
  // The number of elements in the two input vectors must match.
  if (getVec().getType().getSize() !=
      mlir::cast<cir::VectorType>(getIndices().getType()).getSize()) {
    return emitOpError() << ": the number of elements in " << getVec().getType()
                         << " and " << getIndices().getType() << " don't match";
  }
  return success();
}
 
//===----------------------------------------------------------------------===//
// VecTernaryOp
//===----------------------------------------------------------------------===//
 
LogicalResult cir::VecTernaryOp::verify() {
  // Verify that the condition operand has the same number of elements as the
  // other operands.  (The automatic verification already checked that all
  // operands are vector types and that the second and third operands are the
  // same type.)
  if (getCond().getType().getSize() != getLhs().getType().getSize()) {
    return emitOpError() << ": the number of elements in "
                         << getCond().getType() << " and " << getLhs().getType()
                         << " don't match";
  }
  return success();
}
 
OpFoldResult cir::VecTernaryOp::fold(FoldAdaptor adaptor) {
  mlir::Attribute cond = adaptor.getCond();
  mlir::Attribute lhs = adaptor.getLhs();
  mlir::Attribute rhs = adaptor.getRhs();
 
  if (!mlir::isa_and_nonnull<cir::ConstVectorAttr>(cond) ||
      !mlir::isa_and_nonnull<cir::ConstVectorAttr>(lhs) ||
      !mlir::isa_and_nonnull<cir::ConstVectorAttr>(rhs))
    return {};
  auto condVec = mlir::cast<cir::ConstVectorAttr>(cond);
  auto lhsVec = mlir::cast<cir::ConstVectorAttr>(lhs);
  auto rhsVec = mlir::cast<cir::ConstVectorAttr>(rhs);
 
  mlir::ArrayAttr condElts = condVec.getElts();
 
  SmallVector<mlir::Attribute, 16> elements;
  elements.reserve(condElts.size());
 
  for (const auto &[idx, condAttr] :
       llvm::enumerate(condElts.getAsRange<cir::IntAttr>())) {
    if (condAttr.getSInt()) {
      elements.push_back(lhsVec.getElts()[idx]);
    } else {
      elements.push_back(rhsVec.getElts()[idx]);
    }
  }
 
  cir::VectorType vecTy = getLhs().getType();
  return cir::ConstVectorAttr::get(
      vecTy, mlir::ArrayAttr::get(getContext(), elements));
}
 
//===----------------------------------------------------------------------===//
// MatrixTransposeOp
//===----------------------------------------------------------------------===//
 
LogicalResult cir::MatrixTransposeOp::verify() {
  cir::MatrixType valueTy = getValue().getType();
  cir::MatrixType resultTy = getResult().getType();
 
  if ((valueTy.getElementType() != resultTy.getElementType()) ||
      (valueTy.getNumRows() != resultTy.getNumColumns()) ||
      (valueTy.getNumColumns() != resultTy.getNumRows())) {
    auto expectedTy =
        cir::MatrixType::get(valueTy.getElementType(), valueTy.getNumColumns(),
                             valueTy.getNumRows());
    emitOpError() << "operand type " << valueTy << " expects result type of "
                  << expectedTy << " but got " << resultTy;
    return failure();
  }
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// ComplexCreateOp
//===----------------------------------------------------------------------===//
 
LogicalResult cir::ComplexCreateOp::verify() {
  if (getType().getElementType() != getReal().getType()) {
    emitOpError()
        << "operand type of cir.complex.create does not match its result type";
    return failure();
  }
 
  return success();
}
 
OpFoldResult cir::ComplexCreateOp::fold(FoldAdaptor adaptor) {
  mlir::Attribute real = adaptor.getReal();
  mlir::Attribute imag = adaptor.getImag();
  if (!real || !imag)
    return {};
 
  // When both of real and imag are constants, we can fold the operation into an
  // `#cir.const_complex` operation.
  auto realAttr = mlir::cast<mlir::TypedAttr>(real);
  auto imagAttr = mlir::cast<mlir::TypedAttr>(imag);
  return cir::ConstComplexAttr::get(realAttr, imagAttr);
}
 
//===----------------------------------------------------------------------===//
// ComplexRealOp
//===----------------------------------------------------------------------===//
 
LogicalResult cir::ComplexRealOp::verify() {
  mlir::Type operandTy = getOperand().getType();
  if (auto complexOperandTy = mlir::dyn_cast<cir::ComplexType>(operandTy))
    operandTy = complexOperandTy.getElementType();
 
  if (getType() != operandTy) {
    emitOpError() << ": result type does not match operand type";
    return failure();
  }
 
  return success();
}
 
OpFoldResult cir::ComplexRealOp::fold(FoldAdaptor adaptor) {
  if (!mlir::isa<cir::ComplexType>(getOperand().getType()))
    return nullptr;
 
  if (auto complexCreateOp = getOperand().getDefiningOp<cir::ComplexCreateOp>())
    return complexCreateOp.getOperand(0);
 
  auto complex =
      mlir::cast_if_present<cir::ConstComplexAttr>(adaptor.getOperand());
  return complex ? complex.getReal() : nullptr;
}
 
//===----------------------------------------------------------------------===//
// ComplexImagOp
//===----------------------------------------------------------------------===//
 
LogicalResult cir::ComplexImagOp::verify() {
  mlir::Type operandTy = getOperand().getType();
  if (auto complexOperandTy = mlir::dyn_cast<cir::ComplexType>(operandTy))
    operandTy = complexOperandTy.getElementType();
 
  if (getType() != operandTy) {
    emitOpError() << ": result type does not match operand type";
    return failure();
  }
 
  return success();
}
 
OpFoldResult cir::ComplexImagOp::fold(FoldAdaptor adaptor) {
  if (!mlir::isa<cir::ComplexType>(getOperand().getType()))
    return nullptr;
 
  if (auto complexCreateOp = getOperand().getDefiningOp<cir::ComplexCreateOp>())
    return complexCreateOp.getOperand(1);
 
  auto complex =
      mlir::cast_if_present<cir::ConstComplexAttr>(adaptor.getOperand());
  return complex ? complex.getImag() : nullptr;
}
 
//===----------------------------------------------------------------------===//
// ComplexRealPtrOp
//===----------------------------------------------------------------------===//
 
LogicalResult cir::ComplexRealPtrOp::verify() {
  mlir::Type resultPointeeTy = getType().getPointee();
  cir::PointerType operandPtrTy = getOperand().getType();
  auto operandPointeeTy =
      mlir::cast<cir::ComplexType>(operandPtrTy.getPointee());
 
  if (resultPointeeTy != operandPointeeTy.getElementType()) {
    return emitOpError() << ": result type does not match operand type";
  }
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// ComplexImagPtrOp
//===----------------------------------------------------------------------===//
 
LogicalResult cir::ComplexImagPtrOp::verify() {
  mlir::Type resultPointeeTy = getType().getPointee();
  cir::PointerType operandPtrTy = getOperand().getType();
  auto operandPointeeTy =
      mlir::cast<cir::ComplexType>(operandPtrTy.getPointee());
 
  if (resultPointeeTy != operandPointeeTy.getElementType()) {
    return emitOpError()
           << "cir.complex.imag_ptr result type does not match operand type";
  }
  return success();
}
 
//===----------------------------------------------------------------------===//
// Bit manipulation operations
//===----------------------------------------------------------------------===//
 
static OpFoldResult
foldUnaryBitOp(mlir::Attribute inputAttr,
               llvm::function_ref<llvm::APInt(const llvm::APInt &)> func,
               bool poisonZero = false) {
  if (mlir::isa_and_present<cir::PoisonAttr>(inputAttr)) {
    // Propagate poison value
    return inputAttr;
  }
 
  auto input = mlir::dyn_cast_if_present<IntAttr>(inputAttr);
  if (!input)
    return nullptr;
 
  llvm::APInt inputValue = input.getValue();
  if (poisonZero && inputValue.isZero())
    return cir::PoisonAttr::get(input.getType());
 
  llvm::APInt resultValue = func(inputValue);
  return IntAttr::get(input.getType(), resultValue);
}
 
OpFoldResult BitClrsbOp::fold(FoldAdaptor adaptor) {
  return foldUnaryBitOp(adaptor.getInput(), [](const llvm::APInt &inputValue) {
    unsigned resultValue =
        inputValue.getBitWidth() - inputValue.getSignificantBits();
    return llvm::APInt(inputValue.getBitWidth(), resultValue);
  });
}
 
OpFoldResult BitClzOp::fold(FoldAdaptor adaptor) {
  return foldUnaryBitOp(
      adaptor.getInput(),
      [](const llvm::APInt &inputValue) {
        unsigned resultValue = inputValue.countLeadingZeros();
        return llvm::APInt(inputValue.getBitWidth(), resultValue);
      },
      getPoisonZero());
}
 
OpFoldResult BitCtzOp::fold(FoldAdaptor adaptor) {
  return foldUnaryBitOp(
      adaptor.getInput(),
      [](const llvm::APInt &inputValue) {
        return llvm::APInt(inputValue.getBitWidth(),
                           inputValue.countTrailingZeros());
      },
      getPoisonZero());
}
 
OpFoldResult BitFfsOp::fold(FoldAdaptor adaptor) {
  return foldUnaryBitOp(adaptor.getInput(), [](const llvm::APInt &inputValue) {
    unsigned trailingZeros = inputValue.countTrailingZeros();
    unsigned result =
        trailingZeros == inputValue.getBitWidth() ? 0 : trailingZeros + 1;
    return llvm::APInt(inputValue.getBitWidth(), result);
  });
}
 
OpFoldResult BitParityOp::fold(FoldAdaptor adaptor) {
  return foldUnaryBitOp(adaptor.getInput(), [](const llvm::APInt &inputValue) {
    return llvm::APInt(inputValue.getBitWidth(), inputValue.popcount() % 2);
  });
}
 
OpFoldResult BitPopcountOp::fold(FoldAdaptor adaptor) {
  return foldUnaryBitOp(adaptor.getInput(), [](const llvm::APInt &inputValue) {
    return llvm::APInt(inputValue.getBitWidth(), inputValue.popcount());
  });
}
 
OpFoldResult BitReverseOp::fold(FoldAdaptor adaptor) {
  return foldUnaryBitOp(adaptor.getInput(), [](const llvm::APInt &inputValue) {
    return inputValue.reverseBits();
  });
}
 
OpFoldResult ByteSwapOp::fold(FoldAdaptor adaptor) {
  return foldUnaryBitOp(adaptor.getInput(), [](const llvm::APInt &inputValue) {
    return inputValue.byteSwap();
  });
}
 
OpFoldResult RotateOp::fold(FoldAdaptor adaptor) {
  if (mlir::isa_and_present<cir::PoisonAttr>(adaptor.getInput()) ||
      mlir::isa_and_present<cir::PoisonAttr>(adaptor.getAmount())) {
    // Propagate poison values
    return cir::PoisonAttr::get(getType());
  }
 
  auto input = mlir::dyn_cast_if_present<IntAttr>(adaptor.getInput());
  auto amount = mlir::dyn_cast_if_present<IntAttr>(adaptor.getAmount());
  if (!input && !amount)
    return nullptr;
 
  // We could fold cir.rotate even if one of its two operands is not a constant:
  //   - `cir.rotate left/right %0, 0` could be folded into just %0 even if %0
  //     is not a constant.
  //   - `cir.rotate left/right 0/0b111...111, %0` could be folded into 0 or
  //     0b111...111 even if %0 is not a constant.
 
  llvm::APInt inputValue;
  if (input) {
    inputValue = input.getValue();
    if (inputValue.isZero() || inputValue.isAllOnes()) {
      // An input value of all 0s or all 1s will not change after rotation
      return input;
    }
  }
 
  uint64_t amountValue;
  if (amount) {
    amountValue = amount.getValue().urem(getInput().getType().getWidth());
    if (amountValue == 0) {
      // A shift amount of 0 will not change the input value
      return getInput();
    }
  }
 
  if (!input || !amount)
    return nullptr;
 
  assert(inputValue.getBitWidth() == getInput().getType().getWidth() &&
         "input value must have the same bit width as the input type");
 
  llvm::APInt resultValue;
  if (isRotateLeft())
    resultValue = inputValue.rotl(amountValue);
  else
    resultValue = inputValue.rotr(amountValue);
 
  return IntAttr::get(input.getContext(), input.getType(), resultValue);
}
 
//===----------------------------------------------------------------------===//
// InlineAsmOp
//===----------------------------------------------------------------------===//
 
void cir::InlineAsmOp::print(OpAsmPrinter &p) {
  p << '(' << getAsmFlavor() << ", ";
  p.increaseIndent();
  p.printNewline();
 
  llvm::SmallVector<std::string, 3> names{"out", "in", "in_out"};
  auto *nameIt = names.begin();
  auto *attrIt = getOperandAttrs().begin();
 
  for (mlir::OperandRange ops : getAsmOperands()) {
    p << *nameIt << " = ";
 
    p << '[';
    llvm::interleaveComma(llvm::make_range(ops.begin(), ops.end()), p,
                          [&](Value value) {
                            p.printOperand(value);
                            p << " : " << value.getType();
                            if (mlir::isa<mlir::UnitAttr>(*attrIt))
                              p << " (maybe_memory)";
                            attrIt++;
                          });
    p << "],";
    p.printNewline();
    ++nameIt;
  }
 
  p << "{";
  p.printString(getAsmString());
  p << " ";
  p.printString(getConstraints());
  p << "}";
  p.decreaseIndent();
  p << ')';
  if (getSideEffects())
    p << " side_effects";
 
  p.printOptionalAttrDict(
      getOperation()->getDiscardableAttrDictionary().getValue());
 
  if (auto v = getRes())
    p << " -> " << v.getType();
}
 
void cir::InlineAsmOp::build(OpBuilder &odsBuilder, OperationState &odsState,
                             ArrayRef<ValueRange> asmOperands,
                             StringRef asmString, StringRef constraints,
                             bool sideEffects, cir::AsmFlavor asmFlavor,
                             ArrayRef<Attribute> operandAttrs) {
  // Set up the operands_segments for VariadicOfVariadic
  SmallVector<int32_t> segments;
  for (auto operandRange : asmOperands) {
    segments.push_back(operandRange.size());
    odsState.addOperands(operandRange);
  }
 
  odsState.addAttribute(
      "operands_segments",
      DenseI32ArrayAttr::get(odsBuilder.getContext(), segments));
  odsState.addAttribute("asm_string", odsBuilder.getStringAttr(asmString));
  odsState.addAttribute("constraints", odsBuilder.getStringAttr(constraints));
  odsState.addAttribute("asm_flavor",
                        AsmFlavorAttr::get(odsBuilder.getContext(), asmFlavor));
 
  if (sideEffects)
    odsState.addAttribute("side_effects", odsBuilder.getUnitAttr());
 
  odsState.addAttribute("operand_attrs", odsBuilder.getArrayAttr(operandAttrs));
}
 
ParseResult cir::InlineAsmOp::parse(OpAsmParser &parser,
                                    OperationState &result) {
  llvm::SmallVector<mlir::Attribute> operandAttrs;
  llvm::SmallVector<int32_t> operandsGroupSizes;
  std::string asmString, constraints;
  Type resType;
  MLIRContext *ctxt = parser.getBuilder().getContext();
 
  auto error = [&](const Twine &msg) -> LogicalResult {
    return parser.emitError(parser.getCurrentLocation(), msg);
  };
 
  auto expected = [&](const std::string &c) {
    return error("expected '" + c + "'");
  };
 
  if (parser.parseLParen().failed())
    return expected("(");
 
  auto flavor = FieldParser<AsmFlavor, AsmFlavor>::parse(parser);
  if (failed(flavor))
    return error("Unknown AsmFlavor");
 
  if (parser.parseComma().failed())
    return expected(",");
 
  auto parseValue = [&](Value &v) {
    OpAsmParser::UnresolvedOperand op;
 
    if (parser.parseOperand(op) || parser.parseColon())
      return error("can't parse operand");
 
    Type typ;
    if (parser.parseType(typ).failed())
      return error("can't parse operand type");
    llvm::SmallVector<mlir::Value> tmp;
    if (parser.resolveOperand(op, typ, tmp))
      return error("can't resolve operand");
    v = tmp[0];
    return mlir::success();
  };
 
  auto parseOperands = [&](llvm::StringRef name) {
    if (parser.parseKeyword(name).failed())
      return error("expected " + name + " operands here");
    if (parser.parseEqual().failed())
      return expected("=");
    if (parser.parseLSquare().failed())
      return expected("[");
 
    int size = 0;
    if (parser.parseOptionalRSquare().succeeded()) {
      operandsGroupSizes.push_back(size);
      if (parser.parseComma())
        return expected(",");
      return mlir::success();
    }
 
    auto parseOperand = [&]() {
      Value val;
      if (parseValue(val).succeeded()) {
        result.operands.push_back(val);
        size++;
 
        if (parser.parseOptionalLParen().failed()) {
          operandAttrs.push_back(mlir::DictionaryAttr::get(ctxt));
          return mlir::success();
        }
 
        if (parser.parseKeyword("maybe_memory").succeeded()) {
          operandAttrs.push_back(mlir::UnitAttr::get(ctxt));
          if (parser.parseRParen())
            return expected(")");
          return mlir::success();
        } else {
          return expected("maybe_memory");
        }
      }
      return mlir::failure();
    };
 
    if (parser.parseCommaSeparatedList(parseOperand).failed())
      return mlir::failure();
 
    if (parser.parseRSquare().failed() || parser.parseComma().failed())
      return expected("]");
    operandsGroupSizes.push_back(size);
    return mlir::success();
  };
 
  if (parseOperands("out").failed() || parseOperands("in").failed() ||
      parseOperands("in_out").failed())
    return error("failed to parse operands");
 
  if (parser.parseLBrace())
    return expected("{");
  if (parser.parseString(&asmString))
    return error("asm string parsing failed");
  if (parser.parseString(&constraints))
    return error("constraints string parsing failed");
  if (parser.parseRBrace())
    return expected("}");
  if (parser.parseRParen())
    return expected(")");
 
  if (parser.parseOptionalKeyword("side_effects").succeeded())
    result.attributes.set("side_effects", UnitAttr::get(ctxt));
 
  if (parser.parseOptionalAttrDict(result.attributes).failed())
    return mlir::failure();
 
  if (parser.parseOptionalArrow().succeeded() &&
      parser.parseType(resType).failed())
    return mlir::failure();
 
  result.attributes.set("asm_flavor", AsmFlavorAttr::get(ctxt, *flavor));
  result.attributes.set("asm_string", StringAttr::get(ctxt, asmString));
  result.attributes.set("constraints", StringAttr::get(ctxt, constraints));
  result.attributes.set("operand_attrs", ArrayAttr::get(ctxt, operandAttrs));
  result.getOrAddProperties<InlineAsmOp::Properties>().operands_segments =
      parser.getBuilder().getDenseI32ArrayAttr(operandsGroupSizes);
  if (resType)
    result.addTypes(TypeRange{resType});
 
  return mlir::success();
}
 
void InlineAsmOp::getEffects(
    llvm::SmallVectorImpl<mlir::MemoryEffects::EffectInstance> &effects) {
  // If we have any side effects (that is, we're volatile asm), add a read and
  // write memory effect. We do this the same as the llvm dialect InlineAsmOp.
  if (getSideEffects()) {
    effects.emplace_back(mlir::MemoryEffects::Read::get());
    effects.emplace_back(mlir::MemoryEffects::Write::get());
  }
}
 
//===----------------------------------------------------------------------===//
// ThrowOp / TryThrowOp
//===----------------------------------------------------------------------===//
 
template <typename ThrowOpTy>
static mlir::LogicalResult verifyThrowOpImpl(ThrowOpTy op) {
  if (op.rethrows())
    return mlir::success();
 
  if (op.getNumOperands() != 0) {
    if (op.getTypeInfo())
      return mlir::success();
    return op.emitOpError() << "'type_info' symbol attribute missing";
  }
 
  return mlir::failure();
}
 
mlir::LogicalResult cir::ThrowOp::verify() { return verifyThrowOpImpl(*this); }
 
mlir::LogicalResult cir::TryThrowOp::verify() {
  return verifyThrowOpImpl(*this);
}
 
//===----------------------------------------------------------------------===//
// AtomicFetchOp
//===----------------------------------------------------------------------===//
 
LogicalResult cir::AtomicFetchOp::verify() {
  if (getBinop() != cir::AtomicFetchKind::Add &&
      getBinop() != cir::AtomicFetchKind::Sub &&
      getBinop() != cir::AtomicFetchKind::Max &&
      getBinop() != cir::AtomicFetchKind::Min &&
      getBinop() != cir::AtomicFetchKind::Maximum &&
      getBinop() != cir::AtomicFetchKind::Minimum &&
      getBinop() != cir::AtomicFetchKind::MaximumNum &&
      getBinop() != cir::AtomicFetchKind::MinimumNum &&
      !mlir::isa<cir::IntType>(getVal().getType()))
    return emitError("only atomic add, sub, max, min, maximum, minimum, "
                     "maximum_num, and minimum_num operation could operate on "
                     "floating-point values");
 
  if ((getBinop() == cir::AtomicFetchKind::Maximum ||
       getBinop() == cir::AtomicFetchKind::Minimum ||
       getBinop() == cir::AtomicFetchKind::MaximumNum ||
       getBinop() == cir::AtomicFetchKind::MinimumNum) &&
      !mlir::isa<cir::FPTypeInterface>(getVal().getType()))
    return emitError("atomic maximum, minimum, maximum_num, and minimum_num "
                     "operation could only operate on floating-point values");
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// TypeInfoAttr
//===----------------------------------------------------------------------===//
 
LogicalResult cir::TypeInfoAttr::verify(
    ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError,
    ::mlir::Type type, ::mlir::ArrayAttr typeInfoData) {
 
  if (cir::ConstRecordAttr::verify(emitError, type, typeInfoData).failed())
    return failure();
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// TryOp
//===----------------------------------------------------------------------===//
 
void cir::TryOp::getSuccessorRegions(
    mlir::RegionBranchPoint point,
    llvm::SmallVectorImpl<mlir::RegionSuccessor> &regions) {
  // The `try` and the `catchers` region branch back to the parent operation.
  if (!point.isParent()) {
    regions.emplace_back(getOperation());
    return;
  }
 
  regions.push_back(mlir::RegionSuccessor(&getTryRegion()));
 
  // TODO(CIR): If we know a target function never throws a specific type, we
  // can remove the catch handler.
  for (mlir::Region &handlerRegion : this->getHandlerRegions())
    regions.push_back(mlir::RegionSuccessor(&handlerRegion));
}
 
/// Verify that each cir.init_catch_param in \p handlerRegion, outside any
/// nested cir.try, agrees with the handler's leading cir.construct_catch_param
/// \p constructOp, which may be null.  A cir.init_catch_param of kind
/// reference_to_pointer or non_trivial_copy needs \p constructOp on its
/// parameter, and a cir.init_catch_param on the parameter of \p constructOp
/// must have the kind of \p constructOp.
static LogicalResult
verifyCatchParamPairing(mlir::Region &handlerRegion,
                        cir::ConstructCatchParamOp constructOp) {
  mlir::WalkResult result =
      handlerRegion.walk<mlir::WalkOrder::PreOrder>([&](mlir::Operation *op) {
        if (mlir::isa<cir::TryOp>(op))
          return mlir::WalkResult::skip();
        auto initOp = mlir::dyn_cast<cir::InitCatchParamOp>(op);
        if (!initOp)
          return mlir::WalkResult::advance();
 
        cir::InitCatchKind kind = initOp.getKind();
        bool sameParam =
            constructOp && constructOp.getParamAddr() == initOp.getParamAddr();
        if (sameParam && constructOp.getKind() != kind) {
          initOp.emitOpError("kind '")
              << cir::stringifyInitCatchKind(kind)
              << "' does not match the kind '"
              << cir::stringifyInitCatchKind(constructOp.getKind())
              << "' of the preceding 'cir.construct_catch_param'";
          return mlir::WalkResult::interrupt();
        }
        if ((kind == cir::InitCatchKind::ReferenceToPointer ||
             kind == cir::InitCatchKind::NonTrivialCopy) &&
            !sameParam) {
          initOp.emitOpError("'")
              << cir::stringifyInitCatchKind(kind)
              << "' requires a preceding 'cir.construct_catch_param' of the "
                 "same kind on the same parameter";
          return mlir::WalkResult::interrupt();
        }
        return mlir::WalkResult::advance();
      });
  return failure(result.wasInterrupted());
}
 
LogicalResult cir::TryOp::verifyRegions() {
  mlir::ArrayAttr handlerTypes = getHandlerTypes();
  if (!handlerTypes) {
    if (!getHandlerRegions().empty())
      return emitOpError(
          "handler regions must be empty when no handler types are present");
    return success();
  }
 
  mlir::MutableArrayRef<mlir::Region> handlerRegions = getHandlerRegions();
 
  // The parser and builder won't allow this to happen, but the loop below
  // relies on the sizes being the same, so we check it here.
  if (handlerRegions.size() != handlerTypes.size())
    return emitOpError(
        "number of handler regions and handler types must match");
 
  // A filter handler and an unexpected handler together implement a dynamic
  // exception specification. The filter try operation wraps the whole function
  // body and exists only to check the specification, so the two must appear
  // together and must stand alone. A catch-all in the same list would make the
  // filter unreachable, and the filter handler already provides the
  // continue-unwinding path that an unwind handler would supply.
  if (llvm::any_of(handlerTypes, [](mlir::Attribute typeAttr) {
        return mlir::isa<cir::EhFilterAttr, cir::EhUnexpectedAttr>(typeAttr);
      })) {
    if (handlerTypes.size() != 2 ||
        !mlir::isa<cir::EhFilterAttr>(handlerTypes[0]) ||
        !mlir::isa<cir::EhUnexpectedAttr>(handlerTypes[1]))
      return emitOpError("a filter handler must be followed by an unexpected "
                         "handler, and the two must be the only handlers");
  }
 
  for (const auto &[typeAttr, handlerRegion] :
       llvm::zip(handlerTypes, handlerRegions)) {
    // Verify that handler regions have a !cir.eh_token block argument.
    mlir::Block &entryBlock = handlerRegion.front();
    if (entryBlock.getNumArguments() != 1 ||
        !mlir::isa<cir::EhTokenType>(entryBlock.getArgument(0).getType()))
      return emitOpError(
          "handler region must have a single '!cir.eh_token' argument");
 
    // The unwind, filter and unexpected regions do not require a
    // cir.begin_catch. None of them catches the exception.
    if (mlir::isa<cir::UnwindAttr, cir::EhFilterAttr, cir::EhUnexpectedAttr>(
            typeAttr))
      continue;
 
    if (entryBlock.empty())
      return emitOpError("catch handler region must not be empty");
 
    // A terminate scope, which wraps the body of a function that cannot throw,
    // is a catch-all handler that only terminates the program. The exception is
    // caught by the runtime helper that cir.eh.terminate lowers to, so the
    // handler region has no cir.begin_catch of its own.
    if (mlir::isa<cir::EhTerminateOp>(entryBlock.front())) {
      if (!mlir::isa<cir::CatchAllAttr>(typeAttr))
        return emitOpError("'cir.eh.terminate' is only allowed in a catch-all "
                           "handler region");
      continue;
    }
 
    // Nothing may run in a catch handler before cir.begin_catch, so it has to
    // be the handler region's first operation, with two exceptions.
    //
    // When lifetime markers are enabled, the catch parameter's storage can be
    // marked by a cir.lifetime.start. That parameter is the only variable
    // whose lifetime begins before the catch is entered, so there is at most
    // one such marker. Its lifetime-end cleanup has to run after the catch
    // handler is finished (or exited by an exception unwind), so if there is a
    // lifetime begin marker, it is followed by a cir.cleanup.scope that
    // encloses the the rest of the handler with a cir.lifetime.end in its
    // cleanup region.
    //
    // A cir.construct_catch_param may also precede cir.begin_catch, to
    // perform any pre-begin_catch initialization of the catch parameter.
    mlir::Operation *firstOp = &entryBlock.front();
    if (mlir::isa<cir::LifetimeStartOp>(firstOp)) {
      mlir::Operation *next = firstOp->getNextNode();
      auto lifetimeScope = mlir::dyn_cast_if_present<cir::CleanupScopeOp>(next);
      if (!lifetimeScope)
        return emitOpError("'cir.lifetime.start' in a catch handler region "
                           "must be followed by the 'cir.cleanup.scope' of "
                           "its lifetime-end cleanup");
      if (lifetimeScope.getBodyRegion().empty())
        return emitOpError(
            "'cir.lifetime.start' in a catch handler region must be "
            "followed by the 'cir.cleanup.scope' of its lifetime-end "
            "cleanup");
      mlir::Block &scopeBody = lifetimeScope.getBodyRegion().front();
      firstOp = scopeBody.empty() ? nullptr : &scopeBody.front();
    }
 
    auto constructOp =
        mlir::dyn_cast_if_present<cir::ConstructCatchParamOp>(firstOp);
    if (constructOp)
      firstOp = firstOp->getNextNode();
    if (!mlir::isa_and_present<cir::BeginCatchOp>(firstOp))
      return emitOpError(
          "catch handler region must start with 'cir.begin_catch'");
 
    if (failed(verifyCatchParamPairing(handlerRegion, constructOp)))
      return failure();
  }
 
  return success();
}
 
static void
printTryHandlerRegions(mlir::OpAsmPrinter &printer, cir::TryOp op,
                       mlir::MutableArrayRef<mlir::Region> handlerRegions,
                       mlir::ArrayAttr handlerTypes) {
  if (!handlerTypes)
    return;
 
  for (const auto [typeIdx, typeAttr] : llvm::enumerate(handlerTypes)) {
    if (typeIdx)
      printer << " ";
 
    if (mlir::isa<cir::CatchAllAttr>(typeAttr)) {
      printer << "catch all ";
    } else if (mlir::isa<cir::UnwindAttr>(typeAttr)) {
      printer << "unwind ";
    } else if (auto filterAttr = mlir::dyn_cast<cir::EhFilterAttr>(typeAttr)) {
      printer << "filter [";
      llvm::interleaveComma(
          filterAttr.getPermittedTypes(), printer,
          [&](mlir::Attribute sym) { printer.printAttribute(sym); });
      printer << "] ";
    } else if (mlir::isa<cir::EhUnexpectedAttr>(typeAttr)) {
      printer << "unexpected ";
    } else {
      printer << "catch [type ";
      printer.printAttribute(typeAttr);
      printer << "] ";
    }
 
    // Print the handler region's !cir.eh_token block argument.
    mlir::Region &region = handlerRegions[typeIdx];
    if (!region.empty() && region.front().getNumArguments() > 0) {
      printer << "(";
      printer.printRegionArgument(region.front().getArgument(0));
      printer << ") ";
    }
 
    printer.printRegion(region,
                        /*printEntryBLockArgs=*/false,
                        /*printBlockTerminators=*/true);
  }
}
 
static mlir::ParseResult parseTryHandlerRegions(
    mlir::OpAsmParser &parser,
    llvm::SmallVectorImpl<std::unique_ptr<mlir::Region>> &handlerRegions,
    mlir::ArrayAttr &handlerTypes) {
 
  auto parseCheckedCatcherRegion = [&]() -> mlir::ParseResult {
    handlerRegions.emplace_back(new mlir::Region);
 
    mlir::Region &currRegion = *handlerRegions.back();
 
    // Parse the required region argument: (%eh_token : !cir.eh_token)
    llvm::SmallVector<mlir::OpAsmParser::Argument> regionArgs;
    if (parser.parseLParen())
      return failure();
    mlir::OpAsmParser::Argument arg;
    if (parser.parseArgument(arg, /*allowType=*/true))
      return failure();
    regionArgs.push_back(arg);
    if (parser.parseRParen())
      return failure();
 
    mlir::SMLoc regionLoc = parser.getCurrentLocation();
    if (parser.parseRegion(currRegion, regionArgs)) {
      handlerRegions.clear();
      return failure();
    }
 
    if (currRegion.empty())
      return parser.emitError(regionLoc, "handler region shall not be empty");
 
    if (!(currRegion.back().mightHaveTerminator() &&
          currRegion.back().getTerminator()))
      return parser.emitError(
          regionLoc, "blocks are expected to be explicitly terminated");
 
    return success();
  };
 
  bool hasCatchAll = false;
  llvm::SmallVector<mlir::Attribute, 4> catcherAttrs;
  while (parser.parseOptionalKeyword("catch").succeeded()) {
    bool hasLSquare = parser.parseOptionalLSquare().succeeded();
 
    llvm::StringRef attrStr;
    if (parser.parseOptionalKeyword(&attrStr, {"all", "type"}).failed())
      return parser.emitError(parser.getCurrentLocation(),
                              "expected 'all' or 'type' keyword");
 
    bool isCatchAll = attrStr == "all";
    if (isCatchAll) {
      if (hasCatchAll)
        return parser.emitError(parser.getCurrentLocation(),
                                "can't have more than one catch all");
      hasCatchAll = true;
    }
 
    mlir::Attribute exceptionRTTIAttr;
    if (!isCatchAll && parser.parseAttribute(exceptionRTTIAttr).failed())
      return parser.emitError(parser.getCurrentLocation(),
                              "expected valid RTTI info attribute");
 
    catcherAttrs.push_back(isCatchAll
                               ? cir::CatchAllAttr::get(parser.getContext())
                               : exceptionRTTIAttr);
 
    if (hasLSquare && isCatchAll)
      return parser.emitError(parser.getCurrentLocation(),
                              "catch all dosen't need RTTI info attribute");
 
    if (hasLSquare && parser.parseRSquare().failed())
      return parser.emitError(parser.getCurrentLocation(),
                              "expected `]` after RTTI info attribute");
 
    if (parseCheckedCatcherRegion().failed())
      return mlir::failure();
  }
 
  // A filter handler carries the type info symbols permitted by the enclosing
  // function's dynamic exception specification. TryOp::verifyRegions enforces
  // that it is paired with an unexpected handler and that the two stand alone.
  if (parser.parseOptionalKeyword("filter").succeeded()) {
    mlir::SMLoc filterLoc = parser.getCurrentLocation();
    llvm::SmallVector<mlir::Attribute, 4> permittedTypes;
    auto parsePermittedType = [&]() -> mlir::ParseResult {
      mlir::SMLoc typeLoc = parser.getCurrentLocation();
      mlir::Attribute rtti;
      if (parser.parseAttribute(rtti).failed())
        return mlir::failure();
      if (!mlir::isa<cir::GlobalViewAttr>(rtti))
        return parser.emitError(typeLoc, "expected a type info symbol naming a "
                                         "permitted exception type");
      permittedTypes.push_back(rtti);
      return mlir::success();
    };
    if (parser
            .parseCommaSeparatedList(mlir::OpAsmParser::Delimiter::Square,
                                     parsePermittedType)
            .failed())
      return mlir::failure();
 
    auto filterAttr = cir::EhFilterAttr::getChecked(
        [&]() { return parser.emitError(filterLoc); }, parser.getContext(),
        parser.getBuilder().getArrayAttr(permittedTypes));
    if (!filterAttr)
      return mlir::failure();
    catcherAttrs.push_back(filterAttr);
    if (parseCheckedCatcherRegion().failed())
      return mlir::failure();
  }
 
  if (parser.parseOptionalKeyword("unexpected").succeeded()) {
    catcherAttrs.push_back(cir::EhUnexpectedAttr::get(parser.getContext()));
    if (parseCheckedCatcherRegion().failed())
      return mlir::failure();
  }
 
  if (parser.parseOptionalKeyword("unwind").succeeded()) {
    if (hasCatchAll)
      return parser.emitError(parser.getCurrentLocation(),
                              "unwind can't be used with catch all");
 
    catcherAttrs.push_back(cir::UnwindAttr::get(parser.getContext()));
    if (parseCheckedCatcherRegion().failed())
      return mlir::failure();
  }
 
  handlerTypes = parser.getBuilder().getArrayAttr(catcherAttrs);
  return mlir::success();
}
 
//===----------------------------------------------------------------------===//
// EhTypeIdOp
//===----------------------------------------------------------------------===//
 
LogicalResult
cir::EhTypeIdOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
  Operation *op = symbolTable.lookupNearestSymbolFrom(*this, getTypeSymAttr());
  if (!isa_and_nonnull<GlobalOp>(op))
    return emitOpError("'")
           << getTypeSym() << "' does not reference a valid cir.global";
  return success();
}
 
//===----------------------------------------------------------------------===//
// LifetimeStartOp & LifetimeEndOp
//===----------------------------------------------------------------------===//
 
LogicalResult cir::LifetimeStartOp::verify() {
  return verifyProducedBy<cir::AllocaOp>(*this, getPtr(), "ptr");
}
 
LogicalResult cir::LifetimeEndOp::verify() {
  return verifyProducedBy<cir::AllocaOp>(*this, getPtr(), "ptr");
}
 
//===----------------------------------------------------------------------===//
// MemChrOp
//===----------------------------------------------------------------------===//
 
/// Reads a fundamental integer width from a signless i32 attribute.
static std::optional<unsigned> getRecordedIntegerWidth(mlir::Attribute attr) {
  auto intAttr = mlir::dyn_cast<mlir::IntegerAttr>(attr);
  if (!intAttr || !intAttr.getType().isSignlessInteger(32))
    return std::nullopt;
  int64_t width = intAttr.getInt();
  if (width < 0 ||
      !cir::isValidFundamentalIntWidth(static_cast<unsigned>(width)))
    return std::nullopt;
  return static_cast<unsigned>(width);
}
 
LogicalResult cir::MemChrOp::verify() {
  auto moduleOp = (*this)->getParentOfType<mlir::ModuleOp>();
  if (!moduleOp)
    return emitOpError("expects an enclosing module");
 
  // libc memchr uses pointers in the target's default address space.
  if (mlir::cast<cir::PointerType>(getSrc().getType()).getAddrSpace())
    return emitOpError("src must be in the default address space");
 
  auto checkWidth = [&](cir::IntType type, llvm::StringRef operandName,
                        llvm::StringRef attrName) -> LogicalResult {
    mlir::Attribute attr = moduleOp->getAttr(attrName);
    if (!attr)
      return emitOpError("expects the module to record ") << attrName;
    std::optional<unsigned> width = getRecordedIntegerWidth(attr);
    if (!width)
      return emitOpError("requires ")
             << attrName
             << " to be a signless i32 holding a fundamental integer width";
    if (type.getWidth() != *width)
      return emitOpError() << operandName << " must have the width recorded in "
                           << attrName;
    return success();
  };
 
  if (failed(checkWidth(getPattern().getType(), "pattern",
                        cir::CIRDialect::getIntTypeWidthAttrName())))
    return failure();
  return checkWidth(getLen().getType(), "len",
                    cir::CIRDialect::getSizeTypeWidthAttrName());
}
 
//===----------------------------------------------------------------------===//
// InitCatchParamOp
//===----------------------------------------------------------------------===//
 
/// For the reference_to_pointer and reference_to_record_pointer kinds, emit an
/// error on \p op unless \p paramAddrType is the type of the address of a
/// reference to a pointer.  Any other \p kind is accepted.
static LogicalResult verifyCatchParamAddr(mlir::Operation *op,
                                          cir::InitCatchKind kind,
                                          cir::PointerType paramAddrType) {
  if (kind != cir::InitCatchKind::ReferenceToPointer &&
      kind != cir::InitCatchKind::ReferenceToRecordPointer)
    return success();
 
  auto refType = mlir::dyn_cast<cir::PointerType>(paramAddrType.getPointee());
  if (!refType || !mlir::isa<cir::PointerType>(refType.getPointee()))
    return op->emitOpError("'")
           << cir::stringifyInitCatchKind(kind)
           << "' requires 'param_addr' to be the address of a reference to a "
              "pointer";
  return success();
}
 
LogicalResult cir::InitCatchParamOp::verify() {
  return verifyCatchParamAddr(*this, getKind(), getParamAddr().getType());
}
 
//===----------------------------------------------------------------------===//
// ConstructCatchParamOp
//===----------------------------------------------------------------------===//
 
LogicalResult cir::ConstructCatchParamOp::verify() {
  return verifyCatchParamAddr(*this, getKind(), getParamAddr().getType());
}
 
LogicalResult cir::ConstructCatchParamOp::verifySymbolUses(
    SymbolTableCollection &symbolTable) {
  auto copyFnAttr = getCopyFnAttr();
  if (!copyFnAttr)
    return success();
  auto fn =
      symbolTable.lookupNearestSymbolFrom<cir::FuncOp>(*this, getCopyFnAttr());
  if (!fn)
    return emitOpError("'")
           << *getCopyFn() << "' does not reference a valid cir.func";
 
  if (!fn->hasAttr(cir::CIRDialect::getCatchCopyThunkAttrName()))
    return emitOpError("catch-init copy_fn must be tagged with the ")
           << cir::CIRDialect::getCatchCopyThunkAttrName() << " attribute";
 
  cir::FuncType fnType = fn.getFunctionType();
  if (fnType.getNumInputs() != 2 || !fnType.hasVoidReturn())
    return emitOpError("catch-init copy_fn must take two pointer arguments and "
                       "return void");
 
  if (fnType.getInput(0) != getParamAddr().getType())
    return emitOpError("first argument of catch-init copy_fn must match the "
                       "type of 'param_addr'");
 
  if (fnType.getInput(1) != getParamAddr().getType())
    return emitOpError(
        "second argument of catch-init copy_fn must be a pointer "
        "to the catch type");
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// EhDispatchOp
//===----------------------------------------------------------------------===//
 
LogicalResult cir::EhDispatchOp::verify() {
  mlir::ArrayAttr handlerTypes = getCatchTypesAttr();
  if (!handlerTypes)
    return success();
 
  bool hasFilter = false;
  for (mlir::Attribute typeAttr : handlerTypes) {
    if (!mlir::isa<cir::EhFilterAttr>(typeAttr))
      continue;
    if (hasFilter)
      return emitOpError("can't have more than one 'filter' handler");
    hasFilter = true;
  }
 
  // Unlike the other handlers, a filter does not take the place of the default
  // destination. Its destination is taken when the exception is not permitted
  // by the specification, and the default 'unwind' destination is taken when
  // it is. A catch-all default would leave the filter unreachable.
  if (hasFilter && getDefaultIsCatchAll())
    return emitOpError(
        "'filter' handler requires an 'unwind' default destination");
 
  return success();
}
 
static ParseResult
parseEhDispatchDestinations(OpAsmParser &parser, mlir::ArrayAttr &catchTypes,
                            SmallVectorImpl<Block *> &catchDestinations,
                            Block *&defaultDestination,
                            mlir::UnitAttr &defaultIsCatchAll) {
  // Parse: [ ... ]
  if (parser.parseLSquare())
    return failure();
 
  SmallVector<Attribute> handlerTypes;
  bool hasCatchAll = false;
  bool hasUnwind = false;
 
  // Parse handler list.
  auto parseHandler = [&]() -> ParseResult {
    // Check for 'catch_all' or 'unwind' keywords.
    if (succeeded(parser.parseOptionalKeyword("catch_all"))) {
      if (hasCatchAll)
        return parser.emitError(parser.getCurrentLocation(),
                                "duplicate 'catch_all' handler");
      if (hasUnwind)
        return parser.emitError(parser.getCurrentLocation(),
                                "cannot have both 'catch_all' and 'unwind'");
      hasCatchAll = true;
 
      if (parser.parseColon().failed())
        return failure();
 
      if (parser.parseSuccessor(defaultDestination).failed())
        return failure();
 
      return success();
    }
 
    if (succeeded(parser.parseOptionalKeyword("unwind"))) {
      if (hasUnwind)
        return parser.emitError(parser.getCurrentLocation(),
                                "duplicate 'unwind' handler");
      if (hasCatchAll)
        return parser.emitError(parser.getCurrentLocation(),
                                "cannot have both 'catch_all' and 'unwind'");
      hasUnwind = true;
 
      if (parser.parseColon().failed())
        return failure();
 
      if (parser.parseSuccessor(defaultDestination).failed())
        return failure();
      return success();
    }
 
    // A filter handler carries the type info symbols permitted by the
    // enclosing function's dynamic exception specification. Its destination is
    // taken when the exception is *not* one of those types.
    if (succeeded(parser.parseOptionalKeyword("filter"))) {
      SMLoc filterLoc = parser.getCurrentLocation();
      SmallVector<Attribute, 4> permittedTypes;
      auto parsePermittedType = [&]() -> ParseResult {
        mlir::SMLoc typeLoc = parser.getCurrentLocation();
        mlir::Attribute rtti;
        if (parser.parseAttribute(rtti).failed())
          return failure();
        if (!mlir::isa<cir::GlobalViewAttr>(rtti))
          return parser.emitError(typeLoc,
                                  "expected a type info symbol naming a "
                                  "permitted exception type");
        permittedTypes.push_back(rtti);
        return success();
      };
      if (parser
              .parseCommaSeparatedList(OpAsmParser::Delimiter::Paren,
                                       parsePermittedType)
              .failed())
        return failure();
 
      auto filterAttr = cir::EhFilterAttr::getChecked(
          [&]() { return parser.emitError(filterLoc); }, parser.getContext(),
          parser.getBuilder().getArrayAttr(permittedTypes));
      if (!filterAttr)
        return failure();
      handlerTypes.push_back(filterAttr);
 
      if (parser.parseColon().failed())
        return failure();
 
      Block *dest;
      if (parser.parseSuccessor(dest).failed())
        return failure();
      catchDestinations.push_back(dest);
      return success();
    }
 
    // Otherwise, expect 'catch(<attr> : <type>) : ^block'.
    // The 'catch(...)' wrapper allows the attribute to include its type
    // without conflicting with the ':' used for the block destination.
    if (parser.parseKeyword("catch").failed())
      return failure();
 
    if (parser.parseLParen().failed())
      return failure();
 
    mlir::Attribute catchTypeAttr;
    if (parser.parseAttribute(catchTypeAttr).failed())
      return failure();
    handlerTypes.push_back(catchTypeAttr);
 
    if (parser.parseRParen().failed())
      return failure();
 
    if (parser.parseColon().failed())
      return failure();
 
    Block *dest;
    if (parser.parseSuccessor(dest).failed())
      return failure();
    catchDestinations.push_back(dest);
    return success();
  };
 
  if (parser.parseCommaSeparatedList(parseHandler).failed())
    return failure();
 
  if (parser.parseRSquare().failed())
    return failure();
 
  // Verify we have catch_all or unwind.
  if (!hasCatchAll && !hasUnwind)
    return parser.emitError(parser.getCurrentLocation(),
                            "must have either 'catch_all' or 'unwind' handler");
 
  // Add attributes and successors.
  if (!handlerTypes.empty())
    catchTypes = parser.getBuilder().getArrayAttr(handlerTypes);
 
  if (hasCatchAll)
    defaultIsCatchAll = parser.getBuilder().getUnitAttr();
 
  return success();
}
 
static void printEhDispatchDestinations(OpAsmPrinter &p, cir::EhDispatchOp op,
                                        mlir::ArrayAttr catchTypes,
                                        SuccessorRange catchDestinations,
                                        Block *defaultDestination,
                                        mlir::UnitAttr defaultIsCatchAll) {
  p << " [";
  p.printNewline();
 
  // If we have at least one catch type, print them.
  if (catchTypes) {
    // Print type handlers using 'catch(<attr>) : ^block' syntax.
    llvm::interleave(
        llvm::zip(catchTypes, catchDestinations),
        [&](auto i) {
          mlir::Attribute typeAttr = std::get<0>(i);
          if (auto filterAttr = mlir::dyn_cast<cir::EhFilterAttr>(typeAttr)) {
            p << "  filter(";
            llvm::interleaveComma(
                filterAttr.getPermittedTypes(), p,
                [&](mlir::Attribute sym) { p.printAttribute(sym); });
          } else {
            p << "  catch(";
            p.printAttribute(typeAttr);
          }
          p << ") : ";
          p.printSuccessor(std::get<1>(i));
        },
        [&] {
          p << ',';
          p.printNewline();
        });
 
    p << ", ";
    p.printNewline();
  }
 
  // Print catch_all or unwind handler.
  if (defaultIsCatchAll)
    p << "  catch_all : ";
  else
    p << "  unwind : ";
  p.printSuccessor(defaultDestination);
  p.printNewline();
 
  p << "]";
}
 
//===----------------------------------------------------------------------===//
// Standard library op signature matching
//===----------------------------------------------------------------------===//
 
bool cir::StdFindOp::signatureMatches(mlir::TypeRange operands,
                                      mlir::TypeRange results) {
  if (operands.size() != getNumArgs() || results.size() != 1)
    return false;
  mlir::Type iterTy = operands[0];
  return iterTy == operands[1] && iterTy == operands[2] && iterTy == results[0];
}
 
//===----------------------------------------------------------------------===//
// TableGen'd op method definitions
//===----------------------------------------------------------------------===//
 
#define GET_OP_CLASSES
#include "clang/CIR/Dialect/IR/CIROps.cpp.inc"