CIRGenBuilder.h

Go to the documentation of this file.

//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
 
#ifndef LLVM_CLANG_LIB_CIR_CODEGEN_CIRGENBUILDER_H
#define LLVM_CLANG_LIB_CIR_CODEGEN_CIRGENBUILDER_H
 
#include "Address.h"
#include "CIRGenRecordLayout.h"
#include "CIRGenTypeCache.h"
#include "mlir/Dialect/Ptr/IR/MemorySpaceInterfaces.h"
#include "mlir/IR/Attributes.h"
#include "mlir/IR/Builders.h"
#include "mlir/IR/BuiltinAttributes.h"
#include "mlir/Support/LLVM.h"
#include "clang/CIR/Dialect/IR/CIRDataLayout.h"
#include "clang/CIR/MissingFeatures.h"
 
#include "clang/CIR/Dialect/Builder/CIRBaseBuilder.h"
#include "clang/CIR/MissingFeatures.h"
#include "llvm/ADT/APFloat.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/IR/FPEnv.h"
 
namespace clang::CIRGen {
 
class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
  const CIRGenTypeCache &typeCache;
 
  llvm::StringMap<unsigned> recordNames;
  llvm::StringMap<unsigned> globalsVersioning;
 
public:
  CIRGenBuilderTy(mlir::MLIRContext &mlirContext, const CIRGenTypeCache &tc)
      : CIRBaseBuilderTy(mlirContext), typeCache(tc) {}
 
  /// Get a cir::ConstArrayAttr for a string literal.
  /// Note: This is different from what is returned by
  /// mlir::Builder::getStringAttr() which is an mlir::StringAttr.
  mlir::Attribute getString(llvm::StringRef str, mlir::Type eltTy,
                            std::optional<size_t> size,
                            bool ensureNullTerm = true) {
    size_t finalSize = size.value_or(str.size());
 
    size_t lastNonZeroPos = str.find_last_not_of('\0');
    // If the string is full of null bytes, emit a #cir.zero rather than
    // a #cir.const_array.
    if (lastNonZeroPos == llvm::StringRef::npos) {
      auto arrayTy = cir::ArrayType::get(eltTy, finalSize);
      return cir::ZeroAttr::get(arrayTy);
    }
 
    // We emit trailing zeros for all trailing zeros, so the null-terminator in
    // a constant is always in trailing zeros, and the null-terminator is
    // skipped in the CIR representation.
    size_t trailingZerosNum = finalSize - lastNonZeroPos - 1;
    auto truncatedArrayTy =
        cir::ArrayType::get(eltTy, finalSize - trailingZerosNum);
    auto strAttr = mlir::StringAttr::get(str.drop_back(trailingZerosNum),
                                         truncatedArrayTy);
 
    // Most C strings are null terminated, so if we are ensuring there is one,
    // grow the array size by 1 to add a trailing zero if necessary. The 'auto'
    // calculation of trailing zeros (the difference between the provided string
    // and the type) will ensure we get the count correct.
    finalSize += (ensureNullTerm && trailingZerosNum == 0);
 
    auto fullArrayTy = cir::ArrayType::get(eltTy, finalSize);
    return cir::ConstArrayAttr::get(fullArrayTy, strAttr);
  }
 
  cir::ConstArrayAttr getConstArray(mlir::Attribute attrs,
                                    cir::ArrayType arrayTy) const {
    return cir::ConstArrayAttr::get(arrayTy, attrs);
  }
 
  mlir::Attribute getConstRecordOrZeroAttr(mlir::ArrayAttr arrayAttr,
                                           cir::RecordType recordTy);
 
  cir::ConstRecordAttr getAnonConstRecord(mlir::ArrayAttr arrayAttr) {
    llvm::SmallVector<mlir::Type> members;
    for (auto &f : arrayAttr) {
      auto ta = mlir::cast<mlir::TypedAttr>(f);
      members.push_back(ta.getType());
    }
 
    auto sTy = getAnonRecordTy(members, /*packed=*/false,
                               cir::RecordType::getAllDataKinds(members));
    return cir::ConstRecordAttr::get(sTy, arrayAttr);
  }
 
  cir::TypeInfoAttr getTypeInfo(mlir::ArrayAttr fieldsAttr) {
    cir::ConstRecordAttr anonRecord = getAnonConstRecord(fieldsAttr);
    return cir::TypeInfoAttr::get(anonRecord.getType(), fieldsAttr);
  }
 
  std::string getUniqueAnonRecordName() { return getUniqueRecordName("anon"); }
 
  std::string getUniqueRecordName(const std::string &baseName) {
    auto it = recordNames.find(baseName);
    if (it == recordNames.end()) {
      recordNames[baseName] = 0;
      return baseName;
    }
 
    return baseName + "." + std::to_string(recordNames[baseName]++);
  }
 
  cir::LongDoubleType getLongDoubleTy(const llvm::fltSemantics &format) const {
    if (&format == &llvm::APFloat::IEEEdouble())
      return cir::LongDoubleType::get(getContext(), typeCache.doubleTy);
    if (&format == &llvm::APFloat::x87DoubleExtended())
      return cir::LongDoubleType::get(getContext(), typeCache.fP80Ty);
    if (&format == &llvm::APFloat::IEEEquad())
      return cir::LongDoubleType::get(getContext(), typeCache.fP128Ty);
    if (&format == &llvm::APFloat::PPCDoubleDouble())
      llvm_unreachable("NYI: PPC double-double format for long double");
    llvm_unreachable("Unsupported format for long double");
  }
 
  mlir::Type getPtrToVPtrType() {
    return getPointerTo(cir::VPtrType::get(getContext()));
  }
 
  cir::FuncType getFuncType(llvm::ArrayRef<mlir::Type> params, mlir::Type retTy,
                            bool isVarArg = false) {
    return cir::FuncType::get(params, retTy, isVarArg);
  }
 
  /// Get a CIR record kind from a AST declaration tag.
  /// Returns true if the tag kind represents a C++ class (as opposed to a
  /// plain struct).
  static bool tagKindIsClass(const clang::TagTypeKind kind) {
    return kind == clang::TagTypeKind::Class;
  }
 
  /// Returns true if the tag kind is a union.
  static bool tagKindIsUnion(const clang::TagTypeKind kind) {
    return kind == clang::TagTypeKind::Union;
  }
 
  /// Get a CIR named record type.
  ///
  /// If a record already exists and is complete, but the client tries to fetch
  /// it with a different set of attributes, this method will crash.
  cir::RecordType getCompleteNamedRecordType(
      llvm::ArrayRef<mlir::Type> members, bool packed, llvm::StringRef name,
      llvm::ArrayRef<cir::RecordMemberKind> memberKinds) {
    const auto nameAttr = getStringAttr(name);
    assert(!cir::MissingFeatures::astRecordDeclAttr());
 
    // Always a struct at this call site, never a class or a union.
    auto type = cir::StructType::get(getContext(), members, nameAttr, packed,
                                     /*is_class=*/false, memberKinds);
 
    // If we found an existing type, verify that either it is incomplete or
    // it matches the requested attributes.
    assert(!type.isIncomplete() ||
           (type.getMembers() == members && type.getPacked() == packed));
 
    // Complete an incomplete record or ensure the existing complete record
    // matches the requested attributes.
    type.complete(members, packed, memberKinds);
 
    return type;
  }
 
  /// Get an incomplete CIR record type. If we have a complete record
  /// declaration, we may create an incomplete type and then add the
  /// members, so \p rd here may be complete.
  cir::RecordType getIncompleteRecordTy(llvm::StringRef name,
                                        const clang::RecordDecl *rd) {
    const mlir::StringAttr nameAttr = getStringAttr(name);
    if (rd && tagKindIsUnion(rd->getTagKind()))
      return cir::UnionType::get(getContext(), nameAttr);
    bool is_class = rd && tagKindIsClass(rd->getTagKind());
    return cir::StructType::get(getContext(), nameAttr, is_class);
  }
 
  //
  // Operation creation helpers
  // --------------------------
  //
  using CIRBaseBuilderTy::createCopy;
  cir::CopyOp createCopy(Address dst, Address src, bool isVolatile = false,
                         bool skipTailPadding = false) {
    cir::CopyOp op = createCopy(dst.getPointer(), src.getPointer(), isVolatile,
                                skipTailPadding);
    op.setDstAlignment(dst.getAlignment().getQuantity());
    op.setSrcAlignment(src.getAlignment().getQuantity());
    return op;
  }
 
  cir::MemCpyOp createMemCpy(mlir::Location loc, Address dst, Address src,
                             mlir::Value len) {
    return cir::MemCpyOp::create(
        *this, loc, dst.getPointer(), src.getPointer(), len,
        getI64IntegerAttr(dst.getAlignment().getQuantity()),
        getI64IntegerAttr(src.getAlignment().getQuantity()));
  }
 
  cir::MemMoveOp createMemMove(mlir::Location loc, mlir::Value dst,
                               mlir::Value src, mlir::Value len) {
    return cir::MemMoveOp::create(*this, loc, dst, src, len);
  }
 
  cir::MemSetOp createMemSet(mlir::Location loc, mlir::Value dst,
                             mlir::Value val, mlir::Value len) {
    assert(val.getType() == getUInt8Ty());
    return cir::MemSetOp::create(*this, loc, dst, {}, val, len);
  }
 
  cir::MemSetOp createMemSet(mlir::Location loc, Address dst, mlir::Value val,
                             mlir::Value len) {
    mlir::IntegerAttr align = getAlignmentAttr(dst.getAlignment());
    assert(val.getType() == getUInt8Ty());
    return cir::MemSetOp::create(*this, loc, dst.getPointer(), align, val, len);
  }
  // ---------------------------
 
  cir::DataMemberAttr getDataMemberAttr(cir::DataMemberType ty,
                                        llvm::ArrayRef<int32_t> path) {
    return cir::DataMemberAttr::get(ty, path);
  }
 
  cir::DataMemberAttr getNullDataMemberAttr(cir::DataMemberType ty) {
    return cir::DataMemberAttr::get(ty);
  }
 
  // Return true if the value is a null constant such as null pointer, (+0.0)
  // for floating-point or zero initializer
  bool isNullValue(mlir::Attribute attr) const {
    if (mlir::isa<cir::ZeroAttr>(attr))
      return true;
 
    if (const auto ptrVal = mlir::dyn_cast<cir::ConstPtrAttr>(attr))
      return ptrVal.isNullValue();
 
    if (const auto intVal = mlir::dyn_cast<cir::IntAttr>(attr))
      return intVal.isNullValue();
 
    if (const auto boolVal = mlir::dyn_cast<cir::BoolAttr>(attr))
      return !boolVal.getValue();
 
    if (auto fpAttr = mlir::dyn_cast<cir::FPAttr>(attr)) {
      auto fpVal = fpAttr.getValue();
      bool ignored;
      llvm::APFloat fv(+0.0);
      fv.convert(fpVal.getSemantics(), llvm::APFloat::rmNearestTiesToEven,
                 &ignored);
      return fv.bitwiseIsEqual(fpVal);
    }
    if (const auto recordVal = mlir::dyn_cast<cir::ConstRecordAttr>(attr)) {
      for (const auto elt : recordVal.getMembers()) {
        // FIXME(cir): the record's ID should not be considered a member.
        if (mlir::isa<mlir::StringAttr>(elt))
          continue;
        if (!isNullValue(elt))
          return false;
      }
      return true;
    }
 
    if (const auto arrayVal = mlir::dyn_cast<cir::ConstArrayAttr>(attr)) {
      if (mlir::isa<mlir::StringAttr>(arrayVal.getElts()))
        return false;
 
      return llvm::all_of(
          mlir::cast<mlir::ArrayAttr>(arrayVal.getElts()),
          [&](const mlir::Attribute &elt) { return isNullValue(elt); });
    }
    return false;
  }
 
  //
  // Type helpers
  // ------------
  //
  cir::IntType getUIntNTy(int n) {
    switch (n) {
    case 8:
      return getUInt8Ty();
    case 16:
      return getUInt16Ty();
    case 32:
      return getUInt32Ty();
    case 64:
      return getUInt64Ty();
    default:
      return cir::IntType::get(getContext(), n, false);
    }
  }
 
  cir::IntType getSIntNTy(int n) {
    switch (n) {
    case 8:
      return getSInt8Ty();
    case 16:
      return getSInt16Ty();
    case 32:
      return getSInt32Ty();
    case 64:
      return getSInt64Ty();
    default:
      return cir::IntType::get(getContext(), n, true);
    }
  }
 
  cir::VoidType getVoidTy() { return typeCache.voidTy; }
 
  cir::IntType getBoolMemoryTy() { return getUInt8Ty(); }
 
  cir::IntType getSInt8Ty() { return typeCache.sInt8Ty; }
  cir::IntType getSInt16Ty() { return typeCache.sInt16Ty; }
  cir::IntType getSInt32Ty() { return typeCache.sInt32Ty; }
  cir::IntType getSInt64Ty() { return typeCache.sInt64Ty; }
 
  cir::IntType getUInt8Ty() { return typeCache.uInt8Ty; }
  cir::IntType getUInt16Ty() { return typeCache.uInt16Ty; }
  cir::IntType getUInt32Ty() { return typeCache.uInt32Ty; }
  cir::IntType getUInt64Ty() { return typeCache.uInt64Ty; }
 
  cir::FP16Type getFp16Ty() { return typeCache.fP16Ty; }
  cir::BF16Type getBfloat16Ty() { return typeCache.bFloat16Ty; }
  cir::SingleType getSingleTy() { return typeCache.floatTy; }
  cir::DoubleType getDoubleTy() { return typeCache.doubleTy; }
 
  cir::ConstantOp getConstInt(mlir::Location loc, llvm::APSInt intVal);
 
  cir::ConstantOp getConstInt(mlir::Location loc, llvm::APInt intVal,
                              bool isUnsigned = true);
 
  cir::ConstantOp getConstInt(mlir::Location loc, mlir::Type t, uint64_t c);
 
  cir::ConstantOp getConstFP(mlir::Location loc, mlir::Type t,
                             llvm::APFloat fpVal);
 
  bool isInt8Ty(mlir::Type i) {
    return i == typeCache.uInt8Ty || i == typeCache.sInt8Ty;
  }
  bool isInt16Ty(mlir::Type i) {
    return i == typeCache.uInt16Ty || i == typeCache.sInt16Ty;
  }
  bool isInt32Ty(mlir::Type i) {
    return i == typeCache.uInt32Ty || i == typeCache.sInt32Ty;
  }
  bool isInt64Ty(mlir::Type i) {
    return i == typeCache.uInt64Ty || i == typeCache.sInt64Ty;
  }
  bool isInt(mlir::Type i) { return mlir::isa<cir::IntType>(i); }
 
  cir::IntType getExtendedIntTy(cir::IntType ty, bool isSigned) {
    switch (ty.getWidth()) {
    case 8:
      return isSigned ? typeCache.sInt16Ty : typeCache.uInt16Ty;
    case 16:
      return isSigned ? typeCache.sInt32Ty : typeCache.uInt32Ty;
    case 32:
      return isSigned ? typeCache.sInt64Ty : typeCache.uInt64Ty;
    default:
      llvm_unreachable("NYI");
    }
  }
 
  cir::IntType getTruncatedIntTy(cir::IntType ty, bool isSigned) {
    switch (ty.getWidth()) {
    case 16:
      return isSigned ? typeCache.sInt8Ty : typeCache.uInt8Ty;
    case 32:
      return isSigned ? typeCache.sInt16Ty : typeCache.uInt16Ty;
    case 64:
      return isSigned ? typeCache.sInt32Ty : typeCache.uInt32Ty;
    default:
      llvm_unreachable("NYI");
    }
  }
 
  cir::VectorType
  getExtendedOrTruncatedElementVectorType(cir::VectorType vt, bool isExtended,
                                          bool isSigned = false) {
    auto elementTy = mlir::dyn_cast_or_null<cir::IntType>(vt.getElementType());
    assert(elementTy && "expected int vector");
    return cir::VectorType::get(isExtended
                                    ? getExtendedIntTy(elementTy, isSigned)
                                    : getTruncatedIntTy(elementTy, isSigned),
                                vt.getSize());
  }
 
  // Fetch the type representing a pointer to unsigned int8 values.
  cir::PointerType getUInt8PtrTy() { return typeCache.uInt8PtrTy; }
 
  /// Get a CIR anonymous struct type.
  cir::StructType
  getAnonRecordTy(llvm::ArrayRef<mlir::Type> members, bool packed,
                  llvm::ArrayRef<cir::RecordMemberKind> memberKinds) {
    assert(!cir::MissingFeatures::astRecordDeclAttr());
    return cir::StructType::get(getContext(), members, packed,
                                /*is_class=*/false, memberKinds);
  }
 
  //===--------------------------------------------------------------------===//
  // Constant creation helpers
  //===--------------------------------------------------------------------===//
  cir::ConstantOp getSInt32(int32_t c, mlir::Location loc) {
    return getConstantInt(loc, getSInt32Ty(), c);
  }
  cir::ConstantOp getUInt32(uint32_t c, mlir::Location loc) {
    return getConstantInt(loc, getUInt32Ty(), c);
  }
  cir::ConstantOp getSInt64(uint64_t c, mlir::Location loc) {
    return getConstantInt(loc, getSInt64Ty(), c);
  }
  cir::ConstantOp getUInt64(uint64_t c, mlir::Location loc) {
    return getConstantInt(loc, getUInt64Ty(), c);
  }
 
  cir::ConstantOp getZero(mlir::Location loc, mlir::Type ty) {
    // TODO: dispatch creation for primitive types.
    assert((mlir::isa<cir::RecordType>(ty) || mlir::isa<cir::ArrayType>(ty) ||
            mlir::isa<cir::VectorType>(ty)) &&
           "NYI for other types");
    return cir::ConstantOp::create(*this, loc, cir::ZeroAttr::get(ty));
  }
 
  //===--------------------------------------------------------------------===//
  // UnaryOp creation helpers
  //===--------------------------------------------------------------------===//
  mlir::Value createNeg(mlir::Location loc, mlir::Value value,
                        bool nsw = false) {
 
    if (auto intTy = mlir::dyn_cast<cir::IntType>(value.getType())) {
      // Source is a unsigned integer: first cast it to signed.
      if (intTy.isUnsigned())
        value = createIntCast(value, getSIntNTy(intTy.getWidth()));
      return createMinus(loc, value, nsw);
    }
 
    llvm_unreachable("negation for the given type is NYI");
  }
 
  //===--------------------------------------------------------------------===//
  // CastOp creation helpers
  //===--------------------------------------------------------------------===//
 
  // TODO: split this to createFPExt/createFPTrunc when we have dedicated cast
  // operations.
  mlir::Value createFloatingCast(mlir::Value v, mlir::Type destType) {
    return cir::CastOp::create(*this, v.getLoc(), destType,
                               cir::CastKind::floating, v,
                               getConstrainedFPAttr());
  }
 
  mlir::Value createDynCast(mlir::Location loc, mlir::Value src,
                            cir::PointerType destType, bool isRefCast,
                            cir::DynamicCastInfoAttr info) {
    auto castKind =
        isRefCast ? cir::DynamicCastKind::Ref : cir::DynamicCastKind::Ptr;
    return cir::DynamicCastOp::create(*this, loc, destType, castKind, src, info,
                                      /*relative_layout=*/false);
  }
 
  mlir::Value createDynCastToVoid(mlir::Location loc, mlir::Value src,
                                  bool vtableUseRelativeLayout) {
    // TODO(cir): consider address space here.
    assert(!cir::MissingFeatures::addressSpace());
    cir::PointerType destTy = getVoidPtrTy();
    return cir::DynamicCastOp::create(
        *this, loc, destTy, cir::DynamicCastKind::Ptr, src,
        cir::DynamicCastInfoAttr{}, vtableUseRelativeLayout);
  }
 
  //===--------------------------------------------------------------------===//
  // Address creation helpers
  //===--------------------------------------------------------------------===//
  Address createBaseClassAddr(mlir::Location loc, Address addr,
                              mlir::Type destType, unsigned offset,
                              bool assumeNotNull) {
    if (destType == addr.getElementType())
      return addr;
 
    auto ptrTy = getPointerTo(destType, addr.getAddressSpace());
    auto baseAddr =
        cir::BaseClassAddrOp::create(*this, loc, ptrTy, addr.getPointer(),
                                     mlir::APInt(64, offset), assumeNotNull);
    return Address(baseAddr, destType, addr.getAlignment());
  }
 
  Address createDerivedClassAddr(mlir::Location loc, Address addr,
                                 mlir::Type destType, unsigned offset,
                                 bool assumeNotNull) {
    if (destType == addr.getElementType())
      return addr;
 
    cir::PointerType ptrTy = getPointerTo(destType, addr.getAddressSpace());
    auto derivedAddr =
        cir::DerivedClassAddrOp::create(*this, loc, ptrTy, addr.getPointer(),
                                        mlir::APInt(64, offset), assumeNotNull);
    return Address(derivedAddr, destType, addr.getAlignment());
  }
 
  //===--------------------------------------------------------------------===//
  // Virtual Address creation helpers
  //===--------------------------------------------------------------------===//
  mlir::Value createVTTAddrPoint(mlir::Location loc, mlir::Type retTy,
                                 mlir::Value addr, uint64_t offset) {
    return cir::VTTAddrPointOp::create(*this, loc, retTy,
                                       mlir::FlatSymbolRefAttr{}, addr, offset);
  }
 
  mlir::Value createVTTAddrPoint(mlir::Location loc, mlir::Type retTy,
                                 mlir::FlatSymbolRefAttr sym, uint64_t offset) {
    return cir::VTTAddrPointOp::create(*this, loc, retTy, sym, mlir::Value{},
                                       offset);
  }
 
  //===--------------------------------------------------------------------===//
  // Other creation helpers
  //===--------------------------------------------------------------------===//
  cir::IsFPClassOp createIsFPClass(mlir::Location loc, mlir::Value src,
                                   cir::FPClassTest flags) {
    // FPClassTest occupies bits 0-9 (fcAllFlags).  Sema rejects an
    // out-of-range __builtin_isfpclass mask, so any extra bit here is an
    // internal error; assert and mask it off so lowering stays well-formed.
    uint32_t raw = static_cast<uint32_t>(flags);
    uint32_t all = static_cast<uint32_t>(cir::FPClassTest::All);
    assert((raw & ~all) == 0 && "FPClassTest mask has bits outside 0-9");
    flags = static_cast<cir::FPClassTest>(raw & all);
    return cir::IsFPClassOp::create(*this, loc, src, flags);
  }
 
  /// Cast the element type of the given address to a different type,
  /// preserving information like the alignment.
  Address createElementBitCast(mlir::Location loc, Address addr,
                               mlir::Type destType) {
    if (destType == addr.getElementType())
      return addr;
 
    auto srcPtrTy = mlir::cast<cir::PointerType>(addr.getPointer().getType());
    auto ptrTy = getPointerTo(destType, srcPtrTy.getAddrSpace());
    return Address(createBitcast(loc, addr.getPointer(), ptrTy), destType,
                   addr.getAlignment());
  }
 
  cir::LoadOp createLoad(mlir::Location loc, Address addr,
                         bool isVolatile = false, bool isNontemporal = false) {
    mlir::IntegerAttr align = getAlignmentAttr(addr.getAlignment());
    return cir::LoadOp::create(*this, loc, addr.getPointer(), /*isDeref=*/false,
                               isVolatile, isNontemporal,
                               /*alignment=*/align,
                               /*sync_scope=*/cir::SyncScopeKindAttr{},
                               /*mem_order=*/cir::MemOrderAttr{},
                               /*invariant=*/false);
  }
 
  cir::LoadOp createAlignedLoad(mlir::Location loc, mlir::Type ty,
                                mlir::Value ptr, llvm::MaybeAlign align) {
    if (ty != mlir::cast<cir::PointerType>(ptr.getType()).getPointee())
      ptr = createPtrBitcast(ptr, ty);
    uint64_t alignment = align ? align->value() : 0;
    mlir::IntegerAttr alignAttr = getAlignmentAttr(alignment);
    return cir::LoadOp::create(*this, loc, ptr, /*isDeref=*/false,
                               /*isVolatile=*/false, /*isNontemporal=*/false,
                               alignAttr,
                               /*sync_scope=*/cir::SyncScopeKindAttr{},
                               /*mem_order=*/cir::MemOrderAttr{},
                               /*invariant=*/false);
  }
 
  cir::LoadOp
  createAlignedLoad(mlir::Location loc, mlir::Type ty, mlir::Value ptr,
                    clang::CharUnits align = clang::CharUnits::One()) {
    return createAlignedLoad(loc, ty, ptr, align.getAsAlign());
  }
 
  cir::StoreOp createStore(mlir::Location loc, mlir::Value val, Address dst,
                           bool isVolatile = false, bool isNontemporal = false,
                           mlir::IntegerAttr align = {},
                           cir::SyncScopeKindAttr scope = {},
                           cir::MemOrderAttr order = {}) {
    if (!align)
      align = getAlignmentAttr(dst.getAlignment());
    return CIRBaseBuilderTy::createStore(loc, val, dst.getPointer(), isVolatile,
                                         isNontemporal, align, scope, order);
  }
 
  /// Create a cir.complex.real_ptr operation that derives a pointer to the real
  /// part of the complex value pointed to by the specified pointer value.
  mlir::Value createComplexRealPtr(mlir::Location loc, mlir::Value value) {
    auto srcPtrTy = mlir::cast<cir::PointerType>(value.getType());
    auto srcComplexTy = mlir::cast<cir::ComplexType>(srcPtrTy.getPointee());
    return cir::ComplexRealPtrOp::create(
        *this, loc, getPointerTo(srcComplexTy.getElementType()), value);
  }
 
  Address createComplexRealPtr(mlir::Location loc, Address addr) {
    return Address{createComplexRealPtr(loc, addr.getPointer()),
                   addr.getAlignment()};
  }
 
  /// Create a cir.complex.imag_ptr operation that derives a pointer to the
  /// imaginary part of the complex value pointed to by the specified pointer
  /// value.
  mlir::Value createComplexImagPtr(mlir::Location loc, mlir::Value value) {
    auto srcPtrTy = mlir::cast<cir::PointerType>(value.getType());
    auto srcComplexTy = mlir::cast<cir::ComplexType>(srcPtrTy.getPointee());
    return cir::ComplexImagPtrOp::create(
        *this, loc, getPointerTo(srcComplexTy.getElementType()), value);
  }
 
  Address createComplexImagPtr(mlir::Location loc, Address addr) {
    return Address{createComplexImagPtr(loc, addr.getPointer()),
                   addr.getAlignment()};
  }
 
  using CIRBaseBuilderTy::createGetMember;
  Address createGetMember(mlir::Location loc, Address base,
                          llvm::StringRef name, unsigned index) {
    auto recordTy = mlir::cast<cir::RecordType>(base.getElementType());
 
    assert(index < recordTy.getMembers().size() &&
           "member index out of bounds");
    mlir::Type memberTy = recordTy.getMembers()[index];
    mlir::Type memberPtrTy = getPointerTo(memberTy, base.getAddressSpace());
 
    auto moduleOp =
        getInsertionBlock()->getParentOp()->getParentOfType<mlir::ModuleOp>();
    mlir::DataLayout layout(moduleOp);
    auto memberOffset =
        CharUnits::fromQuantity(recordTy.getElementOffset(layout, index));
 
    mlir::Value memberPtr =
        createGetMember(loc, memberPtrTy, base.getBasePointer(), name, index);
    return Address(memberPtr, memberTy,
                   base.getAlignment().alignmentAtOffset(memberOffset),
                   base.isKnownNonNull());
  }
 
  cir::GetRuntimeMemberOp createGetIndirectMember(mlir::Location loc,
                                                  mlir::Value objectPtr,
                                                  mlir::Value memberPtr) {
    auto memberPtrTy = mlir::cast<cir::DataMemberType>(memberPtr.getType());
 
    // TODO(cir): consider address space.
    assert(!cir::MissingFeatures::addressSpace());
    cir::PointerType resultTy = getPointerTo(memberPtrTy.getMemberTy());
 
    return cir::GetRuntimeMemberOp::create(*this, loc, resultTy, objectPtr,
                                           memberPtr);
  }
 
  /// Create a cir.ptr_stride operation to get access to an array element.
  /// \p idx is the index of the element to access, \p shouldDecay is true if
  /// the result should decay to a pointer to the element type.
  mlir::Value getArrayElement(mlir::Location arrayLocBegin,
                              mlir::Location arrayLocEnd, mlir::Value arrayPtr,
                              mlir::Type eltTy, mlir::Value idx,
                              bool shouldDecay);
 
  /// Returns a decayed pointer to the first element of the array
  /// pointed to by \p arrayPtr.
  mlir::Value maybeBuildArrayDecay(mlir::Location loc, mlir::Value arrayPtr,
                                   mlir::Type eltTy);
 
  // Convert byte offset to sequence of high-level indices suitable for
  // GlobalViewAttr. Ideally we shouldn't deal with low-level offsets at all
  // but currently some parts of Clang AST, which we don't want to touch just
  // yet, return them.
  //
  // Returns false if the offset doesn't designate a subelement of \p ty, which
  // happens when it lands outside of the object or in the middle of a scalar
  // member. In that case \p indices is left in an unspecified state and the
  // caller must describe the address with a byte offset, using a
  // GlobalOffsetAttr, instead.
  [[nodiscard]] bool computeGlobalViewIndicesFromFlatOffset(
      int64_t offset, mlir::Type ty, cir::CIRDataLayout layout,
      llvm::SmallVectorImpl<int64_t> &indices);
 
  // Convert high-level indices (e.g. from GlobalViewAttr) to byte offset.
  uint64_t computeOffsetFromGlobalViewIndices(const cir::CIRDataLayout &layout,
                                              mlir::Type ty,
                                              llvm::ArrayRef<int64_t> indices);
 
  /// Creates a versioned global variable. If the symbol is already taken, an ID
  /// will be appended to the symbol. The returned global must always be queried
  /// for its name so it can be referenced correctly.
  [[nodiscard]] cir::GlobalOp
  createVersionedGlobal(mlir::ModuleOp module, mlir::Location loc,
                        mlir::StringRef name, mlir::Type type, bool isConstant,
                        cir::GlobalLinkageKind linkage,
                        mlir::ptr::MemorySpaceAttrInterface addrSpace = {}) {
    // Create a unique name if the given name is already taken.
    std::string uniqueName;
    if (unsigned version = globalsVersioning[name.str()]++)
      uniqueName = name.str() + "." + std::to_string(version);
    else
      uniqueName = name.str();
 
    return createGlobal(module, loc, uniqueName, type, isConstant, linkage,
                        addrSpace);
  }
 
  cir::StackSaveOp createStackSave(mlir::Location loc, mlir::Type ty) {
    return cir::StackSaveOp::create(*this, loc, ty);
  }
 
  cir::StackRestoreOp createStackRestore(mlir::Location loc, mlir::Value v) {
    return cir::StackRestoreOp::create(*this, loc, v);
  }
 
  cir::CmpThreeWayOp createThreeWayCmpTotalOrdering(
      mlir::Location loc, mlir::Value lhs, mlir::Value rhs,
      const llvm::APSInt &ltRes, const llvm::APSInt &eqRes,
      const llvm::APSInt &gtRes, cir::CmpOrdering ordering) {
    assert(ltRes.getBitWidth() == eqRes.getBitWidth() &&
           ltRes.getBitWidth() == gtRes.getBitWidth() &&
           "the three comparison results must have the same bit width");
    assert((ordering == cir::CmpOrdering::Strong ||
            ordering == cir::CmpOrdering::Weak) &&
           "total ordering must be strong or weak");
    cir::IntType cmpResultTy = getSIntNTy(ltRes.getBitWidth());
    auto infoAttr = cir::CmpThreeWayInfoAttr::get(
        getContext(), ordering, ltRes.getSExtValue(), eqRes.getSExtValue(),
        gtRes.getSExtValue());
    return cir::CmpThreeWayOp::create(*this, loc, cmpResultTy, lhs, rhs,
                                      infoAttr);
  }
 
  cir::CmpThreeWayOp createThreeWayCmpPartialOrdering(
      mlir::Location loc, mlir::Value lhs, mlir::Value rhs,
      const llvm::APSInt &ltRes, const llvm::APSInt &eqRes,
      const llvm::APSInt &gtRes, const llvm::APSInt &unorderedRes) {
    assert(ltRes.getBitWidth() == eqRes.getBitWidth() &&
           ltRes.getBitWidth() == gtRes.getBitWidth() &&
           ltRes.getBitWidth() == unorderedRes.getBitWidth() &&
           "the four comparison results must have the same bit width");
    cir::IntType cmpResultTy = getSIntNTy(ltRes.getBitWidth());
    auto infoAttr = cir::CmpThreeWayInfoAttr::get(
        getContext(), ltRes.getSExtValue(), eqRes.getSExtValue(),
        gtRes.getSExtValue(), unorderedRes.getSExtValue());
    return cir::CmpThreeWayOp::create(*this, loc, cmpResultTy, lhs, rhs,
                                      infoAttr);
  }
 
  mlir::Value createSetBitfield(mlir::Location loc, mlir::Type resultType,
                                Address dstAddr, mlir::Type storageType,
                                mlir::Value src, const CIRGenBitFieldInfo &info,
                                bool isLvalueVolatile, bool useVolatile) {
    unsigned offset = useVolatile ? info.volatileOffset : info.offset;
 
    // If using AAPCS and the field is volatile, load with the size of the
    // declared field
    storageType =
        useVolatile ? cir::IntType::get(storageType.getContext(),
                                        info.volatileStorageSize, info.isSigned)
                    : storageType;
    return cir::SetBitfieldOp::create(
        *this, loc, resultType, dstAddr.getPointer(), storageType, src,
        info.name, info.size, offset, info.isSigned, isLvalueVolatile,
        dstAddr.getAlignment().getAsAlign().value());
  }
 
  mlir::Value createGetBitfield(mlir::Location loc, mlir::Type resultType,
                                Address addr, mlir::Type storageType,
                                const CIRGenBitFieldInfo &info,
                                bool isLvalueVolatile, bool useVolatile) {
    unsigned offset = useVolatile ? info.volatileOffset : info.offset;
 
    // If using AAPCS and the field is volatile, load with the size of the
    // declared field
    storageType =
        useVolatile ? cir::IntType::get(storageType.getContext(),
                                        info.volatileStorageSize, info.isSigned)
                    : storageType;
    return cir::GetBitfieldOp::create(*this, loc, resultType, addr.getPointer(),
                                      storageType, info.name, info.size, offset,
                                      info.isSigned, isLvalueVolatile,
                                      addr.getAlignment().getAsAlign().value());
  }
 
  mlir::Value createMaskedLoad(mlir::Location loc, mlir::Type ty,
                               mlir::Value ptr, llvm::Align alignment,
                               mlir::Value mask, mlir::Value passThru) {
    assert(mlir::isa<cir::VectorType>(ty) && "Type should be vector");
    assert(mask && "Mask should not be all-ones (null)");
 
    if (!passThru)
      passThru = this->getConstant(loc, cir::PoisonAttr::get(ty));
 
    auto alignAttr =
        this->getI64IntegerAttr(static_cast<int64_t>(alignment.value()));
 
    return cir::VecMaskedLoadOp::create(*this, loc, ty, ptr, mask, passThru,
                                        alignAttr);
  }
 
  cir::VecShuffleOp
  createVecShuffle(mlir::Location loc, mlir::Value vec1, mlir::Value vec2,
                   llvm::ArrayRef<mlir::Attribute> maskAttrs) {
    auto vecType = mlir::cast<cir::VectorType>(vec1.getType());
    auto resultTy =
        cir::VectorType::get(vecType.getElementType(), maskAttrs.size());
    return cir::VecShuffleOp::create(*this, loc, resultTy, vec1, vec2,
                                     getArrayAttr(maskAttrs));
  }
 
  cir::VecShuffleOp createVecShuffle(mlir::Location loc, mlir::Value vec1,
                                     mlir::Value vec2,
                                     llvm::ArrayRef<int64_t> mask) {
    auto maskAttrs = llvm::to_vector_of<mlir::Attribute>(
        llvm::map_range(mask, [&](int32_t idx) {
          return cir::IntAttr::get(getSInt32Ty(), idx);
        }));
    return createVecShuffle(loc, vec1, vec2, maskAttrs);
  }
 
  cir::VecShuffleOp createVecShuffle(mlir::Location loc, mlir::Value vec1,
                                     llvm::ArrayRef<int64_t> mask) {
    /// Create a unary shuffle. The second vector operand of the IR instruction
    /// is poison.
    cir::ConstantOp poison =
        getConstant(loc, cir::PoisonAttr::get(vec1.getType()));
    return createVecShuffle(loc, vec1, poison, mask);
  }
 
  cir::MatrixExtractOp createMatrixExtract(mlir::Location loc,
                                           mlir::Value matrix,
                                           mlir::Value rowIdx,
                                           mlir::Value columnIdx) {
    return cir::MatrixExtractOp::create(*this, loc, matrix, rowIdx, columnIdx);
  }
 
  cir::MatrixInsertOp createMatrixInsert(mlir::Location loc, mlir::Value value,
                                         mlir::Value matrix, mlir::Value rowIdx,
                                         mlir::Value columnIdx) {
    return cir::MatrixInsertOp::create(*this, loc, value, matrix, rowIdx,
                                       columnIdx);
  }
 
  cir::MatrixColumnMajorLoadOp createMatrixColumnMajorLoad(mlir::Location loc,
                                                           mlir::Type resultTy,
                                                           mlir::Value value,
                                                           mlir::Value stride,
                                                           bool isVolatile) {
    return cir::MatrixColumnMajorLoadOp::create(*this, loc, resultTy, value,
                                                stride, isVolatile);
  }
 
  /// Multiply an MxK matrix by a KxN matrix, producing an MxN matrix.
  cir::MatrixMultiplyOp createMatrixMultiply(mlir::Location loc,
                                             mlir::Value lhs, mlir::Value rhs) {
    auto lhsTy = mlir::cast<cir::MatrixType>(lhs.getType());
    auto rhsTy = mlir::cast<cir::MatrixType>(rhs.getType());
    auto resultTy = cir::MatrixType::get(
        lhsTy.getElementType(), lhsTy.getNumRows(), rhsTy.getNumColumns());
    return cir::MatrixMultiplyOp::create(*this, loc, resultTy, lhs, rhs);
  }
 
  cir::MatrixTransposeOp createMatrixTranspose(mlir::Location loc,
                                               mlir::Value matrix) {
    auto inputTy = mlir::cast<cir::MatrixType>(matrix.getType());
    auto resultTy =
        cir::MatrixType::get(inputTy.getElementType(), inputTy.getNumColumns(),
                             inputTy.getNumRows());
    return cir::MatrixTransposeOp::create(*this, loc, resultTy, matrix);
  }
 
  cir::MatrixColumnMajorStoreOp createMatrixColumnMajorStore(mlir::Location loc,
                                                             mlir::Value matrix,
                                                             mlir::Value data,
                                                             mlir::Value stride,
                                                             bool isVolatile) {
    return cir::MatrixColumnMajorStoreOp::create(*this, loc, matrix, data,
                                                 stride, isVolatile);
  }
 
  std::pair<mlir::Value, mlir::Value>
  splatMatrixOpOperandsIfNecessary(mlir::Location loc, mlir::Value lhs,
                                   mlir::Value rhs) {
    assert(mlir::isa<cir::MatrixType>(lhs.getType()) ||
           mlir::isa<cir::MatrixType>(rhs.getType()));
 
    if (!mlir::isa<cir::MatrixType>(lhs.getType()))
      lhs = cir::VecSplatOp::create(*this, loc, rhs.getType(), lhs);
    else if (!mlir::isa<cir::MatrixType>(rhs.getType()))
      rhs = cir::VecSplatOp::create(*this, loc, lhs.getType(), rhs);
 
    return {lhs, rhs};
  }
 
  template <typename... Operands>
  mlir::Value emitIntrinsicCallOp(mlir::Location loc, const llvm::StringRef str,
                                  const mlir::Type &resTy, Operands &&...op) {
    return cir::LLVMIntrinsicCallOp::create(*this, loc,
                                            this->getStringAttr(str), resTy,
                                            std::forward<Operands>(op)...)
        .getResult();
  }
 
  template <typename... Operands>
  mlir::Value emitIntrinsicCallOp(mlir::Location loc, const llvm::StringRef str,
                                  const mlir::Type &resTy,
                                  cir::FastMathFlagsAttr fastmath,
                                  Operands &&...op) {
    return cir::LLVMIntrinsicCallOp::create(
               *this, loc, this->getStringAttr(str), resTy,
               std::forward<Operands>(op)..., fastmath)
        .getResult();
  }
};
 
} // namespace clang::CIRGen
 
#endif