CIRGenCall.cpp

Go to the documentation of this file.

//===--- CIRGenCall.cpp - Encapsulate calling convention details ----------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// These classes wrap the information about a call or function definition used
// to handle ABI compliancy.
//
//===----------------------------------------------------------------------===//
 
#include "CIRGenCall.h"
#include "CIRGenCXXABI.h"
#include "CIRGenCleanup.h"
#include "CIRGenFunction.h"
#include "CIRGenFunctionInfo.h"
#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
#include "mlir/IR/Attributes.h"
#include "clang/Basic/DiagnosticFrontend.h"
#include "clang/CIR/ABIArgInfo.h"
#include "clang/CIR/MissingFeatures.h"
#include "clang/CodeGenUtils/CallUtils.h"
#include "llvm/ADT/FloatingPointMode.h"
#include "llvm/ADT/StringSet.h"
#include "llvm/Support/TypeSize.h"
 
using namespace clang;
using namespace clang::CIRGen;
 
CIRGenFunctionInfo *
CIRGenFunctionInfo::create(cir::CallingConv cirCC, FunctionType::ExtInfo info,
                           bool isInstanceMethod, CanQualType resultType,
                           llvm::ArrayRef<CanQualType> argTypes,
                           RequiredArgs required) {
  // The first slot allocated for arg type slot is for the return value.
  void *buffer = operator new(
      totalSizeToAlloc<CanQualType>(argTypes.size() + 1));
 
  assert(!cir::MissingFeatures::opCallCIRGenFuncInfoParamInfo());
 
  CIRGenFunctionInfo *fi = new (buffer) CIRGenFunctionInfo();
 
  fi->callingConvention = llvm::to_underlying(cirCC);
  fi->astCallingConvention = info.getCC();
  fi->noReturn = info.getNoReturn();
  fi->instanceMethod = isInstanceMethod;
 
  fi->required = required;
  fi->numArgs = argTypes.size();
 
  // requiredArguments() reads getNumRequiredArgs() entries out of the
  // trailing-object array, so a signature that claims more required arguments
  // than we have types for reads past the end and hands a null QualType to
  // convertType.
  assert((!required.allowsOptionalArgs() ||
          required.getNumRequiredArgs() <= fi->numArgs) &&
         "more required arguments than argument types");
 
  fi->getArgTypes()[0] = resultType;
  std::copy(argTypes.begin(), argTypes.end(), fi->argTypesBegin());
  assert(!cir::MissingFeatures::opCallCIRGenFuncInfoExtParamInfo());
 
  return fi;
}
 
cir::FuncType CIRGenTypes::getFunctionType(GlobalDecl gd) {
  const CIRGenFunctionInfo &fi = arrangeGlobalDeclaration(gd);
  return getFunctionType(fi);
}
 
cir::FuncType CIRGenTypes::getFunctionType(const CIRGenFunctionInfo &info) {
  mlir::Type resultType = convertType(info.getReturnType());
  SmallVector<mlir::Type, 8> argTypes;
  argTypes.reserve(info.getNumRequiredArgs());
 
  for (const CanQualType &argType : info.requiredArguments())
    argTypes.push_back(convertType(argType));
 
  return cir::FuncType::get(argTypes,
                            (resultType ? resultType : builder.getVoidTy()),
                            info.isVariadic());
}
 
CIRGenCallee CIRGenCallee::prepareConcreteCallee(CIRGenFunction &cgf) const {
  if (isVirtual()) {
    const CallExpr *ce = getVirtualCallExpr();
    return cgf.cgm.getCXXABI().getVirtualFunctionPointer(
        cgf, getVirtualMethodDecl(), getThisAddress(), getVirtualFunctionType(),
        ce ? ce->getBeginLoc() : SourceLocation());
  }
  return *this;
}
 
void CIRGenFunction::emitAggregateStore(mlir::Value value, Address dest) {
  // In classic codegen:
  // Function to store a first-class aggregate into memory. We prefer to
  // store the elements rather than the aggregate to be more friendly to
  // fast-isel.
  // In CIR codegen:
  // Emit the most simple cir.store possible (e.g. a store for a whole
  // record), which can later be broken down in other CIR levels (or prior
  // to dialect codegen).
 
  // Stored result for the callers of this function expected to be in the same
  // scope as the value, don't make assumptions about current insertion point.
  mlir::OpBuilder::InsertionGuard guard(builder);
  builder.setInsertionPointAfter(value.getDefiningOp());
  builder.createStore(getLoc(*currSrcLoc), value, dest);
}
 
static void addAttributesFromFunctionProtoType(CIRGenBuilderTy &builder,
                                               mlir::NamedAttrList &attrs,
                                               const FunctionProtoType *fpt) {
  if (!fpt)
    return;
 
  if (!isUnresolvedExceptionSpec(fpt->getExceptionSpecType()) &&
      fpt->isNothrow())
    attrs.set(cir::CIRDialect::getNoThrowAttrName(),
              mlir::UnitAttr::get(builder.getContext()));
}
 
static void addNoBuiltinAttributes(mlir::MLIRContext &ctx,
                                   mlir::NamedAttrList &attrs,
                                   const LangOptions &langOpts,
                                   const NoBuiltinAttr *nba = nullptr) {
  // First, handle the language options passed through -fno-builtin.
  // or, if there is a wildcard in the builtin names specified through the
  // attribute, disable them all.
  if (langOpts.NoBuiltin ||
      (nba && llvm::is_contained(nba->builtinNames(), "*"))) {
    // -fno-builtin disables them all.
    // Empty attribute means 'all'.
    attrs.set(cir::CIRDialect::getNoBuiltinsAttrName(),
              mlir::ArrayAttr::get(&ctx, {}));
    return;
  }
 
  llvm::SetVector<mlir::Attribute> nbFuncs;
  auto addNoBuiltinAttr = [&ctx, &nbFuncs](StringRef builtinName) {
    nbFuncs.insert(mlir::StringAttr::get(&ctx, builtinName));
  };
 
  // Then, add attributes for builtins specified through -fno-builtin-<name>.
  llvm::for_each(langOpts.NoBuiltinFuncs, addNoBuiltinAttr);
 
  // Now, let's check the __attribute__((no_builtin("...")) attribute added to
  // the source.
  if (nba)
    llvm::for_each(nba->builtinNames(), addNoBuiltinAttr);
 
  if (!nbFuncs.empty())
    attrs.set(cir::CIRDialect::getNoBuiltinsAttrName(),
              mlir::ArrayAttr::get(&ctx, nbFuncs.getArrayRef()));
}
 
/// Add denormal-fp-math and denormal-fp-math-f32 as appropriate for the
/// requested denormal behavior, accounting for the overriding behavior of the
/// -f32 case.
static void addDenormalModeAttrs(llvm::DenormalMode fpDenormalMode,
                                 llvm::DenormalMode fp32DenormalMode,
                                 mlir::NamedAttrList &attrs) {
  // TODO(cir): Classic-codegen sets the denormal modes here. There are two
  // values, both with a string, but it seems that perhaps we could combine
  // these into a single attribute?  It seems a little silly to have two so
  // similar named attributes that do the same thing.
}
 
/// Add default attributes to a function, which have merge semantics under
/// -mlink-builtin-bitcode and should not simply overwrite any existing
/// attributes in the linked library.
static void
addMergeableDefaultFunctionAttributes(const CodeGenOptions &codeGenOpts,
                                      mlir::NamedAttrList &attrs) {
  addDenormalModeAttrs(codeGenOpts.FPDenormalMode, codeGenOpts.FP32DenormalMode,
                       attrs);
}
 
static llvm::StringLiteral
getZeroCallUsedRegsKindStr(llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind k) {
  switch (k) {
  case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Skip:
    llvm_unreachable("No string value, shouldn't be able to get here");
  case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPRArg:
    return "used-gpr-arg";
  case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPR:
    return "used-gpr";
  case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedArg:
    return "used-arg";
  case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Used:
    return "used";
  case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPRArg:
    return "all-gpr-arg";
  case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPR:
    return "all-gpr";
  case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllArg:
    return "all-arg";
  case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::All:
    return "all";
  }
 
  llvm_unreachable("Unknown kind?");
}
 
std::optional<cir::FramePointerKind>
CIRGenModule::getFramePointerKind(CodeGenOptions::FramePointerKind kind) {
  switch (kind) {
  case CodeGenOptions::FramePointerKind::None:
    return std::nullopt;
  case CodeGenOptions::FramePointerKind::Reserved:
    return cir::FramePointerKind::Reserved;
  case CodeGenOptions::FramePointerKind::NonLeafNoReserve:
    return cir::FramePointerKind::NonLeafNoReserve;
  case CodeGenOptions::FramePointerKind::NonLeaf:
    return cir::FramePointerKind::NonLeaf;
  case CodeGenOptions::FramePointerKind::All:
    return cir::FramePointerKind::All;
  }
  llvm_unreachable("Unknown frame pointer kind");
}
 
/// Add default attributes to a function, which have merge semantics under
/// -mlink-builtin-bitcode and should not simply overwrite any existing
/// attributes in the linked library.
static void addTrivialDefaultFunctionAttributes(
    mlir::MLIRContext *mlirCtx, StringRef name, bool hasOptNoneAttr,
    const CodeGenOptions &codeGenOpts, const LangOptions &langOpts,
    bool attrOnCallSite, mlir::NamedAttrList &attrs) {
  // TODO(cir): Handle optimize attribute flag here.
  // OptimizeNoneAttr takes precedence over -Os or -Oz. No warning needed.
  if (!hasOptNoneAttr) {
    if (codeGenOpts.OptimizeSize)
      attrs.set(cir::CIRDialect::getOptimizeForSizeAttrName(),
                mlir::UnitAttr::get(mlirCtx));
    if (codeGenOpts.OptimizeSize == 2)
      attrs.set(cir::CIRDialect::getMinSizeAttrName(),
                mlir::UnitAttr::get(mlirCtx));
  }
 
  // TODO(cir): Classic codegen adds 'DisableRedZone', 'indirect-tls-seg-refs'
  // and 'NoImplicitFloat' here.
 
  if (attrOnCallSite) {
    // Add the 'nobuiltin' tag, which is different from 'no-builtins'.
    if (!codeGenOpts.SimplifyLibCalls || langOpts.isNoBuiltinFunc(name))
      attrs.set(cir::CIRDialect::getNoBuiltinAttrName(),
                mlir::UnitAttr::get(mlirCtx));
 
    if (!codeGenOpts.TrapFuncName.empty())
      attrs.set(cir::CIRDialect::getTrapFuncNameAttrName(),
                mlir::StringAttr::get(mlirCtx, codeGenOpts.TrapFuncName));
  } else {
    if (std::optional<cir::FramePointerKind> kind =
            CIRGenModule::getFramePointerKind(codeGenOpts.getFramePointer()))
      attrs.set(cir::CIRDialect::getFramePointerAttrName(),
                cir::FramePointerKindAttr::get(mlirCtx, *kind));
 
    // TODO(cir): a number of other attribute 1-offs based on codegen/lang opts
    // should be done here: less-recise-fpmad null-pointer-is-valid
    // no-trapping-math
    // various inf/nan/nsz/etc work here.
    //
    // TODO(cir): set stack-protector buffer size attribute (sorted oddly in
    // classic compiler inside of the above region, but should be done on its
    // own).
    // TODO(cir): other attributes here:
    // reciprocal estimates, prefer-vector-width, stackrealign, backchain,
    // split-stack, speculative-load-hardening.
 
    if (codeGenOpts.getZeroCallUsedRegs() ==
        llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Skip)
      attrs.erase(cir::CIRDialect::getZeroCallUsedRegsAttrName());
    else
      attrs.set(cir::CIRDialect::getZeroCallUsedRegsAttrName(),
                mlir::StringAttr::get(mlirCtx,
                                      getZeroCallUsedRegsKindStr(
                                          codeGenOpts.getZeroCallUsedRegs())));
  }
 
  if (langOpts.assumeFunctionsAreConvergent()) {
    // Conservatively, mark all functions and calls in CUDA and OpenCL as
    // convergent (meaning, they may call an intrinsically convergent op, such
    // as __syncthreads() / barrier(), and so can't have certain optimizations
    // applied around them).  LLVM will remove this attribute where it safely
    // can.
    attrs.set(cir::CIRDialect::getConvergentAttrName(),
              mlir::UnitAttr::get(mlirCtx));
  }
 
  // Device code cannot unwind. 'nothrow' keeps calls from getting an unwind
  // edge, so 'nounwind' becomes LLVM's nounwind.
  // TODO: OpenMP offload is not covered.
  if ((langOpts.CUDA && langOpts.CUDAIsDevice) || langOpts.OpenCL ||
      langOpts.SYCLIsDevice) {
    attrs.set(cir::CIRDialect::getNoThrowAttrName(),
              mlir::UnitAttr::get(mlirCtx));
    attrs.set(cir::CIRDialect::getNoUnwindAttrName(),
              mlir::UnitAttr::get(mlirCtx));
  }
 
  if (codeGenOpts.SaveRegParams && !attrOnCallSite)
    attrs.set(cir::CIRDialect::getSaveRegParamsAttrName(),
              mlir::UnitAttr::get(mlirCtx));
 
  // These come in the form of an optional equality sign, so make sure we pass
  // these on correctly. These will eventually just be passed through to
  // LLVM-IR, but we want to put them all in 1 array to simplify the
  // LLVM-MLIR dialect.
  SmallVector<mlir::NamedAttribute> defaultFuncAttrs;
  llvm::transform(
      codeGenOpts.DefaultFunctionAttrs, std::back_inserter(defaultFuncAttrs),
      [mlirCtx](llvm::StringRef arg) {
        auto [var, value] = arg.split('=');
        auto valueAttr =
            value.empty()
                ? cast<mlir::Attribute>(mlir::UnitAttr::get(mlirCtx))
                : cast<mlir::Attribute>(mlir::StringAttr::get(mlirCtx, value));
        return mlir::NamedAttribute(var, valueAttr);
      });
 
  if (!defaultFuncAttrs.empty())
    attrs.set(cir::CIRDialect::getDefaultFuncAttrsAttrName(),
              mlir::DictionaryAttr::get(mlirCtx, defaultFuncAttrs));
 
  // TODO(cir): Do branch protection attributes here.
}
 
/// This function matches the behavior of 'getDefaultFunctionAttributes' from
/// classic codegen, despite the similarity of its name to
/// 'addDefaultFunctionDefinitionAttributes', which is a caller of this
/// function.
void CIRGenModule::addDefaultFunctionAttributes(StringRef name,
                                                bool hasOptNoneAttr,
                                                bool attrOnCallSite,
                                                mlir::NamedAttrList &attrs) {
 
  addTrivialDefaultFunctionAttributes(&getMLIRContext(), name, hasOptNoneAttr,
                                      codeGenOpts, langOpts, attrOnCallSite,
                                      attrs);
 
  if (!attrOnCallSite) {
    // TODO(cir): Classic codegen adds pointer-auth attributes here, by calling
    // into TargetCodeGenInfo.  At the moment, we've not looked into this as it
    // is somewhat less used.
    addMergeableDefaultFunctionAttributes(codeGenOpts, attrs);
  }
}
 
/// Construct the CIR attribute list of a function or call.
void CIRGenModule::constructAttributeList(
    llvm::StringRef name, const CIRGenFunctionInfo &info,
    CIRGenCalleeInfo calleeInfo, mlir::NamedAttrList &attrs,
    llvm::MutableArrayRef<mlir::NamedAttrList> argAttrs,
    mlir::NamedAttrList &retAttrs, cir::CallingConv &callingConv,
    bool attrOnCallSite, bool isThunk) {
  callingConv = info.getCallingConvention();
 
  auto addUnitAttr = [&](llvm::StringRef name) {
    attrs.set(name, mlir::UnitAttr::get(&getMLIRContext()));
  };
 
  if (info.isNoReturn())
    addUnitAttr(cir::CIRDialect::getNoReturnAttrName());
 
  // TODO(cir): Implement/check the CSME Nonsecure call attribute here. This
  // requires being in CSME mode.
 
  addAttributesFromFunctionProtoType(getBuilder(), attrs,
                                     calleeInfo.getCalleeFunctionProtoType());
 
  const Decl *targetDecl = calleeInfo.getCalleeDecl().getDecl();
 
  // TODO(cir): OMP Assume Attributes should be here.
 
  const NoBuiltinAttr *nba = nullptr;
 
  // TODO(cir): Some work for arg memory effects can be done here, as it is in
  // classic codegen.
 
  if (targetDecl) {
    if (targetDecl->hasAttr<NoThrowAttr>())
      addUnitAttr(cir::CIRDialect::getNoThrowAttrName());
    // TODO(cir): This is actually only possible if targetDecl isn't a
    // declarator, which ObjCMethodDecl seems to be the only way to get this to
    // happen.  We're including it here for completeness, but we should add a
    // test for this when we start generating ObjectiveC.
    if (targetDecl->hasAttr<NoReturnAttr>())
      addUnitAttr(cir::CIRDialect::getNoReturnAttrName());
    if (targetDecl->hasAttr<ReturnsTwiceAttr>())
      addUnitAttr(cir::CIRDialect::getReturnsTwiceAttrName());
    if (targetDecl->hasAttr<ColdAttr>())
      addUnitAttr(cir::CIRDialect::getColdAttrName());
    if (targetDecl->hasAttr<HotAttr>())
      addUnitAttr(cir::CIRDialect::getHotAttrName());
    if (targetDecl->hasAttr<NoDuplicateAttr>())
      addUnitAttr(cir::CIRDialect::getNoDuplicatesAttrName());
    if (targetDecl->hasAttr<ConvergentAttr>())
      addUnitAttr(cir::CIRDialect::getConvergentAttrName());
 
    if (const FunctionDecl *func = dyn_cast<FunctionDecl>(targetDecl)) {
      addAttributesFromFunctionProtoType(
          getBuilder(), attrs, func->getType()->getAs<FunctionProtoType>());
 
      // TODO(cir): When doing 'return attrs' we need to cover the 'NoAlias' for
      // global allocation functions here.
      assert(!cir::MissingFeatures::opCallAttrs());
 
      const CXXMethodDecl *md = dyn_cast<CXXMethodDecl>(func);
      bool isVirtualCall = md && md->isVirtual();
 
      // Don't use [[noreturn]], _Noreturn or [[no_builtin]] for a call to a
      // virtual function. These attributes are not inherited by overloads.
      if (!(attrOnCallSite && isVirtualCall)) {
        if (func->isNoReturn())
          addUnitAttr(cir::CIRDialect::getNoReturnAttrName());
        nba = func->getAttr<NoBuiltinAttr>();
      }
    }
 
    assert(!cir::MissingFeatures::opCallAttrs());
 
    auto setMemoryEffects = [&](cir::MemoryEffectsAttr effects) {
      attrs.set(cir::CIRDialect::getMemoryEffectsAttrName(), effects);
    };
 
    // 'const', 'pure' and 'noalias' attributed functions are also nounwind.
    if (targetDecl->hasAttr<ConstAttr>()) {
      setMemoryEffects(cir::MemoryEffectsAttr::none(&getMLIRContext()));
      addUnitAttr(cir::CIRDialect::getNoUnwindAttrName());
      // gcc specifies that 'const' functions have greater restrictions than
      // 'pure' functions, so they also cannot have infinite loops.
      addUnitAttr(cir::CIRDialect::getWillReturnAttrName());
    } else if (targetDecl->hasAttr<PureAttr>()) {
      setMemoryEffects(cir::MemoryEffectsAttr::readOnly(&getMLIRContext()));
      addUnitAttr(cir::CIRDialect::getNoUnwindAttrName());
      // gcc specifies that 'pure' functions cannot have infinite loops.
      addUnitAttr(cir::CIRDialect::getWillReturnAttrName());
    } else if (targetDecl->hasAttr<NoAliasAttr>()) {
      setMemoryEffects(
          cir::MemoryEffectsAttr::inaccessibleOrArgMemOnly(&getMLIRContext()));
      addUnitAttr(cir::CIRDialect::getNoUnwindAttrName());
    }
 
    // Both __attribute__((malloc)) and __declspec(restrict) are represented by
    // RestrictAttr. Only forms without a deallocator imply a noalias return.
    if (const auto *restrictAttr = targetDecl->getAttr<RestrictAttr>();
        restrictAttr && restrictAttr->getDeallocator() == nullptr)
      retAttrs.set(mlir::LLVM::LLVMDialect::getNoAliasAttrName(),
                   mlir::UnitAttr::get(&getMLIRContext()));
 
    if (targetDecl->hasAttr<ReturnsNonNullAttr>() &&
        !codeGenOpts.NullPointerIsValid)
      retAttrs.set(mlir::LLVM::LLVMDialect::getNonNullAttrName(),
                   mlir::UnitAttr::get(&getMLIRContext()));
    if (targetDecl->hasAttr<AnyX86NoCallerSavedRegistersAttr>())
      addUnitAttr(cir::CIRDialect::getNoCallerSavedRegsAttrName());
    // TODO(cir): Implement 'NoCFCheck' attribute here.  This requires
    // fcf-protection mode.
    if (targetDecl->hasAttr<LeafAttr>())
      addUnitAttr(cir::CIRDialect::getNoCallbackAttrName());
    // TODO(cir): Implement 'BPFFastCall' attribute here.  This requires C, and
    // the BPF target.
 
    if (auto *allocSizeAttr = targetDecl->getAttr<AllocSizeAttr>()) {
      unsigned size = allocSizeAttr->getElemSizeParam().getLLVMIndex();
 
      if (allocSizeAttr->getNumElemsParam().isValid()) {
        unsigned numElts = allocSizeAttr->getNumElemsParam().getLLVMIndex();
        attrs.set(cir::CIRDialect::getAllocSizeAttrName(),
                  builder.getDenseI32ArrayAttr(
                      {static_cast<int>(size), static_cast<int>(numElts)}));
      } else {
        attrs.set(cir::CIRDialect::getAllocSizeAttrName(),
                  builder.getDenseI32ArrayAttr({static_cast<int>(size)}));
      }
    }
 
    // TODO(cir): Quite a few CUDA and OpenCL attributes are added here.
 
    // -cl-uniform-work-group-size / -foffload-uniform-block: work groups are
    // uniform (global work-size is a multiple of work-group size).
    if (langOpts.OffloadUniformBlock)
      addUnitAttr(cir::CIRDialect::getUniformWorkGroupSizeAttrName());
 
    if (langOpts.CUDA && !langOpts.CUDAIsDevice &&
        targetDecl->hasAttr<CUDAGlobalAttr>()) {
      GlobalDecl kernel(calleeInfo.getCalleeDecl());
      llvm::StringRef kernelName = getMangledName(
          kernel.getWithKernelReferenceKind(KernelReferenceKind::Kernel));
      auto attr = cir::CUDAKernelNameAttr::get(
          &getMLIRContext(),
          mlir::StringAttr::get(&getMLIRContext(), kernelName));
      attrs.set(attr.getMnemonic(), attr);
    }
 
    // TODO(cir): we should also do 'aarch64_pstate_sm_body' here.
 
    if (auto *modularFormat = targetDecl->getAttr<ModularFormatAttr>()) {
      FormatAttr *format = targetDecl->getAttr<FormatAttr>();
      StringRef type = format->getType()->getName();
      std::string formatIdx = std::to_string(format->getFormatIdx());
      std::string firstArg = std::to_string(format->getFirstArg());
      SmallVector<StringRef> args = {
          type, formatIdx, firstArg,
          modularFormat->getModularImplFn()->getName(),
          modularFormat->getImplName()};
      llvm::append_range(args, modularFormat->aspects());
      attrs.set(cir::CIRDialect::getModularFormatAttrName(),
                builder.getStringAttr(llvm::join(args, ",")));
    }
  }
 
  addNoBuiltinAttributes(getMLIRContext(), attrs, getLangOpts(), nba);
 
  bool hasOptNoneAttr = targetDecl && targetDecl->hasAttr<OptimizeNoneAttr>();
  addDefaultFunctionAttributes(name, hasOptNoneAttr, attrOnCallSite, attrs);
  if (targetDecl) {
    // TODO(cir): There is another region of `if (targetDecl)` that handles
    // removing some attributes that are necessary modifications of the
    // default-function attrs. Including:
    // NoSpeculativeLoadHardening
    // SpeculativeLoadHardening
    // NoSplitStack
    // Non-lazy-bind
    // 'sample-profile-suffix-elision-policy'.
 
    if (targetDecl->hasAttr<ZeroCallUsedRegsAttr>()) {
      // A function "__attribute__((...))" overrides the command-line flag.
      auto kind =
          targetDecl->getAttr<ZeroCallUsedRegsAttr>()->getZeroCallUsedRegs();
      attrs.set(
          cir::CIRDialect::getZeroCallUsedRegsAttrName(),
          mlir::StringAttr::get(
              &getMLIRContext(),
              ZeroCallUsedRegsAttr::ConvertZeroCallUsedRegsKindToStr(kind)));
    }
 
    if (targetDecl->hasAttr<NoConvergentAttr>())
      attrs.erase(cir::CIRDialect::getConvergentAttrName());
  }
 
  // Collect non-call-site function IR attributes from declaration-specific
  // information.
  if (!attrOnCallSite) {
    // These functions require the returns_twice attribute for correct
    // codegen, but the attribute may not be added if -fno-builtin is
    // specified. We explicitly add that attribute here.
    static const llvm::StringSet<> returnsTwiceFn{
        "_setjmpex", "setjmp",      "_setjmp", "vfork",
        "sigsetjmp", "__sigsetjmp", "savectx", "getcontext"};
    if (returnsTwiceFn.contains(name))
      addUnitAttr(cir::CIRDialect::getReturnsTwiceAttrName());
 
    llvm::StringMap<std::string> cpuAndFeatures;
    if (getCPUAndFeaturesAttributes(calleeInfo.getCalleeDecl(),
                                    cpuAndFeatures)) {
      for (const auto &[key, val] : cpuAndFeatures)
        attrs.set(key, builder.getStringAttr(val));
    }
  }
 
  // TODO(cir): A bunch of non-call-site function IR attributes from
  // declaration-specific information, including tail calls,
  // cmse_nonsecure_entry, and hotpatch support.
 
  // TODO(cir): Add loader-replaceable attribute here.
 
  constructFunctionReturnAttributes(info, targetDecl, isThunk, retAttrs);
  constructFunctionArgumentAttributes(info, targetDecl, isThunk, attrOnCallSite,
                                      argAttrs);
}
 
bool CIRGenModule::hasStrictReturn(QualType retTy, const Decl *targetDecl) {
  // As-is msan can not tolerate noundef mismatch between caller and
  // implementation. Mismatch is possible for e.g. indirect calls from C-caller
  // into C++. Such mismatches lead to confusing false reports. To avoid
  // expensive workaround on msan we enforce initialization event in uncommon
  // cases where it's allowed.
  if (getLangOpts().Sanitize.has(SanitizerKind::Memory))
    return true;
  // C++ explicitly makes returning undefined values UB. C's rule only applies
  // to used values, so we never mark them noundef for now.
  if (!getLangOpts().CPlusPlus)
    return false;
  if (targetDecl) {
    if (const FunctionDecl *func = dyn_cast<FunctionDecl>(targetDecl)) {
      if (func->isExternC())
        return false;
    } else if (const VarDecl *var = dyn_cast<VarDecl>(targetDecl)) {
      // Function pointer.
      if (var->isExternC())
        return false;
    }
  }
 
  // We don't want to be too aggressive with the return checking, unless
  // it's explicit in the code opts or we're using an appropriate sanitizer.
  // Try to respect what the programmer intended.
  return getCodeGenOpts().StrictReturn ||
         !mayDropFunctionReturn(getASTContext(), retTy) ||
         getLangOpts().Sanitize.has(SanitizerKind::Return);
}
 
bool CIRGenModule::mayDropFunctionReturn(const ASTContext &context,
                                         QualType retTy) {
  // We can't just discard the return value for a record type with a
  // complex destructor or a non-trivially copyable type.
  if (const RecordType *recTy =
          retTy.getCanonicalType()->getAsCanonical<RecordType>()) {
    if (const auto *record = dyn_cast<CXXRecordDecl>(recTy->getDecl()))
      return record->hasTrivialDestructor();
  }
  return retTy.isTriviallyCopyableType(context);
}
 
static bool determineNoUndef(QualType clangTy, CIRGenTypes &types,
                             const cir::CIRDataLayout &layout,
                             const cir::ABIArgInfo &argInfo) {
  mlir::Type ty = types.convertTypeForMem(clangTy);
  assert(!cir::MissingFeatures::abiArgInfo());
  if (argInfo.isIndirect() || argInfo.isIndirectAliased())
    return true;
  if (argInfo.isExtend() && !argInfo.isNoExt())
    return true;
 
  if (cir::isSized(ty) && !layout.typeSizeEqualsStoreSize(ty))
    // TODO: This will result in a modest amount of values not marked noundef
    // when they could be. We care about values that *invisibly* contain undef
    // bits from the perspective of LLVM IR.
    return false;
 
  // Classic CodeGen stores a packed bool vector as an integer with one bit per
  // element, at least a byte wide, so the check above drops noundef when that
  // integer is not a whole number of bytes.  The CIR vector type is sized to a
  // power of two of at least a byte, which always passes that check, so test
  // the storage integer's width directly.
  if (clangTy->isPackedVectorBoolType(types.getASTContext()) &&
      mlir::cast<cir::VectorType>(ty).getBoolStorageWidth() % 8)
    return false;
 
  assert(!cir::MissingFeatures::opCallCallConv());
  // The coerced type is not known until CallConvLowering, which drops noundef
  // when it is wider than the value's memory type, the check classic
  // DetermineNoUndef makes at this point.
  if (clangTy->isBitIntType())
    return true;
  if (clangTy->isReferenceType())
    return true;
  if (clangTy->isNullPtrType())
    return false;
  if (clangTy->isMemberPointerType())
    // TODO: Some member pointers are `noundef`, but it depends on the ABI. For
    // now, never mark them.
    return false;
  if (clangTy->isScalarType()) {
    if (const ComplexType *Complex = dyn_cast<ComplexType>(clangTy))
      return determineNoUndef(Complex->getElementType(), types, layout,
                              argInfo);
    return true;
  }
  if (const VectorType *Vector = dyn_cast<VectorType>(clangTy))
    return determineNoUndef(Vector->getElementType(), types, layout, argInfo);
  if (const MatrixType *Matrix = dyn_cast<MatrixType>(clangTy))
    return determineNoUndef(Matrix->getElementType(), types, layout, argInfo);
  if (const ArrayType *Array = dyn_cast<ArrayType>(clangTy))
    return determineNoUndef(Array->getElementType(), types, layout, argInfo);
 
  // TODO: Some structs may be `noundef`, in specific situations.
  return false;
}
 
void CIRGenModule::constructFunctionReturnAttributes(
    const CIRGenFunctionInfo &info, const Decl *targetDecl, bool isThunk,
    mlir::NamedAttrList &retAttrs) {
  // Collect attributes from arguments and return values.
  QualType retTy = info.getReturnType();
  const cir::ABIArgInfo retInfo = info.getReturnInfo();
  const cir::CIRDataLayout &layout = getDataLayout();
 
  if (codeGenOpts.EnableNoundefAttrs && hasStrictReturn(retTy, targetDecl) &&
      !retTy->isVoidType() &&
      determineNoUndef(retTy, getTypes(), layout, retInfo))
    retAttrs.set(mlir::LLVM::LLVMDialect::getNoUndefAttrName(),
                 mlir::UnitAttr::get(&getMLIRContext()));
 
  if (retTy->hasFloatingRepresentation())
    if (llvm::FPClassTest mask =
            CodeGenUtils::getNoFPClassTestMask(getLangOpts()))
      retAttrs.set(mlir::LLVM::LLVMDialect::getNoFPClassAttrName(),
                   builder.getI64IntegerAttr(mask));
 
  if (!isThunk) {
    // TODO(cir): following comment taken from classic codegen, so if anything
    // happens there, we should reflect it here.
    // FIXME: fix this properly, https://reviews.llvm.org/D100388
    if (const auto *refTy = retTy->getAs<ReferenceType>()) {
      QualType pointeeTy = refTy->getPointeeType();
      if (!pointeeTy->isIncompleteType() && pointeeTy->isConstantSizeType())
        retAttrs.set(mlir::LLVM::LLVMDialect::getDereferenceableAttrName(),
                     builder.getI64IntegerAttr(
                         getMinimumObjectSize(pointeeTy).getQuantity()));
 
      if (getTypes().getTargetAddressSpace(pointeeTy) == 0 &&
          !codeGenOpts.NullPointerIsValid)
        retAttrs.set(mlir::LLVM::LLVMDialect::getNonNullAttrName(),
                     mlir::UnitAttr::get(&getMLIRContext()));
 
      if (pointeeTy->isObjectType())
        retAttrs.set(mlir::LLVM::LLVMDialect::getAlignAttrName(),
                     builder.getI64IntegerAttr(
                         getNaturalPointeeTypeAlignment(retTy).getQuantity()));
    }
  }
}
 
void CIRGenModule::constructFunctionArgumentAttributes(
    const CIRGenFunctionInfo &info, const Decl *targetDecl, bool isThunk,
    bool attrOnCallSite, llvm::MutableArrayRef<mlir::NamedAttrList> argAttrs) {
  assert(!cir::MissingFeatures::abiArgInfo());
  // TODO(cir): classic codegen does a lot of work here based on the ABIArgInfo
  // to set things based on calling convention.
 
  if (info.isInstanceMethod() && !isThunk) {
    QualType thisPtrTy = info.arguments()[0];
    // Member allocation functions are instance methods, but setting attributes
    // on them is nonsensical and not correct. Make sure we skip that here.
    if (!thisPtrTy->isVoidPointerType()) {
      QualType thisTy = thisPtrTy->getPointeeType();
 
      if (!codeGenOpts.NullPointerIsValid &&
          getTypes().getTargetAddressSpace(thisPtrTy) == 0) {
        argAttrs[0].set(mlir::LLVM::LLVMDialect::getDereferenceableAttrName(),
                        builder.getI64IntegerAttr(
                            getMinimumObjectSize(thisTy).getQuantity()));
        argAttrs[0].set(mlir::LLVM::LLVMDialect::getNonNullAttrName(),
                        mlir::UnitAttr::get(&getMLIRContext()));
      } else {
        uint64_t bytes = getMinimumObjectSize(thisTy).getQuantity();
 
        if (bytes != 0)
          argAttrs[0].set(
              mlir::LLVM::LLVMDialect::getDereferenceableOrNullAttrName(),
              builder.getI64IntegerAttr(bytes));
      }
 
      argAttrs[0].set(
          mlir::LLVM::LLVMDialect::getAlignAttrName(),
          builder.getI64IntegerAttr(
              getNaturalPointeeTypeAlignment(thisPtrTy).getQuantity()));
 
      // TODO(cir): the classic codegen has a recently-added bunch of logic for
      // 'dead_on_return' as an attribute. This both doesn't exist in the LLVM
      // dialect, and is 'too new' at the time of writing this to be considered
      // stable enough here.  For now, we'll leave this as a TODO so that when
      // we come back, it is hopefully a more stabilized implementation.
    }
  }
 
  // TODO(cir): the logic between 'this', return, and normal arguments hsould
  // probably be merged at one point, however the logic is unfortunately mildly
  // different between each in classic codegen, so trying to do anything like
  // that seems risky at the moment. At one point we should evaluate if at least
  // dereferenceable, nonnull, and align can be combined.
  const cir::CIRDataLayout &layout = getDataLayout();
  const auto *fd = dyn_cast_or_null<FunctionDecl>(targetDecl);
 
  // Build a parallel array of ParmVarDecls aligned with argAttrs so we can
  // access parameter-level attributes (e.g. restrict, nonnull) without manual
  // index arithmetic in the loop.
  SmallVector<const ParmVarDecl *> parmDecls;
  parmDecls.reserve(argAttrs.size());
  if (fd) {
    if (info.isInstanceMethod())
      parmDecls.push_back(nullptr);
    parmDecls.insert(parmDecls.end(), fd->param_begin(), fd->param_end());
  }
  parmDecls.resize(argAttrs.size(), nullptr);
 
  for (const auto &[argAttrList, argCanType, pvd] :
       llvm::zip_equal(argAttrs, info.arguments(), parmDecls)) {
    assert(!cir::MissingFeatures::abiArgInfo());
    QualType argType = argCanType;
    const cir::ABIArgInfo argInfo = cir::ABIArgInfo::getDirect();
 
    if (codeGenOpts.EnableNoundefAttrs &&
        determineNoUndef(argType, getTypes(), layout, argInfo))
      argAttrList.set(mlir::LLVM::LLVMDialect::getNoUndefAttrName(),
                      mlir::UnitAttr::get(&getMLIRContext()));
 
    assert(!cir::MissingFeatures::abiArgInfo());
    // TODO(cir): there is plenty of other attributes here added due to ABI
    // decisions.  While these probably won't end up here, we note that the
    // classic codegen does it here and perhaps we should pay attention to that.
 
    if (const auto *refTy = argType->getAs<ReferenceType>()) {
      QualType pointeeTy = refTy->getPointeeType();
      if (!pointeeTy->isIncompleteType() && pointeeTy->isConstantSizeType())
        argAttrList.set(mlir::LLVM::LLVMDialect::getDereferenceableAttrName(),
                        builder.getI64IntegerAttr(
                            getMinimumObjectSize(pointeeTy).getQuantity()));
      if (getTypes().getTargetAddressSpace(pointeeTy) == 0 &&
          !codeGenOpts.NullPointerIsValid)
        argAttrList.set(mlir::LLVM::LLVMDialect::getNonNullAttrName(),
                        mlir::UnitAttr::get(&getMLIRContext()));
      if (pointeeTy->isObjectType())
        argAttrList.set(
            mlir::LLVM::LLVMDialect::getAlignAttrName(),
            builder.getI64IntegerAttr(
                getNaturalPointeeTypeAlignment(argType).getQuantity()));
    }
 
    if (argType->hasFloatingRepresentation())
      if (llvm::FPClassTest mask =
              CodeGenUtils::getNoFPClassTestMask(getLangOpts()))
        argAttrList.set(mlir::LLVM::LLVMDialect::getNoFPClassAttrName(),
                        builder.getI64IntegerAttr(mask));
 
    // restrict -> noalias on definitions only (not call sites).  Skip
    // builtins: OGCG applies restrict->noalias in EmitFunctionProlog.
    if (!attrOnCallSite && pvd && pvd->getType()->isPointerType() &&
        pvd->getType().isRestrictQualified() && !fd->getBuiltinID())
      argAttrList.set(mlir::LLVM::LLVMDialect::getNoAliasAttrName(),
                      mlir::UnitAttr::get(&getMLIRContext()));
 
    // __attribute__((nonnull)) on pointer parameters.  Checks both
    // per-parameter and function-level nonnull attributes.
    if (pvd && argType->isAnyPointerType() && !codeGenOpts.NullPointerIsValid) {
      unsigned srcIdx = pvd->getFunctionScopeIndex();
      if (pvd->hasAttr<NonNullAttr>() ||
          (fd->getAttr<NonNullAttr>() &&
           fd->getAttr<NonNullAttr>()->isNonNull(srcIdx)))
        argAttrList.set(mlir::LLVM::LLVMDialect::getNonNullAttrName(),
                        mlir::UnitAttr::get(&getMLIRContext()));
    }
  }
}
 
/// Adds the formal parameters in FPT to the given prefix.  If any parameter in
/// FPT has pass_object_size attrs, then we'll add parameters for those, too.
/// TODO(cir): this should be shared with LLVM codegen
static void appendParameterTypes(const CIRGenTypes &cgt,
                                 SmallVectorImpl<CanQualType> &prefix,
                                 CanQual<FunctionProtoType> fpt) {
  // Fast path: don't touch param info if we don't need to.
  if (!fpt->hasExtParameterInfos()) {
    prefix.append(fpt->param_type_begin(), fpt->param_type_end());
    return;
  }
 
  // In the vast majority of cases, we'll have precisely fpt->getNumParams()
  // parameters; the only thing that can change this is the presence of
  // pass_object_size. So, we preallocate for the common case.
  prefix.reserve(prefix.size() + fpt->getNumParams());
  ArrayRef<FunctionProtoType::ExtParameterInfo> extInfos =
      fpt->getExtParameterInfos();
  assert(extInfos.size() == fpt->getNumParams());
  for (auto [paramType, extInfo] : llvm::zip_equal(
           llvm::make_range(fpt->param_type_begin(), fpt->param_type_end()),
           extInfos)) {
    prefix.push_back(paramType);
    if (extInfo.hasPassObjectSize())
      prefix.push_back(cgt.getASTContext().getCanonicalSizeType());
  }
}
 
const CIRGenFunctionInfo &
CIRGenTypes::arrangeCXXStructorDeclaration(GlobalDecl gd) {
  auto *md = cast<CXXMethodDecl>(gd.getDecl());
 
  llvm::SmallVector<CanQualType, 16> argTypes;
  argTypes.push_back(deriveThisType(md->getParent(), md));
 
  bool passParams = true;
 
  if (auto *cd = dyn_cast<CXXConstructorDecl>(md)) {
    // A base class inheriting constructor doesn't get forwarded arguments
    // needed to construct a virtual base (or base class thereof)
    if (auto inherited = cd->getInheritedConstructor())
      passParams = inheritingCtorHasParams(inherited, gd.getCtorType());
  }
 
  CanQual<FunctionProtoType> fpt = CodeGenUtils::getFormalType(md);
 
  if (passParams)
    appendParameterTypes(*this, argTypes, fpt);
 
  // The structor signature may include implicit parameters.
  [[maybe_unused]] CIRGenCXXABI::AddedStructorArgCounts addedArgs =
      theCXXABI.buildStructorSignature(gd, argTypes);
  assert(!cir::MissingFeatures::opCallCIRGenFuncInfoExtParamInfo());
 
  RequiredArgs required =
      (passParams && md->isVariadic() ? RequiredArgs(argTypes.size())
                                      : RequiredArgs::All);
 
  CanQualType resultType = theCXXABI.hasThisReturn(gd) ? argTypes.front()
                           : theCXXABI.hasMostDerivedReturn(gd)
                               ? astContext.VoidPtrTy
                               : astContext.VoidTy;
 
  assert(!theCXXABI.hasThisReturn(gd) &&
         "Please send PR with a test and remove this");
 
  assert(!cir::MissingFeatures::opCallCIRGenFuncInfoExtParamInfo());
  assert(!cir::MissingFeatures::opCallFnInfoOpts());
 
  return arrangeCIRFunctionInfo(resultType, /*isInstanceMethod=*/true, argTypes,
                                fpt->getExtInfo(), required);
}
 
/// Derives the 'this' type for CIRGen purposes, i.e. ignoring method CVR
/// qualification. Either or both of `rd` and `md` may be null. A null `rd`
/// indicates that there is no meaningful 'this' type, and a null `md` can occur
/// when calling a method pointer.
CanQualType CIRGenTypes::deriveThisType(const CXXRecordDecl *rd,
                                        const CXXMethodDecl *md) {
  CanQualType recTy;
  if (rd) {
    recTy = getASTContext().getCanonicalTagType(rd);
  } else {
    // This can happen with the MS ABI. It shouldn't need anything more than
    // setting recTy to VoidTy here, but we're flagging it for now because we
    // don't have the full handling implemented.
    cgm.errorNYI("deriveThisType: no record decl");
    recTy = getASTContext().VoidTy;
  }
 
  if (md)
    recTy = CanQualType::CreateUnsafe(getASTContext().getAddrSpaceQualType(
        recTy, md->getMethodQualifiers().getAddressSpace()));
  return getASTContext().getPointerType(recTy);
}
 
/// Arrange the CIR function layout for a value of the given function type, on
/// top of any implicit parameters already stored.
static const CIRGenFunctionInfo &
arrangeCIRFunctionInfo(CIRGenTypes &cgt, bool instanceMethod,
                       SmallVectorImpl<CanQualType> &prefix,
                       CanQual<FunctionProtoType> fpt) {
  assert(!cir::MissingFeatures::opCallFnInfoOpts());
  RequiredArgs required =
      RequiredArgs::getFromProtoWithExtraSlots(fpt, prefix.size());
  assert(!cir::MissingFeatures::opCallExtParameterInfo());
  appendParameterTypes(cgt, prefix, fpt);
  CanQualType resultType = fpt->getReturnType().getUnqualifiedType();
  return cgt.arrangeCIRFunctionInfo(resultType, instanceMethod, prefix,
                                    fpt->getExtInfo(), required);
}
 
void CIRGenFunction::emitDelegateCallArg(CallArgList &args,
                                         const VarDecl *param,
                                         SourceLocation loc) {
  // StartFunction converted the ABI-lowered parameter(s) into a local alloca.
  // We need to turn that into an r-value suitable for emitCall
  Address local = getAddrOfLocalVar(param);
 
  QualType type = param->getType();
 
  // GetAddrOfLocalVar returns a pointer-to-pointer for references, but the
  // argument needs to be the original pointer.
  if (type->isReferenceType()) {
    args.add(
        RValue::get(builder.createLoad(getLoc(param->getSourceRange()), local)),
        type);
  } else if (getLangOpts().ObjCAutoRefCount) {
    cgm.errorNYI(param->getSourceRange(),
                 "emitDelegateCallArg: ObjCAutoRefCount");
    // For the most part, we just need to load the alloca, except that aggregate
    // r-values are actually pointers to temporaries.
  } else {
    args.add(convertTempToRValue(local, type, loc), type);
  }
 
  // Deactivate the cleanup for the callee-destructed param that was pushed.
  assert(!cir::MissingFeatures::thunks());
  if (type->isRecordType() &&
      type->castAsRecordDecl()->isParamDestroyedInCallee() &&
      param->needsDestruction(getContext())) {
    cgm.errorNYI(param->getSourceRange(),
                 "emitDelegateCallArg: callee-destructed param");
  }
}
 
static const CIRGenFunctionInfo &
arrangeFreeFunctionLikeCall(CIRGenTypes &cgt, CIRGenModule &cgm,
                            const CallArgList &args,
                            const FunctionType *fnType) {
 
  RequiredArgs required = RequiredArgs::All;
 
  if (const auto *proto = dyn_cast<FunctionProtoType>(fnType)) {
    // A free function call has no extra prefix arguments. Note that
    // getFromProtoWithExtraSlots already accounts for the prototype's
    // pass_object_size parameters; adding them here too would double-count
    // them and make the signature claim more required arguments than `args`
    // actually holds.
    if (proto->isVariadic())
      required = RequiredArgs::getFromProtoWithExtraSlots(proto, 0);
  } else if (cgm.getTargetCIRGenInfo().isNoProtoCallVariadic(
                 cast<FunctionNoProtoType>(fnType)))
    cgm.errorNYI("call to function without a prototype");
 
  SmallVector<CanQualType, 16> argTypes;
  for (const CallArg &arg : args)
    argTypes.push_back(cgt.getASTContext().getCanonicalParamType(arg.ty));
 
  CanQualType retType = fnType->getReturnType()->getCanonicalTypeUnqualified();
 
  assert(!cir::MissingFeatures::opCallFnInfoOpts());
  return cgt.arrangeCIRFunctionInfo(retType, /*isInstanceMethod=*/false,
                                    argTypes, fnType->getExtInfo(), required);
}
 
/// Arrange a call to a C++ method, passing the given arguments.
///
/// extraPrefixArgs is the number of ABI-specific args passed after the `this`
/// parameter.
/// passProtoArgs indicates whether `args` has args for the parameters in the
/// given CXXConstructorDecl.
const CIRGenFunctionInfo &CIRGenTypes::arrangeCXXConstructorCall(
    const CallArgList &args, const CXXConstructorDecl *d, CXXCtorType ctorKind,
    unsigned extraPrefixArgs, unsigned extraSuffixArgs, bool passProtoArgs) {
 
  // FIXME: Kill copy.
  llvm::SmallVector<CanQualType, 16> argTypes;
  for (const auto &arg : args)
    argTypes.push_back(astContext.getCanonicalParamType(arg.ty));
 
  // +1 for implicit this, which should always be args[0]
  unsigned totalPrefixArgs = 1 + extraPrefixArgs;
 
  CanQual<FunctionProtoType> fpt = CodeGenUtils::getFormalType(d);
  RequiredArgs required = passProtoArgs
                              ? RequiredArgs::getFromProtoWithExtraSlots(
                                    fpt, totalPrefixArgs + extraSuffixArgs)
                              : RequiredArgs::All;
 
  GlobalDecl gd(d, ctorKind);
  if (theCXXABI.hasThisReturn(gd))
    cgm.errorNYI(d->getSourceRange(),
                 "arrangeCXXConstructorCall: hasThisReturn");
  if (theCXXABI.hasMostDerivedReturn(gd))
    cgm.errorNYI(d->getSourceRange(),
                 "arrangeCXXConstructorCall: hasMostDerivedReturn");
  CanQualType resultType = astContext.VoidTy;
 
  assert(!cir::MissingFeatures::opCallFnInfoOpts());
  assert(!cir::MissingFeatures::opCallCIRGenFuncInfoExtParamInfo());
 
  return arrangeCIRFunctionInfo(resultType, /*isInstanceMethod=*/true, argTypes,
                                fpt->getExtInfo(), required);
}
 
/// Arrange a call to a C++ method, passing the given arguments.
///
/// numPrefixArgs is the number of the ABI-specific prefix arguments we have. It
/// does not count `this`.
const CIRGenFunctionInfo &CIRGenTypes::arrangeCXXMethodCall(
    const CallArgList &args, const FunctionProtoType *proto,
    RequiredArgs required, unsigned numPrefixArgs) {
  assert(!cir::MissingFeatures::opCallExtParameterInfo());
  assert(numPrefixArgs + 1 <= args.size() &&
         "Emitting a call with less args than the required prefix?");
 
  // FIXME: Kill copy.
  llvm::SmallVector<CanQualType, 16> argTypes;
  for (const CallArg &arg : args)
    argTypes.push_back(astContext.getCanonicalParamType(arg.ty));
 
  assert(!cir::MissingFeatures::opCallFnInfoOpts());
  return arrangeCIRFunctionInfo(
      proto->getReturnType()->getCanonicalTypeUnqualified(),
      /*isInstanceMethod=*/true, argTypes, proto->getExtInfo(), required);
}
 
const CIRGenFunctionInfo &
CIRGenTypes::arrangeFreeFunctionCall(const CallArgList &args,
                                     const FunctionType *fnType) {
  return arrangeFreeFunctionLikeCall(*this, cgm, args, fnType);
}
 
const CIRGenFunctionInfo &
CIRGenTypes::arrangeBuiltinFunctionCall(QualType resultType,
                                        const CallArgList &args) {
  llvm::SmallVector<CanQualType, 16> argTypes;
  for (const CallArg &arg : args)
    argTypes.push_back(astContext.getCanonicalParamType(arg.ty));
 
  CanQualType retType = resultType->getCanonicalTypeUnqualified();
  return arrangeCIRFunctionInfo(retType, /*isInstanceMethod=*/false, argTypes,
                                FunctionType::ExtInfo(), RequiredArgs::All);
}
 
/// Set calling convention for CUDA/HIP kernel.
static void setCUDAKernelCallingConvention(CanQualType &funcTy,
                                           CIRGenModule &cgm,
                                           const FunctionDecl *fd) {
  if (fd->hasAttr<CUDAGlobalAttr>()) {
    const FunctionType *ft = funcTy->getAs<FunctionType>();
    cgm.getTargetCIRGenInfo().setCUDAKernelCallingConvention(ft);
    funcTy = ft->getCanonicalTypeUnqualified();
  }
}
 
/// Arrange the argument and result information for a declaration or definition
/// of the given C++ non-static member function. The member function must be an
/// ordinary function, i.e. not a constructor or destructor.
const CIRGenFunctionInfo &
CIRGenTypes::arrangeCXXMethodDeclaration(const CXXMethodDecl *md) {
  assert(!isa<CXXConstructorDecl>(md) && "wrong method for constructors!");
  assert(!isa<CXXDestructorDecl>(md) && "wrong method for destructors!");
 
  CanQualType funcTy = md->getType()->getCanonicalTypeUnqualified();
  setCUDAKernelCallingConvention(funcTy, cgm, md);
  auto prototype = funcTy.getAs<FunctionProtoType>();
 
  // Mirrors classic CodeGen's check at CGCall.cpp.  C++23 explicit-object
  // member functions (P0847R7, `void f(this Self&&)`) do not receive an
  // implicit `this`; the explicit object parameter takes its place at the
  // AST level and appears as the first parameter of the FunctionProtoType.
  // Arrange them as free functions so we don't prepend a stale implicit
  // `this` to the parameter list, which would produce a CIRGenFunctionInfo
  // with one more argument than the matching cir.func type and trip the
  // assertion in setArgAttrs.
  if (md->isImplicitObjectMemberFunction()) {
    // The abstract case is perfectly fine.
    auto *thisType = theCXXABI.getThisArgumentTypeForMethod(md);
    return arrangeCXXMethodType(thisType, prototype.getTypePtr(), md);
  }
 
  return arrangeFreeFunctionType(prototype);
}
 
/// Arrange the argument and result information for a call to an unknown C++
/// non-static member function of the given abstract type. (A null RD means we
/// don't have any meaningful "this" argument type, so fall back to a generic
/// pointer type). The member fucntion must be an ordinary function, i.e. not a
/// constructor or destructor.
const CIRGenFunctionInfo &
CIRGenTypes::arrangeCXXMethodType(const CXXRecordDecl *rd,
                                  const FunctionProtoType *fpt,
                                  const CXXMethodDecl *md) {
  llvm::SmallVector<CanQualType, 16> argTypes;
 
  // Add the 'this' pointer.
  argTypes.push_back(deriveThisType(rd, md));
 
  assert(!cir::MissingFeatures::opCallFnInfoOpts());
  return ::arrangeCIRFunctionInfo(
      *this, /*isInstanceMethod=*/true, argTypes,
      fpt->getCanonicalTypeUnqualified().getAs<FunctionProtoType>());
}
 
/// Arrange the argument and result information for the declaration or
/// definition of the given function.
const CIRGenFunctionInfo &
CIRGenTypes::arrangeFunctionDeclaration(const FunctionDecl *fd) {
  if (const auto *md = dyn_cast<CXXMethodDecl>(fd))
    if (md->isInstance())
      return arrangeCXXMethodDeclaration(md);
 
  CanQualType funcTy = fd->getType()->getCanonicalTypeUnqualified();
 
  assert(isa<FunctionType>(funcTy));
  setCUDAKernelCallingConvention(funcTy, cgm, fd);
 
  // When declaring a function without a prototype, always use a non-variadic
  // type.
  if (CanQual<FunctionNoProtoType> noProto =
          funcTy.getAs<FunctionNoProtoType>()) {
    assert(!cir::MissingFeatures::opCallCIRGenFuncInfoExtParamInfo());
    assert(!cir::MissingFeatures::opCallFnInfoOpts());
    return arrangeCIRFunctionInfo(noProto->getReturnType(),
                                  /*isInstanceMethod=*/false, {},
                                  noProto->getExtInfo(), RequiredArgs::All);
  }
 
  return arrangeFreeFunctionType(funcTy.castAs<FunctionProtoType>());
}
 
RValue CallArg::getRValue(CIRGenFunction &cgf, mlir::Location loc) const {
  if (!hasLV)
    return rv;
  LValue copy = cgf.makeAddrLValue(cgf.createMemTemp(ty, loc), ty);
  cgf.emitAggregateCopy(copy, lv, ty, AggValueSlot::DoesNotOverlap,
                        lv.isVolatile());
  isUsed = true;
  return RValue::getAggregate(copy.getAddress());
}
 
void CIRGenFunction::emitNonNullArgCheck(RValue rv, QualType argType,
                                         SourceLocation argLoc,
                                         AbstractCallee ac, unsigned paramNum) {
  if (!ac.getDecl() || !(sanOpts.has(SanitizerKind::NonnullAttribute) ||
                         sanOpts.has(SanitizerKind::NullabilityArg)))
    return;
  cgm.errorNYI("non-null arg check is NYI");
}
 
static cir::CIRCallOpInterface emitCallLikeOp(
    CIRGenFunction &cgf, mlir::Location callLoc, cir::FuncType indirectFuncTy,
    mlir::Value indirectFuncVal, cir::FuncOp directFuncOp,
    const SmallVectorImpl<mlir::Value> &cirCallArgs, bool isInvoke,
    const mlir::NamedAttrList &attrs,
    llvm::ArrayRef<mlir::NamedAttrList> argAttrs,
    const mlir::NamedAttrList &retAttrs, cir::CallingConv callingConv) {
  CIRGenBuilderTy &builder = cgf.getBuilder();
 
  assert(!cir::MissingFeatures::opCallSurroundingTry());
 
  assert(builder.getInsertionBlock() && "expected valid basic block");
 
  cir::CallOp op;
  if (indirectFuncTy) {
    op = builder.createIndirectCallOp(callLoc, indirectFuncVal, indirectFuncTy,
                                      cirCallArgs, attrs, argAttrs, retAttrs,
                                      callingConv);
  } else {
    op = builder.createCallOp(callLoc, directFuncOp, cirCallArgs, attrs,
                              argAttrs, retAttrs, callingConv);
  }
 
  return op;
}
 
const CIRGenFunctionInfo &
CIRGenTypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> fpt) {
  SmallVector<CanQualType, 16> argTypes;
  assert(!cir::MissingFeatures::opCallFnInfoOpts());
  return ::arrangeCIRFunctionInfo(*this, /*isInstanceMethod=*/false, argTypes,
                                  fpt);
}
 
const CIRGenFunctionInfo &
CIRGenTypes::arrangeFreeFunctionType(CanQual<FunctionNoProtoType> fnpt) {
  CanQualType resultType = fnpt->getReturnType().getUnqualifiedType();
  assert(!cir::MissingFeatures::opCallFnInfoOpts());
  return arrangeCIRFunctionInfo(resultType, /*isInstanceMethod=*/false, {},
                                fnpt->getExtInfo(), RequiredArgs(0));
}
 
RValue CIRGenFunction::emitCall(const CIRGenFunctionInfo &funcInfo,
                                const CIRGenCallee &callee,
                                ReturnValueSlot returnValue,
                                const CallArgList &args,
                                cir::CIRCallOpInterface *callOp,
                                bool isMustTail, SourceRange clangLoc) {
  QualType retTy = funcInfo.getReturnType();
  cir::FuncType cirFuncTy = getTypes().getFunctionType(funcInfo);
  mlir::Location loc = getLoc(clangLoc);
 
  SmallVector<mlir::Value, 16> cirCallArgs(args.size());
 
  const Decl *targetDecl = callee.getAbstractInfo().getCalleeDecl().getDecl();
 
  if (const FunctionDecl *fd = dyn_cast_or_null<FunctionDecl>(targetDecl)) {
    // We can only guarantee that a function is called from the correct
    // context/function based on the appropriate target attributes,
    // so only check in the case where we have both always_inline and target
    // since otherwise we could be making a conditional call after a check for
    // the proper cpu features (and it won't cause code generation issues due to
    // function based code generation).
    if ((targetDecl->hasAttr<AlwaysInlineAttr>() &&
         (targetDecl->hasAttr<TargetAttr>() ||
          (curFuncDecl && curFuncDecl->hasAttr<TargetAttr>()))) ||
        (curFuncDecl && curFuncDecl->hasAttr<FlattenAttr>() &&
         (curFuncDecl->hasAttr<TargetAttr>() ||
          targetDecl->hasAttr<TargetAttr>())))
      checkTargetFeatures(clangLoc.getBegin(), fd);
  }
 
  const FunctionDecl *callerDecl = dyn_cast_or_null<FunctionDecl>(curCodeDecl);
  const FunctionDecl *calleeDecl = dyn_cast_or_null<FunctionDecl>(targetDecl);
 
  assert(!cir::MissingFeatures::emitLifetimeMarkers());
 
  // Translate all of the arguments as necessary to match the CIR lowering.
  for (auto [argNo, arg, canQualArgType] :
       llvm::enumerate(args, funcInfo.argTypes())) {
 
    // Insert a padding argument to ensure proper alignment.
    assert(!cir::MissingFeatures::opCallPaddingArgs());
 
    mlir::Type argType = convertType(canQualArgType);
    if (!mlir::isa<cir::RecordType>(argType) &&
        !mlir::isa<cir::ComplexType>(argType)) {
      mlir::Value v;
      if (arg.isAggregate())
        cgm.errorNYI(loc, "emitCall: aggregate call argument");
      v = arg.getKnownRValue().getValue();
 
      // We might have to widen integers, but we should never truncate.
      if (argType != v.getType() && mlir::isa<cir::IntType>(v.getType()))
        cgm.errorNYI(loc, "emitCall: widening integer call argument");
 
      // If we have a pointer argument and there's an address space mismatch,
      // insert an address_space cast to match the expected function signature.
      if (argType != v.getType()) {
        auto argPtrTy = mlir::dyn_cast<cir::PointerType>(argType);
        auto vPtrTy = mlir::dyn_cast<cir::PointerType>(v.getType());
        if (argPtrTy && vPtrTy &&
            argPtrTy.getPointee() == vPtrTy.getPointee() &&
            argPtrTy.getAddrSpace() != vPtrTy.getAddrSpace()) {
          v = performAddrSpaceCast(v, argPtrTy);
        }
      }
 
      // If the argument doesn't match, perform a bitcast to coerce it. This
      // can happen due to trivial type mismatches.
      // TODO(cir): When getFunctionType is added, assert that this isn't
      // needed.
      assert(!cir::MissingFeatures::opCallBitcastArg());
      cirCallArgs[argNo] = v;
    } else {
      Address src = Address::invalid();
      if (!arg.isAggregate()) {
        src = createMemTemp(arg.ty, loc, "coerce");
        arg.copyInto(*this, src, loc);
      } else {
        src = arg.hasLValue() ? arg.getKnownLValue().getAddress()
                              : arg.getKnownRValue().getAggregateAddress();
      }
 
      // Fast-isel and the optimizer generally like scalar values better than
      // FCAs, so we flatten them if this is safe to do for this argument.
      mlir::Type srcTy = src.getElementType();
      // FIXME(cir): get proper location for each argument.
      mlir::Location argLoc = loc;
 
      // If the source type is smaller than the destination type of the
      // coerce-to logic, copy the source value into a temp alloca the size
      // of the destination type to allow loading all of it. The bits past
      // the source value are left undef.
      // FIXME(cir): add data layout info and compare sizes instead of
      // matching the types.
      //
      // uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(SrcTy);
      // uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(STy);
      // if (SrcSize < DstSize) {
      assert(!cir::MissingFeatures::dataLayoutTypeAllocSize());
      if (srcTy != argType) {
        cgm.errorNYI(loc, "emitCall: source type does not match argument type");
      } else {
        // FIXME(cir): this currently only runs when the types are exactly the
        // same, but should be when alloc sizes are the same, fix this as soon
        // as datalayout gets introduced.
        assert(!cir::MissingFeatures::dataLayoutTypeAllocSize());
      }
 
      // assert(NumCIRArgs == STy.getMembers().size());
      // In LLVMGen: Still only pass the struct without any gaps but mark it
      // as such somehow.
      //
      // In CIRGen: Emit a load from the "whole" struct,
      // which shall be broken later by some lowering step into multiple
      // loads.
      assert(!cir::MissingFeatures::lowerAggregateLoadStore());
      cirCallArgs[argNo] = builder.createLoad(argLoc, src);
    }
  }
 
  const CIRGenCallee &concreteCallee = callee.prepareConcreteCallee(*this);
  mlir::Operation *calleePtr = concreteCallee.getFunctionPointer();
 
  assert(!cir::MissingFeatures::opCallInAlloca());
 
  mlir::NamedAttrList attrs;
  std::vector<mlir::NamedAttrList> argAttrs(funcInfo.arguments().size());
  mlir::NamedAttrList retAttrs;
  StringRef funcName;
  if (auto calleeFuncOp = dyn_cast<cir::FuncOp>(calleePtr))
    funcName = calleeFuncOp.getName();
 
  assert(!cir::MissingFeatures::opCallAttrs());
  cir::CallingConv callingConv;
  cgm.constructAttributeList(funcName, funcInfo, callee.getAbstractInfo(),
                             attrs, argAttrs, retAttrs, callingConv,
                             /*attrOnCallSite=*/true, /*isThunk=*/false);
 
  // TODO(cir): Classic CodeGen redirects calls to OpenCL kernels to a
  // non-kernel stub (__clang_ocl_kern_imp_*), since kernel calling conventions
  // do not permit calls.
  if (callingConv == cir::CallingConv::SpirKernel ||
      callingConv == cir::CallingConv::AMDGPUKernel ||
      callingConv == cir::CallingConv::PTXKernel)
    cgm.errorNYI(loc, "emitCall: call to kernel function");
 
  auto resolvedFuncOpFromGlobal = [&](mlir::Operation *op) -> cir::FuncOp {
    if (auto fnOp = dyn_cast<cir::FuncOp>(op))
      return fnOp;
    if (auto getGlobalOp = dyn_cast<cir::GetGlobalOp>(op)) {
      // FIXME(cir): This peephole optimization avoids indirect calls for
      // builtins. This should be fixed in the builtin declaration instead by
      // not emitting an unecessary get_global in the first place. However,
      // this is also used for no-prototype functions.
      mlir::Operation *globalOp = cgm.getGlobalValue(getGlobalOp.getName());
      assert(globalOp && "undefined global function");
      return cast<cir::FuncOp>(globalOp);
    }
    return nullptr;
  };
 
  cir::FuncType indirectFuncTy;
  mlir::Value indirectFuncVal;
  cir::FuncOp directFuncOp;
 
  // If the callee resolves to a FuncOp whose stored signature differs from
  // this call site's expected signature, the CIR verifier would reject the
  // mismatched types. This happens, for example, when two declarations share a
  // mangled name via __asm__ renaming (glibc's __REDIRECT_NTH pattern) but
  // disagree about a struct argument type. If that happens, we demote the
  // direct call to an indirect call through a function-pointer bitcast typed
  // at the call site.
  if (cir::FuncOp candidate = resolvedFuncOpFromGlobal(calleePtr)) {
    if (candidate.getFunctionType() == cirFuncTy) {
      directFuncOp = candidate;
    } else {
      mlir::Value addr = cir::GetGlobalOp::create(
          builder, loc, cir::PointerType::get(candidate.getFunctionType()),
          candidate.getSymName());
      indirectFuncTy = cirFuncTy;
      indirectFuncVal =
          builder.createBitcast(addr, cir::PointerType::get(cirFuncTy));
    }
  } else {
    [[maybe_unused]] mlir::ValueTypeRange<mlir::ResultRange> resultTypes =
        calleePtr->getResultTypes();
    [[maybe_unused]] auto funcPtrTy =
        mlir::dyn_cast<cir::PointerType>(resultTypes.front());
    assert(funcPtrTy && mlir::isa<cir::FuncType>(funcPtrTy.getPointee()) &&
           "expected pointer to function");
 
    indirectFuncTy = cirFuncTy;
    indirectFuncVal = calleePtr->getResult(0);
  }
 
  assert(!cir::MissingFeatures::msvcCXXPersonality());
  assert(!cir::MissingFeatures::functionUsesSEHTry());
  assert(!cir::MissingFeatures::nothrowAttr());
 
  bool cannotThrow = attrs.getNamed("nothrow").has_value();
  bool isInvoke = !cannotThrow && isCatchOrCleanupRequired();
 
  mlir::Location callLoc = loc;
  cir::CIRCallOpInterface theCall = emitCallLikeOp(
      *this, loc, indirectFuncTy, indirectFuncVal, directFuncOp, cirCallArgs,
      isInvoke, attrs, argAttrs, retAttrs, callingConv);
 
  if (callOp)
    *callOp = theCall;
 
  // Add srcloc if we have [[gnu::error/warning]] or ShowInliningChain.
  if (calleeDecl) {
    bool needSrcLoc = calleeDecl->hasAttr<ErrorAttr>();
    if (!needSrcLoc && cgm.getCodeGenOpts().ShowInliningChain)
      needSrcLoc = calleeDecl->isInlined() ||
                   calleeDecl->hasAttr<AlwaysInlineAttr>() ||
                   calleeDecl->getStorageClass() == SC_Static ||
                   calleeDecl->isInAnonymousNamespace();
    if (needSrcLoc)
      theCall->setAttr(
          cir::CIRDialect::getSrcLocAttrName(),
          builder.getI64IntegerAttr(clangLoc.getBegin().getRawEncoding()));
  }
 
  // Sema/emitAttributedStmt (see
  // https://github.com/llvm/llvm-project/issues/214764) should one-day enforce
  // that only one of these is valid at a time. For now, we have the same 'bug'
  // as classic codegen where we can end up having BOTH of these.
  if (inNoInlineAttributedStmt)
    theCall.setInlineKind(cir::InlineKind::NoInline);
  if (inAlwaysInlineAttributedStmt &&
      !cgm.getTargetCIRGenInfo().wouldInliningViolateFunctionCallABI(
          callerDecl, calleeDecl))
    theCall.setInlineKind(cir::InlineKind::AlwaysInline);
 
  if (isMustTail) {
    // PPC/MIPS have some diagnostics for classic-codegen, but we don't support
    // them yet.
    const llvm::Triple &triple = getTarget().getTriple();
    if (triple.isPPC() || triple.isMIPS()) {
      cgm.errorNYI(mustTailCall->getBeginLoc(),
                   "musttail call target legality checks");
      return getUndefRValue(retTy);
    }
 
    // This call replaces the frame that the handler enforcing the exception
    // specification belongs to, taking that handler with it.
    if (ehSpecTryOp) {
      if (!inEHSpecTerminateScope()) {
        // The filter of a dynamic specification has to run while an exception
        // is leaving the function, and the tail call skips over it.
        cgm.errorUnsupported(mustTailCall, "tail call skipping over cleanups");
        return getUndefRValue(retTy);
      }
 
      // Losing a terminate handler is safe when the callee cannot throw,
      // because then the callee's own handler stands in for it.
      if (!cannotThrow) {
        cgm.getDiags().Report(mustTailCall->getBeginLoc(),
                              diag::err_musttail_noexcept_mismatch);
        return getUndefRValue(retTy);
      }
    }
 
    // Musttail is required to return immediately, so no cleanup can run
    // between the call and the return. Cleanups that the return makes
    // unnecessary are simply skipped. Anything else cannot be expressed.
    assert(!cir::MissingFeatures::fakeUseCleanup());
    for (EHScopeStack::iterator it = ehStack.begin(),
                                end = ehStack.find(prologueCleanupDepth);
         it != end; ++it) {
      auto *cleanupScope = dyn_cast<EHCleanupScope>(&*it);
      if (!cleanupScope ||
          !cleanupScope->getCleanup()->isRedundantBeforeReturn()) {
        cgm.errorUnsupported(mustTailCall, "tail call skipping over cleanups");
        return getUndefRValue(retTy);
      }
    }
 
    theCall->setAttr(cir::CIRDialect::getMustTailAttrName(),
                     builder.getUnitAttr());
 
    if (isa<cir::VoidType>(convertType(retTy)))
      cir::ReturnOp::create(builder, loc);
    else
      cir::ReturnOp::create(builder, loc, theCall->getResult(0));
 
    // Musttail must return immediately, so we just do that.  All of the below
    // stuff is effectively UB if this is a musttail, so just do a return
    // immediately.
    builder.createBlock(builder.getBlock()->getParent());
    return getUndefRValue(retTy);
  }
 
  assert(!cir::MissingFeatures::opCallReturn());
 
  mlir::Type retCIRTy = convertType(retTy);
  if (isa<cir::VoidType>(retCIRTy))
    return getUndefRValue(retTy);
  switch (getEvaluationKind(retTy)) {
  case cir::TEK_Aggregate: {
    Address destPtr = returnValue.getValue();
 
    if (!destPtr.isValid())
      destPtr = createMemTemp(retTy, callLoc, getCounterAggTmpAsString());
 
    mlir::ResultRange results = theCall->getOpResults();
    assert(results.size() <= 1 && "multiple returns from a call");
 
    SourceLocRAIIObject loc{*this, clangLoc};
    emitAggregateStore(results[0], destPtr);
    return RValue::getAggregate(destPtr);
  }
  case cir::TEK_Scalar: {
    mlir::ResultRange results = theCall->getOpResults();
    assert(results.size() == 1 && "unexpected number of returns");
 
    // If the argument doesn't match, perform a bitcast to coerce it. This
    // can happen due to trivial type mismatches.
    if (results[0].getType() != retCIRTy)
      cgm.errorNYI(loc, "bitcast on function return value");
 
    mlir::Region *region = builder.getBlock()->getParent();
    if (region != theCall->getParentRegion())
      cgm.errorNYI(loc, "function calls with cleanup");
 
    return RValue::get(results[0]);
  }
  case cir::TEK_Complex: {
    mlir::ResultRange results = theCall->getOpResults();
    assert(!results.empty() &&
           "Expected at least one result for complex rvalue");
    return RValue::getComplex(results[0]);
  }
  }
  llvm_unreachable("Invalid evaluation kind");
}
 
void CallArg::copyInto(CIRGenFunction &cgf, Address addr,
                       mlir::Location loc) const {
  LValue dst = cgf.makeAddrLValue(addr, ty);
  if (!hasLV && rv.isScalar())
    cgf.cgm.errorNYI(loc, "copyInto scalar value");
  else if (!hasLV && rv.isComplex())
    cgf.emitStoreOfComplex(loc, rv.getComplexValue(), dst, /*isInit=*/true);
  else
    cgf.cgm.errorNYI(loc, "copyInto hasLV");
  isUsed = true;
}
 
mlir::Value CIRGenFunction::emitRuntimeCall(mlir::Location loc,
                                            cir::FuncOp callee,
                                            ArrayRef<mlir::Value> args,
                                            mlir::NamedAttrList attrs) {
 
  // TODO(cir): set the calling convention to this runtime call.
  assert(!cir::MissingFeatures::opFuncCallingConv());
 
  cir::CallOp call = builder.createCallOp(loc, callee, args);
  assert(call->getNumResults() <= 1 &&
         "runtime functions have at most 1 result");
 
  if (!attrs.empty())
    call->setAttrs(attrs);
 
  if (call->getNumResults() == 0)
    return nullptr;
 
  return call->getResult(0);
}
 
void CIRGenFunction::emitCallArg(CallArgList &args, const clang::Expr *e,
                                 clang::QualType argType) {
  assert(argType->isReferenceType() == e->isGLValue() &&
         "reference binding to unmaterialized r-value!");
 
  if (e->isGLValue()) {
    assert(e->getObjectKind() == OK_Ordinary);
    return args.add(emitReferenceBindingToExpr(e), argType);
  }
 
  bool hasAggregateEvalKind = hasAggregateEvaluationKind(argType);
 
  // For callee-destructed parameters (trivial_abi, MS ABI), create an
  // aggregate temp and let the callee destroy it.
  if (argType->isRecordType() &&
      argType->castAsRecordDecl()->isParamDestroyedInCallee()) {
    AggValueSlot slot = createAggTemp(argType, getLoc(e->getSourceRange()),
                                      getCounterAggTmpAsString());
 
    bool destroyedInCallee = true;
    if (const auto *rd = argType->getAsCXXRecordDecl())
      destroyedInCallee = rd->hasNonTrivialDestructor();
 
    if (destroyedInCallee)
      slot.setExternallyDestructed();
 
    emitAggExpr(e, slot);
    RValue rv = slot.asRValue();
    args.add(rv, argType);
 
    if (destroyedInCallee && getLangOpts().Exceptions)
      cgm.errorNYI(e->getSourceRange(),
                   "callee-destructed param with exceptions");
    return;
  }
 
  if (hasAggregateEvalKind && isa<ImplicitCastExpr>(e) &&
      cast<CastExpr>(e)->getCastKind() == CK_LValueToRValue) {
    LValue lv = emitLValue(cast<CastExpr>(e)->getSubExpr());
    assert(lv.isSimple());
    args.addUncopiedAggregate(lv, argType);
    return;
  }
 
  args.add(emitAnyExprToTemp(e), argType);
}
 
QualType CIRGenFunction::getVarArgType(const Expr *arg) {
  // System headers on Windows define NULL to 0 instead of 0LL on Win64. MSVC
  // implicitly widens null pointer constants that are arguments to varargs
  // functions to pointer-sized ints.
  if (!getTarget().getTriple().isOSWindows())
    return arg->getType();
 
  assert(!cir::MissingFeatures::msabi());
  cgm.errorNYI(arg->getSourceRange(), "getVarArgType: NYI for Windows target");
  return arg->getType();
}
 
/// Similar to emitAnyExpr(), however, the result will always be accessible
/// even if no aggregate location is provided.
RValue CIRGenFunction::emitAnyExprToTemp(const Expr *e) {
  AggValueSlot aggSlot = AggValueSlot::ignored();
 
  if (hasAggregateEvaluationKind(e->getType()))
    aggSlot = createAggTemp(e->getType(), getLoc(e->getSourceRange()),
                            getCounterAggTmpAsString());
 
  return emitAnyExpr(e, aggSlot);
}
 
void CIRGenFunction::emitCallArgs(
    CallArgList &args, PrototypeWrapper prototype,
    llvm::iterator_range<clang::CallExpr::const_arg_iterator> argRange,
    AbstractCallee callee, unsigned paramsToSkip) {
  llvm::SmallVector<QualType, 16> argTypes;
 
  assert(!cir::MissingFeatures::opCallCallConv());
 
  // First, if a prototype was provided, use those argument types.
  bool isVariadic = false;
  if (prototype.p) {
    assert(!cir::MissingFeatures::opCallObjCMethod());
 
    const auto *fpt = cast<const FunctionProtoType *>(prototype.p);
    isVariadic = fpt->isVariadic();
    assert(!cir::MissingFeatures::opCallCallConv());
    argTypes.assign(fpt->param_type_begin() + paramsToSkip,
                    fpt->param_type_end());
  }
 
  // If we still have any arguments, emit them using the type of the argument.
  for (const clang::Expr *a : llvm::drop_begin(argRange, argTypes.size()))
    argTypes.push_back(isVariadic ? getVarArgType(a) : a->getType());
  assert(argTypes.size() == (size_t)(argRange.end() - argRange.begin()));
 
  // We must evaluate arguments from right to left in the MS C++ ABI, because
  // arguments are destroyed left to right in the callee. As a special case,
  // there are certain language constructs taht require left-to-right
  // evaluation, and in those cases we consider the evaluation order requirement
  // to trump the "destruction order is reverse construction order" guarantee.
  auto leftToRight = true;
  assert(!cir::MissingFeatures::msabi());
 
  auto maybeEmitImplicitObjectSize = [&](size_t i, const Expr *arg,
                                         RValue emittedArg) {
    if (!callee.hasFunctionDecl() || i >= callee.getNumParams())
      return;
    auto *ps = callee.getParamDecl(i)->getAttr<PassObjectSizeAttr>();
    if (!ps)
      return;
 
    QualType sizeTy = getContext().getSizeType();
    assert(emittedArg.getValue() && "We emitted nothing for the arg?");
    mlir::Value v = evaluateOrEmitBuiltinObjectSize(
        arg, ps->getType(), cast<cir::IntType>(cgm.sizeTy),
        emittedArg.getValue(), ps->isDynamic());
    args.add(RValue::get(v), sizeTy);
    // When emitting right-to-left, the size arg was appended after the
    // pointer arg; swap them so the size follows the pointer in the final
    // argument list after the outer reverse.
    if (!leftToRight)
      std::iter_swap(args.rbegin(), std::next(args.rbegin()));
  };
 
  // Evaluate each argument in the appropriate order.
  size_t callArgsStart = args.size();
  for (size_t i = 0; i != argTypes.size(); ++i) {
    size_t idx = leftToRight ? i : argTypes.size() - i - 1;
    CallExpr::const_arg_iterator currentArg = argRange.begin() + idx;
    size_t initialArgSize = args.size();
 
    emitCallArg(args, *currentArg, argTypes[idx]);
 
    // In particular, we depend on it being the last arg in Args, and the
    // objectsize bits depend on there only being one arg if !LeftToRight.
    assert(initialArgSize + 1 == args.size() &&
           "The code below depends on only adding one arg per emitCallArg");
    (void)initialArgSize;
 
    // Since pointer argument are never emitted as LValue, it is safe to emit
    // non-null argument check for r-value only.
    if (!args.back().hasLValue()) {
      RValue rvArg = args.back().getKnownRValue();
      assert(!cir::MissingFeatures::sanitizers());
      maybeEmitImplicitObjectSize(idx, *currentArg, rvArg);
    }
 
    if (!leftToRight)
      std::reverse(args.begin() + callArgsStart, args.end());
  }
}
 
// FIXME(cir): This is identical to the version from classic-codegen, we should
// figure out how to move this to a common location.
bool CIRGenTypes::inheritingCtorHasParams(const InheritedConstructor &inherited,
                                          CXXCtorType type) {
  // Parameters are unnecessary if we're constructing a base class subobject
  // and the inherited constructor lives in a virtual base.
  return type == Ctor_Complete ||
         !inherited.getShadowDecl()->constructsVirtualBase() ||
         !getASTContext().getTargetInfo().getCXXABI().hasConstructorVariants();
}