CIRGenFunction.cpp

Go to the documentation of this file.

//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// Internal per-function state used for AST-to-ClangIR code gen
//
//===----------------------------------------------------------------------===//
 
#include "CIRGenFunction.h"
 
#include "CIRGenCXXABI.h"
#include "CIRGenCall.h"
#include "CIRGenValue.h"
#include "mlir/IR/Location.h"
#include "clang/AST/Attr.h"
#include "clang/AST/ExprCXX.h"
#include "clang/AST/GlobalDecl.h"
#include "clang/CIR/Dialect/IR/CIRDialect.h"
#include "clang/CIR/MissingFeatures.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/IR/FPEnv.h"
 
#include <cassert>
 
namespace clang::CIRGen {
 
CIRGenFunction::CIRGenFunction(CIRGenModule &cgm, CIRGenBuilderTy &builder,
                               bool suppressNewContext)
    : CIRGenTypeCache(cgm), cgm{cgm}, builder(builder) {
  ehStack.setCGF(this);
}
 
CIRGenFunction::~CIRGenFunction() {}
 
// This is copied from clang/lib/CodeGen/CodeGenFunction.cpp
cir::TypeEvaluationKind CIRGenFunction::getEvaluationKind(QualType type) {
  type = type.getCanonicalType();
  while (true) {
    switch (type->getTypeClass()) {
#define TYPE(name, parent)
#define ABSTRACT_TYPE(name, parent)
#define NON_CANONICAL_TYPE(name, parent) case Type::name:
#define DEPENDENT_TYPE(name, parent) case Type::name:
#define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(name, parent) case Type::name:
#include "clang/AST/TypeNodes.inc"
      llvm_unreachable("non-canonical or dependent type in IR-generation");
 
    case Type::Auto:
    case Type::DeducedTemplateSpecialization:
      llvm_unreachable("undeduced type in IR-generation");
 
    // Various scalar types.
    case Type::Builtin:
    case Type::Pointer:
    case Type::BlockPointer:
    case Type::LValueReference:
    case Type::RValueReference:
    case Type::MemberPointer:
    case Type::Vector:
    case Type::ExtVector:
    case Type::ConstantMatrix:
    case Type::FunctionProto:
    case Type::FunctionNoProto:
    case Type::Enum:
    case Type::ObjCObjectPointer:
    case Type::Pipe:
    case Type::BitInt:
    case Type::OverflowBehavior:
    case Type::HLSLAttributedResource:
    case Type::HLSLInlineSpirv:
      return cir::TEK_Scalar;
 
    // Complexes.
    case Type::Complex:
      return cir::TEK_Complex;
 
    // Arrays, records, and Objective-C objects.
    case Type::ConstantArray:
    case Type::IncompleteArray:
    case Type::VariableArray:
    case Type::Record:
    case Type::ObjCObject:
    case Type::ObjCInterface:
    case Type::ArrayParameter:
      return cir::TEK_Aggregate;
 
    // We operate on atomic values according to their underlying type.
    case Type::Atomic:
      type = cast<AtomicType>(type)->getValueType();
      continue;
    }
    llvm_unreachable("unknown type kind!");
  }
}
 
mlir::Type CIRGenFunction::convertTypeForMem(QualType t) {
  return cgm.getTypes().convertTypeForMem(t);
}
 
mlir::Type CIRGenFunction::convertType(QualType t) {
  return cgm.getTypes().convertType(t);
}
 
mlir::Location CIRGenFunction::getLoc(SourceLocation srcLoc) {
  // Some AST nodes might contain invalid source locations (e.g.
  // CXXDefaultArgExpr), workaround that to still get something out.
  if (srcLoc.isValid()) {
    const SourceManager &sm = getContext().getSourceManager();
    PresumedLoc pLoc = sm.getPresumedLoc(srcLoc);
    StringRef filename = pLoc.getFilename();
    return mlir::FileLineColLoc::get(builder.getStringAttr(filename),
                                     pLoc.getLine(), pLoc.getColumn());
  }
  // We expect to have a currSrcLoc set, so we assert here, but it isn't
  // critical for the correctness of compilation, so in non-assert builds
  // we fallback on using an unknown location.
  assert(currSrcLoc && "expected to inherit some source location");
  if (currSrcLoc)
    return *currSrcLoc;
  // We're brave, but time to give up.
  return builder.getUnknownLoc();
}
 
mlir::Location CIRGenFunction::getLoc(SourceRange srcLoc) {
  // Some AST nodes might contain invalid source locations (e.g.
  // CXXDefaultArgExpr), workaround that to still get something out.
  if (srcLoc.isValid()) {
    mlir::Location beg = getLoc(srcLoc.getBegin());
    mlir::Location end = getLoc(srcLoc.getEnd());
    SmallVector<mlir::Location, 2> locs = {beg, end};
    mlir::Attribute metadata;
    return mlir::FusedLoc::get(locs, metadata, &getMLIRContext());
  }
  // We expect to have a currSrcLoc set, so we assert here, but it isn't
  // critical for the correctness of compilation, so in non-assert builds
  // we fallback on using an unknown location.
  assert(currSrcLoc && "expected to inherit some source location");
  if (currSrcLoc)
    return *currSrcLoc;
  // We're brave, but time to give up.
  return builder.getUnknownLoc();
}
 
mlir::Location CIRGenFunction::getLoc(mlir::Location lhs, mlir::Location rhs) {
  SmallVector<mlir::Location, 2> locs = {lhs, rhs};
  mlir::Attribute metadata;
  return mlir::FusedLoc::get(locs, metadata, &getMLIRContext());
}
 
bool CIRGenFunction::containsLabel(const Stmt *s, bool ignoreCaseStmts) {
  // Null statement, not a label!
  if (!s)
    return false;
 
  // If this is a label, we have to emit the code, consider something like:
  // if (0) {  ...  foo:  bar(); }  goto foo;
  //
  // TODO: If anyone cared, we could track __label__'s, since we know that you
  // can't jump to one from outside their declared region.
  if (isa<LabelStmt>(s))
    return true;
 
  // If this is a case/default statement, and we haven't seen a switch, we
  // have to emit the code.
  if (isa<SwitchCase>(s) && !ignoreCaseStmts)
    return true;
 
  // If this is a switch statement, we want to ignore case statements when we
  // recursively process the sub-statements of the switch. If we haven't
  // encountered a switch statement, we treat case statements like labels, but
  // if we are processing a switch statement, case statements are expected.
  if (isa<SwitchStmt>(s))
    ignoreCaseStmts = true;
 
  // Scan subexpressions for verboten labels.
  return std::any_of(s->child_begin(), s->child_end(),
                     [=](const Stmt *subStmt) {
                       return containsLabel(subStmt, ignoreCaseStmts);
                     });
}
 
/// If the specified expression does not fold to a constant, or if it does but
/// contains a label, return false.  If it constant folds return true and set
/// the boolean result in Result.
bool CIRGenFunction::constantFoldsToBool(const Expr *cond, bool &resultBool,
                                         bool allowLabels) {
  llvm::APSInt resultInt;
  if (!constantFoldsToSimpleInteger(cond, resultInt, allowLabels))
    return false;
 
  resultBool = resultInt.getBoolValue();
  return true;
}
 
/// If the specified expression does not fold to a constant, or if it does
/// fold but contains a label, return false. If it constant folds, return
/// true and set the folded value.
bool CIRGenFunction::constantFoldsToSimpleInteger(const Expr *cond,
                                                  llvm::APSInt &resultInt,
                                                  bool allowLabels) {
  // FIXME: Rename and handle conversion of other evaluatable things
  // to bool.
  Expr::EvalResult result;
  if (!cond->EvaluateAsInt(result, getContext()))
    return false; // Not foldable, not integer or not fully evaluatable.
 
  llvm::APSInt intValue = result.Val.getInt();
  if (!allowLabels && containsLabel(cond))
    return false; // Contains a label.
 
  resultInt = intValue;
  return true;
}
 
void CIRGenFunction::emitAndUpdateRetAlloca(QualType type, mlir::Location loc,
                                            CharUnits alignment) {
  if (!type->isVoidType()) {
    Address allocaAddr = Address::invalid();
    returnValue = createMemTemp(type, alignment, loc, "__retval", &allocaAddr);
    fnRetAlloca = allocaAddr.getPointer();
  }
}
 
void CIRGenFunction::declare(mlir::Value addrVal, const Decl *var, QualType ty,
                             mlir::Location loc, CharUnits alignment,
                             bool isParam) {
  assert(isa<NamedDecl>(var) && "Needs a named decl");
  assert(!symbolTable.count(var) && "not supposed to be available just yet");
 
  Address addr(addrVal, alignment);
  cir::AllocaOp allocaOp = addr.getUnderlyingAllocaOp();
  assert(allocaOp && "expected cir::AllocaOp");
 
  if (isParam)
    allocaOp.setInitAttr(mlir::UnitAttr::get(&getMLIRContext()));
  if (ty->isReferenceType() || ty.isConstQualified())
    allocaOp.setConstantAttr(mlir::UnitAttr::get(&getMLIRContext()));
 
  symbolTable.insert(var, addrVal);
}
 
void CIRGenFunction::LexicalScope::cleanup() {
  CIRGenBuilderTy &builder = cgf.builder;
  LexicalScope *localScope = cgf.curLexScope;
 
  // Process all return blocks — emit cir.return ops.
  // TODO(cir): Handle returning from a switch statement through a cleanup
  // block. We can't simply jump to the cleanup block, because the cleanup block
  // is not part of the case region. Either reemit all cleanups in the return
  // block or wait for MLIR structured control flow to support early exits.
  llvm::SmallVector<mlir::Block *> retBlocks;
  for (mlir::Block *retBlock : localScope->getRetBlocks()) {
    mlir::OpBuilder::InsertionGuard guard(builder);
    builder.setInsertionPointToEnd(retBlock);
    retBlocks.push_back(retBlock);
    mlir::Location retLoc = localScope->getRetLoc(retBlock);
    emitReturn(retLoc);
  }
 
  // Pop cleanup scopes from the EH stack. In CIR, this emits cleanup code
  // into the cleanup regions of cir.cleanup.scope ops — no CFG-level cleanup
  // blocks or branches are needed.
  if (performCleanup) {
    assert(!cir::MissingFeatures::generateDebugInfo());
    forceCleanup();
  }
 
  mlir::Block *curBlock = builder.getBlock();
  if (isGlobalInit() && !curBlock)
    return;
  if (curBlock->mightHaveTerminator() && curBlock->getTerminator())
    return;
 
  // Get rid of any empty block at the end of the scope. An empty non-entry
  // block is created when a terminator (return/break/continue) is followed
  // by unreachable code.
  bool isEntryBlock = builder.getInsertionBlock()->isEntryBlock();
  if (!isEntryBlock && curBlock->empty()) {
    curBlock->erase();
    for (mlir::Block *retBlock : retBlocks) {
      if (retBlock->getUses().empty())
        retBlock->erase();
    }
    return;
  }
 
  if (localScope->depth == 0) {
    // Reached the end of the function.
    if (localScope->getRetBlocks().size() == 1) {
      mlir::Block *retBlock = localScope->getRetBlocks()[0];
      mlir::Location retLoc = localScope->getRetLoc(retBlock);
      if (retBlock->getUses().empty()) {
        retBlock->erase();
      } else {
        cir::BrOp::create(builder, retLoc, retBlock);
        return;
      }
    }
    emitImplicitReturn();
    return;
  }
 
  // End of any local scope != function.
  // Ternary ops have to deal with matching arms for yielding types
  // and do return a value, it must do its own cir.yield insertion.
  if (!localScope->isTernary() && !curBlock->mightHaveTerminator()) {
    !retVal ? cir::YieldOp::create(builder, localScope->endLoc)
            : cir::YieldOp::create(builder, localScope->endLoc, retVal);
  }
}
 
cir::ReturnOp CIRGenFunction::LexicalScope::emitReturn(mlir::Location loc) {
  CIRGenBuilderTy &builder = cgf.getBuilder();
 
  auto fn = dyn_cast<cir::FuncOp>(cgf.curFn);
  assert(fn && "emitReturn from non-function");
 
  if (!fn.getFunctionType().hasVoidReturn()) {
    // Load the value from `__retval` and return it via the `cir.return` op.
    auto value = cir::LoadOp::create(
        builder, loc, fn.getFunctionType().getReturnType(), *cgf.fnRetAlloca);
    return cir::ReturnOp::create(builder, loc,
                                 llvm::ArrayRef(value.getResult()));
  }
  return cir::ReturnOp::create(builder, loc);
}
 
// This is copied from CodeGenModule::MayDropFunctionReturn.  This is a
// candidate for sharing between CIRGen and CodeGen.
static bool mayDropFunctionReturn(const ASTContext &astContext,
                                  QualType returnType) {
  // We can't just discard the return value for a record type with a complex
  // destructor or a non-trivially copyable type.
  if (const auto *classDecl = returnType->getAsCXXRecordDecl())
    return classDecl->hasTrivialDestructor();
  return returnType.isTriviallyCopyableType(astContext);
}
 
static bool previousOpIsNonYieldingCleanup(mlir::Block *block) {
  if (block->empty())
    return false;
  mlir::Operation *op = &block->back();
  auto cleanupScopeOp = mlir::dyn_cast<cir::CleanupScopeOp>(op);
  if (!cleanupScopeOp)
    return false;
 
  // Check whether the body region of the cleanup scope exits via cir.yield.
  // Exits via cir.return or cir.goto do not fall through to the operation
  // following the cleanup scope, and exits via break, continue, and resume
  // are not expected here.
  for (mlir::Block &bodyBlock : cleanupScopeOp.getBodyRegion()) {
    if (bodyBlock.mightHaveTerminator()) {
      if (mlir::isa<cir::YieldOp>(bodyBlock.getTerminator()))
        return false;
      assert(!mlir::isa<cir::BreakOp>(bodyBlock.getTerminator()) &&
             !mlir::isa<cir::ContinueOp>(bodyBlock.getTerminator()) &&
             !mlir::isa<cir::ResumeOp>(bodyBlock.getTerminator()));
    }
  }
  return true;
}
 
void CIRGenFunction::LexicalScope::emitImplicitReturn() {
  CIRGenBuilderTy &builder = cgf.getBuilder();
  LexicalScope *localScope = cgf.curLexScope;
 
  const auto *fd = cast<clang::FunctionDecl>(cgf.curGD.getDecl());
 
  // In C++, flowing off the end of a non-void function is always undefined
  // behavior. In C, flowing off the end of a non-void function is undefined
  // behavior only if the non-existent return value is used by the caller.
  // That influences whether the terminating op is trap, unreachable, or
  // return.
  if (cgf.getLangOpts().CPlusPlus && !fd->hasImplicitReturnZero() &&
      !cgf.sawAsmBlock && !fd->getReturnType()->isVoidType() &&
      builder.getInsertionBlock() &&
      !previousOpIsNonYieldingCleanup(builder.getInsertionBlock())) {
    bool shouldEmitUnreachable =
        cgf.cgm.getCodeGenOpts().StrictReturn ||
        !mayDropFunctionReturn(fd->getASTContext(), fd->getReturnType());
 
    if (shouldEmitUnreachable) {
      assert(!cir::MissingFeatures::sanitizers());
      if (cgf.cgm.getCodeGenOpts().OptimizationLevel == 0)
        cir::TrapOp::create(builder, localScope->endLoc);
      else
        cir::UnreachableOp::create(builder, localScope->endLoc);
      builder.clearInsertionPoint();
      return;
    }
  }
 
  (void)emitReturn(localScope->endLoc);
}
 
cir::TryOp CIRGenFunction::LexicalScope::getClosestTryParent() {
  LexicalScope *scope = this;
  while (scope) {
    if (scope->isTry())
      return scope->getTry();
    scope = scope->parentScope;
  }
  return nullptr;
}
 
/// An argument came in as a promoted argument; demote it back to its
/// declared type.
static mlir::Value emitArgumentDemotion(CIRGenFunction &cgf, const VarDecl *var,
                                        mlir::Value value) {
  mlir::Type ty = cgf.convertType(var->getType());
 
  // This can happen with promotions that actually don't change the
  // underlying type, like the enum promotions.
  if (value.getType() == ty)
    return value;
 
  assert((mlir::isa<cir::IntType>(ty) || cir::isAnyFloatingPointType(ty)) &&
         "unexpected promotion type");
 
  if (mlir::isa<cir::IntType>(ty))
    return cgf.getBuilder().CIRBaseBuilderTy::createIntCast(value, ty);
 
  return cgf.getBuilder().createFloatingCast(value, ty);
}
 
void CIRGenFunction::emitFunctionProlog(const FunctionArgList &args,
                                        mlir::Block *entryBB,
                                        const FunctionDecl *fd,
                                        SourceLocation bodyBeginLoc) {
  // Naked functions don't have prologues.
  if (fd && fd->hasAttr<NakedAttr>()) {
    cgm.errorNYI(bodyBeginLoc, "naked function decl");
  }
 
  // Declare all the function arguments in the symbol table.
  for (const auto nameValue : llvm::zip(args, entryBB->getArguments())) {
    const VarDecl *paramVar = std::get<0>(nameValue);
    mlir::Value paramVal = std::get<1>(nameValue);
    CharUnits alignment = getContext().getDeclAlign(paramVar);
    mlir::Location paramLoc = getLoc(paramVar->getSourceRange());
    paramVal.setLoc(paramLoc);
 
    mlir::Value addrVal =
        emitAlloca(cast<NamedDecl>(paramVar)->getName(),
                   convertType(paramVar->getType()), paramLoc, alignment,
                   /*insertIntoFnEntryBlock=*/true);
 
    declare(addrVal, paramVar, paramVar->getType(), paramLoc, alignment,
            /*isParam=*/true);
 
    setAddrOfLocalVar(paramVar, Address(addrVal, alignment));
 
    bool isPromoted = isa<ParmVarDecl>(paramVar) &&
                      cast<ParmVarDecl>(paramVar)->isKNRPromoted();
    assert(!cir::MissingFeatures::constructABIArgDirectExtend());
    if (isPromoted)
      paramVal = emitArgumentDemotion(*this, paramVar, paramVal);
 
    // Location of the store to the param storage tracked as beginning of
    // the function body.
    mlir::Location fnBodyBegin = getLoc(bodyBeginLoc);
    builder.CIRBaseBuilderTy::createStore(fnBodyBegin, paramVal, addrVal);
  }
  assert(builder.getInsertionBlock() && "Should be valid");
}
 
void CIRGenFunction::startFunction(GlobalDecl gd, QualType returnType,
                                   cir::FuncOp fn, cir::FuncType funcType,
                                   FunctionArgList args, SourceLocation loc,
                                   SourceLocation startLoc) {
  assert(!curFn &&
         "CIRGenFunction can only be used for one function at a time");
 
  curFn = fn;
 
  const Decl *d = gd.getDecl();
 
  didCallStackSave = false;
  curCodeDecl = d;
  const auto *fd = dyn_cast_or_null<FunctionDecl>(d);
  curFuncDecl = (d ? d->getNonClosureContext() : nullptr);
 
  prologueCleanupDepth = ehStack.stable_begin();
 
  mlir::Block *entryBB = &fn.getBlocks().front();
  builder.setInsertionPointToStart(entryBB);
 
  // Determine the function body begin location for the prolog.
  // If fd is null or has no body, use startLoc as fallback.
  SourceLocation bodyBeginLoc = startLoc;
  if (fd) {
    if (Stmt *body = fd->getBody())
      bodyBeginLoc = body->getBeginLoc();
    else
      bodyBeginLoc = fd->getLocation();
  }
 
  emitFunctionProlog(args, entryBB, fd, bodyBeginLoc);
 
  // When the current function is not void, create an address to store the
  // result value.
  if (!returnType->isVoidType()) {
    // Determine the function body end location.
    // If fd is null or has no body, use loc as fallback.
    SourceLocation bodyEndLoc = loc;
    if (fd) {
      if (Stmt *body = fd->getBody())
        bodyEndLoc = body->getEndLoc();
      else
        bodyEndLoc = fd->getLocation();
    }
    emitAndUpdateRetAlloca(returnType, getLoc(bodyEndLoc),
                           getContext().getTypeAlignInChars(returnType));
 
    // If this is an implicit-return-zero function, initialize the return
    // value. This mirrors the implicit-return-zero handling in classic
    // codegen's EmitFunctionProlog (CGCall.cpp). It is done here, after
    // emitAndUpdateRetAlloca, because in CIR the return slot is created
    // after the prolog (the opposite of classic codegen, where ReturnValue
    // is set up before EmitFunctionProlog runs).
    // TODO(cir): Align prolog handling with classic codegen.
    if (fd && fd->hasImplicitReturnZero()) {
      mlir::Type cirRetTy = convertType(returnType.getUnqualifiedType());
      mlir::Location bodyBeginMLIRLoc = getLoc(bodyBeginLoc);
      mlir::Value zero = builder.getNullValue(cirRetTy, bodyBeginMLIRLoc);
      builder.CIRBaseBuilderTy::createStore(bodyBeginMLIRLoc, zero,
                                            returnValue.getPointer());
    }
  }
 
  // Only implicit-object member functions (without an explicit `this`
  // parameter) receive an implicit `this` argument that the CXXABI prolog has
  // to set up. C++23 explicit-object members (P0847R7) carry their object via a
  // regular parameter and use the standard parameter prolog instead.
  if (isa_and_nonnull<CXXMethodDecl>(d) &&
      cast<CXXMethodDecl>(d)->isImplicitObjectMemberFunction()) {
    cgm.getCXXABI().emitInstanceFunctionProlog(loc, *this);
 
    const auto *md = cast<CXXMethodDecl>(d);
    if (md->getParent()->isLambda() && md->getOverloadedOperator() == OO_Call) {
      // We're in a lambda.
      auto fn = dyn_cast<cir::FuncOp>(curFn);
      assert(fn && "lambda in non-function region");
      fn.setLambda(true);
 
      // Figure out the captures.
      md->getParent()->getCaptureFields(lambdaCaptureFields,
                                        lambdaThisCaptureField);
      if (lambdaThisCaptureField) {
        // If the lambda captures the object referred to by '*this' - either by
        // value or by reference, make sure CXXThisValue points to the correct
        // object.
 
        // Get the lvalue for the field (which is a copy of the enclosing object
        // or contains the address of the enclosing object).
        LValue thisFieldLValue =
            emitLValueForLambdaField(lambdaThisCaptureField);
        if (!lambdaThisCaptureField->getType()->isPointerType()) {
          // If the enclosing object was captured by value, just use its
          // address. Sign this pointer.
          cxxThisValue = thisFieldLValue.getPointer();
        } else {
          // Load the lvalue pointed to by the field, since '*this' was captured
          // by reference.
          cxxThisValue =
              emitLoadOfLValue(thisFieldLValue, SourceLocation()).getValue();
        }
      }
      for (auto *fd : md->getParent()->fields()) {
        if (fd->hasCapturedVLAType())
          cgm.errorNYI(loc, "lambda captured VLA type");
      }
    } else {
      // Not in a lambda; just use 'this' from the method.
      // FIXME: Should we generate a new load for each use of 'this'? The fast
      // register allocator would be happier...
      cxxThisValue = cxxabiThisValue;
    }
 
    assert(!cir::MissingFeatures::sanitizers());
    assert(!cir::MissingFeatures::emitTypeCheck());
  }
 
  // If any of the arguments have a variably modified type, make sure to
  // emit the type size, but only if the function is not naked. Naked functions
  // have no prolog to run this evaluation.
  if (!fd || !fd->hasAttr<NakedAttr>()) {
    for (const VarDecl *vd : args) {
      // Dig out the type as written from ParmVarDecls; it's unclear whether
      // the standard (C99 6.9.1p10) requires this, but we're following the
      // precedent set by gcc.
      QualType ty;
      if (const auto *pvd = dyn_cast<ParmVarDecl>(vd))
        ty = pvd->getOriginalType();
      else
        ty = vd->getType();
      if (ty->isVariablyModifiedType())
        emitVariablyModifiedType(ty);
    }
  }
}
 
void CIRGenFunction::finishIndirectBranch() {
  // The block is created on the first `goto *expr`, so if it is absent the
  // function has no indirect goto and nothing needs wiring -- a label whose
  // address is merely taken still emits its address constant on its own.
  if (!indirectGotoBlock)
    return;
 
  // Every label is emitted by now, so each address-taken label resolves to its
  // LabelOp.  A label may be named more than once (a dispatch table can list it
  // twice), but a block only needs to appear once in the successor list, so
  // duplicates are dropped.
  llvm::SmallVector<mlir::Block *> successors;
  llvm::SmallVector<mlir::ValueRange> rangeOperands;
  llvm::SmallPtrSet<mlir::Block *, 8> seen;
  for (cir::BlockAddrInfoAttr info : indirectGotoTargets) {
    cir::LabelOp labelOp = cgm.lookupBlockAddressInfo(info);
    assert(labelOp && "expected cir.label to be emitted for block address");
    mlir::Block *dest = labelOp->getBlock();
    if (!seen.insert(dest).second)
      continue;
    successors.push_back(dest);
    rangeOperands.push_back(dest->getArguments());
  }
 
  mlir::OpBuilder::InsertionGuard guard(builder);
  builder.setInsertionPointToEnd(indirectGotoBlock);
  cir::IndirectBrOp::create(builder, builder.getUnknownLoc(),
                            indirectGotoBlock->getArgument(0), false,
                            rangeOperands, successors);
  indirectGotoTargets.clear();
}
 
void CIRGenFunction::finishFunction(SourceLocation endLoc) {
  // Emit the indirect branch with all resolved label destinations.
  finishIndirectBranch();
 
  // Pop any cleanups that might have been associated with the
  // parameters.  Do this in whatever block we're currently in; it's
  // important to do this before we enter the return block or return
  // edges will be *really* confused.
  // TODO(cir): Use prologueCleanupDepth here.
  bool hasCleanups = ehStack.stable_begin() != prologueCleanupDepth;
  if (hasCleanups) {
    assert(!cir::MissingFeatures::generateDebugInfo());
    // FIXME(cir): should we clearInsertionPoint? breaks many testcases
    popCleanupBlocks(prologueCleanupDepth);
  }
 
  assert(deferredConditionalCleanupStack.empty() &&
         "deferred conditional cleanups were not consumed by a "
         "FullExprCleanupScope");
}
 
mlir::LogicalResult CIRGenFunction::emitFunctionBody(const clang::Stmt *body) {
  // We start with function level scope for variables.
  SymTableScopeTy varScope(symbolTable);
 
  if (const CompoundStmt *block = dyn_cast<CompoundStmt>(body))
    return emitCompoundStmtWithoutScope(*block);
 
  return emitStmt(body, /*useCurrentScope=*/true);
}
 
static void eraseEmptyAndUnusedBlocks(cir::FuncOp func) {
  // Remove any leftover blocks that are unreachable and empty, since they do
  // not represent unreachable code useful for warnings nor anything deemed
  // useful in general.
  SmallVector<mlir::Block *> blocksToDelete;
  for (mlir::Block &block : func.getBlocks()) {
    if (block.empty() && block.getUses().empty())
      blocksToDelete.push_back(&block);
  }
  for (mlir::Block *block : blocksToDelete)
    block->erase();
}
 
cir::FuncOp CIRGenFunction::generateCode(clang::GlobalDecl gd, cir::FuncOp fn,
                                         cir::FuncType funcType) {
  const auto *funcDecl = cast<FunctionDecl>(gd.getDecl());
  curGD = gd;
 
  if (funcDecl->isInlineBuiltinDeclaration()) {
    // When generating code for a builtin with an inline declaration, use a
    // mangled name to hold the actual body, while keeping an external
    // declaration in case the function pointer is referenced somewhere.
    std::string fdInlineName = (cgm.getMangledName(funcDecl) + ".inline").str();
    cir::FuncOp clone =
        mlir::cast_or_null<cir::FuncOp>(cgm.getGlobalValue(fdInlineName));
    if (!clone) {
      mlir::OpBuilder::InsertionGuard guard(builder);
      builder.setInsertionPoint(fn);
      clone = cir::FuncOp::create(builder, fn.getLoc(), fdInlineName,
                                  fn.getFunctionType());
      cgm.insertGlobalSymbol(clone);
      clone.setLinkage(cir::GlobalLinkageKind::InternalLinkage);
      clone.setSymVisibility("private");
      clone.setInlineKind(cir::InlineKind::AlwaysInline);
    }
    fn.setLinkage(cir::GlobalLinkageKind::ExternalLinkage);
    fn.setSymVisibility("private");
    fn = clone;
  } else {
    // Detect the unusual situation where an inline version is shadowed by a
    // non-inline version. In that case we should pick the external one
    // everywhere. That's GCC behavior too.
    for (const FunctionDecl *pd = funcDecl->getPreviousDecl(); pd;
         pd = pd->getPreviousDecl()) {
      if (LLVM_UNLIKELY(pd->isInlineBuiltinDeclaration())) {
        std::string inlineName = funcDecl->getName().str() + ".inline";
        if (auto inlineFn = mlir::cast_or_null<cir::FuncOp>(
                cgm.getGlobalValue(inlineName))) {
          // Replace all uses of the .inline function with the regular function
          // FIXME: This performs a linear walk over the module. Introduce some
          // caching here.
          if (inlineFn
                  .replaceAllSymbolUses(fn.getSymNameAttr(), cgm.getModule())
                  .failed())
            llvm_unreachable("Failed to replace inline builtin symbol uses");
          cgm.eraseGlobalSymbol(inlineFn);
          inlineFn.erase();
        }
        break;
      }
    }
  }
 
  SourceLocation loc = funcDecl->getLocation();
  Stmt *body = funcDecl->getBody();
  SourceRange bodyRange =
      body ? body->getSourceRange() : funcDecl->getLocation();
 
  SourceLocRAIIObject fnLoc{*this, loc.isValid() ? getLoc(loc)
                                                 : builder.getUnknownLoc()};
 
  auto validMLIRLoc = [&](clang::SourceLocation clangLoc) {
    return clangLoc.isValid() ? getLoc(clangLoc) : builder.getUnknownLoc();
  };
  const mlir::Location fusedLoc = mlir::FusedLoc::get(
      &getMLIRContext(),
      {validMLIRLoc(bodyRange.getBegin()), validMLIRLoc(bodyRange.getEnd())});
  mlir::Block *entryBB = fn.addEntryBlock();
 
  FunctionArgList args;
  QualType retTy = buildFunctionArgList(gd, args);
 
  // Create a scope in the symbol table to hold variable declarations.
  SymTableScopeTy varScope(symbolTable);
  {
    LexicalScope lexScope(*this, fusedLoc, entryBB);
 
    // Emit the standard function prologue.
    startFunction(gd, retTy, fn, funcType, args, loc, bodyRange.getBegin());
    if (funcDecl->UsesFPIntrin() || funcDecl->hasAttr<StrictFPAttr>()) {
      cgm.errorNYI(loc, "STDC FENV_ACCESS");
      return fn;
    }
 
    // Save parameters for coroutine function.
    if (body && isa_and_nonnull<CoroutineBodyStmt>(body))
      llvm::append_range(fnArgs, funcDecl->parameters());
 
    if (isa<CXXDestructorDecl>(funcDecl)) {
      emitDestructorBody(args);
    } else if (isa<CXXConstructorDecl>(funcDecl)) {
      emitConstructorBody(args);
    } else if (getLangOpts().CUDA && !getLangOpts().CUDAIsDevice &&
               funcDecl->hasAttr<CUDAGlobalAttr>()) {
      cgm.getCUDARuntime().emitDeviceStub(*this, fn, args);
    } else if (isa<CXXMethodDecl>(funcDecl) &&
               cast<CXXMethodDecl>(funcDecl)->isLambdaStaticInvoker()) {
      // The lambda static invoker function is special, because it forwards or
      // clones the body of the function call operator (but is actually
      // static).
      emitLambdaStaticInvokeBody(cast<CXXMethodDecl>(funcDecl));
    } else if (funcDecl->isDefaulted() && isa<CXXMethodDecl>(funcDecl) &&
               (cast<CXXMethodDecl>(funcDecl)->isCopyAssignmentOperator() ||
                cast<CXXMethodDecl>(funcDecl)->isMoveAssignmentOperator())) {
      // Implicit copy-assignment gets the same special treatment as implicit
      // copy-constructors.
      emitImplicitAssignmentOperatorBody(args);
    } else if (body) {
      // Emit standard function body.
      if (mlir::failed(emitFunctionBody(body))) {
        return nullptr;
      }
    } else {
      // Anything without a body should have been handled above.
      llvm_unreachable("no definition for normal function");
    }
 
    if (mlir::failed(fn.verifyBody()))
      return nullptr;
 
    finishFunction(bodyRange.getEnd());
  }
 
  eraseEmptyAndUnusedBlocks(fn);
  return fn;
}
 
void CIRGenFunction::emitConstructorBody(FunctionArgList &args) {
  assert(!cir::MissingFeatures::sanitizers());
  const auto *ctor = cast<CXXConstructorDecl>(curGD.getDecl());
  CXXCtorType ctorType = curGD.getCtorType();
 
  assert((cgm.getTarget().getCXXABI().hasConstructorVariants() ||
          ctorType == Ctor_Complete) &&
         "can only generate complete ctor for this ABI");
 
  cgm.setCXXSpecialMemberAttr(cast<cir::FuncOp>(curFn), ctor);
 
  if (ctorType == Ctor_Complete && isConstructorDelegationValid(ctor) &&
      cgm.getTarget().getCXXABI().hasConstructorVariants()) {
    emitDelegateCXXConstructorCall(ctor, Ctor_Base, args, ctor->getEndLoc());
    return;
  }
 
  const FunctionDecl *definition = nullptr;
  Stmt *body = ctor->getBody(definition);
  assert(definition == ctor && "emitting wrong constructor body");
 
  bool isTryBody = isa_and_nonnull<CXXTryStmt>(body);
 
  // A type that handles the emission of the constructor body, that can be
  // called directly for cases where we don't have a try-body, or passed to
  // emitCXXTryStmt.
  struct ctorTryBodyEmitter final : cxxTryBodyEmitter {
    const CXXConstructorDecl *ctor = nullptr;
    CXXCtorType ctorType;
    FunctionArgList &args;
    Stmt *emitterBody = nullptr;
    ctorTryBodyEmitter(const CXXConstructorDecl *ctor, CXXCtorType ctorType,
                       FunctionArgList &args, bool isTryBody, Stmt *b)
        : ctor(ctor), ctorType(ctorType), args(args),
          emitterBody(isTryBody ? cast<CXXTryStmt>(b)->getTryBlock() : b) {}
    ~ctorTryBodyEmitter() override = default;
 
    mlir::LogicalResult operator()(CIRGenFunction &cgf) override {
      assert(!cir::MissingFeatures::incrementProfileCounter());
      assert(!cir::MissingFeatures::runCleanupsScope());
 
      //// TODO: in restricted cases, we can emit the vbase initializers of a
      //// complete ctor and then delegate to the base ctor.
 
      cgf.emitCtorPrologue(ctor, ctorType, args);
      return cgf.emitStmt(emitterBody, /*useCurrentScope=*/true);
    }
  };
 
  ctorTryBodyEmitter emitter{ctor, ctorType, args, isTryBody, body};
  mlir::LogicalResult bodyRes =
      isTryBody ? emitCXXTryStmt(*cast<CXXTryStmt>(body), emitter)
                : emitter(*this);
 
  // TODO(cir): propagate this result via mlir::logical result. Just
  // unreachable now just to have it handled.
  if (bodyRes.failed())
    cgm.errorNYI(ctor->getSourceRange(),
                 "emitConstructorBody: emit body statement failed.");
}
 
/// Emits the body of the current destructor.
void CIRGenFunction::emitDestructorBody(FunctionArgList &args) {
  const CXXDestructorDecl *dtor = cast<CXXDestructorDecl>(curGD.getDecl());
  CXXDtorType dtorType = curGD.getDtorType();
 
  cgm.setCXXSpecialMemberAttr(cast<cir::FuncOp>(curFn), dtor);
 
  // For an abstract class, non-base destructors are never used (and can't
  // be emitted in general, because vbase dtors may not have been validated
  // by Sema), but the Itanium ABI doesn't make them optional and Clang may
  // in fact emit references to them from other compilations, so emit them
  // as functions containing a trap instruction.
  if (dtorType != Dtor_Base && dtor->getParent()->isAbstract()) {
    SourceLocation loc =
        dtor->hasBody() ? dtor->getBody()->getBeginLoc() : dtor->getLocation();
    emitTrap(getLoc(loc), true);
    return;
  }
 
  Stmt *body = dtor->getBody();
  assert(body && !cir::MissingFeatures::incrementProfileCounter());
 
  // The call to operator delete in a deleting destructor happens
  // outside of the function-try-block, which means it's always
  // possible to delegate the destructor body to the complete
  // destructor.  Do so.
  if (dtorType == Dtor_Deleting || dtorType == Dtor_VectorDeleting) {
    if (cxxStructorImplicitParamValue && dtorType == Dtor_VectorDeleting)
      cgm.errorNYI(dtor->getSourceRange(), "emitConditionalArrayDtorCall");
    RunCleanupsScope dtorEpilogue(*this);
    enterDtorCleanups(dtor, Dtor_Deleting);
    if (haveInsertPoint()) {
      QualType thisTy = dtor->getFunctionObjectParameterType();
      emitCXXDestructorCall(dtor, Dtor_Complete, /*forVirtualBase=*/false,
                            /*delegating=*/false, loadCXXThisAddress(), thisTy);
    }
    return;
  }
 
  // If the body is a function-try-block, enter the try before
  // anything else.
  const bool isTryBody = isa_and_nonnull<CXXTryStmt>(body);
  if (isTryBody)
    cgm.errorNYI(dtor->getSourceRange(), "function-try-block destructor");
 
  assert(!cir::MissingFeatures::sanitizers());
 
  // Enter the epilogue cleanups.
  RunCleanupsScope dtorEpilogue(*this);
 
  // If this is the complete variant, just invoke the base variant;
  // the epilogue will destruct the virtual bases.  But we can't do
  // this optimization if the body is a function-try-block, because
  // we'd introduce *two* handler blocks.  In the Microsoft ABI, we
  // always delegate because we might not have a definition in this TU.
  switch (dtorType) {
  case Dtor_Unified:
    llvm_unreachable("not expecting a unified dtor");
  case Dtor_Comdat:
    llvm_unreachable("not expecting a COMDAT");
  case Dtor_Deleting:
  case Dtor_VectorDeleting:
    llvm_unreachable("already handled deleting case");
 
  case Dtor_Complete:
    assert((body || getTarget().getCXXABI().isMicrosoft()) &&
           "can't emit a dtor without a body for non-Microsoft ABIs");
 
    // Enter the cleanup scopes for virtual bases.
    enterDtorCleanups(dtor, Dtor_Complete);
 
    if (!isTryBody) {
      QualType thisTy = dtor->getFunctionObjectParameterType();
      emitCXXDestructorCall(dtor, Dtor_Base, /*forVirtualBase=*/false,
                            /*delegating=*/false, loadCXXThisAddress(), thisTy);
      break;
    }
 
    // Fallthrough: act like we're in the base variant.
    [[fallthrough]];
 
  case Dtor_Base:
    assert(body);
 
    // Enter the cleanup scopes for fields and non-virtual bases.
    enterDtorCleanups(dtor, Dtor_Base);
 
    assert(!cir::MissingFeatures::vtableInitialization());
 
    if (isTryBody) {
      cgm.errorNYI(dtor->getSourceRange(), "function-try-block destructor");
    } else if (body) {
      (void)emitStmt(body, /*useCurrentScope=*/true);
    } else {
      assert(dtor->isImplicit() && "bodyless dtor not implicit");
      // nothing to do besides what's in the epilogue
    }
    // -fapple-kext must inline any call to this dtor into
    // the caller's body.
    assert(!cir::MissingFeatures::appleKext());
 
    break;
  }
 
  // Jump out through the epilogue cleanups.
  dtorEpilogue.forceCleanup();
 
  // Exit the try if applicable.
  if (isTryBody)
    cgm.errorNYI(dtor->getSourceRange(), "function-try-block destructor");
}
 
/// Given a value of type T* that may not be to a complete object, construct
/// an l-vlaue withi the natural pointee alignment of T.
LValue CIRGenFunction::makeNaturalAlignPointeeAddrLValue(mlir::Value val,
                                                         QualType ty) {
  // FIXME(cir): is it safe to assume Op->getResult(0) is valid? Perhaps
  // assert on the result type first.
  LValueBaseInfo baseInfo;
  assert(!cir::MissingFeatures::opTBAA());
  CharUnits align = cgm.getNaturalTypeAlignment(ty, &baseInfo);
  return makeAddrLValue(Address(val, align), ty, baseInfo);
}
 
LValue CIRGenFunction::makeNaturalAlignAddrLValue(mlir::Value val,
                                                  QualType ty) {
  LValueBaseInfo baseInfo;
  CharUnits alignment = cgm.getNaturalTypeAlignment(ty, &baseInfo);
  Address addr(val, convertTypeForMem(ty), alignment);
  assert(!cir::MissingFeatures::opTBAA());
  return makeAddrLValue(addr, ty, baseInfo);
}
 
// Map the LangOption for exception behavior into the corresponding enum in
// the IR.
static llvm::fp::ExceptionBehavior
toConstrainedExceptMd(LangOptions::FPExceptionModeKind kind) {
  switch (kind) {
  case LangOptions::FPE_Ignore:
    return llvm::fp::ebIgnore;
  case LangOptions::FPE_MayTrap:
    return llvm::fp::ebMayTrap;
  case LangOptions::FPE_Strict:
    return llvm::fp::ebStrict;
  case LangOptions::FPE_Default:
    llvm_unreachable("expected explicitly initialized exception behavior");
  }
  llvm_unreachable("unsupported FP exception behavior");
}
 
clang::QualType CIRGenFunction::buildFunctionArgList(clang::GlobalDecl gd,
                                                     FunctionArgList &args) {
  const auto *fd = cast<FunctionDecl>(gd.getDecl());
  QualType retTy = fd->getReturnType();
 
  // Only implicit-object member functions need the CXXABI-supplied `this`
  // parameter prepended to the arg list.  Explicit-object members carry the
  // object as a regular parameter that fd->parameters() already enumerates.
  const auto *md = dyn_cast<CXXMethodDecl>(fd);
  if (md && md->isImplicitObjectMemberFunction()) {
    if (cgm.getCXXABI().hasThisReturn(gd))
      cgm.errorNYI(fd->getSourceRange(), "this return");
    else if (cgm.getCXXABI().hasMostDerivedReturn(gd))
      cgm.errorNYI(fd->getSourceRange(), "most derived return");
    cgm.getCXXABI().buildThisParam(*this, args);
  }
 
  bool passedParams = true;
  if (const auto *cd = dyn_cast<CXXConstructorDecl>(fd))
    if (auto inherited = cd->getInheritedConstructor())
      passedParams =
          getTypes().inheritingCtorHasParams(inherited, gd.getCtorType());
 
  if (passedParams) {
    for (auto *param : fd->parameters()) {
      args.push_back(param);
      if (!param->hasAttr<PassObjectSizeAttr>())
        continue;
 
      auto *implicit = ImplicitParamDecl::Create(
          getContext(), param->getDeclContext(), param->getLocation(),
          /*Id=*/nullptr, getContext().getSizeType(), ImplicitParamKind::Other);
      sizeArguments[param] = implicit;
      args.push_back(implicit);
    }
  }
 
  if (md && (isa<CXXConstructorDecl>(md) || isa<CXXDestructorDecl>(md)))
    cgm.getCXXABI().addImplicitStructorParams(*this, retTy, args);
 
  return retTy;
}
 
LValue CIRGenFunction::emitInitListLValue(const InitListExpr *e) {
  // Initializing an aggregate temporary in C++11: T{...}.
  if (!e->isGLValue())
    return emitAggExprToLValue(e);
 
  // An lvalue initializer list must be initializing a reference.
  assert(e->isTransparent() && "non-transparent glvalue init list");
  return emitLValue(e->getInit(0));
}
 
static std::variant<LValue, RValue>
emitPseudoObjectExpr(CIRGenFunction &cgf, const PseudoObjectExpr *e,
                     bool forLValue, AggValueSlot slot) {
  using OVMD = CIRGenFunction::OpaqueValueMappingData;
  SmallVector<OVMD> opaques;
  llvm::scope_exit opaque_cleanup{
      [&]() { llvm::for_each(opaques, [&](OVMD &o) { o.unbind(cgf); }); }};
 
  // Find the result expression, if any.
  const Expr *resultExpr = e->getResultExpr();
  std::variant<LValue, RValue> result;
 
  for (const Expr *semantic : e->semantics()) {
    // If this semantic expression is an opaque value, bind it
    // to the result of its source expression.
    if (const auto *ov = dyn_cast<OpaqueValueExpr>(semantic)) {
 
      // Skip unique OVEs.
      if (ov->isUnique()) {
        assert(ov != resultExpr &&
               "A unique OVE cannot be used as the result expression");
        continue;
      }
 
      // If this is the result expression, we may need to evaluate
      // directly into the slot.
      OVMD opaqueData;
      if (ov == resultExpr && ov->isPRValue() && !forLValue &&
          CIRGenFunction::hasAggregateEvaluationKind(ov->getType())) {
        cgf.cgm.errorNYI(e->getSourceRange(),
                         "emitPseudoObjectExpr for RValue & aggregate kind");
      } else {
        opaqueData = OVMD::bind(cgf, ov, ov->getSourceExpr());
 
        // If this is the result, also evaluate the result now.
        if (ov == resultExpr) {
          // FIXME: This doesn't really affect anything, but I cannot find a
          // test for this, so leave an ErrorNYI here until we can find one.
          cgf.cgm.errorNYI(e->getSourceRange(),
                           "emitPseudoObjectExpr as result");
          if (forLValue)
            result = cgf.emitLValue(ov);
          else
            cgf.cgm.errorNYI(e->getSourceRange(),
                             "emitPseudoObjectExpr as an RValue");
        }
      }
      opaques.push_back(opaqueData);
    } else if (semantic == resultExpr) {
      // Otherwise, if the expression is the result, evaluate it
      // and remember the result.
      if (forLValue)
        result = cgf.emitLValue(semantic);
      else
        result = cgf.emitAnyExpr(semantic, slot);
    } else {
      // FIXME: best I can tell, this is only reachable as an r-value, so this
      // isn't properly tested.
      cgf.cgm.errorNYI(e->getSourceRange(),
                       "emitPseudoObjectExpr as an ignored value");
      // Otherwise, evaluate the expression in an ignored context.
      cgf.emitIgnoredExpr(semantic);
    }
  }
 
  return result;
}
 
RValue CIRGenFunction::emitPseudoObjectRValue(const PseudoObjectExpr *e,
                                              AggValueSlot slot) {
  return std::get<RValue>(
      emitPseudoObjectExpr(*this, e, /*forLValue=*/false, slot));
}
 
LValue CIRGenFunction::emitPseudoObjectLValue(const PseudoObjectExpr *e) {
  return std::get<LValue>(emitPseudoObjectExpr(*this, e, /*forLValue=*/true,
                                               AggValueSlot::ignored()));
}
 
/// Emit code to compute a designator that specifies the location
/// of the expression.
/// FIXME: document this function better.
LValue CIRGenFunction::emitLValue(const Expr *e) {
  assert(!cir::MissingFeatures::generateDebugInfo());
  switch (e->getStmtClass()) {
  default:
    getCIRGenModule().errorNYI(e->getSourceRange(),
                               "emitLValue: unsupported l-value class");
    return LValue();
 
  case Expr::ObjCPropertyRefExprClass:
    llvm_unreachable("cannot emit a property reference directly");
 
  case Expr::ObjCSelectorExprClass:
    getCIRGenModule().errorNYI(e->getSourceRange(),
                               "emitLValue: ObjCSelectorExpr");
    return LValue();
  case Expr::ObjCIsaExprClass:
    getCIRGenModule().errorNYI(e->getSourceRange(), "emitLValue: ObjCIsaExpr");
    return LValue();
  case Expr::BinaryOperatorClass:
    return emitBinaryOperatorLValue(cast<BinaryOperator>(e));
  case Expr::CompoundAssignOperatorClass: {
    QualType ty = e->getType();
    if (const AtomicType *at = ty->getAs<AtomicType>())
      ty = at->getValueType();
    if (!ty->isAnyComplexType())
      return emitCompoundAssignmentLValue(cast<CompoundAssignOperator>(e));
 
    return emitComplexCompoundAssignmentLValue(cast<CompoundAssignOperator>(e));
  }
  case Expr::CallExprClass:
  case Expr::CXXMemberCallExprClass:
  case Expr::CXXOperatorCallExprClass:
  case Expr::UserDefinedLiteralClass:
    return emitCallExprLValue(cast<CallExpr>(e));
  case Expr::CXXRewrittenBinaryOperatorClass:
    getCIRGenModule().errorNYI(e->getSourceRange(),
                               "emitLValue: CXXRewrittenBinaryOperator");
    return LValue();
  case Expr::VAArgExprClass:
    getCIRGenModule().errorNYI(e->getSourceRange(), "emitLValue: VAArgExpr");
    return LValue();
  case Expr::DeclRefExprClass:
    return emitDeclRefLValue(cast<DeclRefExpr>(e));
  case Expr::ConstantExprClass:
    getCIRGenModule().errorNYI(e->getSourceRange(), "emitLValue: ConstantExpr");
    return LValue();
  case Expr::ParenExprClass:
    return emitLValue(cast<ParenExpr>(e)->getSubExpr());
  case Expr::GenericSelectionExprClass:
    return emitLValue(cast<GenericSelectionExpr>(e)->getResultExpr());
  case Expr::PredefinedExprClass:
    return emitPredefinedLValue(cast<PredefinedExpr>(e));
  case Expr::StringLiteralClass:
    return emitStringLiteralLValue(cast<StringLiteral>(e));
  case Expr::ObjCEncodeExprClass:
    getCIRGenModule().errorNYI(e->getSourceRange(),
                               "emitLValue: ObjCEncodeExpr");
    return LValue();
  case Expr::PseudoObjectExprClass:
    return emitPseudoObjectLValue(cast<PseudoObjectExpr>(e));
  case Expr::InitListExprClass:
    return emitInitListLValue(cast<InitListExpr>(e));
  case Expr::CXXTemporaryObjectExprClass:
  case Expr::CXXConstructExprClass:
    return emitCXXConstructLValue(cast<CXXConstructExpr>(e));
  case Expr::CXXBindTemporaryExprClass:
    return emitCXXBindTemporaryLValue(cast<CXXBindTemporaryExpr>(e));
  case Expr::CXXUuidofExprClass:
    getCIRGenModule().errorNYI(e->getSourceRange(),
                               "emitLValue: CXXUuidofExpr");
    return LValue();
  case Expr::LambdaExprClass:
    getCIRGenModule().errorNYI(e->getSourceRange(), "emitLValue: LambdaExpr");
    return LValue();
  case Expr::ExprWithCleanupsClass: {
    const auto *cleanups = cast<ExprWithCleanups>(e);
    FullExprCleanupScope scope(*this, cleanups->getSubExpr());
    LValue lv = emitLValue(cleanups->getSubExpr());
    if (lv.isSimple()) {
      // Defend against branches out of gnu statement expressions surrounded by
      // cleanups.
      Address addr = lv.getAddress();
      mlir::Value v = addr.getPointer();
      scope.exit({&v});
      return LValue::makeAddr(addr.withPointer(v), lv.getType(),
                              lv.getBaseInfo());
    }
    // FIXME: Is it possible to create an ExprWithCleanups that produces a
    // bitfield lvalue or some other non-simple lvalue?
    return lv;
  }
  case Expr::CXXDefaultArgExprClass: {
    auto *dae = cast<CXXDefaultArgExpr>(e);
    CXXDefaultArgExprScope scope(*this, dae);
    return emitLValue(dae->getExpr());
  }
  case Expr::CXXDefaultInitExprClass: {
    auto *die = cast<CXXDefaultInitExpr>(e);
    CXXDefaultInitExprScope scope(*this, die);
    return emitLValue(die->getExpr());
  }
  case Expr::CXXTypeidExprClass:
    return emitCXXTypeidLValue(cast<CXXTypeidExpr>(e));
  case Expr::ObjCMessageExprClass:
    getCIRGenModule().errorNYI(e->getSourceRange(),
                               "emitLValue: ObjCMessageExpr");
    return LValue();
  case Expr::ObjCIvarRefExprClass:
    getCIRGenModule().errorNYI(e->getSourceRange(),
                               "emitLValue: ObjCIvarRefExpr");
    return LValue();
  case Expr::StmtExprClass:
    getCIRGenModule().errorNYI(e->getSourceRange(), "emitLValue: StmtExpr");
    return LValue();
  case Expr::UnaryOperatorClass:
    return emitUnaryOpLValue(cast<UnaryOperator>(e));
  case Expr::ArraySubscriptExprClass:
    return emitArraySubscriptExpr(cast<ArraySubscriptExpr>(e));
  case Expr::MatrixSingleSubscriptExprClass:
    getCIRGenModule().errorNYI(e->getSourceRange(),
                               "emitLValue: MatrixSingleSubscriptExpr");
    return LValue();
  case Expr::MatrixSubscriptExprClass:
    getCIRGenModule().errorNYI(e->getSourceRange(),
                               "emitLValue: MatrixSubscriptExpr");
    return LValue();
  case Expr::ArraySectionExprClass:
    getCIRGenModule().errorNYI(e->getSourceRange(),
                               "emitLValue: ArraySectionExpr");
    return LValue();
  case Expr::ExtVectorElementExprClass:
    return emitExtVectorElementExpr(cast<ExtVectorElementExpr>(e));
  case Expr::MatrixElementExprClass:
    getCIRGenModule().errorNYI(e->getSourceRange(),
                               "emitLValue: MatrixElementExpr");
    return LValue();
  case Expr::CXXThisExprClass:
    getCIRGenModule().errorNYI(e->getSourceRange(), "emitLValue: CXXThisExpr");
    return LValue();
  case Expr::MemberExprClass:
    return emitMemberExpr(cast<MemberExpr>(e));
  case Expr::CompoundLiteralExprClass:
    return emitCompoundLiteralLValue(cast<CompoundLiteralExpr>(e));
  case Expr::ConditionalOperatorClass:
    return emitConditionalOperatorLValue(cast<ConditionalOperator>(e));
  case Expr::BinaryConditionalOperatorClass:
    return emitConditionalOperatorLValue(cast<BinaryConditionalOperator>(e));
  case Expr::ChooseExprClass:
    return emitLValue(cast<ChooseExpr>(e)->getChosenSubExpr());
  case Expr::OpaqueValueExprClass:
    return emitOpaqueValueLValue(cast<OpaqueValueExpr>(e));
  case Expr::SubstNonTypeTemplateParmExprClass:
    return emitLValue(cast<SubstNonTypeTemplateParmExpr>(e)->getReplacement());
  case Expr::ImplicitCastExprClass:
  case Expr::CStyleCastExprClass:
  case Expr::CXXFunctionalCastExprClass:
  case Expr::CXXStaticCastExprClass:
  case Expr::CXXDynamicCastExprClass:
  case Expr::CXXReinterpretCastExprClass:
  case Expr::CXXConstCastExprClass:
    return emitCastLValue(cast<CastExpr>(e));
  case Expr::CXXAddrspaceCastExprClass:
  case Expr::ObjCBridgedCastExprClass:
    // TODO(cir): These can just be moved into the cast handling above, but
    //            they need test cases.
    getCIRGenModule().errorNYI(e->getSourceRange(),
                               "emitLValue: addrspace or ObjC bridged cast");
    return LValue();
  case Expr::MaterializeTemporaryExprClass:
    return emitMaterializeTemporaryExpr(cast<MaterializeTemporaryExpr>(e));
  case Expr::CoawaitExprClass:
    getCIRGenModule().errorNYI(e->getSourceRange(), "emitLValue: CoawaitExpr");
    return LValue();
  case Expr::CoyieldExprClass:
    getCIRGenModule().errorNYI(e->getSourceRange(), "emitLValue: CoyieldExpr");
    return LValue();
  case Expr::PackIndexingExprClass:
    getCIRGenModule().errorNYI(e->getSourceRange(),
                               "emitLValue: PackIndexingExpr");
    return LValue();
  case Expr::HLSLOutArgExprClass:
    llvm_unreachable("cannot emit a HLSL out argument directly");
  }
}
 
static std::string getVersionedTmpName(llvm::StringRef name, unsigned cnt) {
  SmallString<256> buffer;
  llvm::raw_svector_ostream out(buffer);
  out << name << cnt;
  return std::string(out.str());
}
 
std::string CIRGenFunction::getCounterRefTmpAsString() {
  return getVersionedTmpName("ref.tmp", counterRefTmp++);
}
 
std::string CIRGenFunction::getCounterAggTmpAsString() {
  return getVersionedTmpName("agg.tmp", counterAggTmp++);
}
 
void CIRGenFunction::emitNullInitialization(mlir::Location loc, Address destPtr,
                                            QualType ty) {
  // Ignore empty classes in C++.
  if (getLangOpts().CPlusPlus)
    if (const auto *rd = ty->getAsCXXRecordDecl(); rd && rd->isEmpty())
      return;
 
  // Cast the dest ptr to the appropriate i8 pointer type.
  if (builder.isInt8Ty(destPtr.getElementType())) {
    cgm.errorNYI(loc, "Cast the dest ptr to the appropriate i8 pointer type");
  }
 
  // Get size and alignment info for this aggregate.
  const CharUnits size = getContext().getTypeSizeInChars(ty);
  if (size.isZero()) {
    // But note that getTypeInfo returns 0 for a VLA.
    if (isa<VariableArrayType>(getContext().getAsArrayType(ty))) {
      cgm.errorNYI(loc,
                   "emitNullInitialization for zero size VariableArrayType");
    } else {
      return;
    }
  }
 
  // If the type contains a pointer to data member we can't memset it to zero.
  // Instead, create a null constant and copy it to the destination.
  // TODO: there are other patterns besides zero that we can usefully memset,
  // like -1, which happens to be the pattern used by member-pointers.
  if (!cgm.getTypes().isZeroInitializable(ty)) {
    // Only the pointer-to-data-member case is tested here; emitNullConstant
    // owns the NYIs for shapes it cannot build (virtual bases, non-zero-init
    // arrays).
    assert((ty->isMemberDataPointerType() || ty->isRecordType()) &&
           "emitNullInitialization: only pointer-to-data-member (directly or "
           "within a record) null initialization is implemented");
    mlir::Value nullVal = cgm.emitNullConstant(ty, loc);
    builder.createStore(loc, nullVal, destPtr);
    return;
  }
 
  // In LLVM Codegen: otherwise, just memset the whole thing to zero using
  // Builder.CreateMemSet. In CIR just emit a store of #cir.zero to the
  // respective address.
  // Builder.CreateMemSet(DestPtr, Builder.getInt8(0), SizeVal, false);
  const mlir::Value zeroValue = builder.getNullValue(convertType(ty), loc);
  builder.createStore(loc, zeroValue, destPtr);
}
 
CIRGenFunction::CIRGenFPOptionsRAII::CIRGenFPOptionsRAII(CIRGenFunction &cgf,
                                                         const clang::Expr *e)
    : cgf(cgf) {
  ConstructorHelper(e->getFPFeaturesInEffect(cgf.getLangOpts()));
}
 
CIRGenFunction::CIRGenFPOptionsRAII::CIRGenFPOptionsRAII(CIRGenFunction &cgf,
                                                         FPOptions fpFeatures)
    : cgf(cgf) {
  ConstructorHelper(fpFeatures);
}
 
void CIRGenFunction::CIRGenFPOptionsRAII::ConstructorHelper(
    FPOptions fpFeatures) {
  oldFPFeatures = cgf.curFPFeatures;
  cgf.curFPFeatures = fpFeatures;
 
  oldExcept = cgf.builder.getDefaultConstrainedExcept();
  oldRounding = cgf.builder.getDefaultConstrainedRounding();
 
  if (oldFPFeatures == fpFeatures)
    return;
 
  // TODO(cir): create guard to restore fast math configurations.
  assert(!cir::MissingFeatures::fastMathGuard());
 
  [[maybe_unused]] llvm::RoundingMode newRoundingBehavior =
      fpFeatures.getRoundingMode();
  // TODO(cir): override rounding behaviour once FM configs are guarded.
  [[maybe_unused]] llvm::fp::ExceptionBehavior newExceptionBehavior =
      toConstrainedExceptMd(static_cast<LangOptions::FPExceptionModeKind>(
          fpFeatures.getExceptionMode()));
  // TODO(cir): override exception behaviour once FM configs are guarded.
 
  // TODO(cir): override FP flags once FM configs are guarded.
  assert(!cir::MissingFeatures::fastMathFlags());
 
  assert((cgf.curFuncDecl == nullptr || cgf.builder.getIsFPConstrained() ||
          isa<CXXConstructorDecl>(cgf.curFuncDecl) ||
          isa<CXXDestructorDecl>(cgf.curFuncDecl) ||
          (newExceptionBehavior == llvm::fp::ebIgnore &&
           newRoundingBehavior == llvm::RoundingMode::NearestTiesToEven)) &&
         "FPConstrained should be enabled on entire function");
 
  // TODO(cir): mark CIR function with fast math attributes.
  assert(!cir::MissingFeatures::fastMathFuncAttributes());
}
 
CIRGenFunction::CIRGenFPOptionsRAII::~CIRGenFPOptionsRAII() {
  cgf.curFPFeatures = oldFPFeatures;
  cgf.builder.setDefaultConstrainedExcept(oldExcept);
  cgf.builder.setDefaultConstrainedRounding(oldRounding);
}
 
// TODO(cir): should be shared with LLVM codegen.
bool CIRGenFunction::shouldNullCheckClassCastValue(const CastExpr *ce) {
  const Expr *e = ce->getSubExpr();
 
  if (ce->getCastKind() == CK_UncheckedDerivedToBase)
    return false;
 
  if (isa<CXXThisExpr>(e->IgnoreParens())) {
    // We always assume that 'this' is never null.
    return false;
  }
 
  if (const ImplicitCastExpr *ice = dyn_cast<ImplicitCastExpr>(ce)) {
    // And that glvalue casts are never null.
    if (ice->isGLValue())
      return false;
  }
 
  return true;
}
 
/// Computes the length of an array in elements, as well as the base
/// element type and a properly-typed first element pointer.
mlir::Value
CIRGenFunction::emitArrayLength(const clang::ArrayType *origArrayType,
                                QualType &baseType, Address &addr) {
  const clang::ArrayType *arrayType = origArrayType;
 
  // If it's a VLA, we have to load the stored size.  Note that
  // this is the size of the VLA in bytes, not its size in elements.
  mlir::Value numVLAElements = nullptr;
  if (isa<VariableArrayType>(arrayType)) {
    numVLAElements = getVLASize(cast<VariableArrayType>(arrayType)).numElts;
 
    // Walk into all VLAs.  This doesn't require changes to addr,
    // which has type T* where T is the first non-VLA element type.
    do {
      QualType elementType = arrayType->getElementType();
      arrayType = getContext().getAsArrayType(elementType);
 
      // If we only have VLA components, 'addr' requires no adjustment.
      if (!arrayType) {
        baseType = elementType;
        return numVLAElements;
      }
    } while (isa<VariableArrayType>(arrayType));
 
    // We get out here only if we find a constant array type
    // inside the VLA.
  }
 
  // Classic codegen emits an all-zero inbounds GEP to convert addr from
  // [M x [N x T]]* to T*. CIR doesn't need this because callers handle
  // the array-to-element pointer conversion themselves (via array_to_ptrdecay
  // casts, ptr_bitcast, or manual array type peeling).
 
  uint64_t countFromCLAs = 1;
  QualType eltType;
 
  auto cirArrayType = mlir::dyn_cast<cir::ArrayType>(addr.getElementType());
 
  while (cirArrayType) {
    assert(isa<ConstantArrayType>(arrayType));
    countFromCLAs *= cirArrayType.getSize();
    eltType = arrayType->getElementType();
 
    cirArrayType =
        mlir::dyn_cast<cir::ArrayType>(cirArrayType.getElementType());
 
    arrayType = getContext().getAsArrayType(arrayType->getElementType());
    assert((!cirArrayType || arrayType) &&
           "CIR and Clang types are out-of-sync");
  }
 
  if (arrayType) {
    // From this point onwards, the Clang array type has been emitted
    // as some other type (probably a packed struct). Compute the array
    // size, and just emit the 'begin' expression as a bitcast.
    cgm.errorNYI(*currSrcLoc, "length for non-array underlying types");
  }
 
  baseType = eltType;
 
  mlir::Value numElements =
      builder.getConstInt(*currSrcLoc, sizeTy, countFromCLAs);
 
  // If we had any VLA dimensions, factor them in.
  if (numVLAElements)
    numElements =
        builder.createMul(numVLAElements.getLoc(), numVLAElements, numElements,
                          cir::OverflowBehavior::NoUnsignedWrap);
 
  return numElements;
}
 
void CIRGenFunction::instantiateIndirectGotoBlock() {
  // If we already made the indirect branch for indirect goto, return its block.
  if (indirectGotoBlock)
    return;
 
  mlir::OpBuilder::InsertionGuard guard(builder);
  indirectGotoBlock =
      builder.createBlock(builder.getBlock()->getParent(), {}, {voidPtrTy},
                          {builder.getUnknownLoc()});
}
 
mlir::Value CIRGenFunction::emitAlignmentAssumption(
    mlir::Value ptrValue, QualType ty, SourceLocation loc,
    SourceLocation assumptionLoc, int64_t alignment, mlir::Value offsetValue) {
  assert(!cir::MissingFeatures::sanitizers());
  mlir::Location assumeLoc = getLoc(assumptionLoc);
  mlir::Value alignValue = builder.getUInt64(alignment, assumeLoc);
  mlir::Value cond = builder.getBool(true, assumeLoc);
  llvm::SmallVector<mlir::Value> bundleArgs{ptrValue, alignValue};
  if (offsetValue)
    bundleArgs.push_back(offsetValue);
  cir::AssumeOp::create(builder, assumeLoc, cond, cir::AssumeBundleKind::Align,
                        bundleArgs);
  return ptrValue;
}
 
mlir::Value CIRGenFunction::emitAlignmentAssumption(
    mlir::Value ptrValue, const Expr *expr, SourceLocation assumptionLoc,
    int64_t alignment, mlir::Value offsetValue) {
  QualType ty = expr->getType();
  SourceLocation loc = expr->getExprLoc();
  return emitAlignmentAssumption(ptrValue, ty, loc, assumptionLoc, alignment,
                                 offsetValue);
}
 
CIRGenFunction::VlaSizePair CIRGenFunction::getVLASize(QualType type) {
  const VariableArrayType *vla =
      cgm.getASTContext().getAsVariableArrayType(type);
  assert(vla && "type was not a variable array type!");
  return getVLASize(vla);
}
 
CIRGenFunction::VlaSizePair
CIRGenFunction::getVLASize(const VariableArrayType *type) {
  // The number of elements so far; always size_t.
  mlir::Value numElements;
 
  QualType elementType;
  do {
    elementType = type->getElementType();
    mlir::Value vlaSize = vlaSizeMap[type->getSizeExpr()];
    assert(vlaSize && "no size for VLA!");
    assert(vlaSize.getType() == sizeTy);
 
    if (!numElements) {
      numElements = vlaSize;
    } else {
      // It's undefined behavior if this wraps around, so mark it that way.
      // FIXME: Teach -fsanitize=undefined to trap this.
 
      numElements =
          builder.createMul(numElements.getLoc(), numElements, vlaSize,
                            cir::OverflowBehavior::NoUnsignedWrap);
    }
  } while ((type = getContext().getAsVariableArrayType(elementType)));
 
  assert(numElements && "Undefined elements number");
  return {numElements, elementType};
}
 
CIRGenFunction::VlaSizePair
CIRGenFunction::getVLAElements1D(const VariableArrayType *vla) {
  mlir::Value vlaSize = vlaSizeMap[vla->getSizeExpr()];
  assert(vlaSize && "no size for VLA!");
  assert(vlaSize.getType() == sizeTy);
  return {vlaSize, vla->getElementType()};
}
 
// TODO(cir): Most of this function can be shared between CIRGen
// and traditional LLVM codegen
void CIRGenFunction::emitVariablyModifiedType(QualType type) {
  assert(type->isVariablyModifiedType() &&
         "Must pass variably modified type to EmitVLASizes!");
 
  // We're going to walk down into the type and look for VLA
  // expressions.
  do {
    assert(type->isVariablyModifiedType());
 
    const Type *ty = type.getTypePtr();
    switch (ty->getTypeClass()) {
    case Type::CountAttributed:
    case Type::PackIndexing:
    case Type::ArrayParameter:
    case Type::HLSLAttributedResource:
    case Type::HLSLInlineSpirv:
    case Type::PredefinedSugar:
      cgm.errorNYI("CIRGenFunction::emitVariablyModifiedType");
      break;
 
#define TYPE(Class, Base)
#define ABSTRACT_TYPE(Class, Base)
#define NON_CANONICAL_TYPE(Class, Base)
#define DEPENDENT_TYPE(Class, Base) case Type::Class:
#define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base)
#include "clang/AST/TypeNodes.inc"
      llvm_unreachable(
          "dependent type must be resolved before the CIR codegen");
 
    // These types are never variably-modified.
    case Type::Builtin:
    case Type::Complex:
    case Type::Vector:
    case Type::ExtVector:
    case Type::ConstantMatrix:
    case Type::Record:
    case Type::Enum:
    case Type::Using:
    case Type::TemplateSpecialization:
    case Type::ObjCTypeParam:
    case Type::ObjCObject:
    case Type::ObjCInterface:
    case Type::ObjCObjectPointer:
    case Type::BitInt:
    case Type::OverflowBehavior:
      llvm_unreachable("type class is never variably-modified!");
 
    case Type::Adjusted:
      type = cast<clang::AdjustedType>(ty)->getAdjustedType();
      break;
 
    case Type::Decayed:
      type = cast<clang::DecayedType>(ty)->getPointeeType();
      break;
 
    case Type::Pointer:
      type = cast<clang::PointerType>(ty)->getPointeeType();
      break;
 
    case Type::BlockPointer:
      type = cast<clang::BlockPointerType>(ty)->getPointeeType();
      break;
 
    case Type::LValueReference:
    case Type::RValueReference:
      type = cast<clang::ReferenceType>(ty)->getPointeeType();
      break;
 
    case Type::MemberPointer:
      type = cast<clang::MemberPointerType>(ty)->getPointeeType();
      break;
 
    case Type::ConstantArray:
    case Type::IncompleteArray:
      // Losing element qualification here is fine.
      type = cast<clang::ArrayType>(ty)->getElementType();
      break;
 
    case Type::VariableArray: {
      // Losing element qualification here is fine.
      const VariableArrayType *vat = cast<clang::VariableArrayType>(ty);
 
      // Unknown size indication requires no size computation.
      // Otherwise, evaluate and record it.
      if (const Expr *sizeExpr = vat->getSizeExpr()) {
        // It's possible that we might have emitted this already,
        // e.g. with a typedef and a pointer to it.
        mlir::Value &entry = vlaSizeMap[sizeExpr];
        if (!entry) {
          mlir::Value size = emitScalarExpr(sizeExpr);
          assert(!cir::MissingFeatures::sanitizers());
 
          // Always zexting here would be wrong if it weren't
          // undefined behavior to have a negative bound.
          // FIXME: What about when size's type is larger than size_t?
          entry = builder.createBoolIntToIntCast(size, sizeTy);
        }
      }
      type = vat->getElementType();
      break;
    }
 
    case Type::FunctionProto:
    case Type::FunctionNoProto:
      type = cast<clang::FunctionType>(ty)->getReturnType();
      break;
 
    case Type::Paren:
    case Type::TypeOf:
    case Type::UnaryTransform:
    case Type::Attributed:
    case Type::BTFTagAttributed:
    case Type::SubstTemplateTypeParm:
    case Type::MacroQualified:
      // Keep walking after single level desugaring.
      type = type.getSingleStepDesugaredType(getContext());
      break;
 
    case Type::Typedef:
    case Type::Decltype:
    case Type::Auto:
    case Type::DeducedTemplateSpecialization:
      // Stop walking: nothing to do.
      return;
 
    case Type::TypeOfExpr:
      // Stop walking: emit typeof expression.
      emitIgnoredExpr(cast<clang::TypeOfExprType>(ty)->getUnderlyingExpr());
      return;
 
    case Type::Atomic:
      type = cast<clang::AtomicType>(ty)->getValueType();
      break;
 
    case Type::Pipe:
      type = cast<clang::PipeType>(ty)->getElementType();
      break;
    }
  } while (type->isVariablyModifiedType());
}
 
Address CIRGenFunction::emitVAListRef(const Expr *e) {
  if (getContext().getBuiltinVaListType()->isArrayType())
    return emitPointerWithAlignment(e);
  return emitLValue(e).getAddress();
}
 
} // namespace clang::CIRGen