CIRGenDecl.cpp

Go to the documentation of this file.

//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This contains code to emit Decl nodes as CIR code.
//
//===----------------------------------------------------------------------===//
 
#include "CIRGenCleanup.h"
#include "CIRGenConstantEmitter.h"
#include "CIRGenFunction.h"
#include "mlir/IR/Location.h"
#include "clang/AST/Attr.h"
#include "clang/AST/Attrs.inc"
#include "clang/AST/Decl.h"
#include "clang/AST/DeclOpenACC.h"
#include "clang/AST/Expr.h"
#include "clang/AST/ExprCXX.h"
#include "clang/Basic/Cuda.h"
#include "clang/CIR/Dialect/IR/CIRAttrs.h"
#include "clang/CIR/MissingFeatures.h"
 
using namespace clang;
using namespace clang::CIRGen;
 
CIRGenFunction::AutoVarEmission
CIRGenFunction::emitAutoVarAlloca(const VarDecl &d,
                                  mlir::OpBuilder::InsertPoint ip) {
  QualType ty = d.getType();
  if (ty.getAddressSpace() != LangAS::Default)
    cgm.errorNYI(d.getSourceRange(), "emitAutoVarAlloca: address space");
 
  mlir::Location loc = getLoc(d.getSourceRange());
  bool nrvo =
      getContext().getLangOpts().ElideConstructors && d.isNRVOVariable();
 
  CIRGenFunction::AutoVarEmission emission(d);
  emission.isEscapingByRef = d.isEscapingByref();
  if (emission.isEscapingByRef)
    cgm.errorNYI(d.getSourceRange(),
                 "emitAutoVarAlloca: decl escaping by reference");
 
  CharUnits alignment = getContext().getDeclAlign(&d);
 
  // If the type is variably-modified, emit all the VLA sizes for it.
  if (ty->isVariablyModifiedType())
    emitVariablyModifiedType(ty);
 
  assert(!cir::MissingFeatures::openMP());
 
  Address address = Address::invalid();
  if (ty->isConstantSizeType()) {
    // If this value is an array, struct, or vector with a statically
    // determinable constant initializer, there are optimizations we can do.
    //
    // TODO: We should constant-evaluate the initializer of any variable,
    // as long as it is initialized by a constant expression. Currently,
    // isConstantInitializer produces wrong answers for structs with
    // reference or bitfield members, and a few other cases, and checking
    // for POD-ness protects us from some of these.
    if (d.getInit() &&
        (ty->isArrayType() || ty->isRecordType() || ty->isVectorType()) &&
        (d.isConstexpr() ||
         ((ty.isPODType(getContext()) ||
           getContext().getBaseElementType(ty)->isObjCObjectPointerType()) &&
          d.getInit()->isConstantInitializer(getContext())))) {
 
      // If the variable's a const type, and it's neither an NRVO
      // candidate nor a __block variable and has no mutable members,
      // emit it as a global instead.
      // Exception is if a variable is located in non-constant address space
      // in OpenCL.
      // TODO(cir): perhaps we don't need this at all at CIR since this can
      // be done as part of lowering down to LLVM.
      bool needsDtor =
          d.needsDestruction(getContext()) == QualType::DK_cxx_destructor;
      if ((!getContext().getLangOpts().OpenCL ||
           ty.getAddressSpace() == LangAS::opencl_constant) &&
          (cgm.getCodeGenOpts().MergeAllConstants && !nrvo &&
           !d.isEscapingByref() &&
           ty.isConstantStorage(getContext(), true, !needsDtor))) {
        cgm.errorNYI(d.getSourceRange(), "emitAutoVarAlloca: type constant");
      }
      // Otherwise, tell the initialization code that we're in this case.
      emission.isConstantAggregate = true;
    }
 
    // A normal fixed sized variable becomes an alloca in the entry block,
    // unless:
    // - it's an NRVO variable.
    // - we are compiling OpenMP and it's an OpenMP local variable.
    if (nrvo) {
      // The named return value optimization: allocate this variable in the
      // return slot, so that we can elide the copy when returning this
      // variable (C++0x [class.copy]p34).
      address = returnValue;
 
      if (const RecordDecl *rd = ty->getAsRecordDecl()) {
        if (const auto *cxxrd = dyn_cast<CXXRecordDecl>(rd);
            (cxxrd && !cxxrd->hasTrivialDestructor()) ||
            rd->isNonTrivialToPrimitiveDestroy()) {
          // In LLVM: Create a flag that is used to indicate when the NRVO was
          // applied to this variable. Set it to zero to indicate that NRVO was
          // not applied. For now, use the same approach for CIRGen until we can
          // be sure it's worth doing something more aggressive.
          cir::ConstantOp falseNVRO = builder.getFalse(loc);
          Address nrvoFlag = createTempAlloca(falseNVRO.getType(),
                                              CharUnits::One(), loc, "nrvo",
                                              /*arraySize=*/nullptr);
          assert(builder.getInsertionBlock());
          builder.createStore(loc, falseNVRO, nrvoFlag);
 
          // Record the NRVO flag for this variable.
          nrvoFlags[&d] = nrvoFlag.getPointer();
          emission.nrvoFlag = nrvoFlag.getPointer();
        }
      }
    } else {
      // A normal fixed sized variable becomes an alloca in the entry block,
      mlir::Type allocaTy = convertTypeForMem(ty);
      // Create the temp alloca and declare variable using it.
      address = createTempAlloca(allocaTy, alignment, loc, d.getName(),
                                 /*arraySize=*/nullptr, /*alloca=*/nullptr, ip);
      declare(address.getPointer(), &d, ty, getLoc(d.getSourceRange()),
              alignment);
    }
  } else {
    // Non-constant size type
    assert(!cir::MissingFeatures::openMP());
    if (!didCallStackSave) {
      // Save the stack.
      cir::PointerType defaultTy = allocaInt8PtrTy;
      CharUnits align = CharUnits::fromQuantity(
          cgm.getDataLayout().getAlignment(defaultTy, false));
      Address stack = createTempAlloca(defaultTy, align, loc, "saved_stack");
 
      mlir::Value v = builder.createStackSave(loc, defaultTy);
      assert(v.getType() == allocaInt8PtrTy);
      builder.createStore(loc, v, stack);
 
      didCallStackSave = true;
 
      // Push a cleanup block and restore the stack there.
      // FIXME: in general circumstances, this should be an EH cleanup.
      pushStackRestore(NormalCleanup, stack);
    }
 
    VlaSizePair vlaSize = getVLASize(ty);
    mlir::Type memTy = convertTypeForMem(vlaSize.type);
 
    // Allocate memory for the array.
    address =
        createTempAlloca(memTy, alignment, loc, d.getName(), vlaSize.numElts,
                         /*alloca=*/nullptr, builder.saveInsertionPoint());
 
    // If we have debug info enabled, properly describe the VLA dimensions for
    // this type by registering the vla size expression for each of the
    // dimensions.
    assert(!cir::MissingFeatures::generateDebugInfo());
  }
 
  emission.addr = address;
  setAddrOfLocalVar(&d, address);
 
  return emission;
}
 
/// Determine whether the given initializer is trivial in the sense
/// that it requires no code to be generated.
bool CIRGenFunction::isTrivialInitializer(const Expr *init) {
  if (!init)
    return true;
 
  if (const CXXConstructExpr *construct = dyn_cast<CXXConstructExpr>(init))
    if (CXXConstructorDecl *constructor = construct->getConstructor())
      if (constructor->isTrivial() && constructor->isDefaultConstructor() &&
          !construct->requiresZeroInitialization())
        return true;
 
  return false;
}
 
static void emitStoresForConstant(CIRGenModule &cgm, const VarDecl &d,
                                  Address addr, bool isVolatile,
                                  CIRGenBuilderTy &builder,
                                  mlir::TypedAttr constant) {
  mlir::Type ty = constant.getType();
  cir::CIRDataLayout layout{cgm.getModule()};
  uint64_t constantSize = layout.getTypeAllocSize(ty);
  if (!constantSize)
    return;
  assert(!cir::MissingFeatures::addAutoInitAnnotation());
  assert(!cir::MissingFeatures::vectorConstants());
 
  if (addr.getElementType() != ty)
    addr = addr.withElementType(builder, ty);
 
  // If the address is an alloca, set the init attribute.
  // The address is usually and alloca, but there is at least one case where
  // emitAutoVarInit is called from the OpenACC codegen with an address that
  // is not an alloca.
  auto allocaOp = addr.getDefiningOp<cir::AllocaOp>();
  if (allocaOp)
    allocaOp.setInitAttr(mlir::UnitAttr::get(&cgm.getMLIRContext()));
 
  // There are cases where OpenACC codegen calls emitAutoVarInit with a
  // temporary decl that doesn't have a source range set.
  mlir::Location loc = builder.getUnknownLoc();
  if (d.getSourceRange().isValid())
    loc = cgm.getLoc(d.getSourceRange());
 
  // Emit cir.const + cir.store, preserving source-level semantics. For
  // aggregate types (arrays, records), LoweringPrepare implements the OG
  // optimization tiers (shouldCreateMemCpyFromGlobal, shouldUseBZeroPlusStores,
  // shouldUseMemSetToInitialize, shouldSplitConstantStore) by transforming
  // into cir.global + cir.get_global + cir.copy when appropriate.
  builder.createStore(loc, builder.getConstant(loc, constant), addr);
}
 
void CIRGenFunction::emitAutoVarInit(
    const CIRGenFunction::AutoVarEmission &emission) {
  assert(emission.variable && "emission was not valid!");
 
  // If this was emitted as a global constant, we're done.
  if (emission.wasEmittedAsGlobal())
    return;
 
  const VarDecl &d = *emission.variable;
 
  QualType type = d.getType();
 
  // If this local has an initializer, emit it now.
  const Expr *init = d.getInit();
 
  // Initialize the variable here if it doesn't have a initializer and it is a
  // C struct that is non-trivial to initialize or an array containing such a
  // struct.
  if (!init && type.isNonTrivialToPrimitiveDefaultInitialize() ==
                   QualType::PDIK_Struct) {
    cgm.errorNYI(d.getSourceRange(),
                 "emitAutoVarInit: non-trivial to default initialize");
    return;
  }
 
  const Address addr = emission.addr;
 
  // Check whether this is a byref variable that's potentially
  // captured and moved by its own initializer.  If so, we'll need to
  // emit the initializer first, then copy into the variable.
  assert(!cir::MissingFeatures::opAllocaCaptureByInit());
 
  // Note: constexpr already initializes everything correctly.
  LangOptions::TrivialAutoVarInitKind trivialAutoVarInit =
      (d.isConstexpr()
           ? LangOptions::TrivialAutoVarInitKind::Uninitialized
           : (d.getAttr<UninitializedAttr>()
                  ? LangOptions::TrivialAutoVarInitKind::Uninitialized
                  : getContext().getLangOpts().getTrivialAutoVarInit()));
 
  auto initializeWhatIsTechnicallyUninitialized = [&](Address addr) {
    if (trivialAutoVarInit ==
        LangOptions::TrivialAutoVarInitKind::Uninitialized)
      return;
 
    cgm.errorNYI(d.getSourceRange(), "emitAutoVarInit: trivial initialization");
  };
 
  if (isTrivialInitializer(init)) {
    initializeWhatIsTechnicallyUninitialized(addr);
    return;
  }
 
  mlir::Attribute constant;
  if (emission.isConstantAggregate ||
      d.mightBeUsableInConstantExpressions(getContext())) {
    // FIXME: Differently from LLVM we try not to emit / lower too much
    // here for CIR since we are interested in seeing the ctor in some
    // analysis later on. So CIR's implementation of ConstantEmitter will
    // frequently return an empty Attribute, to signal we want to codegen
    // some trivial ctor calls and whatnots.
    constant = ConstantEmitter(*this).tryEmitAbstractForInitializer(d);
    if (constant && !mlir::isa<cir::ZeroAttr>(constant) &&
        (trivialAutoVarInit !=
         LangOptions::TrivialAutoVarInitKind::Uninitialized)) {
      cgm.errorNYI(d.getSourceRange(), "emitAutoVarInit: constant aggregate");
      return;
    }
  }
 
  // NOTE(cir): In case we have a constant initializer, we can just emit a
  // store. But, in CIR, we wish to retain any ctor calls, so if it is a
  // CXX temporary object creation, we ensure the ctor call is used deferring
  // its removal/optimization to the CIR lowering.
  if (!constant || isa<CXXTemporaryObjectExpr>(init)) {
    initializeWhatIsTechnicallyUninitialized(addr);
    LValue lv = makeAddrLValue(addr, type, AlignmentSource::Decl);
    emitExprAsInit(init, &d, lv);
 
    if (!emission.wasEmittedAsOffloadClause()) {
      // In case lv has uses it means we indeed initialized something
      // out of it while trying to build the expression, mark it as such.
      mlir::Value val = lv.getAddress().getPointer();
      assert(val && "Should have an address");
      auto allocaOp = val.getDefiningOp<cir::AllocaOp>();
      assert(allocaOp && "Address should come straight out of the alloca");
 
      if (!allocaOp.use_empty())
        allocaOp.setInitAttr(mlir::UnitAttr::get(&getMLIRContext()));
    }
 
    return;
  }
 
  // FIXME(cir): migrate most of this file to use mlir::TypedAttr directly.
  auto typedConstant = mlir::dyn_cast<mlir::TypedAttr>(constant);
  assert(typedConstant && "expected typed attribute");
  if (!emission.isConstantAggregate) {
    // For simple scalar/complex initialization, store the value directly.
    LValue lv = makeAddrLValue(addr, type);
    assert(init && "expected initializer");
    mlir::Location initLoc = getLoc(init->getSourceRange());
    // lv.setNonGC(true);
    return emitStoreThroughLValue(
        RValue::get(builder.getConstant(initLoc, typedConstant)), lv);
  }
 
  emitStoresForConstant(cgm, d, addr, type.isVolatileQualified(), builder,
                        typedConstant);
}
 
void CIRGenFunction::emitAutoVarCleanups(
    const CIRGenFunction::AutoVarEmission &emission) {
  const VarDecl &d = *emission.variable;
 
  // Check the type for a cleanup.
  if (QualType::DestructionKind dtorKind = d.needsDestruction(getContext()))
    emitAutoVarTypeCleanup(emission, dtorKind);
 
  assert(!cir::MissingFeatures::opAllocaPreciseLifetime());
 
  // Handle the cleanup attribute.
  if (d.hasAttr<CleanupAttr>())
    cgm.errorNYI(d.getSourceRange(), "emitAutoVarCleanups: CleanupAttr");
}
 
/// Emit code and set up symbol table for a variable declaration with auto,
/// register, or no storage class specifier. These turn into simple stack
/// objects, globals depending on target.
void CIRGenFunction::emitAutoVarDecl(const VarDecl &d) {
  CIRGenFunction::AutoVarEmission emission = emitAutoVarAlloca(d);
  emitAutoVarInit(emission);
  emitAutoVarCleanups(emission);
}
 
void CIRGenFunction::emitVarDecl(const VarDecl &d) {
  // If the declaration has external storage, don't emit it now, allow it to be
  // emitted lazily on its first use.
  if (d.hasExternalStorage())
    return;
 
  if (d.getStorageDuration() != SD_Automatic) {
    // Static sampler variables translated to function calls.
    if (d.getType()->isSamplerT()) {
      // Nothing needs to be done here, but let's flag it as an error until we
      // have a test. It requires OpenCL support.
      cgm.errorNYI(d.getSourceRange(), "emitVarDecl: static sampler type");
      return;
    }
 
    cir::GlobalLinkageKind linkage =
        cgm.getCIRLinkageVarDefinition(&d, /*IsConstant=*/false);
 
    // FIXME: We need to force the emission/use of a guard variable for
    // some variables even if we can constant-evaluate them because
    // we can't guarantee every translation unit will constant-evaluate them.
 
    return emitStaticVarDecl(d, linkage);
  }
 
  if (d.getType().getAddressSpace() == LangAS::opencl_local)
    cgm.errorNYI(d.getSourceRange(), "emitVarDecl: openCL address space");
 
  assert(d.hasLocalStorage());
 
  CIRGenFunction::VarDeclContext varDeclCtx{*this, &d};
  return emitAutoVarDecl(d);
}
 
static std::string getStaticDeclName(CIRGenModule &cgm, const VarDecl &d) {
  if (cgm.getLangOpts().CPlusPlus)
    return cgm.getMangledName(&d).str();
 
  // If this isn't C++, we don't need a mangled name, just a pretty one.
  assert(!d.isExternallyVisible() && "name shouldn't matter");
  std::string contextName;
  const DeclContext *dc = d.getDeclContext();
  if (auto *cd = dyn_cast<CapturedDecl>(dc))
    dc = cast<DeclContext>(cd->getNonClosureContext());
  if (const auto *fd = dyn_cast<FunctionDecl>(dc))
    contextName = std::string(cgm.getMangledName(fd));
  else if (isa<BlockDecl>(dc))
    cgm.errorNYI(d.getSourceRange(),
                 "getStaticDeclName: block decl context for static var");
  else if (isa<ObjCMethodDecl>(dc))
    cgm.errorNYI(d.getSourceRange(),
                 "getStaticDeclName: ObjC decl context for static var");
  else
    cgm.errorNYI(d.getSourceRange(),
                 "getStaticDeclName: Unknown context for static var decl");
 
  contextName += "." + d.getNameAsString();
  return contextName;
}
 
// TODO(cir): LLVM uses a Constant base class. Maybe CIR could leverage an
// interface for all constants?
cir::GlobalOp
CIRGenModule::getOrCreateStaticVarDecl(const VarDecl &d,
                                       cir::GlobalLinkageKind linkage) {
  // In general, we don't always emit static var decls once before we reference
  // them. It is possible to reference them before emitting the function that
  // contains them, and it is possible to emit the containing function multiple
  // times.
  if (cir::GlobalOp existingGV = getStaticLocalDeclAddress(&d))
    return existingGV;
 
  QualType ty = d.getType();
  assert(ty->isConstantSizeType() && "VLAs can't be static");
 
  // Use the label if the variable is renamed with the asm-label extension.
  if (d.hasAttr<AsmLabelAttr>())
    errorNYI(d.getSourceRange(), "getOrCreateStaticVarDecl: asm label");
 
  std::string name = getStaticDeclName(*this, d);
 
  mlir::Type lty = getTypes().convertTypeForMem(ty);
  assert(!cir::MissingFeatures::addressSpace());
 
  // OpenCL variables in local address space and CUDA shared
  // variables cannot have an initializer.
  mlir::Attribute init = nullptr;
  if (ty.getAddressSpace() == LangAS::opencl_local ||
      d.hasAttr<CUDASharedAttr>() || d.hasAttr<LoaderUninitializedAttr>())
    init = cir::UndefAttr::get(lty);
  else
    init = builder.getZeroInitAttr(convertType(ty));
 
  cir::GlobalOp gv = builder.createVersionedGlobal(
      getModule(), getLoc(d.getLocation()), name, lty, false, linkage);
  // TODO(cir): infer visibility from linkage in global op builder.
  gv.setVisibility(getMLIRVisibilityFromCIRLinkage(linkage));
  gv.setInitialValueAttr(init);
  gv.setAlignment(getASTContext().getDeclAlign(&d).getAsAlign().value());
 
  if (supportsCOMDAT() && gv.isWeakForLinker())
    gv.setComdat(true);
 
  if (d.getTLSKind())
    errorNYI(d.getSourceRange(), "getOrCreateStaticVarDecl: TLS");
 
  setGVProperties(gv, &d);
 
  // OG checks if the expected address space, denoted by the type, is the
  // same as the actual address space indicated by attributes. If they aren't
  // the same, an addrspacecast is emitted when this variable is accessed.
  // In CIR however, cir.get_global already carries that information in
  // !cir.ptr type - if this global is in OpenCL local address space, then its
  // type would be !cir.ptr<..., addrspace(offload_local)>. Therefore we don't
  // need an explicit address space cast in CIR: they will get emitted when
  // lowering to LLVM IR.
 
  // Ensure that the static local gets initialized by making sure the parent
  // function gets emitted eventually.
  const Decl *dc = cast<Decl>(d.getDeclContext());
 
  // We can't name blocks or captured statements directly, so try to emit their
  // parents.
  if (isa<BlockDecl>(dc) || isa<CapturedDecl>(dc)) {
    dc = dc->getNonClosureContext();
    // FIXME: Ensure that global blocks get emitted.
    if (!dc)
      errorNYI(d.getSourceRange(), "non-closure context");
  }
 
  GlobalDecl gd;
  if (isa<CXXConstructorDecl>(dc))
    errorNYI(d.getSourceRange(), "C++ constructors static var context");
  else if (isa<CXXDestructorDecl>(dc))
    errorNYI(d.getSourceRange(), "C++ destructors static var context");
  else if (const auto *fd = dyn_cast<FunctionDecl>(dc))
    gd = GlobalDecl(fd);
  else {
    // Don't do anything for Obj-C method decls or global closures. We should
    // never defer them.
    assert(isa<ObjCMethodDecl>(dc) && "unexpected parent code decl");
  }
  if (gd.getDecl() && cir::MissingFeatures::openMP()) {
    // Disable emission of the parent function for the OpenMP device codegen.
    errorNYI(d.getSourceRange(), "OpenMP");
  }
 
  return gv;
}
 
Address CIRGenModule::createUnnamedGlobalFrom(const VarDecl &d,
                                              mlir::Attribute constAttr,
                                              CharUnits align) {
  auto functionName = [&](const DeclContext *dc) -> std::string {
    if (const auto *fd = dyn_cast<FunctionDecl>(dc)) {
      if (const auto *cc = dyn_cast<CXXConstructorDecl>(fd))
        return cc->getNameAsString();
      if (const auto *cd = dyn_cast<CXXDestructorDecl>(fd))
        return cd->getNameAsString();
      return std::string(getMangledName(fd));
    } else if (const auto *om = dyn_cast<ObjCMethodDecl>(dc)) {
      return om->getNameAsString();
    } else if (isa<BlockDecl>(dc)) {
      return "<block>";
    } else if (isa<CapturedDecl>(dc)) {
      return "<captured>";
    } else {
      llvm_unreachable("expected a function or method");
    }
  };
 
  // Form a simple per-variable cache of these values in case we find we
  // want to reuse them.
  cir::GlobalOp &cacheEntry = initializerConstants[&d];
  if (!cacheEntry || cacheEntry.getInitialValue() != constAttr) {
    auto ty = mlir::cast<mlir::TypedAttr>(constAttr).getType();
    bool isConstant = true;
 
    std::string name;
    if (d.hasGlobalStorage())
      name = getMangledName(&d).str() + ".const";
    else if (const DeclContext *dc = d.getParentFunctionOrMethod())
      name = ("__const." + functionName(dc) + "." + d.getName()).str();
    else
      llvm_unreachable("local variable has no parent function or method");
 
    assert(!cir::MissingFeatures::addressSpace());
    cir::GlobalOp gv = builder.createVersionedGlobal(
        getModule(), getLoc(d.getLocation()), name, ty, isConstant,
        cir::GlobalLinkageKind::PrivateLinkage);
    // TODO(cir): infer visibility from linkage in global op builder.
    gv.setVisibility(getMLIRVisibilityFromCIRLinkage(
        cir::GlobalLinkageKind::PrivateLinkage));
    gv.setInitialValueAttr(constAttr);
    gv.setAlignment(align.getAsAlign().value());
    // TODO(cir): Set unnamed address attribute when available in CIR
 
    cacheEntry = gv;
  } else if (cacheEntry.getAlignment() < align.getQuantity()) {
    cacheEntry.setAlignment(align.getAsAlign().value());
  }
 
  // Create a GetGlobalOp to get a pointer to the global
  assert(!cir::MissingFeatures::addressSpace());
  mlir::Type eltTy = mlir::cast<mlir::TypedAttr>(constAttr).getType();
  auto ptrTy = builder.getPointerTo(cacheEntry.getSymType());
  mlir::Value globalPtr = cir::GetGlobalOp::create(
      builder, getLoc(d.getLocation()), ptrTy, cacheEntry.getSymName());
  return Address(globalPtr, eltTy, align);
}
 
/// Add the initializer for 'd' to the global variable that has already been
/// created for it. If the initializer has a different type than gv does, this
/// may free gv and return a different one. Otherwise it just returns gv.
cir::GlobalOp CIRGenFunction::addInitializerToStaticVarDecl(
    const VarDecl &d, cir::GlobalOp gv, cir::GetGlobalOp gvAddr) {
  ConstantEmitter emitter(*this);
  mlir::TypedAttr init = mlir::dyn_cast_if_present<mlir::TypedAttr>(
      emitter.tryEmitForInitializer(d));
 
  // If constant emission failed, then this should be a C++ static
  // initializer.
  if (!init) {
    if (!getLangOpts().CPlusPlus) {
      cgm.errorNYI(d.getInit()->getSourceRange(),
                   "constant l-value expression");
    } else if (d.hasFlexibleArrayInit(getContext())) {
      cgm.errorNYI(d.getInit()->getSourceRange(), "flexible array initializer");
    } else {
      // Since we have a static initializer, this global variable can't
      // be constant.
      gv.setConstant(false);
      emitCXXGuardedInit(d, gv, /*performInit*/ true);
      gvAddr.setStaticLocal(true);
    }
    return gv;
  }
 
  // TODO(cir): There should be debug code here to assert that the decl size
  // matches the CIR data layout type alloc size, but the code for calculating
  // the type alloc size is not implemented yet.
  assert(!cir::MissingFeatures::dataLayoutTypeAllocSize());
 
  // The initializer may differ in type from the global. Rewrite
  // the global to match the initializer.  (We have to do this
  // because some types, like unions, can't be completely represented
  // in the LLVM type system.)
  if (gv.getSymType() != init.getType()) {
    gv.setSymType(init.getType());
 
    // Normally this should be done with a call to cgm.replaceGlobal(oldGV, gv),
    // but since at this point the current block hasn't been really attached,
    // there's no visibility into the GetGlobalOp corresponding to this Global.
    // Given those constraints, thread in the GetGlobalOp and update it
    // directly.
    assert(!cir::MissingFeatures::addressSpace());
    gvAddr.getAddr().setType(builder.getPointerTo(init.getType()));
  }
 
  bool needsDtor =
      d.needsDestruction(getContext()) == QualType::DK_cxx_destructor;
 
  gv.setConstant(d.getType().isConstantStorage(
      getContext(), /*ExcludeCtor=*/true, !needsDtor));
  gv.setInitialValueAttr(init);
 
  emitter.finalize(gv);
 
  if (needsDtor) {
    // We have a constant initializer, but a nontrivial destructor. We still
    // need to perform a guarded "initialization" in order to register the
    // destructor.
    cgm.errorNYI(d.getSourceRange(), "C++ guarded init");
  }
 
  return gv;
}
 
void CIRGenFunction::emitStaticVarDecl(const VarDecl &d,
                                       cir::GlobalLinkageKind linkage) {
  // Check to see if we already have a global variable for this
  // declaration.  This can happen when double-emitting function
  // bodies, e.g. with complete and base constructors.
  cir::GlobalOp globalOp = cgm.getOrCreateStaticVarDecl(d, linkage);
  // TODO(cir): we should have a way to represent global ops as values without
  // having to emit a get global op. Sometimes these emissions are not used.
  mlir::Value addr = builder.createGetGlobal(globalOp);
  auto getAddrOp = addr.getDefiningOp<cir::GetGlobalOp>();
  assert(getAddrOp && "expected cir::GetGlobalOp");
 
  CharUnits alignment = getContext().getDeclAlign(&d);
 
  // Store into LocalDeclMap before generating initializer to handle
  // circular references.
  mlir::Type elemTy = convertTypeForMem(d.getType());
  setAddrOfLocalVar(&d, Address(addr, elemTy, alignment));
 
  // We can't have a VLA here, but we can have a pointer to a VLA,
  // even though that doesn't really make any sense.
  // Make sure to evaluate VLA bounds now so that we have them for later.
  if (d.getType()->isVariablyModifiedType()) {
    cgm.errorNYI(d.getSourceRange(),
                 "emitStaticVarDecl: variably modified type");
  }
 
  // Save the type in case adding the initializer forces a type change.
  mlir::Type expectedType = addr.getType();
 
  cir::GlobalOp var = globalOp;
 
  assert(!cir::MissingFeatures::cudaSupport());
 
  // If this value has an initializer, emit it.
  if (d.getInit())
    var = addInitializerToStaticVarDecl(d, var, getAddrOp);
 
  var.setAlignment(alignment.getAsAlign().value());
 
  // There are a lot of attributes that need to be handled here. Until
  // we start to support them, we just report an error if there are any.
  if (d.hasAttr<AnnotateAttr>())
    cgm.errorNYI(d.getSourceRange(), "emitStaticVarDecl: Global annotations");
  if (d.getAttr<PragmaClangBSSSectionAttr>())
    cgm.errorNYI(d.getSourceRange(),
                 "emitStaticVarDecl: CIR global BSS section attribute");
  if (d.getAttr<PragmaClangDataSectionAttr>())
    cgm.errorNYI(d.getSourceRange(),
                 "emitStaticVarDecl: CIR global Data section attribute");
  if (d.getAttr<PragmaClangRodataSectionAttr>())
    cgm.errorNYI(d.getSourceRange(),
                 "emitStaticVarDecl: CIR global Rodata section attribute");
  if (d.getAttr<PragmaClangRelroSectionAttr>())
    cgm.errorNYI(d.getSourceRange(),
                 "emitStaticVarDecl: CIR global Relro section attribute");
 
  if (d.getAttr<SectionAttr>())
    cgm.errorNYI(d.getSourceRange(),
                 "emitStaticVarDecl: CIR global object file section attribute");
 
  if (cgm.getCodeGenOpts().KeepPersistentStorageVariables)
    cgm.errorNYI(d.getSourceRange(), "static var keep persistent storage");
 
  // From traditional codegen:
  // We may have to cast the constant because of the initializer
  // mismatch above.
  //
  // FIXME: It is really dangerous to store this in the map; if anyone
  // RAUW's the GV uses of this constant will be invalid.
  mlir::Value castedAddr =
      builder.createBitcast(getAddrOp.getAddr(), expectedType);
  localDeclMap.find(&d)->second = Address(castedAddr, elemTy, alignment);
  cgm.setStaticLocalDeclAddress(&d, var);
 
  assert(!cir::MissingFeatures::sanitizers());
  assert(!cir::MissingFeatures::generateDebugInfo());
}
 
void CIRGenFunction::emitScalarInit(const Expr *init, mlir::Location loc,
                                    LValue lvalue, bool capturedByInit) {
  assert(!cir::MissingFeatures::objCLifetime());
 
  SourceLocRAIIObject locRAII{*this, loc};
  mlir::Value value = emitScalarExpr(init);
  if (capturedByInit) {
    cgm.errorNYI(init->getSourceRange(), "emitScalarInit: captured by init");
    return;
  }
  assert(!cir::MissingFeatures::emitNullabilityCheck());
  emitStoreThroughLValue(RValue::get(value), lvalue, true);
}
 
void CIRGenFunction::emitExprAsInit(const Expr *init, const ValueDecl *d,
                                    LValue lvalue, bool capturedByInit) {
  SourceLocRAIIObject loc{*this, getLoc(init->getSourceRange())};
  if (capturedByInit) {
    cgm.errorNYI(init->getSourceRange(), "emitExprAsInit: captured by init");
    return;
  }
 
  QualType type = d->getType();
 
  if (type->isReferenceType()) {
    RValue rvalue = emitReferenceBindingToExpr(init);
    if (capturedByInit)
      cgm.errorNYI(init->getSourceRange(), "emitExprAsInit: captured by init");
    emitStoreThroughLValue(rvalue, lvalue);
    return;
  }
  switch (CIRGenFunction::getEvaluationKind(type)) {
  case cir::TEK_Scalar:
    emitScalarInit(init, getLoc(d->getSourceRange()), lvalue);
    return;
  case cir::TEK_Complex: {
    mlir::Value complex = emitComplexExpr(init);
    if (capturedByInit)
      cgm.errorNYI(init->getSourceRange(),
                   "emitExprAsInit: complex type captured by init");
    mlir::Location loc = getLoc(init->getExprLoc());
    emitStoreOfComplex(loc, complex, lvalue,
                       /*isInit*/ true);
    return;
  }
  case cir::TEK_Aggregate:
    // The overlap flag here should be calculated.
    assert(!cir::MissingFeatures::aggValueSlotMayOverlap());
    emitAggExpr(init,
                AggValueSlot::forLValue(lvalue, AggValueSlot::IsDestructed,
                                        AggValueSlot::IsNotAliased,
                                        AggValueSlot::MayOverlap));
    return;
  }
  llvm_unreachable("bad evaluation kind");
}
 
void CIRGenFunction::emitDecl(const Decl &d, bool evaluateConditionDecl) {
  switch (d.getKind()) {
  case Decl::BuiltinTemplate:
  case Decl::TranslationUnit:
  case Decl::ExternCContext:
  case Decl::Namespace:
  case Decl::UnresolvedUsingTypename:
  case Decl::ClassTemplateSpecialization:
  case Decl::ClassTemplatePartialSpecialization:
  case Decl::VarTemplateSpecialization:
  case Decl::VarTemplatePartialSpecialization:
  case Decl::TemplateTypeParm:
  case Decl::UnresolvedUsingValue:
  case Decl::NonTypeTemplateParm:
  case Decl::CXXDeductionGuide:
  case Decl::CXXMethod:
  case Decl::CXXConstructor:
  case Decl::CXXDestructor:
  case Decl::CXXConversion:
  case Decl::Field:
  case Decl::MSProperty:
  case Decl::IndirectField:
  case Decl::ObjCIvar:
  case Decl::ObjCAtDefsField:
  case Decl::ParmVar:
  case Decl::ImplicitParam:
  case Decl::ClassTemplate:
  case Decl::VarTemplate:
  case Decl::FunctionTemplate:
  case Decl::TypeAliasTemplate:
  case Decl::TemplateTemplateParm:
  case Decl::ObjCMethod:
  case Decl::ObjCCategory:
  case Decl::ObjCProtocol:
  case Decl::ObjCInterface:
  case Decl::ObjCCategoryImpl:
  case Decl::ObjCImplementation:
  case Decl::ObjCProperty:
  case Decl::ObjCCompatibleAlias:
  case Decl::PragmaComment:
  case Decl::PragmaDetectMismatch:
  case Decl::AccessSpec:
  case Decl::LinkageSpec:
  case Decl::Export:
  case Decl::ObjCPropertyImpl:
  case Decl::FileScopeAsm:
  case Decl::Friend:
  case Decl::FriendTemplate:
  case Decl::Block:
  case Decl::OutlinedFunction:
  case Decl::Captured:
  case Decl::UsingShadow:
  case Decl::ConstructorUsingShadow:
  case Decl::ObjCTypeParam:
  case Decl::Binding:
  case Decl::UnresolvedUsingIfExists:
  case Decl::HLSLBuffer:
  case Decl::HLSLRootSignature:
    llvm_unreachable("Declaration should not be in declstmts!");
 
  case Decl::Function:     // void X();
  case Decl::EnumConstant: // enum ? { X = ? }
  case Decl::StaticAssert: // static_assert(X, ""); [C++0x]
  case Decl::Label:        // __label__ x;
  case Decl::Import:
  case Decl::MSGuid: // __declspec(uuid("..."))
  case Decl::TemplateParamObject:
  case Decl::Empty:
  case Decl::Concept:
  case Decl::LifetimeExtendedTemporary:
  case Decl::RequiresExprBody:
  case Decl::UnnamedGlobalConstant:
    // None of these decls require codegen support.
    return;
 
  case Decl::Enum:      // enum X;
  case Decl::Record:    // struct/union/class X;
  case Decl::CXXRecord: // struct/union/class X; [C++]
  case Decl::NamespaceAlias:
  case Decl::Using:          // using X; [C++]
  case Decl::UsingEnum:      // using enum X; [C++]
  case Decl::UsingDirective: // using namespace X; [C++]
    assert(!cir::MissingFeatures::generateDebugInfo());
    return;
  case Decl::Var:
  case Decl::Decomposition: {
    const VarDecl &vd = cast<VarDecl>(d);
    assert(vd.isLocalVarDecl() &&
           "Should not see file-scope variables inside a function!");
    emitVarDecl(vd);
    if (evaluateConditionDecl)
      maybeEmitDeferredVarDeclInit(&vd);
    return;
  }
  case Decl::OpenACCDeclare:
    emitOpenACCDeclare(cast<OpenACCDeclareDecl>(d));
    return;
  case Decl::OpenACCRoutine:
    emitOpenACCRoutine(cast<OpenACCRoutineDecl>(d));
    return;
  case Decl::OMPThreadPrivate:
    emitOMPThreadPrivateDecl(cast<OMPThreadPrivateDecl>(d));
    return;
  case Decl::OMPGroupPrivate:
    emitOMPGroupPrivateDecl(cast<OMPGroupPrivateDecl>(d));
    return;
  case Decl::OMPAllocate:
    emitOMPAllocateDecl(cast<OMPAllocateDecl>(d));
    return;
  case Decl::OMPCapturedExpr:
    emitOMPCapturedExpr(cast<OMPCapturedExprDecl>(d));
    return;
  case Decl::OMPRequires:
    emitOMPRequiresDecl(cast<OMPRequiresDecl>(d));
    return;
  case Decl::OMPDeclareMapper:
    emitOMPDeclareMapper(cast<OMPDeclareMapperDecl>(d));
    return;
  case Decl::OMPDeclareReduction:
    emitOMPDeclareReduction(cast<OMPDeclareReductionDecl>(d));
    return;
  case Decl::Typedef:     // typedef int X;
  case Decl::TypeAlias: { // using X = int; [C++0x]
    QualType ty = cast<TypedefNameDecl>(d).getUnderlyingType();
    assert(!cir::MissingFeatures::generateDebugInfo());
    if (ty->isVariablyModifiedType())
      cgm.errorNYI(d.getSourceRange(), "emitDecl: variably modified type");
    return;
  }
  case Decl::ImplicitConceptSpecialization:
  case Decl::TopLevelStmt:
  case Decl::UsingPack:
    cgm.errorNYI(d.getSourceRange(),
                 std::string("emitDecl: unhandled decl type: ") +
                     d.getDeclKindName());
  }
}
 
void CIRGenFunction::emitNullabilityCheck(LValue lhs, mlir::Value rhs,
                                          SourceLocation loc) {
  if (!sanOpts.has(SanitizerKind::NullabilityAssign))
    return;
 
  assert(!cir::MissingFeatures::sanitizers());
}
 
namespace {
struct DestroyObject final : EHScopeStack::Cleanup {
  DestroyObject(Address addr, QualType type,
                CIRGenFunction::Destroyer *destroyer)
      : addr(addr), type(type), destroyer(destroyer) {
    assert(!cir::MissingFeatures::useEHCleanupForArray());
  }
 
  Address addr;
  QualType type;
  CIRGenFunction::Destroyer *destroyer;
 
  void emit(CIRGenFunction &cgf, Flags flags) override {
    assert(!cir::MissingFeatures::useEHCleanupForArray());
    cgf.emitDestroy(addr, type, destroyer);
  }
};
 
template <class Derived> struct DestroyNRVOVariable : EHScopeStack::Cleanup {
  DestroyNRVOVariable(Address addr, QualType type, mlir::Value nrvoFlag)
      : nrvoFlag(nrvoFlag), addr(addr), ty(type) {}
 
  mlir::Value nrvoFlag;
  Address addr;
  QualType ty;
 
  void emit(CIRGenFunction &cgf, Flags flags) override {
    // Along the exceptions path we always execute the dtor.
    bool nrvo = flags.isForNormalCleanup() && nrvoFlag;
 
    CIRGenBuilderTy &builder = cgf.getBuilder();
    mlir::OpBuilder::InsertionGuard guard(builder);
    if (nrvo) {
      // If we exited via NRVO, we skip the destructor call.
      mlir::Location loc = addr.getPointer().getLoc();
      mlir::Value didNRVO = builder.createFlagLoad(loc, nrvoFlag);
      mlir::Value notNRVO = builder.createNot(didNRVO);
      cir::IfOp::create(builder, loc, notNRVO, /*withElseRegion=*/false,
                        [&](mlir::OpBuilder &b, mlir::Location) {
                          static_cast<Derived *>(this)->emitDestructorCall(cgf);
                          builder.createYield(loc);
                        });
    } else {
      static_cast<Derived *>(this)->emitDestructorCall(cgf);
    }
  }
 
  virtual ~DestroyNRVOVariable() = default;
};
 
struct DestroyNRVOVariableCXX final
    : DestroyNRVOVariable<DestroyNRVOVariableCXX> {
  DestroyNRVOVariableCXX(Address addr, QualType type,
                         const CXXDestructorDecl *dtor, mlir::Value nrvoFlag)
      : DestroyNRVOVariable<DestroyNRVOVariableCXX>(addr, type, nrvoFlag),
        dtor(dtor) {}
 
  const CXXDestructorDecl *dtor;
 
  void emitDestructorCall(CIRGenFunction &cgf) {
    cgf.emitCXXDestructorCall(dtor, Dtor_Complete,
                              /*forVirtualBase=*/false,
                              /*delegating=*/false, addr, ty);
  }
};
 
struct CallStackRestore final : EHScopeStack::Cleanup {
  Address stack;
  CallStackRestore(Address stack) : stack(stack) {}
  void emit(CIRGenFunction &cgf, Flags flags) override {
    mlir::Location loc = stack.getPointer().getLoc();
    mlir::Value v = cgf.getBuilder().createLoad(loc, stack);
    cgf.getBuilder().createStackRestore(loc, v);
  }
};
 
/// A cleanup which performs a partial array destroy where the end pointer is
/// irregularly determined and must be loaded from a local.
struct IrregularPartialArrayDestroy final : EHScopeStack::Cleanup {
  mlir::Value arrayBegin;
  Address arrayEndPointer;
  QualType elementType;
  CharUnits elementAlign;
  CIRGenFunction::Destroyer *destroyer;
 
  IrregularPartialArrayDestroy(mlir::Value arrayBegin, Address arrayEndPointer,
                               QualType elementType, CharUnits elementAlign,
                               CIRGenFunction::Destroyer *destroyer)
      : arrayBegin(arrayBegin), arrayEndPointer(arrayEndPointer),
        elementType(elementType), elementAlign(elementAlign),
        destroyer(destroyer) {}
 
  void emit(CIRGenFunction &cgf, Flags flags) override {
    CIRGenBuilderTy &builder = cgf.getBuilder();
    mlir::Location loc = arrayBegin.getLoc();
 
    mlir::Value arrayEnd = builder.createLoad(loc, arrayEndPointer);
 
    // The cleanup is destroying elements in reverse from arrayEnd back to
    // arrayBegin, but only if arrayEnd != arrayBegin (i.e. something was
    // constructed).
    mlir::Type cirElementType = cgf.convertTypeForMem(elementType);
    cir::PointerType ptrToElmType = builder.getPointerTo(cirElementType);
 
    mlir::Value ne = cir::CmpOp::create(builder, loc, cir::CmpOpKind::ne,
                                        arrayEnd, arrayBegin);
    cir::IfOp::create(
        builder, loc, ne, /*withElseRegion=*/false,
        [&](mlir::OpBuilder &b, mlir::Location loc) {
          Address iterAddr = cgf.createTempAlloca(
              ptrToElmType, cgf.getPointerAlign(), loc, "__array_idx");
          builder.createStore(loc, arrayEnd, iterAddr);
          builder.createDoWhile(
              loc,
              /*condBuilder=*/
              [&](mlir::OpBuilder &b, mlir::Location loc) {
                mlir::Value cur = builder.createLoad(loc, iterAddr);
                mlir::Value cmp = cir::CmpOp::create(
                    builder, loc, cir::CmpOpKind::ne, cur, arrayBegin);
                builder.createCondition(cmp);
              },
              /*bodyBuilder=*/
              [&](mlir::OpBuilder &b, mlir::Location loc) {
                mlir::Value cur = builder.createLoad(loc, iterAddr);
                cir::ConstantOp negOne = builder.getConstInt(
                    loc, mlir::cast<cir::IntType>(cgf.ptrDiffTy), -1);
                mlir::Value prev = cir::PtrStrideOp::create(
                    builder, loc, ptrToElmType, cur, negOne);
                builder.createStore(loc, prev, iterAddr);
                Address elemAddr = Address(prev, cirElementType, elementAlign);
                destroyer(cgf, elemAddr, elementType);
                builder.createYield(loc);
              });
          builder.createYield(loc);
        });
  }
};
} // namespace
 
/// Push an EH cleanup to destroy already-constructed elements of the given
/// array.  The cleanup may be popped with deactivateCleanupBlock or
/// popCleanupBlock.
///
/// \param elementType - the immediate element type of the array;
///   possibly still an array type
void CIRGenFunction::pushIrregularPartialArrayCleanup(mlir::Value arrayBegin,
                                                      Address arrayEndPointer,
                                                      QualType elementType,
                                                      CharUnits elementAlign,
                                                      Destroyer *destroyer) {
  ehStack.pushCleanup<IrregularPartialArrayDestroy>(
      EHCleanup, arrayBegin, arrayEndPointer, elementType, elementAlign,
      destroyer);
}
 
/// pushEHDestroyIfNeeded - Push the standard destructor for the given type as
/// an EH-only cleanup. If EH cleanup is not needed, just return.
void CIRGenFunction::pushEHDestroyIfNeeded(QualType::DestructionKind dtorKind,
                                           Address addr, QualType type) {
  if (!needsEHCleanup(dtorKind))
    return;
 
  assert(!cir::MissingFeatures::useEHCleanupForArray());
  pushDestroy(EHCleanup, addr, type, getDestroyer(dtorKind));
}
 
/// Push the standard destructor for the given type as
/// at least a normal cleanup.
void CIRGenFunction::pushDestroy(QualType::DestructionKind dtorKind,
                                 Address addr, QualType type) {
  assert(dtorKind && "cannot push destructor for trivial type");
 
  CleanupKind cleanupKind = getCleanupKind(dtorKind);
  pushDestroy(cleanupKind, addr, type, getDestroyer(dtorKind));
}
 
void CIRGenFunction::pushDestroy(CleanupKind cleanupKind, Address addr,
                                 QualType type, Destroyer *destroyer) {
  pushFullExprCleanup<DestroyObject>(cleanupKind, addr, type, destroyer);
}
 
void CIRGenFunction::pushDestroyAndDeferDeactivation(
    QualType::DestructionKind dtorKind, Address addr, QualType type) {
  assert(dtorKind && "cannot push destructor for trivial type");
 
  CleanupKind cleanupKind = getCleanupKind(dtorKind);
  pushDestroyAndDeferDeactivation(
      cleanupKind, addr, type, getDestroyer(dtorKind), cleanupKind & EHCleanup);
}
 
void CIRGenFunction::pushDestroyAndDeferDeactivation(
    CleanupKind cleanupKind, Address addr, QualType type, Destroyer *destroyer,
    bool useEHCleanupForArray) {
  assert(!cir::MissingFeatures::useEHCleanupForArray());
  pushCleanupAndDeferDeactivation<DestroyObject>(cleanupKind, addr, type,
                                                 destroyer);
}
 
void CIRGenFunction::pushLifetimeExtendedDestroy(CleanupKind cleanupKind,
                                                 Address addr, QualType type,
                                                 Destroyer *destroyer,
                                                 bool useEHCleanupForArray) {
  if (isInConditionalBranch()) {
    cgm.errorNYI("conditional lifetime-extended destroy");
    return;
  }
 
  // Add the cleanup to the EHStack. After the full-expr, this would be
  // deactivated before being popped from the stack.
  pushDestroyAndDeferDeactivation(cleanupKind, addr, type, destroyer,
                                  useEHCleanupForArray);
 
  assert(!cir::MissingFeatures::useEHCleanupForArray());
 
  pushCleanupAfterFullExpr(cleanupKind, addr, type, destroyer);
}
 
void CIRGenFunction::pushPendingCleanupToEHStack(
    const PendingCleanupEntry &entry) {
  ehStack.pushCleanup<DestroyObject>(entry.kind, entry.addr, entry.type,
                                     entry.destroyer);
 
  if (entry.activeFlag.isValid()) {
    EHCleanupScope &scope = cast<EHCleanupScope>(*ehStack.begin());
    scope.setActiveFlag(entry.activeFlag);
    if (scope.isNormalCleanup())
      scope.setTestFlagInNormalCleanup();
    if (scope.isEHCleanup())
      scope.setTestFlagInEHCleanup();
  }
}
 
/// Destroys all the elements of the given array, beginning from last to first.
///
/// \param begin - a type* denoting the first element of the array
/// \param numElements - the number of elements in the array
/// \param elementType - the element type of the array
/// \param destroyer - the function to call to destroy elements
void CIRGenFunction::emitArrayDestroy(mlir::Value begin,
                                      mlir::Value numElements,
                                      QualType elementType,
                                      CharUnits elementAlign,
                                      Destroyer *destroyer) {
  assert(!elementType->isArrayType());
 
  // Differently from LLVM traditional codegen, use a higher level
  // representation instead of lowering directly to a loop.
  mlir::Type cirElementType = convertTypeForMem(elementType);
  cir::PointerType ptrToElmType = builder.getPointerTo(cirElementType);
 
  auto regionBuilder = [&](mlir::OpBuilder &b, mlir::Location loc) {
    mlir::BlockArgument arg =
        b.getInsertionBlock()->addArgument(ptrToElmType, loc);
    Address curAddr = Address(arg, cirElementType, elementAlign);
    assert(!cir::MissingFeatures::dtorCleanups());
 
    // Perform the actual destruction there.
    destroyer(*this, curAddr, elementType);
 
    cir::YieldOp::create(b, loc);
  };
 
  // For a constant array size, use the static form of ArrayDtor.
  if (auto constantCount = numElements.getDefiningOp<cir::ConstantOp>()) {
    uint64_t size = 0;
    if (auto constIntAttr = constantCount.getValueAttr<cir::IntAttr>())
      size = constIntAttr.getUInt();
    auto arrayTy = cir::ArrayType::get(cirElementType, size);
    mlir::Value arrayOp = builder.createPtrBitcast(begin, arrayTy);
    cir::ArrayDtor::create(builder, *currSrcLoc, arrayOp, regionBuilder);
    return;
  }
 
  // For a dynamic array size (VLA), use the dynamic form of ArrayDtor.
  mlir::Value elemBegin = builder.createPtrBitcast(begin, cirElementType);
  cir::ArrayDtor::create(builder, *currSrcLoc, elemBegin, numElements,
                         regionBuilder);
}
 
/// Immediately perform the destruction of the given object.
///
/// \param addr - the address of the object; a type*
/// \param type - the type of the object; if an array type, all
///   objects are destroyed in reverse order
/// \param destroyer - the function to call to destroy individual
///   elements
void CIRGenFunction::emitDestroy(Address addr, QualType type,
                                 Destroyer *destroyer) {
  const ArrayType *arrayType = getContext().getAsArrayType(type);
  if (!arrayType)
    return destroyer(*this, addr, type);
 
  mlir::Value length = emitArrayLength(arrayType, type, addr);
 
  CharUnits elementAlign = addr.getAlignment().alignmentOfArrayElement(
      getContext().getTypeSizeInChars(type));
 
  // If the array length is constant, we can check for zero at compile time.
  auto constantCount = length.getDefiningOp<cir::ConstantOp>();
  if (constantCount) {
    auto constIntAttr = mlir::dyn_cast<cir::IntAttr>(constantCount.getValue());
    if (constIntAttr && constIntAttr.getUInt() == 0)
      return;
  }
 
  mlir::Value begin = addr.getPointer();
  assert(!cir::MissingFeatures::useEHCleanupForArray());
  emitArrayDestroy(begin, length, type, elementAlign, destroyer);
 
  // If the array destroy didn't use the length op, we can erase it.
  if (constantCount && constantCount.use_empty())
    constantCount.erase();
}
 
CIRGenFunction::Destroyer *
CIRGenFunction::getDestroyer(QualType::DestructionKind kind) {
  switch (kind) {
  case QualType::DK_none:
    llvm_unreachable("no destroyer for trivial dtor");
  case QualType::DK_cxx_destructor:
    return destroyCXXObject;
  case QualType::DK_objc_strong_lifetime:
  case QualType::DK_objc_weak_lifetime:
  case QualType::DK_nontrivial_c_struct:
    cgm.errorNYI("getDestroyer: other destruction kind");
    return nullptr;
  }
  llvm_unreachable("Unknown DestructionKind");
}
 
void CIRGenFunction::pushStackRestore(CleanupKind kind, Address spMem) {
  ehStack.pushCleanup<CallStackRestore>(kind, spMem);
}
 
/// Enter a destroy cleanup for the given local variable.
void CIRGenFunction::emitAutoVarTypeCleanup(
    const CIRGenFunction::AutoVarEmission &emission,
    QualType::DestructionKind dtorKind) {
  assert(dtorKind != QualType::DK_none);
 
  // Note that for __block variables, we want to destroy the
  // original stack object, not the possibly forwarded object.
  Address addr = emission.getObjectAddress(*this);
 
  const VarDecl *var = emission.variable;
  QualType type = var->getType();
 
  CleanupKind cleanupKind = NormalAndEHCleanup;
  CIRGenFunction::Destroyer *destroyer = nullptr;
 
  switch (dtorKind) {
  case QualType::DK_none:
    llvm_unreachable("no cleanup for trivially-destructible variable");
 
  case QualType::DK_cxx_destructor:
    // If there's an NRVO flag on the emission, we need a different
    // cleanup.
    if (emission.nrvoFlag) {
      assert(!type->isArrayType());
      CXXDestructorDecl *dtor = type->getAsCXXRecordDecl()->getDestructor();
      ehStack.pushCleanup<DestroyNRVOVariableCXX>(cleanupKind, addr, type, dtor,
                                                  emission.nrvoFlag);
      return;
    }
    // Otherwise, this is handled below.
    break;
 
  case QualType::DK_objc_strong_lifetime:
  case QualType::DK_objc_weak_lifetime:
  case QualType::DK_nontrivial_c_struct:
    cgm.errorNYI(var->getSourceRange(),
                 "emitAutoVarTypeCleanup: other dtor kind");
    return;
  }
 
  // If we haven't chosen a more specific destroyer, use the default.
  if (!destroyer)
    destroyer = getDestroyer(dtorKind);
 
  assert(!cir::MissingFeatures::useEHCleanupForArray());
  ehStack.pushCleanup<DestroyObject>(cleanupKind, addr, type, destroyer);
}
 
void CIRGenFunction::maybeEmitDeferredVarDeclInit(const VarDecl *vd) {
  if (auto *dd = dyn_cast_if_present<DecompositionDecl>(vd)) {
    for (auto *b : dd->flat_bindings())
      if (auto *hd = b->getHoldingVar())
        emitVarDecl(*hd);
  }
}