LoweringPrepare.cpp

Go to the documentation of this file.

//===- LoweringPrepare.cpp - pareparation work for LLVM lowering ----------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
 
#include "PassDetail.h"
#include "TargetLowering/LowerModule.h"
#include "mlir/IR/Attributes.h"
#include "mlir/IR/BuiltinAttributeInterfaces.h"
#include "mlir/IR/IRMapping.h"
#include "mlir/IR/Location.h"
#include "mlir/IR/Value.h"
#include "clang/Basic/Cuda.h"
#include "clang/Basic/SourceManager.h"
#include "clang/Basic/Specifiers.h"
#include "clang/Basic/TargetCXXABI.h"
#include "clang/Basic/TargetInfo.h"
#include "clang/CIR/Dialect/Builder/CIRBaseBuilder.h"
#include "clang/CIR/Dialect/IR/CIRAttrs.h"
#include "clang/CIR/Dialect/IR/CIRDataLayout.h"
#include "clang/CIR/Dialect/IR/CIRDialect.h"
#include "clang/CIR/Dialect/IR/CIROpsEnums.h"
#include "clang/CIR/Dialect/IR/CIRTypes.h"
#include "clang/CIR/Dialect/Passes.h"
#include "clang/CIR/Dialect/Transforms/CIRTransformUtils.h"
#include "clang/CIR/Interfaces/ASTAttrInterfaces.h"
#include "clang/CIR/MissingFeatures.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/TypeSwitch.h"
#include "llvm/IR/Instructions.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/VersionTuple.h"
 
#include <map>
#include <memory>
#include <optional>
 
using namespace mlir;
using namespace cir;
 
namespace mlir {
#define GEN_PASS_DEF_LOWERINGPREPARE
#include "clang/CIR/Dialect/Passes.h.inc"
} // namespace mlir
 
static SmallString<128> getTransformedFileName(mlir::ModuleOp mlirModule) {
  SmallString<128> fileName;
 
  if (mlirModule.getSymName())
    fileName = llvm::sys::path::filename(mlirModule.getSymName()->str());
 
  if (fileName.empty())
    fileName = "<null>";
 
  for (size_t i = 0; i < fileName.size(); ++i) {
    // Replace everything that's not [a-zA-Z0-9._] with a _. This set happens
    // to be the set of C preprocessing numbers.
    if (!clang::isPreprocessingNumberBody(fileName[i]))
      fileName[i] = '_';
  }
 
  return fileName;
}
 
namespace {
struct LoweringPreparePass
    : public impl::LoweringPrepareBase<LoweringPreparePass> {
  LoweringPreparePass() = default;
 
  // `mlir::SymbolTableCollection` is move-only (it owns lazily-created
  // `unique_ptr<SymbolTable>` entries), which makes the implicit copy
  // constructor ill-formed.  MLIR's `clonePass()` requires copy
  // construction, so define one explicitly.  Per-run state members
  // (dynamic initializers, guard maps, symbol-table cache, etc.) all
  // start fresh in the cloned pass, which matches MLIR convention for
  // pass clones and is more correct than the previous default-generated
  // behavior that silently copied them.
  LoweringPreparePass(const LoweringPreparePass &other)
      : impl::LoweringPrepareBase<LoweringPreparePass>(other) {}
 
  void runOnOperation() override;
 
  void runOnOp(mlir::Operation *op);
  void lowerCastOp(cir::CastOp op);
  void lowerComplexConjOp(cir::ComplexConjOp op);
  void lowerComplexDivOp(cir::ComplexDivOp op);
  void lowerComplexMulOp(cir::ComplexMulOp op);
  void lowerComplexFDivOp(cir::ComplexFDivOp op);
  void lowerComplexFMulOp(cir::ComplexFMulOp op);
  void lowerGetGlobalOp(cir::GetGlobalOp op);
  void lowerGlobalOp(cir::GlobalOp op);
  void lowerThreeWayCmpOp(cir::CmpThreeWayOp op);
  void lowerArrayDtor(cir::ArrayDtor op);
  void lowerArrayCtor(cir::ArrayCtor op);
  void lowerTrivialCopyCall(cir::CallOp op);
  void lowerStoreOfConstAggregate(cir::StoreOp op);
  void lowerLocalInitOp(cir::LocalInitOp op);
  void lowerRegisterExitDtorOp(cir::RegisterExitDtorOp op);
  void lowerStdOp(cir::StdOpInterface op);
 
  /// Return the FuncOp called by `callOp`.  Uses the cached `symbolTables`
  /// member to avoid the O(M) module-wide scan that the static
  /// `mlir::SymbolTable::lookupNearestSymbolFrom` would do per call.
  cir::FuncOp getCalledFunction(cir::CallOp callOp);
 
  /// Return a private constant cir::GlobalOp with the given type and initial
  /// value, suitable for backing a memcpy-initialized local aggregate.
  ///
  /// If a global with `baseName` (or one of its `.<n>` versioned siblings)
  /// already has a matching type and initial value, that global is reused.
  /// Otherwise a new global is created with the next available `.<n>` suffix
  /// (matching CIRGenBuilder::createVersionedGlobal and OGCG behavior).
  cir::GlobalOp
  getOrCreateConstAggregateGlobal(CIRBaseBuilderTy &builder, mlir::Location loc,
                                  llvm::StringRef baseName, mlir::Type ty,
                                  mlir::TypedAttr constant, uint64_t alignment);
 
  /// Build the function that initializes the specified global
  cir::FuncOp buildCXXGlobalVarDeclInitFunc(cir::GlobalOp op);
 
  /// When looking at the 'global' op, create the wrapper function.
  void defineGlobalThreadLocalWrapper(cir::GlobalOp op, cir::FuncOp initAlias,
                                      bool isVarDefinition);
  /// Create an initialization alias for a thread-local variable.
  cir::FuncOp defineGlobalThreadLocalInitAlias(cir::GlobalOp op,
                                               cir::FuncOp aliasee);
  /// Get the declaration for the 'wrapper' function for a global-TLS variable.
  cir::FuncOp getOrCreateThreadLocalWrapper(CIRBaseBuilderTy &builder,
                                            cir::GlobalOp op);
  // Function that generates the get-global and 'if' condition on a guard
  // variable for an init function that must run once: the global TLS init
  // function, or the initializer of a C++20 named-module interface unit. This
  // inserts an 'if' with the store at the beginning of the 'then' region, so
  // inserts into the body should happen after that.
  cir::IfOp buildInitGuardCheck(CIRBaseBuilderTy &builder, mlir::Location loc,
                                cir::GlobalOp guard, bool threadLocal);
  /// Handle the dtor region by registering destructor with __cxa_atexit
  cir::FuncOp getOrCreateDtorFunc(CIRBaseBuilderTy &builder, cir::GlobalOp op,
                                  mlir::Region &dtorRegion,
                                  cir::CallOp &dtorCall);
  /// Create a `void __cxx_global_array_dtor[.N](void *)` helper after `op`,
  /// with an empty entry block.
  cir::FuncOp buildGlobalArrayDtorFunc(CIRBaseBuilderTy &builder,
                                       mlir::Operation *op);
 
  /// Build a function named `fnName` with the given linkage that calls each
  /// of `initializers` in order, then returns, and register it in
  /// `globalCtorList` under `priority`. With a `guard` byte, the calls run
  /// only when it is still zero and set it first. Shared by the default
  /// `_GLOBAL__sub_I_*` initializer function, the C++20 named-module
  /// initializer, and the per-priority `_GLOBAL__I_<priority>` initializer
  /// functions.
  cir::FuncOp
  buildGlobalInitCallerFunc(llvm::StringRef fnName,
                            cir::GlobalLinkageKind linkage,
                            llvm::ArrayRef<cir::FuncOp> initializers,
                            uint32_t priority, cir::GlobalOp guard = {});
 
  /// Build a module init function that calls all the dynamic initializers.
  void buildCXXGlobalInitFunc();
  /// Build one `_GLOBAL__I_<priority>` function per distinct priority found
  /// in `prioritizedDynamicInitializers`, in ascending priority order, and
  /// register each with `globalCtorList`. The first of them, the highest
  /// priority, calls `importedInits` before its own initializers and takes
  /// them out of the vector, as classic CodeGen does.
  void buildCXXGlobalPriorityInitFuncs(
      llvm::SmallVectorImpl<cir::FuncOp> &importedInits);
  // Build an init function for all of the ordered global thread local storage
  // variables.
  void buildCXXGlobalTlsFunc();
 
  /// Materialize global ctor/dtor list
  void buildGlobalCtorDtorList();
 
  cir::FuncOp buildRuntimeFunction(
      mlir::OpBuilder &builder, llvm::StringRef name, mlir::Location loc,
      cir::FuncType type,
      cir::GlobalLinkageKind linkage = cir::GlobalLinkageKind::ExternalLinkage);
 
  cir::GlobalOp getOrCreateRuntimeVariable(
      mlir::OpBuilder &builder, llvm::StringRef name, mlir::Location loc,
      mlir::Type type,
      cir::GlobalLinkageKind linkage = cir::GlobalLinkageKind::ExternalLinkage,
      cir::VisibilityKind visibility = cir::VisibilityKind::Default);
 
  /// ------------
  /// CUDA registration related
  /// ------------
 
  llvm::StringMap<FuncOp> cudaKernelMap;
  llvm::SmallVector<std::pair<cir::GlobalOp, cir::CUDAVarRegistrationInfoAttr>>
      cudaDeviceVars;
 
  /// Build the CUDA module constructor that registers the fat binary
  /// with the CUDA runtime.
  void buildCUDAModuleCtor();
  std::optional<FuncOp> buildCUDAModuleDtor();
  std::optional<FuncOp> buildHIPModuleDtor();
  std::optional<FuncOp> buildCUDARegisterGlobals();
  void buildCUDARegisterVars(cir::CIRBaseBuilderTy &builder,
                             FuncOp regGlobalFunc);
  void buildCUDARegisterGlobalFunctions(cir::CIRBaseBuilderTy &builder,
                                        FuncOp regGlobalFunc);
 
  /// Handle static local variable initialization with guard variables.
  void handleStaticLocal(cir::GlobalOp globalOp, cir::LocalInitOp localInitOp);
 
  /// Handle the guard-protected init for static local/weak/linkonce-odr
  /// variables.
  void emitGuardedInit(CIRBaseBuilderTy &builder, cir::GlobalOp globalOp,
                       mlir::Region &ctorRegion, mlir::Region &dtorRegion);
 
  /// Get or create __cxa_guard_acquire function.
  cir::FuncOp getGuardAcquireFn(cir::PointerType guardPtrTy);
 
  /// Get or create __cxa_guard_release function.
  cir::FuncOp getGuardReleaseFn(cir::PointerType guardPtrTy);
 
  /// Get or create __cxa_guard_abort function.
  cir::FuncOp getGuardAbortFn(cir::PointerType guardPtrTy);
 
  /// Get or create the __init_tls function.
  cir::FuncOp getTlsInitFn();
 
  // Create the __tls_guard variable.
  cir::GlobalOp createGlobalThreadLocalGuard(CIRBaseBuilderTy &builder,
                                             mlir::Location loc);
 
  /// Create a guard global variable for a static local.
  cir::GlobalOp createGuardGlobalOp(CIRBaseBuilderTy &builder,
                                    mlir::Location loc, llvm::StringRef name,
                                    cir::IntType guardTy,
                                    cir::GlobalLinkageKind linkage);
 
  /// Append 'global' to the module's llvm.used list to make sure it doesn't get
  /// garbage-collected by the linker.
  void addUsedGlobal(cir::GlobalOp global);
 
  /// Get the guard variable for a static local declaration.
  cir::GlobalOp getStaticLocalDeclGuardAddress(llvm::StringRef globalSymName) {
    auto it = staticLocalDeclGuardMap.find(globalSymName);
    if (it != staticLocalDeclGuardMap.end())
      return it->second;
    return nullptr;
  }
 
  /// Set the guard variable for a static local declaration.
  void setStaticLocalDeclGuardAddress(llvm::StringRef globalSymName,
                                      cir::GlobalOp guard) {
    staticLocalDeclGuardMap[globalSymName] = guard;
  }
 
  /// Get or create the guard variable for a static local declaration.
  cir::GlobalOp getOrCreateStaticLocalDeclGuardAddress(
      CIRBaseBuilderTy &builder, cir::GlobalOp globalOp, StringRef guardName,
      bool isLocalVarDecl, bool useInt8GuardVariable) {
 
    cir::CIRDataLayout dataLayout(mlirModule);
    cir::IntType guardTy;
    clang::CharUnits guardAlignment;
    // Guard variables are 64 bits in the generic ABI and size width on ARM
    // (i.e. 32-bit on AArch32, 64-bit on AArch64).
    if (useInt8GuardVariable) {
      guardTy = cir::IntType::get(&getContext(), 8, /*isSigned=*/true);
      guardAlignment = clang::CharUnits::One();
    } else if (useARMGuardVarABI()) {
      // Guard variables are size width on ARM (32-bit AArch32, 64-bit AArch64).
      const unsigned sizeTypeSize =
          getTargetInfo().getTypeWidth(getTargetInfo().getSignedSizeType());
      guardTy =
          cir::IntType::get(&getContext(), sizeTypeSize, /*isSigned=*/true);
      guardAlignment =
          clang::CharUnits::fromQuantity(dataLayout.getABITypeAlign(guardTy));
    } else {
      guardTy = cir::IntType::get(&getContext(), 64, /*isSigned=*/true);
      guardAlignment =
          clang::CharUnits::fromQuantity(dataLayout.getABITypeAlign(guardTy));
    }
    assert(guardTy && guardAlignment.getQuantity() != 0);
 
    llvm::StringRef globalSymName = globalOp.getSymName();
    cir::GlobalOp guard = getStaticLocalDeclGuardAddress(globalSymName);
    if (!guard) {
      // Create the guard variable with a zero-initializer.
      guard = createGuardGlobalOp(builder, globalOp->getLoc(), guardName,
                                  guardTy, globalOp.getLinkage());
      guard.setInitialValueAttr(cir::IntAttr::get(guardTy, 0));
      guard.setDSOLocal(globalOp.getDsoLocal());
      guard.setAlignment(guardAlignment.getAsAlign().value());
      guard.setTlsModel(globalOp.getTlsModel());
      guard.setGlobalVisibility(globalOp.getGlobalVisibility());
 
      // The ABI says: "It is suggested that it be emitted in the same COMDAT
      // group as the associated data object." In practice, this doesn't work
      // for non-ELF and non-Wasm object formats, so only do it for ELF and
      // Wasm.
      std::optional<llvm::StringRef> comdat = globalOp.getComdat();
      const llvm::Triple &triple = getTargetInfo().getTriple();
      if (!isLocalVarDecl && comdat.has_value() &&
          (triple.isOSBinFormatELF() || triple.isOSBinFormatWasm())) {
        guard.setComdat(comdat->empty() ? globalOp.getSymName() : *comdat);
      } else if (triple.supportsCOMDAT() && guard.isWeakForLinker()) {
        // A weak guard that isn't joining the variable's COMDAT gets its own,
        // even if the variable has none (such as a plain weak variable).
        guard.setSelfComdat();
      }
 
      setStaticLocalDeclGuardAddress(globalSymName, guard);
    }
    return guard;
  }
 
  /// Target/ABI facts sourced from the module's own attributes.
  std::unique_ptr<cir::LowerModule> lowerModule;
 
  const clang::TargetInfo &getTargetInfo() const {
    assert(lowerModule && "LoweringPrepare requires a module with a triple");
    return lowerModule->getTarget();
  }
 
  const clang::LangOptions &getLangOpts() const {
    assert(lowerModule && "LoweringPrepare requires a module with LangOptions");
    return lowerModule->getLangOpts();
  }
 
  /// Platform SDK version recorded on the module by CIRGen. An absent attribute
  /// yields an empty VersionTuple, which is what a compilation without
  /// -target-sdk-version behaves like: no version-gated feature is enabled.
  /// Returns std::nullopt if the attribute is present but unparseable; an error
  /// has been emitted and the pass marked as failed, so the caller must bail
  /// out.
  std::optional<llvm::VersionTuple> getSDKVersion() {
    auto sdkVersionAttr = mlirModule->getAttrOfType<mlir::StringAttr>(
        CIRDialect::getSDKVersionAttrName());
    if (!sdkVersionAttr)
      return llvm::VersionTuple();
 
    llvm::VersionTuple sdkVersion;
    if (sdkVersion.tryParse(sdkVersionAttr.getValue())) {
      mlirModule->emitError("cannot parse platform SDK version from ")
          << CIRDialect::getSDKVersionAttrName() << " = '"
          << sdkVersionAttr.getValue() << "'";
      signalPassFailure();
      return std::nullopt;
    }
    return sdkVersion;
  }
 
  /// Appends the CUID hash recorded on the module by CIRGen, if any, to
  /// \p name.
  std::string addCUIDSuffix(llvm::StringRef name) {
    std::string result = name.str();
    if (auto cuidHash = mlirModule->getAttrOfType<mlir::StringAttr>(
            CIRDialect::getCUIDHashAttrName()))
      result += ("_" + cuidHash.getValue()).str();
    return result;
  }
 
  /// Tracks current module.
  mlir::ModuleOp mlirModule;
 
  /// Cached symbol tables used to avoid repeated O(M) module-wide scans
  /// during per-call/per-global symbol lookups.  Lazily populated on first
  /// use.  Pass methods access this directly rather than threading it
  /// through helper signatures (see PR feedback on #195919).
  ///
  /// Invariant: every site that mutates the module's symbol table either
  /// (a) keeps `symbolTables` in sync via
  /// `symbolTables.getSymbolTable(mlirModule).insert(...)` (as
  /// `getOrCreateConstAggregateGlobal` does), or (b) creates a symbol
  /// that is never resolved through the cache later.  Today
  /// `buildRuntimeFunction` and `getOrCreateRuntimeVariable` fall in the
  /// (b) bucket: their callers either use a separate map
  /// (`cudaKernelMap`, `staticLocalDeclGuardMap`, `dynamicInitializers`)
  /// or the static `mlir::SymbolTable::lookupNearestSymbolFrom`, never
  /// the cached path.  If a future change adds a cached lookup of a
  /// freshly created symbol, the corresponding create site MUST move
  /// to bucket (a) (insert into the cache or call
  /// `invalidateSymbolTable`).
  mlir::SymbolTableCollection symbolTables;
 
  /// Tracks existing dynamic initializers.
  llvm::StringMap<uint32_t> dynamicInitializerNames;
  llvm::SmallVector<cir::FuncOp> dynamicInitializers;
  /// Dynamic initializers with an explicit `init_priority` attribute,
  /// grouped by priority (keys kept in ascending order). Each group is
  /// emitted into its own `_GLOBAL__I_<priority>` function instead of being
  /// folded into the single default-priority `_GLOBAL__sub_I_*` function.
  std::map<unsigned, llvm::SmallVector<cir::FuncOp, 4>>
      prioritizedDynamicInitializers;
  llvm::SmallVector<cir::FuncOp> globalThreadLocalInitializers;
  llvm::StringMap<cir::FuncOp> threadLocalWrappers;
  llvm::StringMap<cir::FuncOp> threadLocalInitAliases;
 
  /// Tracks guard variables for static locals (keyed by global symbol name).
  llvm::StringMap<cir::GlobalOp> staticLocalDeclGuardMap;
 
  llvm::StringMap<llvm::SmallVector<cir::GlobalOp, 1>> constAggregateGlobals;
 
  /// List of ctors and their priorities to be called before main()
  llvm::SmallVector<cir::GlobalCtorDtorEntry, 4> globalCtorList;
  /// List of dtors and their priorities to be called when unloading module.
  llvm::SmallVector<cir::GlobalCtorDtorEntry, 4> globalDtorList;
 
  /// Returns true if the target uses ARM-style guard variables for static
  /// local initialization (32-bit guard, check bit 0 only).
  bool useARMGuardVarABI() const {
    switch (lowerModule->getCXXABIKind()) {
    case clang::TargetCXXABI::GenericARM:
    case clang::TargetCXXABI::iOS:
    case clang::TargetCXXABI::WatchOS:
    case clang::TargetCXXABI::GenericAArch64:
    case clang::TargetCXXABI::WebAssembly:
      return true;
    default:
      return false;
    }
  }
 
  /// Register `dtorFunc` to be called with `arg` when the program exits, or
  /// when the current thread exits if `tls` is set.
  cir::CallOp emitAtExitCall(CIRBaseBuilderTy &builder, mlir::Location loc,
                             cir::FuncOp dtorFunc, mlir::Value arg, bool tls) {
    cir::GlobalOp handle;
    cir::FuncOp fnAtExit;
    cir::PointerType voidPtrTy = builder.getVoidPtrTy();
    cir::PointerType voidFnPtrTy = builder.getVoidFnPtrTy({voidPtrTy});
    {
      mlir::OpBuilder::InsertionGuard guard(builder);
      // Create a variable that binds the atexit to this shared object.
      builder.setInsertionPointToStart(&mlirModule.getBodyRegion().front());
      handle = getOrCreateRuntimeVariable(
          builder, "__dso_handle", loc, builder.getUIntNTy(8),
          cir::GlobalLinkageKind::ExternalLinkage, cir::VisibilityKind::Hidden);
 
      // Create a runtime helper function:
      //    extern "C" int __cxa_atexit(void (*f)(void *), void *p, void *d);
      cir::PointerType handlePtrTy = builder.getPointerTo(handle.getSymType());
      IntType intTy = builder.getSIntNTy(32);
      auto fnAtExitType =
          cir::FuncType::get({voidFnPtrTy, voidPtrTy, handlePtrTy}, intTy);
 
      llvm::StringLiteral nameAtExit = "__cxa_atexit";
      if (tls)
        nameAtExit = getTargetInfo().getTriple().isOSDarwin()
                         ? llvm::StringLiteral("_tlv_atexit")
                         : llvm::StringLiteral("__cxa_thread_atexit");
 
      fnAtExit = buildRuntimeFunction(builder, nameAtExit, loc, fnAtExitType);
    }
 
    mlir::Value args[3];
    auto dtorPtrTy = cir::PointerType::get(dtorFunc.getFunctionType());
    args[0] = cir::GetGlobalOp::create(builder, loc, dtorPtrTy,
                                       dtorFunc.getSymName());
    args[0] = cir::CastOp::create(builder, loc, voidFnPtrTy,
                                  cir::CastKind::bitcast, args[0]);
    args[1] = cir::CastOp::create(builder, loc, voidPtrTy,
                                  cir::CastKind::bitcast, arg);
    args[2] = cir::GetGlobalOp::create(
        builder, loc, builder.getPointerTo(handle.getSymType()),
        handle.getSymName());
    // TODO(cir): set the runtime calling convention on the __cxa_atexit call.
    assert(!cir::MissingFeatures::opFuncCallingConv());
    return builder.createCallOp(loc, fnAtExit, args);
  }
 
  void emitGlobalGuardedDtorRegion(CIRBaseBuilderTy &builder,
                                   cir::GlobalOp global,
                                   mlir::Region &dtorRegion, bool tls,
                                   mlir::Block &entryBB) {
    // If this is a simple call to a destructor, get the called function.
    // Otherwise, create a helper function for the entire dtor region,
    // replacing the current dtor region body with a call to the helper
    // function.
    cir::CallOp dtorCall;
    cir::FuncOp dtorFunc =
        getOrCreateDtorFunc(builder, global, dtorRegion, dtorCall);
 
    // Replace the dtor (or helper) call with a call to
    //   __cxa_atexit(&dtor, &var, &__dso_handle)
    builder.setInsertionPointAfter(dtorCall);
    emitAtExitCall(builder, dtorCall.getLoc(), dtorFunc,
                   dtorCall.getArgOperand(0), tls);
    dtorCall->erase();
    mlir::Block &dtorBlock = dtorRegion.front();
    entryBB.getOperations().splice(entryBB.end(), dtorBlock.getOperations(),
                                   dtorBlock.begin(),
                                   std::prev(dtorBlock.end()));
    // make sure we leave the insert location after the operations we just
    // inserted.
    builder.setInsertionPointToEnd(&entryBB);
  }
 
  /// Emit the guarded initialization for a static local variable.
  /// This handles the if/else structure after the guard byte check,
  /// following OG's ItaniumCXXABI::EmitGuardedInit skeleton.
  void emitCXXGuardedInitIf(CIRBaseBuilderTy &builder, cir::GlobalOp globalOp,
                            mlir::Region &ctorRegion, mlir::Region &dtorRegion,
                            bool isLocalVarDecl, mlir::Value guardPtr,
                            cir::PointerType guardPtrTy, bool threadsafe) {
    auto loc = globalOp->getLoc();
 
    // TODO(cir): set the runtime calling convention on the __cxa_guard_* calls.
    assert(!cir::MissingFeatures::opFuncCallingConv());
 
    // The semantics of dynamic initialization of variables with static or
    // thread storage duration depends on whether they are declared at
    // block-scope. The initialization of such variables at block-scope can be
    // aborted with an exception and later retried (per C++20 [stmt.dcl]p4),
    // and recursive entry to their initialization has undefined behavior (also
    // per C++20 [stmt.dcl]p4). For such variables declared at non-block scope,
    // exceptions lead to termination (per C++20 [except.terminate]p1), and
    // recursive references to the variables are governed only by the lifetime
    // rules (per C++20 [class.cdtor]p2), which means such references are
    // perfectly fine as long as they avoid touching memory. As a result,
    // block-scope variables must not be marked as initialized until after
    // initialization completes (unless the mark is reverted following an
    // exception), but non-block-scope variables must be marked prior to
    // initialization so that recursive accesses during initialization do not
    // restart initialization.
 
    auto emitBody = [&]() {
      // Emit the initializer and add a global destructor if appropriate.
      mlir::Block *insertBlock = builder.getInsertionBlock();
      if (!ctorRegion.empty()) {
        assert(ctorRegion.hasOneBlock() && "Enforced by MaxSizedRegion<1>");
 
        mlir::Block &block = ctorRegion.front();
        insertBlock->getOperations().splice(
            insertBlock->end(), block.getOperations(), block.begin(),
            std::prev(block.end()));
      }
 
      if (!dtorRegion.empty()) {
        assert(dtorRegion.hasOneBlock() && "Enforced by MaxSizedRegion<1>");
 
        // Whether the destructor is registered with __cxa_thread_atexit depends
        // on the variable being thread_local, not on the guard being
        // thread-safe, so make sure we use getTlsModel here instead of
        // threadsafe.
        emitGlobalGuardedDtorRegion(builder, globalOp, dtorRegion,
                                    globalOp.getTlsModel().has_value(),
                                    *insertBlock);
      }
      builder.setInsertionPointToEnd(insertBlock);
      ctorRegion.getBlocks().clear();
    };
 
    // Variables used when coping with thread-safe statics and exceptions.
    if (threadsafe) {
      // Call __cxa_guard_acquire.
      cir::CallOp acquireCall = builder.createCallOp(
          loc, getGuardAcquireFn(guardPtrTy), mlir::ValueRange{guardPtr});
      mlir::Value acquireResult = acquireCall.getResult();
 
      auto acquireZero = builder.getConstantInt(
          loc, mlir::cast<cir::IntType>(acquireResult.getType()), 0);
      auto shouldInit = builder.createCompare(loc, cir::CmpOpKind::ne,
                                              acquireResult, acquireZero);
 
      // Create the IfOp for the shouldInit check.
      // Pass an empty callback to avoid auto-creating a yield terminator.
      auto ifOp =
          cir::IfOp::create(builder, loc, shouldInit, /*withElseRegion=*/false,
                            [](mlir::OpBuilder &, mlir::Location) {});
      mlir::OpBuilder::InsertionGuard insertGuard(builder);
      builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
 
      // An exception out of the initializer leaves the variable uninitialized
      // and the next entry has to run the initializer again ([stmt.dcl]p4), so
      // the guard has to be given back along the exceptional edge. Without
      // this the guard stays held and that retry terminates instead.
      //
      // OG pushes an EH cleanup here and pops it after the initializer, which
      // costs nothing when there is no unwind path. A cir.cleanup.scope is
      // structural, so it is only worth building when there can be one.
      // OG: CGF.EHStack.pushCleanup<CallGuardAbort>(EHCleanup, guard);
      //     ... CGF.PopCleanupBlock();
      if (getLangOpts().Exceptions) {
        cir::CleanupScopeOp::create(
            builder, loc, cir::CleanupKind::EH,
            [&](mlir::OpBuilder &, mlir::Location bodyLoc) {
              emitBody();
              builder.createYield(bodyLoc);
            },
            [&](mlir::OpBuilder &, mlir::Location cleanupLoc) {
              cir::CallOp abortCall =
                  builder.createCallOp(cleanupLoc, getGuardAbortFn(guardPtrTy),
                                       mlir::ValueRange{guardPtr});
              // __cxa_guard_abort is declared noexcept, and marking it so keeps
              // this out of an invoke: a cleanup that can throw while unwinding
              // needs a terminate edge, which is dead weight here.
              abortCall.setNothrowAttr(builder.getUnitAttr());
              builder.createYield(cleanupLoc);
            });
        builder.setInsertionPointToEnd(&ifOp.getThenRegion().front());
      } else {
        emitBody();
      }
 
      // Call __cxa_guard_release. This cannot throw.
      builder.createCallOp(loc, getGuardReleaseFn(guardPtrTy),
                           mlir::ValueRange{guardPtr});
 
      builder.createYield(loc);
    } else if (!isLocalVarDecl) {
      // For non-local variables, store 1 into the first byte of the guard
      // variable before the object initialization begins so that references
      // to the variable during initialization don't restart initialization.
      mlir::Value byteGuardPtr = builder.createBitcast(
          guardPtr, cir::PointerType::get(builder.getSIntNTy(8)));
      builder.createStore(loc,
                          builder.getConstantInt(loc, builder.getSIntNTy(8), 1),
                          byteGuardPtr);
      emitBody();
    } else {
      emitBody();
      // For local variables, store 1 into the first byte of the guard variable
      // after the object initialization completes so that initialization is
      // retried if initialization is interrupted by an exception.
      mlir::Value byteGuardPtr = builder.createBitcast(
          guardPtr, cir::PointerType::get(builder.getSIntNTy(8)));
      builder.createStore(loc,
                          builder.getConstantInt(loc, builder.getSIntNTy(8), 1),
                          byteGuardPtr);
    }
 
    builder.createYield(loc); // Outermost IfOp
  }
};
 
} // namespace
 
cir::GlobalOp LoweringPreparePass::getOrCreateRuntimeVariable(
    mlir::OpBuilder &builder, llvm::StringRef name, mlir::Location loc,
    mlir::Type type, cir::GlobalLinkageKind linkage,
    cir::VisibilityKind visibility) {
  cir::GlobalOp g = dyn_cast_or_null<cir::GlobalOp>(
      mlir::SymbolTable::lookupNearestSymbolFrom(
          mlirModule, mlir::StringAttr::get(mlirModule->getContext(), name)));
  if (!g) {
    g = cir::GlobalOp::create(builder, loc, name, type);
    g.setLinkageAttr(
        cir::GlobalLinkageKindAttr::get(builder.getContext(), linkage));
    mlir::SymbolTable::setSymbolVisibility(
        g, mlir::SymbolTable::Visibility::Private);
    g.setGlobalVisibility(visibility);
  }
  return g;
}
 
cir::FuncOp LoweringPreparePass::buildRuntimeFunction(
    mlir::OpBuilder &builder, llvm::StringRef name, mlir::Location loc,
    cir::FuncType type, cir::GlobalLinkageKind linkage) {
  cir::FuncOp f = dyn_cast_or_null<FuncOp>(SymbolTable::lookupNearestSymbolFrom(
      mlirModule, StringAttr::get(mlirModule->getContext(), name)));
  if (!f) {
    f = cir::FuncOp::create(builder, loc, name, type);
    f.setLinkageAttr(
        cir::GlobalLinkageKindAttr::get(builder.getContext(), linkage));
    mlir::SymbolTable::setSymbolVisibility(
        f, mlir::SymbolTable::Visibility::Private);
 
    assert(!cir::MissingFeatures::opFuncExtraAttrs());
  }
  return f;
}
 
static mlir::Value lowerScalarToComplexCast(mlir::MLIRContext &ctx,
                                            cir::CastOp op) {
  cir::CIRBaseBuilderTy builder(ctx);
  builder.setInsertionPoint(op);
 
  mlir::Value src = op.getSrc();
  mlir::Value imag = builder.getNullValue(src.getType(), op.getLoc());
  return builder.createComplexCreate(op.getLoc(), src, imag);
}
 
static mlir::Value lowerComplexToScalarCast(mlir::MLIRContext &ctx,
                                            cir::CastOp op,
                                            cir::CastKind elemToBoolKind) {
  cir::CIRBaseBuilderTy builder(ctx);
  builder.setInsertionPoint(op);
 
  mlir::Value src = op.getSrc();
  if (!mlir::isa<cir::BoolType>(op.getType()))
    return builder.createComplexReal(op.getLoc(), src);
 
  // Complex cast to bool: (bool)(a+bi) => (bool)a || (bool)b
  mlir::Value srcReal = builder.createComplexReal(op.getLoc(), src);
  mlir::Value srcImag = builder.createComplexImag(op.getLoc(), src);
 
  cir::BoolType boolTy = builder.getBoolTy();
  mlir::Value srcRealToBool =
      builder.createCast(op.getLoc(), elemToBoolKind, srcReal, boolTy);
  mlir::Value srcImagToBool =
      builder.createCast(op.getLoc(), elemToBoolKind, srcImag, boolTy);
  return builder.createLogicalOr(op.getLoc(), srcRealToBool, srcImagToBool);
}
 
static mlir::Value lowerComplexToComplexCast(mlir::MLIRContext &ctx,
                                             cir::CastOp op,
                                             cir::CastKind scalarCastKind) {
  CIRBaseBuilderTy builder(ctx);
  builder.setInsertionPoint(op);
 
  mlir::Value src = op.getSrc();
  auto dstComplexElemTy =
      mlir::cast<cir::ComplexType>(op.getType()).getElementType();
 
  mlir::Value srcReal = builder.createComplexReal(op.getLoc(), src);
  mlir::Value srcImag = builder.createComplexImag(op.getLoc(), src);
 
  mlir::Value dstReal = builder.createCast(op.getLoc(), scalarCastKind, srcReal,
                                           dstComplexElemTy);
  mlir::Value dstImag = builder.createCast(op.getLoc(), scalarCastKind, srcImag,
                                           dstComplexElemTy);
  return builder.createComplexCreate(op.getLoc(), dstReal, dstImag);
}
 
void LoweringPreparePass::lowerCastOp(cir::CastOp op) {
  mlir::MLIRContext &ctx = getContext();
  mlir::Value loweredValue = [&]() -> mlir::Value {
    switch (op.getKind()) {
    case cir::CastKind::float_to_complex:
    case cir::CastKind::int_to_complex:
      return lowerScalarToComplexCast(ctx, op);
    case cir::CastKind::float_complex_to_real:
    case cir::CastKind::int_complex_to_real:
      return lowerComplexToScalarCast(ctx, op, op.getKind());
    case cir::CastKind::float_complex_to_bool:
      return lowerComplexToScalarCast(ctx, op, cir::CastKind::float_to_bool);
    case cir::CastKind::int_complex_to_bool:
      return lowerComplexToScalarCast(ctx, op, cir::CastKind::int_to_bool);
    case cir::CastKind::float_complex:
      return lowerComplexToComplexCast(ctx, op, cir::CastKind::floating);
    case cir::CastKind::float_complex_to_int_complex:
      return lowerComplexToComplexCast(ctx, op, cir::CastKind::float_to_int);
    case cir::CastKind::int_complex:
      return lowerComplexToComplexCast(ctx, op, cir::CastKind::integral);
    case cir::CastKind::int_complex_to_float_complex:
      return lowerComplexToComplexCast(ctx, op, cir::CastKind::int_to_float);
    default:
      return nullptr;
    }
  }();
 
  if (loweredValue) {
    op.replaceAllUsesWith(loweredValue);
    op.erase();
  }
}
 
static mlir::Value buildComplexBinOpLibCall(
    LoweringPreparePass &pass, CIRBaseBuilderTy &builder,
    llvm::StringRef (*libFuncNameGetter)(llvm::APFloat::Semantics),
    mlir::Location loc, cir::ComplexType ty, mlir::Value lhsReal,
    mlir::Value lhsImag, mlir::Value rhsReal, mlir::Value rhsImag) {
  cir::FPTypeInterface elementTy =
      mlir::cast<cir::FPTypeInterface>(ty.getElementType());
 
  llvm::StringRef libFuncName = libFuncNameGetter(
      llvm::APFloat::SemanticsToEnum(elementTy.getFloatSemantics()));
  llvm::SmallVector<mlir::Type, 4> libFuncInputTypes(4, elementTy);
 
  cir::FuncType libFuncTy = cir::FuncType::get(libFuncInputTypes, ty);
 
  // Insert a declaration for the runtime function to be used in Complex
  // multiplication and division when needed
  cir::FuncOp libFunc;
  {
    mlir::OpBuilder::InsertionGuard ipGuard{builder};
    builder.setInsertionPointToStart(pass.mlirModule.getBody());
    libFunc = pass.buildRuntimeFunction(builder, libFuncName, loc, libFuncTy);
  }
 
  // TODO(cir): set the runtime calling convention to this call.
  assert(!cir::MissingFeatures::opFuncCallingConv());
  cir::CallOp call =
      builder.createCallOp(loc, libFunc, {lhsReal, lhsImag, rhsReal, rhsImag});
  return call.getResult();
}
 
static llvm::StringRef
getComplexDivLibCallName(llvm::APFloat::Semantics semantics) {
  switch (semantics) {
  case llvm::APFloat::S_IEEEhalf:
    return "__divhc3";
  case llvm::APFloat::S_IEEEsingle:
    return "__divsc3";
  case llvm::APFloat::S_IEEEdouble:
    return "__divdc3";
  case llvm::APFloat::S_PPCDoubleDouble:
    return "__divtc3";
  case llvm::APFloat::S_x87DoubleExtended:
    return "__divxc3";
  case llvm::APFloat::S_IEEEquad:
    return "__divtc3";
  default:
    llvm_unreachable("unsupported floating point type");
  }
}
 
static mlir::Value
buildAlgebraicComplexDiv(CIRBaseBuilderTy &builder, mlir::Location loc,
                         mlir::Value lhsReal, mlir::Value lhsImag,
                         mlir::Value rhsReal, mlir::Value rhsImag) {
  // (a+bi) / (c+di) = ((ac+bd)/(cc+dd)) + ((bc-ad)/(cc+dd))i
  mlir::Value &a = lhsReal;
  mlir::Value &b = lhsImag;
  mlir::Value &c = rhsReal;
  mlir::Value &d = rhsImag;
 
  // The element type of the complex (lhs/rhs) determines whether floating
  // point or integer ops are needed.
  bool isFP = cir::isFPOrVectorOfFPType(a.getType());
  auto mul = [&](mlir::Location l, mlir::Value x, mlir::Value y) {
    return isFP ? builder.createFMul(l, x, y) : builder.createMul(l, x, y);
  };
  auto add = [&](mlir::Location l, mlir::Value x, mlir::Value y) {
    return isFP ? builder.createFAdd(l, x, y) : builder.createAdd(l, x, y);
  };
  auto sub = [&](mlir::Location l, mlir::Value x, mlir::Value y) {
    return isFP ? builder.createFSub(l, x, y) : builder.createSub(l, x, y);
  };
  auto div = [&](mlir::Location l, mlir::Value x, mlir::Value y) {
    return isFP ? builder.createFDiv(l, x, y) : builder.createDiv(l, x, y);
  };
 
  mlir::Value ac = mul(loc, a, c);     // a*c
  mlir::Value bd = mul(loc, b, d);     // b*d
  mlir::Value cc = mul(loc, c, c);     // c*c
  mlir::Value dd = mul(loc, d, d);     // d*d
  mlir::Value acbd = add(loc, ac, bd); // ac+bd
  mlir::Value ccdd = add(loc, cc, dd); // cc+dd
  mlir::Value resultReal = div(loc, acbd, ccdd);
 
  mlir::Value bc = mul(loc, b, c);     // b*c
  mlir::Value ad = mul(loc, a, d);     // a*d
  mlir::Value bcad = sub(loc, bc, ad); // bc-ad
  mlir::Value resultImag = div(loc, bcad, ccdd);
  return builder.createComplexCreate(loc, resultReal, resultImag);
}
 
static mlir::Value
buildRangeReductionComplexDiv(CIRBaseBuilderTy &builder, mlir::Location loc,
                              mlir::Value lhsReal, mlir::Value lhsImag,
                              mlir::Value rhsReal, mlir::Value rhsImag) {
  // Implements Smith's algorithm for complex division.
  // SMITH, R. L. Algorithm 116: Complex division. Commun. ACM 5, 8 (1962).
 
  // Let:
  //   - lhs := a+bi
  //   - rhs := c+di
  //   - result := lhs / rhs = e+fi
  //
  // The algorithm pseudocode looks like follows:
  //   if fabs(c) >= fabs(d):
  //     r := d / c
  //     tmp := c + r*d
  //     e = (a + b*r) / tmp
  //     f = (b - a*r) / tmp
  //   else:
  //     r := c / d
  //     tmp := d + r*c
  //     e = (a*r + b) / tmp
  //     f = (b*r - a) / tmp
 
  mlir::Value &a = lhsReal;
  mlir::Value &b = lhsImag;
  mlir::Value &c = rhsReal;
  mlir::Value &d = rhsImag;
 
  // Smith's algorithm is only used for floating-point complex division.
  assert(cir::isFPOrVectorOfFPType(a.getType()) &&
         "range-reduction complex divide expects floating-point operands");
 
  auto trueBranchBuilder = [&](mlir::OpBuilder &, mlir::Location) {
    mlir::Value r = builder.createFDiv(loc, d, c);    // r := d / c
    mlir::Value rd = builder.createFMul(loc, r, d);   // r*d
    mlir::Value tmp = builder.createFAdd(loc, c, rd); // tmp := c + r*d
 
    mlir::Value br = builder.createFMul(loc, b, r);   // b*r
    mlir::Value abr = builder.createFAdd(loc, a, br); // a + b*r
    mlir::Value e = builder.createFDiv(loc, abr, tmp);
 
    mlir::Value ar = builder.createFMul(loc, a, r);   // a*r
    mlir::Value bar = builder.createFSub(loc, b, ar); // b - a*r
    mlir::Value f = builder.createFDiv(loc, bar, tmp);
 
    mlir::Value result = builder.createComplexCreate(loc, e, f);
    builder.createYield(loc, result);
  };
 
  auto falseBranchBuilder = [&](mlir::OpBuilder &, mlir::Location) {
    mlir::Value r = builder.createFDiv(loc, c, d);    // r := c / d
    mlir::Value rc = builder.createFMul(loc, r, c);   // r*c
    mlir::Value tmp = builder.createFAdd(loc, d, rc); // tmp := d + r*c
 
    mlir::Value ar = builder.createFMul(loc, a, r);   // a*r
    mlir::Value arb = builder.createFAdd(loc, ar, b); // a*r + b
    mlir::Value e = builder.createFDiv(loc, arb, tmp);
 
    mlir::Value br = builder.createFMul(loc, b, r);   // b*r
    mlir::Value bra = builder.createFSub(loc, br, a); // b*r - a
    mlir::Value f = builder.createFDiv(loc, bra, tmp);
 
    mlir::Value result = builder.createComplexCreate(loc, e, f);
    builder.createYield(loc, result);
  };
 
  auto cFabs = cir::FAbsOp::create(builder, loc, c);
  auto dFabs = cir::FAbsOp::create(builder, loc, d);
  cir::CmpOp cmpResult =
      builder.createCompare(loc, cir::CmpOpKind::ge, cFabs, dFabs);
  auto ternary = cir::TernaryOp::create(builder, loc, cmpResult,
                                        trueBranchBuilder, falseBranchBuilder);
 
  return ternary.getResult();
}
 
static mlir::Type higherPrecisionElementTypeForComplexArithmetic(
    mlir::MLIRContext &context, const clang::TargetInfo &targetInfo,
    const clang::LangOptions &langOpts, CIRBaseBuilderTy &builder,
    mlir::Type elementType) {
 
  auto getHigherPrecisionFPType = [&context,
                                   &targetInfo](mlir::Type type) -> mlir::Type {
    if (mlir::isa<cir::FP16Type>(type))
      return cir::SingleType::get(&context);
 
    if (mlir::isa<cir::SingleType>(type) || mlir::isa<cir::BF16Type>(type))
      return cir::DoubleType::get(&context);
 
    if (mlir::isa<cir::DoubleType>(type)) {
      const llvm::fltSemantics &format = targetInfo.getLongDoubleFormat();
      mlir::Type underlying;
      if (&format == &llvm::APFloat::IEEEdouble())
        underlying = cir::DoubleType::get(&context);
      else if (&format == &llvm::APFloat::x87DoubleExtended())
        underlying = cir::FP80Type::get(&context);
      else if (&format == &llvm::APFloat::IEEEquad())
        underlying = cir::FP128Type::get(&context);
      else
        llvm_unreachable("NYI: unsupported long double format");
      return cir::LongDoubleType::get(&context, underlying);
    }
 
    return type;
  };
 
  auto getFloatTypeSemantics =
      [&langOpts, &targetInfo](mlir::Type type) -> const llvm::fltSemantics & {
    const clang::TargetInfo &info = targetInfo;
    if (mlir::isa<cir::FP16Type>(type))
      return info.getHalfFormat();
 
    if (mlir::isa<cir::BF16Type>(type))
      return info.getBFloat16Format();
 
    if (mlir::isa<cir::SingleType>(type))
      return info.getFloatFormat();
 
    if (mlir::isa<cir::DoubleType>(type))
      return info.getDoubleFormat();
 
    if (mlir::isa<cir::LongDoubleType>(type)) {
      if (langOpts.OpenMP && langOpts.OpenMPIsTargetDevice)
        llvm_unreachable("NYI Float type semantics with OpenMP");
      return info.getLongDoubleFormat();
    }
 
    if (mlir::isa<cir::FP128Type>(type)) {
      if (langOpts.OpenMP && langOpts.OpenMPIsTargetDevice)
        llvm_unreachable("NYI Float type semantics with OpenMP");
      return info.getFloat128Format();
    }
 
    llvm_unreachable("Unsupported float type semantics");
  };
 
  const mlir::Type higherElementType = getHigherPrecisionFPType(elementType);
  const llvm::fltSemantics &elementTypeSemantics =
      getFloatTypeSemantics(elementType);
  const llvm::fltSemantics &higherElementTypeSemantics =
      getFloatTypeSemantics(higherElementType);
 
  // Check that the promoted type can handle the intermediate values without
  // overflowing. This can be interpreted as:
  // (SmallerType.LargestFiniteVal * SmallerType.LargestFiniteVal) * 2 <=
  //      LargerType.LargestFiniteVal.
  // In terms of exponent it gives this formula:
  // (SmallerType.LargestFiniteVal * SmallerType.LargestFiniteVal
  // doubles the exponent of SmallerType.LargestFiniteVal)
  if (llvm::APFloat::semanticsMaxExponent(elementTypeSemantics) * 2 + 1 <=
      llvm::APFloat::semanticsMaxExponent(higherElementTypeSemantics)) {
    return higherElementType;
  }
 
  // The intermediate values can't be represented in the promoted type
  // without overflowing.
  return {};
}
 
static mlir::Value lowerComplexFDiv(LoweringPreparePass &pass,
                                    CIRBaseBuilderTy &builder,
                                    mlir::Location loc, cir::ComplexFDivOp op,
                                    mlir::Value lhsReal, mlir::Value lhsImag,
                                    mlir::Value rhsReal, mlir::Value rhsImag,
                                    mlir::MLIRContext &mlirCx,
                                    const clang::TargetInfo &targetInfo) {
  cir::ComplexType complexTy = op.getType();
  cir::ComplexRangeKind range = op.getRange();
  switch (range) {
  case ComplexRangeKind::Full: {
    return buildComplexBinOpLibCall(pass, builder, &getComplexDivLibCallName,
                                    loc, complexTy, lhsReal, lhsImag, rhsReal,
                                    rhsImag);
  }
  case ComplexRangeKind::Improved: {
    return buildRangeReductionComplexDiv(builder, loc, lhsReal, lhsImag,
                                         rhsReal, rhsImag);
  }
  case ComplexRangeKind::Promoted: {
    mlir::Type originalElementType = complexTy.getElementType();
    mlir::Type higherPrecisionElementType =
        higherPrecisionElementTypeForComplexArithmetic(
            mlirCx, targetInfo, pass.getLangOpts(), builder,
            originalElementType);
 
    if (!higherPrecisionElementType)
      return buildRangeReductionComplexDiv(builder, loc, lhsReal, lhsImag,
                                           rhsReal, rhsImag);
 
    cir::CastKind floatingCastKind = cir::CastKind::floating;
    lhsReal = builder.createCast(floatingCastKind, lhsReal,
                                 higherPrecisionElementType);
    lhsImag = builder.createCast(floatingCastKind, lhsImag,
                                 higherPrecisionElementType);
    rhsReal = builder.createCast(floatingCastKind, rhsReal,
                                 higherPrecisionElementType);
    rhsImag = builder.createCast(floatingCastKind, rhsImag,
                                 higherPrecisionElementType);
 
    mlir::Value algebraicResult = buildAlgebraicComplexDiv(
        builder, loc, lhsReal, lhsImag, rhsReal, rhsImag);
 
    mlir::Value resultReal = builder.createComplexReal(loc, algebraicResult);
    mlir::Value resultImag = builder.createComplexImag(loc, algebraicResult);
 
    mlir::Value finalReal =
        builder.createCast(floatingCastKind, resultReal, originalElementType);
    mlir::Value finalImag =
        builder.createCast(floatingCastKind, resultImag, originalElementType);
    return builder.createComplexCreate(loc, finalReal, finalImag);
  }
  case ComplexRangeKind::Basic: {
    return buildAlgebraicComplexDiv(builder, loc, lhsReal, lhsImag, rhsReal,
                                    rhsImag);
  }
  }
  llvm_unreachable("fully covered switch");
}
 
void LoweringPreparePass::lowerComplexFDivOp(cir::ComplexFDivOp op) {
  cir::CIRBaseBuilderTy builder(getContext());
  builder.setInsertionPointAfter(op);
  mlir::Location loc = op.getLoc();
  mlir::TypedValue<cir::ComplexType> lhs = op.getLhs();
  mlir::TypedValue<cir::ComplexType> rhs = op.getRhs();
  mlir::Value lhsReal = builder.createComplexReal(loc, lhs);
  mlir::Value lhsImag = builder.createComplexImag(loc, lhs);
  mlir::Value rhsReal = builder.createComplexReal(loc, rhs);
  mlir::Value rhsImag = builder.createComplexImag(loc, rhs);
 
  mlir::Value loweredResult =
      lowerComplexFDiv(*this, builder, loc, op, lhsReal, lhsImag, rhsReal,
                       rhsImag, getContext(), getTargetInfo());
  op.replaceAllUsesWith(loweredResult);
  op.erase();
}
 
void LoweringPreparePass::lowerComplexDivOp(cir::ComplexDivOp op) {
  cir::CIRBaseBuilderTy builder(getContext());
  builder.setInsertionPointAfter(op);
  mlir::Location loc = op.getLoc();
  mlir::TypedValue<cir::ComplexType> lhs = op.getLhs();
  mlir::TypedValue<cir::ComplexType> rhs = op.getRhs();
  mlir::Value lhsReal = builder.createComplexReal(loc, lhs);
  mlir::Value lhsImag = builder.createComplexImag(loc, lhs);
  mlir::Value rhsReal = builder.createComplexReal(loc, rhs);
  mlir::Value rhsImag = builder.createComplexImag(loc, rhs);
 
  mlir::Value loweredResult = buildAlgebraicComplexDiv(
      builder, loc, lhsReal, lhsImag, rhsReal, rhsImag);
  op.replaceAllUsesWith(loweredResult);
  op.erase();
}
 
static llvm::StringRef
getComplexMulLibCallName(llvm::APFloat::Semantics semantics) {
  switch (semantics) {
  case llvm::APFloat::S_IEEEhalf:
    return "__mulhc3";
  case llvm::APFloat::S_IEEEsingle:
    return "__mulsc3";
  case llvm::APFloat::S_IEEEdouble:
    return "__muldc3";
  case llvm::APFloat::S_PPCDoubleDouble:
    return "__multc3";
  case llvm::APFloat::S_x87DoubleExtended:
    return "__mulxc3";
  case llvm::APFloat::S_IEEEquad:
    return "__multc3";
  default:
    llvm_unreachable("unsupported floating point type");
  }
}
 
static mlir::Value lowerComplexFMul(LoweringPreparePass &pass,
                                    CIRBaseBuilderTy &builder,
                                    mlir::Location loc, cir::ComplexFMulOp op,
                                    mlir::Value lhsReal, mlir::Value lhsImag,
                                    mlir::Value rhsReal, mlir::Value rhsImag) {
  // (a+bi) * (c+di) = (ac-bd) + (ad+bc)i
  mlir::Value resultRealLhs = builder.createFMul(loc, lhsReal, rhsReal); // ac
  mlir::Value resultRealRhs = builder.createFMul(loc, lhsImag, rhsImag); // bd
  mlir::Value resultImagLhs = builder.createFMul(loc, lhsReal, rhsImag); // ad
  mlir::Value resultImagRhs = builder.createFMul(loc, lhsImag, rhsReal); // bc
  mlir::Value resultReal =
      builder.createFSub(loc, resultRealLhs, resultRealRhs);
  mlir::Value resultImag =
      builder.createFAdd(loc, resultImagLhs, resultImagRhs);
  mlir::Value algebraicResult =
      builder.createComplexCreate(loc, resultReal, resultImag);
 
  cir::ComplexType complexTy = op.getType();
  cir::ComplexRangeKind rangeKind = op.getRange();
  if (rangeKind == cir::ComplexRangeKind::Basic ||
      rangeKind == cir::ComplexRangeKind::Improved ||
      rangeKind == cir::ComplexRangeKind::Promoted)
    return algebraicResult;
 
  assert(!cir::MissingFeatures::fastMathFlags());
 
  // Check whether the real part and the imaginary part of the result are both
  // NaN. If so, emit a library call to compute the multiplication instead.
  // We check a value against NaN by comparing the value against itself.
  mlir::Value resultRealIsNaN = builder.createIsNaN(loc, resultReal);
  mlir::Value resultImagIsNaN = builder.createIsNaN(loc, resultImag);
  mlir::Value resultRealAndImagAreNaN =
      builder.createLogicalAnd(loc, resultRealIsNaN, resultImagIsNaN);
 
  return cir::TernaryOp::create(
             builder, loc, resultRealAndImagAreNaN,
             [&](mlir::OpBuilder &, mlir::Location) {
               mlir::Value libCallResult = buildComplexBinOpLibCall(
                   pass, builder, &getComplexMulLibCallName, loc, complexTy,
                   lhsReal, lhsImag, rhsReal, rhsImag);
               builder.createYield(loc, libCallResult);
             },
             [&](mlir::OpBuilder &, mlir::Location) {
               builder.createYield(loc, algebraicResult);
             })
      .getResult();
}
 
void LoweringPreparePass::lowerComplexFMulOp(cir::ComplexFMulOp op) {
  cir::CIRBaseBuilderTy builder(getContext());
  builder.setInsertionPointAfter(op);
  mlir::Location loc = op.getLoc();
  mlir::TypedValue<cir::ComplexType> lhs = op.getLhs();
  mlir::TypedValue<cir::ComplexType> rhs = op.getRhs();
 
  // (a+bi) * (c+di) = (ac-bd) + (ad+bc)i
  mlir::Value lhsReal = builder.createComplexReal(loc, lhs);
  mlir::Value lhsImag = builder.createComplexImag(loc, lhs);
  mlir::Value rhsReal = builder.createComplexReal(loc, rhs);
  mlir::Value rhsImag = builder.createComplexImag(loc, rhs);
  mlir::Value loweredResult = lowerComplexFMul(*this, builder, loc, op, lhsReal,
                                               lhsImag, rhsReal, rhsImag);
  op.replaceAllUsesWith(loweredResult);
  op.erase();
}
 
void LoweringPreparePass::lowerComplexMulOp(cir::ComplexMulOp op) {
  cir::CIRBaseBuilderTy builder(getContext());
  builder.setInsertionPointAfter(op);
  mlir::Location loc = op.getLoc();
  mlir::TypedValue<cir::ComplexType> lhs = op.getLhs();
  mlir::TypedValue<cir::ComplexType> rhs = op.getRhs();
  mlir::Value lhsReal = builder.createComplexReal(loc, lhs);
  mlir::Value lhsImag = builder.createComplexImag(loc, lhs);
  mlir::Value rhsReal = builder.createComplexReal(loc, rhs);
  mlir::Value rhsImag = builder.createComplexImag(loc, rhs);
 
  mlir::Value resultRealLhs = builder.createMul(loc, lhsReal, rhsReal); // ac
  mlir::Value resultRealRhs = builder.createMul(loc, lhsImag, rhsImag); // bd
  mlir::Value resultImagLhs = builder.createMul(loc, lhsReal, rhsImag); // ad
  mlir::Value resultImagRhs = builder.createMul(loc, lhsImag, rhsReal); // bc
  mlir::Value resultReal = builder.createSub(loc, resultRealLhs, resultRealRhs);
  mlir::Value resultImag = builder.createAdd(loc, resultImagLhs, resultImagRhs);
 
  mlir::Value loweredResult =
      builder.createComplexCreate(loc, resultReal, resultImag);
 
  op.replaceAllUsesWith(loweredResult);
  op.erase();
}
 
void LoweringPreparePass::lowerComplexConjOp(cir::ComplexConjOp op) {
  mlir::Location loc = op.getLoc();
  CIRBaseBuilderTy builder(getContext());
  builder.setInsertionPointAfter(op);
 
  mlir::Value operand = op.getOperand();
  mlir::Value operandReal = builder.createComplexReal(loc, operand);
  mlir::Value operandImag = builder.createComplexImag(loc, operand);
 
  // The complex conjugate is formed by negating the imaginary component.
  const bool isFP = cir::isFPOrVectorOfFPType(operandReal.getType());
  mlir::Value resultImag = isFP ? builder.createFNeg(loc, operandImag)
                                : builder.createMinus(loc, operandImag);
 
  mlir::Value result =
      builder.createComplexCreate(loc, operandReal, resultImag);
  op->replaceAllUsesWith(mlir::ValueRange{result});
  op->erase();
}
 
cir::FuncOp
LoweringPreparePass::buildGlobalArrayDtorFunc(CIRBaseBuilderTy &builder,
                                              mlir::Operation *op) {
  mlir::OpBuilder::InsertionGuard guard(builder);
  builder.setInsertionPointAfter(op);
  SmallString<256> fnName("__cxx_global_array_dtor");
  uint32_t cnt = dynamicInitializerNames[fnName]++;
  if (cnt)
    fnName += "." + std::to_string(cnt);
 
  auto fnType =
      cir::FuncType::get({builder.getVoidPtrTy()}, builder.getVoidTy());
  cir::FuncOp dtorFunc =
      buildRuntimeFunction(builder, fnName, op->getLoc(), fnType,
                           cir::GlobalLinkageKind::InternalLinkage);
 
  SmallVector<mlir::NamedAttribute> paramAttrs;
  paramAttrs.push_back(
      builder.getNamedAttr("llvm.noundef", builder.getUnitAttr()));
  SmallVector<mlir::Attribute> argAttrDicts;
  argAttrDicts.push_back(
      mlir::DictionaryAttr::get(builder.getContext(), paramAttrs));
  dtorFunc.setArgAttrsAttr(
      mlir::ArrayAttr::get(builder.getContext(), argAttrDicts));
 
  dtorFunc.addEntryBlock();
  return dtorFunc;
}
 
void LoweringPreparePass::lowerRegisterExitDtorOp(cir::RegisterExitDtorOp op) {
  CIRBaseBuilderTy builder(getContext());
  mlir::Location loc = op.getLoc();
  auto object = symbolTables.lookupNearestSymbolFrom<cir::GlobalOp>(
      op, op.getObjectAttr());
  assert(object && "register_exit_dtor must reference a cir.global");
  mlir::Block &body = op.getBody().front();
  builder.setInsertionPoint(op);
 
  // When the body is a single call on the object's address, register the
  // callee directly with that address.
  cir::FuncOp dtorFunc;
  mlir::Value arg;
  if (body.getOperations().size() == 3) {
    auto getGlobal = mlir::dyn_cast<cir::GetGlobalOp>(body.front());
    auto call = mlir::dyn_cast<cir::CallOp>(*std::next(body.begin()));
    if (getGlobal && call && getGlobal.getName() == op.getObject() &&
        call.getNumArgOperands() == 1 &&
        call.getArgOperand(0) == getGlobal.getResult()) {
      dtorFunc = getCalledFunction(call);
      if (dtorFunc)
        arg = builder.clone(*getGlobal)->getResult(0);
    }
  }
 
  // Otherwise the body moves into a helper that ignores its argument, and the
  // helper is registered with a null argument.
  if (!dtorFunc) {
    dtorFunc = buildGlobalArrayDtorFunc(builder, object);
    mlir::Block *entryBB = &dtorFunc.getBody().front();
    entryBB->getOperations().splice(entryBB->end(), body.getOperations());
    auto yieldOp = mlir::cast<cir::YieldOp>(entryBB->getTerminator());
    {
      mlir::OpBuilder::InsertionGuard guard(builder);
      builder.setInsertionPoint(yieldOp);
      cir::ReturnOp::create(builder, yieldOp.getLoc());
    }
    yieldOp->erase();
    arg = builder.getNullPtr(builder.getVoidPtrTy(), loc);
  }
 
  cir::CallOp atExitCall = emitAtExitCall(builder, loc, dtorFunc, arg,
                                          object.getTlsModel().has_value());
  // The atexit functions do not throw, and marking the call so keeps it out of
  // an invoke inside the initializer.
  atExitCall.setNothrowAttr(builder.getUnitAttr());
  op->erase();
}
 
cir::FuncOp LoweringPreparePass::getOrCreateDtorFunc(CIRBaseBuilderTy &builder,
                                                     cir::GlobalOp op,
                                                     mlir::Region &dtorRegion,
                                                     cir::CallOp &dtorCall) {
  mlir::OpBuilder::InsertionGuard guard(builder);
  assert(!cir::MissingFeatures::astVarDeclInterface());
 
  // Look for operations in dtorBlock
  mlir::Block &dtorBlock = dtorRegion.front();
 
  // The first operation should be a get_global to retrieve the address
  // of the global variable we're destroying.
  auto opIt = dtorBlock.getOperations().begin();
  cir::GetGlobalOp ggop = mlir::cast<cir::GetGlobalOp>(*opIt);
 
  // The simple case is just a call to a destructor, like this:
  //
  //   %0 = cir.get_global %globalS : !cir.ptr<!rec_S>
  //   cir.call %_ZN1SD1Ev(%0) : (!cir.ptr<!rec_S>) -> ()
  //   (implicit cir.yield)
  //
  // That is, if the second operation is a call that takes the get_global
  // result as its only operand, and the only other operation is a yield, then
  // we can just return the called function.
  if (dtorBlock.getOperations().size() == 3) {
    auto callOp = mlir::dyn_cast<cir::CallOp>(&*(++opIt));
    auto yieldOp = mlir::dyn_cast<cir::YieldOp>(&*(++opIt));
    if (yieldOp && callOp && callOp.getNumOperands() == 1 &&
        callOp.getArgOperand(0) == ggop) {
      dtorCall = callOp;
      return getCalledFunction(callOp);
    }
  }
 
  // Otherwise, we need to create a helper function to replace the dtor region.
  cir::FuncOp dtorFunc = buildGlobalArrayDtorFunc(builder, op);
  mlir::Block *entryBB = &dtorFunc.getBody().front();
 
  // Move everything from the dtor region into the helper function.
  entryBB->getOperations().splice(entryBB->begin(), dtorBlock.getOperations(),
                                  dtorBlock.begin(), dtorBlock.end());
 
  // Before erasing this, clone it back into the dtor region
  cir::GetGlobalOp dtorGGop =
      mlir::cast<cir::GetGlobalOp>(entryBB->getOperations().front());
  builder.setInsertionPointToStart(&dtorBlock);
  builder.clone(*dtorGGop.getOperation());
 
  // Replace all uses of the help function's get_global with the function
  // argument.
  mlir::Value dtorArg = entryBB->getArgument(0);
  // The arg is a void*, but the destructor needs the variables type, so we
  // might need to introduce a cast.
  if (dtorArg.getType() != dtorGGop.getType()) {
    builder.setInsertionPointToStart(entryBB);
    dtorArg =
        builder.createBitcast(dtorGGop.getLoc(), dtorArg, dtorGGop.getType());
  }
  dtorGGop.replaceAllUsesWith(dtorArg);
  dtorGGop.erase();
 
  // Replace the yield in the final block with a return
  mlir::Block &finalBlock = dtorFunc.getBody().back();
  auto yieldOp = cast<cir::YieldOp>(finalBlock.getTerminator());
  builder.setInsertionPoint(yieldOp);
  cir::ReturnOp::create(builder, yieldOp->getLoc());
  yieldOp->erase();
 
  // Create a call to the helper function, passing the original get_global op
  // as the argument.
  cir::GetGlobalOp origGGop =
      mlir::cast<cir::GetGlobalOp>(dtorBlock.getOperations().front());
  builder.setInsertionPointAfter(origGGop);
  mlir::Value ggopResult = origGGop.getResult();
  dtorCall = builder.createCallOp(op.getLoc(), dtorFunc, ggopResult,
                                  /*attrs=*/{}, /*argAttrs=*/{},
                                  /*resAttrs=*/{}, dtorFunc.getCallingConv());
 
  // Add a yield after the call.
  auto finalYield = cir::YieldOp::create(builder, op.getLoc());
 
  // Erase everything after the yield.
  dtorBlock.getOperations().erase(std::next(mlir::Block::iterator(finalYield)),
                                  dtorBlock.end());
  dtorRegion.getBlocks().erase(std::next(dtorRegion.begin()), dtorRegion.end());
 
  return dtorFunc;
}
 
cir::FuncOp
LoweringPreparePass::buildCXXGlobalVarDeclInitFunc(cir::GlobalOp op) {
  // TODO(cir): Store this in the GlobalOp.
  // This should come from the MangleContext, but for now I'm hardcoding it.
  SmallString<256> fnName("__cxx_global_var_init");
  // Get a unique name
  uint32_t cnt = dynamicInitializerNames[fnName]++;
  if (cnt)
    fnName += "." + std::to_string(cnt);
 
  // Create a variable initialization function.
  CIRBaseBuilderTy builder(getContext());
  builder.setInsertionPointAfter(op);
  cir::VoidType voidTy = builder.getVoidTy();
  auto fnType = cir::FuncType::get({}, voidTy);
  FuncOp f = buildRuntimeFunction(builder, fnName, op.getLoc(), fnType,
                                  cir::GlobalLinkageKind::InternalLinkage);
 
  // Forward the constrained floating-point marker recorded on the global by
  // CodeGen onto the generated initializer function. The marker on the global
  // is no longer meaningful once its regions have been moved out, so clear it.
  if (op.getStrictfp()) {
    f->setAttr(cir::CIRDialect::getStrictFPAttrName(),
               mlir::UnitAttr::get(&getContext()));
    op.setStrictfp(false);
  }
 
  // Move over the initialization code of the ctor region.
  // The ctor region may have multiple blocks when exception handling
  // scaffolding creates extra blocks (e.g., unreachable/trap blocks).
  // We move all operations from the first block (minus the yield) into
  // the function entry, and discard extra blocks (which contain only
  // unreachable terminators from EH cleanup paths).
  mlir::Block *entryBB = f.addEntryBlock();
  builder.setInsertionPointToStart(entryBB);
 
  // If this is a global TLS variable (that is, declared at namespace scope), we
  // have to emit the guard variable here.
  bool needsTlsGuard = op.getTlsRefs() && op.getTlsRefs()->getGuardName();
 
  // Non-TLS globals (such as weak/linkonce_odr/inline-ns scope variables) use
  // the __cxa_guard_acquire/release instead of the TLS guard.
  if (!needsTlsGuard && op.getDynamicInitGuard()) {
    if (!op.getDynamicInitInfo()) {
      op->emitError(
          "guarded global with a guard is missing 'dynamic_init_info'");
      return {};
    }
 
    if (op.getDynamicInitInfo()->getLocal()) {
      op->emitError("guarded global is marked local");
      return {};
    }
    emitGuardedInit(builder, op, op.getCtorRegion(), op.getDtorRegion());
    builder.setInsertionPointToEnd(entryBB);
    cir::ReturnOp::create(builder, op.getLoc());
    return f;
  }
 
  cir::IfOp guardIf;
  if (needsTlsGuard) {
    guardIf = buildInitGuardCheck(
        builder, op.getLoc(),
        getOrCreateStaticLocalDeclGuardAddress(
            builder, op, op.getTlsRefs()->getGuardName().getValue(),
            /*isLocalVarDecl=*/false,
            /*useInt8GuardVariable=*/op.hasInternalLinkage()),
        /*threadLocal=*/true);
    builder.setInsertionPointToEnd(&guardIf.getThenRegion().front());
  }
 
  if (!op.getCtorRegion().empty()) {
    mlir::Block &block = op.getCtorRegion().front();
    mlir::Block *insertBlock = builder.getBlock();
    insertBlock->getOperations().splice(insertBlock->end(),
                                        block.getOperations(), block.begin(),
                                        std::prev(block.end()));
  }
 
  // Register the destructor call with __cxa_atexit
  mlir::Region &dtorRegion = op.getDtorRegion();
  if (!dtorRegion.empty()) {
    assert(!cir::MissingFeatures::astVarDeclInterface());
 
    emitGlobalGuardedDtorRegion(builder, op, dtorRegion,
                                op.getTlsModel().has_value(),
                                *builder.getBlock());
  }
 
  // If we're actually in the 'if' above, create a yield.
  if (needsTlsGuard) {
    builder.setInsertionPointToEnd(&guardIf.getThenRegion().back());
    cir::YieldOp::create(builder, op.getLoc());
  }
 
  // Replace cir.yield with cir.return
  builder.setInsertionPointToEnd(entryBB);
  mlir::Operation *yieldOp = nullptr;
  if (!op.getCtorRegion().empty()) {
    mlir::Block &block = op.getCtorRegion().front();
    yieldOp = &block.getOperations().back();
  } else {
    assert(!dtorRegion.empty());
    mlir::Block &block = dtorRegion.front();
    yieldOp = &block.getOperations().back();
  }
 
  assert(isa<cir::YieldOp>(*yieldOp));
  cir::ReturnOp::create(builder, yieldOp->getLoc());
  return f;
}
 
cir::FuncOp
LoweringPreparePass::getGuardAcquireFn(cir::PointerType guardPtrTy) {
  // int __cxa_guard_acquire(__guard *guard_object);
  CIRBaseBuilderTy builder(getContext());
  mlir::OpBuilder::InsertionGuard ipGuard{builder};
  builder.setInsertionPointToStart(mlirModule.getBody());
  mlir::Location loc = mlirModule.getLoc();
  cir::IntType intTy = cir::IntType::get(&getContext(), 32, /*isSigned=*/true);
  auto fnType = cir::FuncType::get({guardPtrTy}, intTy);
  return buildRuntimeFunction(builder, "__cxa_guard_acquire", loc, fnType);
}
 
cir::FuncOp
LoweringPreparePass::getGuardReleaseFn(cir::PointerType guardPtrTy) {
  // void __cxa_guard_release(__guard *guard_object);
  CIRBaseBuilderTy builder(getContext());
  mlir::OpBuilder::InsertionGuard ipGuard{builder};
  builder.setInsertionPointToStart(mlirModule.getBody());
  mlir::Location loc = mlirModule.getLoc();
  cir::VoidType voidTy = cir::VoidType::get(&getContext());
  auto fnType = cir::FuncType::get({guardPtrTy}, voidTy);
  return buildRuntimeFunction(builder, "__cxa_guard_release", loc, fnType);
}
 
cir::FuncOp LoweringPreparePass::getGuardAbortFn(cir::PointerType guardPtrTy) {
  // void __cxa_guard_abort(__guard *guard_object);
  CIRBaseBuilderTy builder(getContext());
  mlir::OpBuilder::InsertionGuard ipGuard{builder};
  builder.setInsertionPointToStart(mlirModule.getBody());
  mlir::Location loc = mlirModule.getLoc();
  cir::VoidType voidTy = cir::VoidType::get(&getContext());
  auto fnType = cir::FuncType::get({guardPtrTy}, voidTy);
  return buildRuntimeFunction(builder, "__cxa_guard_abort", loc, fnType);
}
 
cir::FuncOp LoweringPreparePass::getTlsInitFn() {
  // void __tls_init(void);
  CIRBaseBuilderTy builder(getContext());
  mlir::OpBuilder::InsertionGuard _{builder};
  builder.setInsertionPointToStart(mlirModule.getBody());
  mlir::Location loc = mlirModule.getLoc();
  auto fnType = builder.getVoidFnTy();
  return buildRuntimeFunction(builder, "__tls_init", loc, fnType,
                              cir::GlobalLinkageKind::InternalLinkage);
}
 
cir::GlobalOp LoweringPreparePass::createGuardGlobalOp(
    CIRBaseBuilderTy &builder, mlir::Location loc, llvm::StringRef name,
    cir::IntType guardTy, cir::GlobalLinkageKind linkage) {
  mlir::OpBuilder::InsertionGuard guard(builder);
  builder.setInsertionPointToStart(mlirModule.getBody());
  cir::GlobalOp g = cir::GlobalOp::create(builder, loc, name, guardTy);
  g.setLinkageAttr(
      cir::GlobalLinkageKindAttr::get(builder.getContext(), linkage));
  mlir::SymbolTable::setSymbolVisibility(
      g, mlir::SymbolTable::Visibility::Private);
  return g;
}
 
void LoweringPreparePass::addUsedGlobal(cir::GlobalOp global) {
  CIRBaseBuilderTy builder(getContext());
  mlir::ptr::MemorySpaceAttrInterface eltAS;
  if (unsigned targetAS =
          getTargetInfo().getTargetAddressSpace(clang::LangAS::Default))
    eltAS = cir::TargetAddressSpaceAttr::get(&getContext(), targetAS);
  cir::PointerType voidPtrTy = builder.getVoidPtrTy(eltAS);
  mlir::Attribute entry = cir::GlobalViewAttr::get(
      voidPtrTy, mlir::FlatSymbolRefAttr::get(global.getSymNameAttr()));
 
  mlir::StringAttr usedName = builder.getStringAttr("llvm.used");
  cir::GlobalOp used =
      symbolTables.lookupSymbolIn<cir::GlobalOp>(mlirModule, usedName);
 
  llvm::SmallVector<mlir::Attribute, 4> elements;
  if (used) {
    auto existing = mlir::cast<cir::ConstArrayAttr>(*used.getInitialValue());
    llvm::append_range(elements,
                       mlir::cast<mlir::ArrayAttr>(existing.getElts()));
  }
  elements.push_back(entry);
 
  cir::ArrayType arrayTy = cir::ArrayType::get(voidPtrTy, elements.size());
  cir::ConstArrayAttr initAttr = cir::ConstArrayAttr::get(
      arrayTy, mlir::ArrayAttr::get(&getContext(), elements));
 
  if (used) {
    used.setSymType(arrayTy);
    used.setInitialValueAttr(initAttr);
    return;
  }
 
  builder.setInsertionPointToStart(mlirModule.getBody());
  used = cir::GlobalOp::create(
      builder, mlirModule.getLoc(), usedName.getValue(), arrayTy,
      /*isConstant=*/false,
      cir::CIRDataLayout(mlirModule).getDefaultGlobalsAddrSpace(&getContext()));
  used.setLinkage(cir::GlobalLinkageKind::AppendingLinkage);
  used.setInitialValueAttr(initAttr);
  used.setSectionAttr(builder.getStringAttr("llvm.metadata"));
  symbolTables.getSymbolTable(mlirModule).insert(used);
}
 
void LoweringPreparePass::handleStaticLocal(cir::GlobalOp globalOp,
                                            cir::LocalInitOp localInitOp) {
  CIRBaseBuilderTy builder(getContext());
 
  builder.setInsertionPointAfter(localInitOp);
  mlir::Block *localInitBlock = builder.getInsertionBlock();
 
  // Remove the terminator temporarily - we'll add it back at the end.
  mlir::Operation *ret = localInitBlock->getTerminator();
  ret->remove();
  // Note: These two insert-point-after sets are necessary, as the 'trailing'
  // operation has changed thanks to the terminator removal.
  builder.setInsertionPointAfter(localInitOp);
 
  emitGuardedInit(builder, globalOp, localInitOp.getCtorRegion(),
                  localInitOp.getDtorRegion());
 
  // Insert the removed terminator back.
  builder.getInsertionBlock()->push_back(ret);
}
 
void LoweringPreparePass::emitGuardedInit(CIRBaseBuilderTy &builder,
                                          cir::GlobalOp globalOp,
                                          mlir::Region &ctorRegion,
                                          mlir::Region &dtorRegion) {
  // Dynamic-initialization facts are materialized into a serializable
  // attribute by CIRGen, so this pass does not need a live ASTContext to
  // read them.
  std::optional<cir::DynamicInitInfoAttr> infoOption =
      globalOp.getDynamicInitInfo();
 
  // The GlobalOp verifier requires 'dynamic_init_info' whenever a
  // 'dynamic_init_guard' is present, so verified IR reaching here has it.
  // Emit an error instead of asserting so malformed .cir gets a diagnostic.
  if (!infoOption.has_value()) {
    globalOp->emitError(
        "dynamic-init-guarded global is missing 'dynamic_init_info'");
    return;
  }
 
  cir::DynamicInitInfoAttr info = infoOption.value();
 
  // Inline variables that weren't instantiated from variable templates have
  // partially-ordered initialization within their translation unit.
  bool nonTemplateInline =
      info.getIsInline() && !info.isTemplateInstantiation();
 
  // We only need to use thread-safe statics for local non-TLS variables and
  // inline variables; other global initialization is always single-threaded
  // or (through lazy dynamic loading in multiple threads) unsequenced.
  bool threadsafe = getLangOpts().ThreadsafeStatics &&
                    (info.getLocal() || nonTemplateInline) &&
                    info.getTls() == cir::TLSKind::None;
 
  // If we have a global variable with internal linkage and thread-safe statics
  // are disabled, we can just let the guard variable be of type i8.
  bool useInt8GuardVariable = !threadsafe && globalOp.hasInternalLinkage();
 
  // Create the guard variable if we don't already have it.
  cir::GlobalOp guard = getOrCreateStaticLocalDeclGuardAddress(
      builder, globalOp, globalOp.getDynamicInitGuard()->getName().getValue(),
      info.getLocal(), useInt8GuardVariable);
  if (!guard) {
    return;
  }
 
  mlir::Value guardPtr =
      builder.createGetGlobal(guard, info.getTls() != cir::TLSKind::None);
 
  // Test whether the variable has completed initialization.
  //
  // Itanium C++ ABI 3.3.2:
  //   The following is pseudo-code showing how these functions can be used:
  //     if (obj_guard.first_byte == 0) {
  //       if ( __cxa_guard_acquire (&obj_guard) ) {
  //         try {
  //           ... initialize the object ...;
  //         } catch (...) {
  //            __cxa_guard_abort (&obj_guard);
  //            throw;
  //         }
  //         ... queue object destructor with __cxa_atexit() ...;
  //         __cxa_guard_release (&obj_guard);
  //       }
  //     }
  //
  // If threadsafe statics are enabled, but we don't have inline atomics, just
  // call __cxa_guard_acquire unconditionally. The "inline" check isn't
  // actually inline, and the user might not expect calls to __atomic libcalls.
  unsigned maxInlineWidthInBits = getTargetInfo().getMaxAtomicInlineWidth();
 
  if (!threadsafe || maxInlineWidthInBits) {
    // Load the first byte of the guard variable.
    auto bytePtrTy = cir::PointerType::get(builder.getSIntNTy(8));
    mlir::Value bytePtr = builder.createBitcast(guardPtr, bytePtrTy);
    mlir::Value guardLoad = builder.createAlignedLoad(
        globalOp.getLoc(), bytePtr, *guard.getAlignment());
 
    // Itanium ABI:
    //   An implementation supporting thread-safety on multiprocessor
    //   systems must also guarantee that references to the initialized
    //   object do not occur before the load of the initialization flag.
    //
    // In LLVM, we do this by marking the load Acquire.
    if (threadsafe) {
      auto loadOp = mlir::cast<cir::LoadOp>(guardLoad.getDefiningOp());
      loadOp.setMemOrder(cir::MemOrder::Acquire);
      loadOp.setSyncScope(cir::SyncScopeKind::System);
    }
 
    // For ARM, we should only check the first bit, rather than the entire byte:
    //
    // ARM C++ ABI 3.2.3.1:
    //   To support the potential use of initialization guard variables
    //   as semaphores that are the target of ARM SWP and LDREX/STREX
    //   synchronizing instructions we define a static initialization
    //   guard variable to be a 4-byte aligned, 4-byte word with the
    //   following inline access protocol.
    //     #define INITIALIZED 1
    //     if ((obj_guard & INITIALIZED) != INITIALIZED) {
    //       if (__cxa_guard_acquire(&obj_guard))
    //         ...
    //     }
    //
    // and similarly for ARM64:
    //
    // ARM64 C++ ABI 3.2.2:
    //   This ABI instead only specifies the value bit 0 of the static guard
    //   variable; all other bits are platform defined. Bit 0 shall be 0 when
    //   the variable is not initialized and 1 when it is.
    if (useARMGuardVarABI() && !useInt8GuardVariable) {
      auto one = builder.getConstantInt(
          globalOp.getLoc(), mlir::cast<cir::IntType>(guardLoad.getType()), 1);
      guardLoad = builder.createAnd(globalOp.getLoc(), guardLoad, one);
    }
 
    // Check if the first byte of the guard variable is zero.
    auto zero = builder.getConstantInt(
        globalOp.getLoc(), mlir::cast<cir::IntType>(guardLoad.getType()), 0);
    auto needsInit = builder.createCompare(globalOp.getLoc(),
                                           cir::CmpOpKind::eq, guardLoad, zero);
 
    // Build the guarded initialization inside an if block.
    cir::IfOp::create(
        builder, globalOp.getLoc(), needsInit,
        /*withElseRegion=*/false, [&](mlir::OpBuilder &, mlir::Location) {
          emitCXXGuardedInitIf(
              builder, globalOp, ctorRegion, dtorRegion, info.getLocal(),
              guardPtr, builder.getPointerTo(guard.getSymType()), threadsafe);
        });
  } else {
    // Threadsafe statics without inline atomics - call __cxa_guard_acquire
    // unconditionally without the initial guard byte check.
    globalOp->emitError("NYI: guarded init without inline atomics support");
    return;
  }
}
 
void LoweringPreparePass::lowerLocalInitOp(cir::LocalInitOp initOp) {
 
  // If we don't actually need to initialize anything anymore, we're done here.
  if (initOp.getCtorRegion().empty() && initOp.getDtorRegion().empty()) {
    initOp.erase();
    return;
  }
 
  cir::GlobalOp globalOp = initOp.getReferencedGlobal(symbolTables);
  assert(globalOp && "No global-op found");
 
  handleStaticLocal(globalOp, initOp);
 
  // Remove the init local op, now that we've done everything we need with it.
  initOp.erase();
}
static bool isThreadWrapperReplaceable(const clang::TargetInfo &targetInfo) {
  // Note: Classic codegen needs to check that the VarDecl.getTLSKind() ==
  // TLS_Dynamic, but we don't attempt to emit the thread wrapper unless that is
  // already the case.  So the only thing that matters here is whether it is
  // darwin.
  return targetInfo.getTriple().isOSDarwin();
}
 
static cir::GlobalLinkageKind
getThreadLocalWrapperLinkage(GlobalOp op, const clang::TargetInfo &targetInfo) {
  if (isLocalLinkage(op.getLinkage()))
    return op.getLinkage();
 
  if (isThreadWrapperReplaceable(targetInfo))
    if (!isLinkOnceLinkage(op.getLinkage()) &&
        !isWeakODRLinkage(op.getLinkage()))
      return op.getLinkage();
 
  // If this isn't a TU in which this variable is defined, the thread wrapper is
  // discardable.
  if (op.isDeclaration())
    return cir::GlobalLinkageKind::LinkOnceODRLinkage;
  return cir::GlobalLinkageKind::WeakODRLinkage;
}
 
cir::FuncOp
LoweringPreparePass::getOrCreateThreadLocalWrapper(CIRBaseBuilderTy &builder,
                                                   GlobalOp op) {
  mlir::OpBuilder::InsertionGuard insertGuard(builder);
  builder.setInsertionPointToStart(&mlirModule.getBodyRegion().front());
 
  mlir::StringAttr wrapperName = op.getTlsRefs()->getWrapperName();
 
  auto existingWrapperIter = threadLocalWrappers.find(wrapperName.getValue());
  if (existingWrapperIter != threadLocalWrappers.end())
    return existingWrapperIter->second;
 
  // type is ptr-to-global-type(void);
  auto funcType = cir::FuncType::get({}, builder.getPointerTo(op.getSymType()));
  cir::FuncOp func =
      cir::FuncOp::create(builder, op.getLoc(), wrapperName, funcType);
 
  cir::GlobalLinkageKind linkageKind =
      getThreadLocalWrapperLinkage(op, getTargetInfo());
  func.setLinkageAttr(
      cir::GlobalLinkageKindAttr::get(&getContext(), linkageKind));
 
  if (getTargetInfo().getTriple().supportsCOMDAT() && func.isWeakForLinker())
    func.setSelfComdat();
 
  mlir::SymbolTable::setSymbolVisibility(
      func, mlir::SymbolTable::Visibility::Private);
 
  if (!isLocalLinkage(linkageKind)) {
    if (!isThreadWrapperReplaceable(getTargetInfo()) ||
        isLinkOnceLinkage(linkageKind) || isWeakODRLinkage(linkageKind) ||
        op.getGlobalVisibility() == cir::VisibilityKind::Hidden)
      func.setGlobalVisibility(cir::VisibilityKind::Hidden);
  }
  if (isThreadWrapperReplaceable(getTargetInfo()))
    op->emitError("Unhandled thread wrapper attributes for CC and Nounwind");
 
  threadLocalWrappers.insert({wrapperName.getValue(), func});
  return func;
}
 
void LoweringPreparePass::defineGlobalThreadLocalWrapper(cir::GlobalOp op,
                                                         cir::FuncOp initAlias,
                                                         bool isVarDefinition) {
  CIRBaseBuilderTy builder(getContext());
  cir::FuncOp wrapper = getOrCreateThreadLocalWrapper(builder, op);
  mlir::Block *entryBB = wrapper.addEntryBlock();
  builder.setInsertionPointToStart(entryBB);
  // If we are a situation where we have/need one, emit a call to the init
  // function.
  if (initAlias) {
    mlir::Location aliasLoc = initAlias.getLoc();
    if (!isVarDefinition) {
      // If this isn't a definition, we have to check that the alias exists.
      mlir::Value funcLoad = cir::GetGlobalOp::create(
          builder, aliasLoc, cir::PointerType::get(initAlias.getFunctionType()),
          initAlias.getSymName());
      mlir::Value nullCheck =
          builder.getNullValue(funcLoad.getType(), aliasLoc);
      mlir::Value cmp = cir::CmpOp::create(
          builder, aliasLoc, cir::CmpOpKind::ne, funcLoad, nullCheck);
      cir::IfOp::create(builder, aliasLoc, cmp, /*withElseRegion=*/false,
                        [&](mlir::OpBuilder &, mlir::Location loc) {
                          builder.createCallOp(aliasLoc, initAlias, {},
                                               /*attrs=*/{},
                                               /*argAttrs=*/{}, /*resAttrs=*/{},
                                               initAlias.getCallingConv());
                          cir::YieldOp::create(builder, aliasLoc);
                        });
    } else {
      // If this IS a definition, we know the alias exists, so we can just emit
      // a call to it.
      builder.createCallOp(aliasLoc, initAlias, {}, /*attrs=*/{},
                           /*argAttrs=*/{}, /*resAttrs=*/{},
                           initAlias.getCallingConv());
    }
  }
  cir::GetGlobalOp get = builder.createGetGlobal(op, /*tls=*/true);
  cir::ReturnOp::create(builder, op.getLoc(), {get});
}
 
cir::FuncOp
LoweringPreparePass::defineGlobalThreadLocalInitAlias(cir::GlobalOp op,
                                                      cir::FuncOp aliasee) {
  CIRBaseBuilderTy builder(getContext());
  mlir::OpBuilder::InsertionGuard insertGuard(builder);
  builder.setInsertionPointToStart(&mlirModule.getBodyRegion().front());
  mlir::StringAttr aliasName = op.getTlsRefs()->getInitName();
  auto existingAliasIter = threadLocalInitAliases.find(aliasName.getValue());
 
  if (existingAliasIter != threadLocalInitAliases.end())
    return existingAliasIter->second;
 
  auto funcType = builder.getVoidFnTy();
  cir::FuncOp alias =
      cir::FuncOp::create(builder, op.getLoc(), aliasName, funcType);
  alias.setLinkage(op.getLinkage());
 
  if (aliasee) {
    alias.setAliasee(aliasee.getSymName());
  } else {
    // If we don't have anything to alias (because this isn't a variable
    // definition!), we set this as just a function definition with no alias,
    // and extern-weak.
    alias.setLinkage(cir::GlobalLinkageKind::ExternalWeakLinkage);
    mlir::SymbolTable::setSymbolVisibility(
        alias, mlir::SymbolTable::Visibility::Private);
  }
 
  threadLocalInitAliases.insert({aliasName.getValue(), alias});
  return alias;
}
 
void LoweringPreparePass::lowerGlobalOp(GlobalOp op) {
  // Static locals are handled separately via guard variables.
  if (op.getDynamicInitGuard() &&
      (!op.getDynamicInitInfo() || op.getDynamicInitInfo()->getLocal()))
    return;
 
  mlir::Region &ctorRegion = op.getCtorRegion();
  mlir::Region &dtorRegion = op.getDtorRegion();
  cir::FuncOp initAlias;
 
  if (!ctorRegion.empty() || !dtorRegion.empty()) {
    // Build a variable initialization function and move the initialzation code
    // in the ctor region over.
    cir::FuncOp f = buildCXXGlobalVarDeclInitFunc(op);
 
    // Clear the ctor and dtor region
    ctorRegion.getBlocks().clear();
    dtorRegion.getBlocks().clear();
 
    assert(!cir::MissingFeatures::astVarDeclInterface());
    if (op.getTlsModel()) {
      // There are two types of global TLS variables: 'ordered' and 'unordered'.
      // 'ordered' are the common case. A call to any of them causes all of the
      // initializers for all other 'ordered' ones to be called, via a
      // `__tls_init` function. So the 'init alias' that gets called in the
      // wrapper for these goes directly to `__tls_init`.
 
      // 'Unordered' values are the case for variable templates. In this case,
      // their init alias goes directly to their init function. The FE generates
      // a guard variable for them (since they cannot use the global guard), so
      // we differentiate them that way.
 
      if (op.getTlsRefs()->getGuardName()) {
        // Unordered: the alias is the function we just generated.
        initAlias = defineGlobalThreadLocalInitAlias(op, f);
      } else {
        // Ordered: Get the __tls_init, and make the alias to that.
        initAlias = defineGlobalThreadLocalInitAlias(op, getTlsInitFn());
        // Ordered inits also need to get called from the __tls_init function,
        // so we add the init function to the list, so that we can add them to
        // it later.
        globalThreadLocalInitializers.push_back(f);
      }
    } else if (std::optional<uint32_t> priority = op.getInitPriority()) {
      prioritizedDynamicInitializers[*priority].push_back(f);
    } else if (op.getDynamicInitInfo() &&
               (op.getDynamicInitInfo()->getIsInline() ||
                op.getDynamicInitInfo()->isTemplateInstantiation()) &&
               getTargetInfo().getTriple().supportsCOMDAT() &&
               !op.hasInternalLinkage()) {
      // Vague-linkage globals (inline variables and template instantiations)
      // get their own `llvm.global_ctors` entry keyed to the variable via
      // COMDAT associated data, rather than being folded into
      // `_GLOBAL__sub_I_*`. This ensures at most one initializer per DSO runs,
      // and lets the linker drop the initializer call along with a discarded
      // duplicate definition of the variable. Other guarded globals, such as a
      // plain weak variable, stay ordered with the rest of the translation
      // unit's initializers. Matches classic CodeGen's
      // EmitCXXGlobalVarDeclInitFunc.
      assert(!cir::MissingFeatures::msabi());
      globalCtorList.emplace_back(f.getSymName(),
                                  cir::GlobalCtorAttr::getDefaultPriority(),
                                  op.getSymName());
      // When COMDAT is used on ELF or in the MS C++ ABI, the key must be in
      // `llvm.used` to prevent the linker from garbage-collecting it (and,
      // with it, the global ctor entry keyed to it). Matches classic
      // CodeGen's EmitCXXGlobalVarDeclInitFunc.
      if (getTargetInfo().getTriple().isOSBinFormatELF() ||
          getTargetInfo().getCXXABI().isMicrosoft())
        addUsedGlobal(op);
 
      // Since the global ctor entry is keyed to the variable, the init function
      // can be discarded along with it, so put it in the variable's COMDAT.
      std::optional<llvm::StringRef> comdat = op.getComdat();
      if (comdat && (getTargetInfo().getTriple().isOSBinFormatELF() ||
                     getTargetInfo().getTriple().isOSBinFormatWasm()))
        f.setComdat(comdat->empty() ? op.getSymName() : *comdat);
    } else {
      dynamicInitializers.push_back(f);
    }
  } else if (op.getTlsModel() && op.getTlsRefs() && op.isDeclaration()) {
    // If this is a declaration and has no init function, we probably DO have to
    // create an alias that needs checking, so create it as extern-weak.
    initAlias = defineGlobalThreadLocalInitAlias(op, {});
  }
 
  // We need a wrapper for TLS globals that MIGHT have a non-constant
  // initialization. The FE will have generated the TlsRefs for any with
  // known dynamic init, or unknown (extern) init.
  if (op.getTlsModel() && op.getTlsRefs())
    defineGlobalThreadLocalWrapper(op, initAlias, !op.isDeclaration());
 
  assert(!cir::MissingFeatures::opGlobalAnnotations());
}
 
void LoweringPreparePass::lowerGetGlobalOp(GetGlobalOp op) {
  if (!op.getTls())
    return;
  auto globalOp = mlir::cast<cir::GlobalOp>(
      symbolTables.lookupNearestSymbolFrom(op, op.getNameAttr()));
 
  // Only global/namespace scope thread local variables need to have their
  // get-global operations rewritten to be calls to a wrapper function.  If
  // we're not in a dynamic TLS (or one without the TLS markers), we can leave
  // this one as a get-global and return early.
  if (!globalOp.getTlsModel() || !globalOp.getTlsRefs())
    return;
 
  // If this is a global TLS, we need to replace the call to 'get_global' with a
  // call to the wrapper function.  Classic codegen figures out some cases where
  // we can omit this, but for now we're going to always put it in, as it is
  // effectively a no-op.
 
  // The first 'GetGlobalOp' at the beginning of a ctor/dtor region on one of
  // these is for the purpose of creating/destroying.  We want to skip replacing
  // THAT one, but leave all other get-global-ops in place, else
  // self-referential ops won't work right.
 
  // Note that ctors/dtors are removed during this pass. We get away with these
  // checks because the only time that these situations can actually be true
  // (that is, the ctor/dtor region exist) is if we're in the process of
  // converting the ctor/dtor for this. If we're NOT doing that, the ctor/dtor
  // will have already disappeared.
  mlir::Operation *parentOp = op->getParentOp();
  if (parentOp == globalOp) {
    mlir::Region *ctorRegion = &globalOp.getCtorRegion();
    mlir::Region *dtorRegion = &globalOp.getDtorRegion();
 
    if (!ctorRegion->empty() && &*ctorRegion->op_begin() == op.getOperation())
      return;
    if (!dtorRegion->empty() && &*dtorRegion->op_begin() == op.getOperation())
      return;
  }
 
  CIRBaseBuilderTy builder(getContext());
  cir::FuncOp wrapperFunc = getOrCreateThreadLocalWrapper(builder, globalOp);
 
  builder.setInsertionPoint(op);
  cir::CallOp call = builder.createCallOp(
      wrapperFunc.getLoc(),
      mlir::FlatSymbolRefAttr::get(wrapperFunc.getSymNameAttr()),
      wrapperFunc.getFunctionType().getReturnType(), {}, /*attrs=*/{},
      /*argAttrs=*/{}, /*resAttrs=*/{}, wrapperFunc.getCallingConv());
  op->replaceAllUsesWith(call);
  op.erase();
}
 
void LoweringPreparePass::lowerThreeWayCmpOp(CmpThreeWayOp op) {
  CIRBaseBuilderTy builder(getContext());
  builder.setInsertionPointAfter(op);
 
  mlir::Location loc = op->getLoc();
  cir::CmpThreeWayInfoAttr cmpInfo = op.getInfo();
 
  mlir::Value ltRes =
      builder.getConstantInt(loc, op.getType(), cmpInfo.getLt());
  mlir::Value eqRes =
      builder.getConstantInt(loc, op.getType(), cmpInfo.getEq());
  mlir::Value gtRes =
      builder.getConstantInt(loc, op.getType(), cmpInfo.getGt());
 
  mlir::Value transformedResult;
  if (cmpInfo.getOrdering() != CmpOrdering::Partial) {
    // Total ordering
    mlir::Value lt =
        builder.createCompare(loc, CmpOpKind::lt, op.getLhs(), op.getRhs());
    mlir::Value selectOnLt = builder.createSelect(loc, lt, ltRes, gtRes);
    mlir::Value eq =
        builder.createCompare(loc, CmpOpKind::eq, op.getLhs(), op.getRhs());
    transformedResult = builder.createSelect(loc, eq, eqRes, selectOnLt);
  } else {
    // Partial ordering
    cir::ConstantOp unorderedRes = builder.getConstantInt(
        loc, op.getType(), cmpInfo.getUnordered().value());
 
    mlir::Value eq =
        builder.createCompare(loc, CmpOpKind::eq, op.getLhs(), op.getRhs());
    mlir::Value selectOnEq = builder.createSelect(loc, eq, eqRes, unorderedRes);
    mlir::Value gt =
        builder.createCompare(loc, CmpOpKind::gt, op.getLhs(), op.getRhs());
    mlir::Value selectOnGt = builder.createSelect(loc, gt, gtRes, selectOnEq);
    mlir::Value lt =
        builder.createCompare(loc, CmpOpKind::lt, op.getLhs(), op.getRhs());
    transformedResult = builder.createSelect(loc, lt, ltRes, selectOnGt);
  }
 
  op.replaceAllUsesWith(transformedResult);
  op.erase();
}
 
template <typename AttributeTy>
static llvm::SmallVector<mlir::Attribute>
prepareCtorDtorAttrList(mlir::MLIRContext *context,
                        llvm::ArrayRef<cir::GlobalCtorDtorEntry> list) {
  llvm::SmallVector<mlir::Attribute> attrs;
  for (const auto &entry : list) {
    mlir::FlatSymbolRefAttr associated;
    if (!entry.associated.empty())
      associated = mlir::FlatSymbolRefAttr::get(context, entry.associated);
    attrs.push_back(
        AttributeTy::get(context, entry.name, entry.priority, associated));
  }
  return attrs;
}
 
void LoweringPreparePass::buildGlobalCtorDtorList() {
  if (!globalCtorList.empty()) {
    llvm::SmallVector<mlir::Attribute> globalCtors =
        prepareCtorDtorAttrList<cir::GlobalCtorAttr>(&getContext(),
                                                     globalCtorList);
 
    mlirModule->setAttr(cir::CIRDialect::getGlobalCtorsAttrName(),
                        mlir::ArrayAttr::get(&getContext(), globalCtors));
  }
 
  if (!globalDtorList.empty()) {
    llvm::SmallVector<mlir::Attribute> globalDtors =
        prepareCtorDtorAttrList<cir::GlobalDtorAttr>(&getContext(),
                                                     globalDtorList);
    mlirModule->setAttr(cir::CIRDialect::getGlobalDtorsAttrName(),
                        mlir::ArrayAttr::get(&getContext(), globalDtors));
  }
}
 
cir::GlobalOp
LoweringPreparePass::createGlobalThreadLocalGuard(CIRBaseBuilderTy &builder,
                                                  mlir::Location loc) {
  mlir::OpBuilder::InsertionGuard guard(builder);
  builder.setInsertionPointToStart(mlirModule.getBody());
 
  // The TLS Guard is always an Int8Ty.
  cir::IntType guardTy = builder.getSIntNTy(8);
  auto g = cir::GlobalOp::create(builder, loc, "__tls_guard", guardTy);
  g.setLinkageAttr(cir::GlobalLinkageKindAttr::get(
      builder.getContext(), cir::GlobalLinkageKind::InternalLinkage));
  g.setAlignment(clang::CharUnits::One().getAsAlign().value());
 
  if (auto defTlsModel = mlirModule->getAttrOfType<TLSModelAttr>(
          cir::CIRDialect::getDefaultTlsModelAttrName())) {
    g.setTlsModel(defTlsModel.getValue());
  } else {
    // Default value, unless overridden in the IR/by the frontend.
    g.setTlsModel(TLSModel::GeneralDynamic);
  }
 
  g.setInitialValueAttr(cir::IntAttr::get(guardTy, 0));
  return g;
}
 
cir::IfOp LoweringPreparePass::buildInitGuardCheck(CIRBaseBuilderTy &builder,
                                                   mlir::Location loc,
                                                   cir::GlobalOp guard,
                                                   bool threadLocal) {
  cir::GetGlobalOp getGuard = builder.createGetGlobal(guard, threadLocal);
  mlir::Value getGuardValue = getGuard;
 
  // Classic codegen always just loads the first byte of the guard instead of
  // the whole thing. __tls_guard is already only 8 bits, but for the case of
  // unordered TLS, it gets created as 64 bits.
  if (guard.getSymType() != builder.getSIntNTy(8))
    getGuardValue = builder.createBitcast(
        getGuard, cir::PointerType::get(builder.getSIntNTy(8)));
 
  mlir::Value guardLoad =
      builder.createAlignedLoad(loc, getGuardValue, *guard.getAlignment());
  auto zero = builder.getConstantInt(loc, builder.getSIntNTy(8), 0);
  cir::CmpOp compare =
      builder.createCompare(loc, cir::CmpOpKind::eq, guardLoad, zero);
  return cir::IfOp::create(
      builder, loc, compare,
      /*withElseRegion=*/false, [&](mlir::OpBuilder &, mlir::Location loc) {
        // Classic codegen still does this store as a i8, but it doesn't seem
        // reasonable to do an i8 store into a 64 bit value?
        builder.createStore(
            loc, builder.getConstantInt(loc, guard.getSymType(), 1), getGuard);
      });
}
 
void LoweringPreparePass::buildCXXGlobalTlsFunc() {
  if (globalThreadLocalInitializers.empty())
    return;
 
  // The global-ordered-init function for TLS variables just calls each of the
  // init-functions in order after doing a guard.
 
  cir::FuncOp tlsInit = getTlsInitFn();
  mlir::Location loc = tlsInit.getLoc();
  CIRBaseBuilderTy builder(getContext());
  mlir::Block *entryBB = tlsInit.addEntryBlock();
  builder.setInsertionPointToStart(entryBB);
 
  cir::IfOp ifOperation = buildInitGuardCheck(
      builder, loc, createGlobalThreadLocalGuard(builder, loc),
      /*threadLocal=*/true);
 
  // Emit the body of the guarded spot.
  builder.setInsertionPointToEnd(&ifOperation.getThenRegion().front());
  for (cir::FuncOp initFunc : globalThreadLocalInitializers)
    builder.createCallOp(loc, initFunc, {}, /*attrs=*/{}, /*argAttrs=*/{},
                         /*resAttrs=*/{}, initFunc.getCallingConv());
  cir::YieldOp::create(builder, loc);
 
  builder.setInsertionPointAfter(ifOperation);
  cir::ReturnOp::create(builder, loc);
}
 
/// Compute the zero-padded priority suffix used to name priority-specific
/// global init functions, so that the function names also sort in priority
/// order (e.g. 200 -> "000200").
static std::string getPrioritySuffix(unsigned priority) {
  assert(priority <= 65535 && "Priority should always be <= 65535.");
  std::string prioritySuffix = llvm::utostr(priority);
  assert(prioritySuffix.size() < 6);
  prioritySuffix = std::string(6 - prioritySuffix.size(), '0') + prioritySuffix;
  return prioritySuffix;
}
 
cir::FuncOp LoweringPreparePass::buildGlobalInitCallerFunc(
    llvm::StringRef fnName, cir::GlobalLinkageKind linkage,
    llvm::ArrayRef<cir::FuncOp> initializers, uint32_t priority,
    cir::GlobalOp guard) {
  CIRBaseBuilderTy builder(getContext());
  builder.setInsertionPointToEnd(&mlirModule.getBodyRegion().back());
  auto fnType = cir::FuncType::get({}, builder.getVoidTy());
  cir::FuncOp fn = buildRuntimeFunction(builder, fnName, mlirModule.getLoc(),
                                        fnType, linkage);
  builder.setInsertionPointToStart(fn.addEntryBlock());
  mlir::Location loc = fn.getLoc();
  auto emitCalls = [&] {
    for (cir::FuncOp init : initializers)
      builder.createCallOp(init.getLoc(), init, {}, /*attrs=*/{},
                           /*argAttrs=*/{}, /*resAttrs=*/{},
                           init.getCallingConv());
  };
  if (guard) {
    cir::IfOp guardIf =
        buildInitGuardCheck(builder, loc, guard, /*threadLocal=*/false);
    builder.setInsertionPointToEnd(&guardIf.getThenRegion().front());
    emitCalls();
    cir::YieldOp::create(builder, loc);
    builder.setInsertionPointAfter(guardIf);
  } else {
    emitCalls();
  }
  cir::ReturnOp::create(builder, loc);
  globalCtorList.emplace_back(fnName, priority);
  return fn;
}
 
void LoweringPreparePass::buildCXXGlobalPriorityInitFuncs(
    llvm::SmallVectorImpl<cir::FuncOp> &importedInits) {
  // std::map keeps priorities in ascending order, so each group is already
  // ready to emit into its own function, named after its priority so that
  // the functions are naturally ordered relative to one another.
  for (const auto &[priority, initializers] : prioritizedDynamicInitializers) {
    SmallString<256> fnName;
    fnName += "_GLOBAL__I_";
    fnName += getPrioritySuffix(priority);
 
    llvm::SmallVector<cir::FuncOp> calls(importedInits.begin(),
                                         importedInits.end());
    calls.append(initializers.begin(), initializers.end());
    importedInits.clear();
    buildGlobalInitCallerFunc(fnName, cir::GlobalLinkageKind::InternalLinkage,
                              calls, priority);
  }
}
 
void LoweringPreparePass::buildCXXGlobalInitFunc() {
  CIRBaseBuilderTy builder(getContext());
  builder.setInsertionPointToEnd(&mlirModule.getBodyRegion().back());
  mlir::Location loc = mlirModule.getLoc();
 
  // The initializers of the imported C++20 named modules, recorded by CIRGen
  // as the mangled names of functions this translation unit declares, at the
  // location of the import, and calls before its own initializers.
  llvm::SmallVector<cir::FuncOp> importedInits;
  if (auto inits = mlirModule->getAttrOfType<mlir::ArrayAttr>(
          cir::CIRDialect::getCXXModuleImportedInitsAttrName()))
    for (auto init : inits.getAsRange<cir::CXXModuleInitAttr>())
      importedInits.push_back(
          buildRuntimeFunction(builder, init.getName().getValue(),
                               init.getLoc(), builder.getVoidFnTy()));
 
  // The C++20 named-module init function name is precomputed by CIRGen and
  // stored as a module-level attribute.  Its presence is what marks this
  // module as a named-module interface unit, so the name and the external
  // linkage that goes with it both come from the attribute and this pass needs
  // no live ASTContext.  Modules built directly from textual CIR can opt in to
  // the module-init form by setting the same attribute.
  if (auto moduleInitFnName = mlirModule->getAttrOfType<mlir::StringAttr>(
          cir::CIRDialect::getCXXModuleInitFnNameAttrName())) {
    // As in classic CodeGen's EmitCXXModuleInitFunc, an interface unit gets
    // one initializer function, even with nothing to run, since importing
    // translation units may call it. It runs the imported modules'
    // initializers, then the unit's own in priority order, which get no
    // `_GLOBAL__I_<priority>` function of their own. The global ctor entry
    // and every importing translation unit call it, so a guard byte makes
    // the initializers run once; there is none when there is nothing to run.
    llvm::SmallVector<cir::FuncOp> initializers(importedInits.begin(),
                                                importedInits.end());
    for (const auto &[priority, inits] : prioritizedDynamicInitializers)
      initializers.append(inits.begin(), inits.end());
    initializers.append(dynamicInitializers.begin(), dynamicInitializers.end());
 
    llvm::StringRef fnName = moduleInitFnName.getValue();
    cir::GlobalOp guard;
    if (!initializers.empty()) {
      cir::IntType guardTy = builder.getSIntNTy(8);
      guard =
          createGuardGlobalOp(builder, loc, (fnName + "__in_chrg").str(),
                              guardTy, cir::GlobalLinkageKind::InternalLinkage);
      guard.setAlignment(clang::CharUnits::One().getAsAlign().value());
      guard.setInitialValueAttr(cir::IntAttr::get(guardTy, 0));
    }
    buildGlobalInitCallerFunc(fnName, cir::GlobalLinkageKind::ExternalLinkage,
                              initializers,
                              cir::GlobalCtorAttr::getDefaultPriority(), guard);
    return;
  }
 
  // Any other translation unit, as in classic CodeGen's
  // EmitCXXGlobalInitFunc: one function per priority, the imported modules'
  // initializers in front of the first, then the default-priority function
  // for the rest, only when there is something left to call.
  buildCXXGlobalPriorityInitFuncs(importedInits);
  if (importedInits.empty() && dynamicInitializers.empty())
    return;
 
  llvm::SmallVector<cir::FuncOp> initializers(importedInits.begin(),
                                              importedInits.end());
  initializers.append(dynamicInitializers.begin(), dynamicInitializers.end());
 
  // Include the filename in the symbol name. Including "sub_" matches gcc
  // and makes sure these symbols appear lexicographically behind the symbols
  // with priority (TBD).  Module implementation units behave the same
  // way as a non-modular TU with imports.
  SmallString<256> fnName;
  fnName += "_GLOBAL__sub_I_";
  fnName += getTransformedFileName(mlirModule);
  buildGlobalInitCallerFunc(fnName, cir::GlobalLinkageKind::InternalLinkage,
                            initializers,
                            cir::GlobalCtorAttr::getDefaultPriority());
}
 
/// Lower a cir.array.ctor or cir.array.dtor into a do-while loop that
/// iterates over every element.  For cir.array.ctor ops whose partial_dtor
/// region is non-empty, the ctor loop is wrapped in a cir.cleanup.scope whose
/// EH cleanup performs a reverse destruction loop using the partial dtor body.
static void lowerArrayDtorCtorIntoLoop(cir::CIRBaseBuilderTy &builder,
                                       const clang::TargetInfo &targetInfo,
                                       mlir::Operation *op, mlir::Type eltTy,
                                       mlir::Value addr,
                                       mlir::Value numElements,
                                       uint64_t arrayLen, bool isCtor) {
  mlir::Location loc = op->getLoc();
  bool isDynamic = numElements != nullptr;
 
  // TODO: instead of getting the size from the target, create alias for
  // PtrDiffTy and unify with CIRGen stuff.
  const unsigned sizeTypeSize =
      targetInfo.getTypeWidth(targetInfo.getSignedSizeType());
 
  // Both constructors and destructors use end = begin + numElements.
  // Constructors iterate forward [begin, end).  Destructors iterate backward
  // from end, decrementing before calling the destructor on each element.
  mlir::Value begin, end;
  if (isDynamic) {
    begin = addr;
    end = cir::PtrStrideOp::create(builder, loc, eltTy, begin, numElements);
  } else {
    mlir::Value endOffsetVal =
        builder.getUnsignedInt(loc, arrayLen, sizeTypeSize);
    begin = cir::CastOp::create(builder, loc, eltTy,
                                cir::CastKind::array_to_ptrdecay, addr);
    end = cir::PtrStrideOp::create(builder, loc, eltTy, begin, endOffsetVal);
  }
 
  mlir::Value start = isCtor ? begin : end;
  mlir::Value stop = isCtor ? end : begin;
 
  // For dynamic destructors, guard against zero elements.
  // This places the destructor loop emitted below inside the if block.
  cir::IfOp ifOp;
  if (isDynamic) {
    mlir::Value guardCond;
    if (isCtor) {
      mlir::Value zero = builder.getUnsignedInt(loc, 0, sizeTypeSize);
      guardCond = cir::CmpOp::create(builder, loc, cir::CmpOpKind::ne,
                                     numElements, zero);
    } else {
      // We could check for numElements != 0 in this case too, but this matches
      // what classic codegen does.
      guardCond =
          cir::CmpOp::create(builder, loc, cir::CmpOpKind::ne, start, stop);
    }
    ifOp = cir::IfOp::create(builder, loc, guardCond,
                             /*withElseRegion=*/false,
                             [&](mlir::OpBuilder &, mlir::Location) {});
    builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
  }
 
  cir::CIRDataLayout dataLayout(op->getParentOfType<mlir::ModuleOp>());
  cir::PointerType tmpAddrTy = builder.getPointerTo(
      eltTy, dataLayout.getAllocaAddrSpace(builder.getContext()));
  mlir::Value tmpAddr = builder.createAlloca(
      loc, /*addr type*/ tmpAddrTy, "__array_idx",
      builder.getAlignmentAttr(dataLayout.getABITypeAlign(eltTy).value()));
  builder.createStore(loc, start, tmpAddr);
 
  mlir::Block *bodyBlock = &op->getRegion(0).front();
 
  // Clone the region body (ctor/dtor call and any setup ops like per-element
  // zero-init) into the loop, remapping the block argument to the current
  // element pointer.
  auto cloneRegionBodyInto = [&](mlir::Block *srcBlock,
                                 mlir::Value replacement) {
    mlir::IRMapping map;
    map.map(srcBlock->getArgument(0), replacement);
    for (mlir::Operation &regionOp : *srcBlock) {
      if (!mlir::isa<cir::YieldOp>(&regionOp))
        builder.clone(regionOp, map);
    }
  };
 
  mlir::Block *partialDtorBlock = nullptr;
  if (auto arrayCtor = mlir::dyn_cast<cir::ArrayCtor>(op)) {
    mlir::Region &partialDtor = arrayCtor.getPartialDtor();
    if (!partialDtor.empty())
      partialDtorBlock = &partialDtor.front();
  } else if (auto arrayDtor = mlir::dyn_cast<cir::ArrayDtor>(op)) {
    // When the element destructor may throw, reuse the body block as the
    // partial-dtor block so that an exception thrown by an element's dtor
    // continues the reverse-destruction loop in the EH cleanup region. The
    // body block already stores the next element pointer to `tmpAddr`
    // before invoking the dtor, so when an exception unwinds from the
    // dtor call `tmpAddr` already points at the element that threw, and
    // the cleanup loop picks up from `tmpAddr - 1` and walks back to
    // `begin`.
    if (arrayDtor.getDtorMayThrow())
      partialDtorBlock = bodyBlock;
  }
 
  auto emitCtorDtorLoop = [&]() {
    builder.createDoWhile(
        loc,
        /*condBuilder=*/
        [&](mlir::OpBuilder &b, mlir::Location loc) {
          auto currentElement = cir::LoadOp::create(b, loc, eltTy, tmpAddr);
          auto cmp = cir::CmpOp::create(builder, loc, cir::CmpOpKind::ne,
                                        currentElement, stop);
          builder.createCondition(cmp);
        },
        /*bodyBuilder=*/
        [&](mlir::OpBuilder &b, mlir::Location loc) {
          auto currentElement = cir::LoadOp::create(b, loc, eltTy, tmpAddr);
          if (isCtor) {
            cloneRegionBodyInto(bodyBlock, currentElement);
            mlir::Value stride = builder.getUnsignedInt(loc, 1, sizeTypeSize);
            auto nextElement = cir::PtrStrideOp::create(builder, loc, eltTy,
                                                        currentElement, stride);
            builder.createStore(loc, nextElement, tmpAddr);
          } else {
            mlir::Value stride = builder.getSignedInt(loc, -1, sizeTypeSize);
            auto prevElement = cir::PtrStrideOp::create(builder, loc, eltTy,
                                                        currentElement, stride);
            builder.createStore(loc, prevElement, tmpAddr);
            cloneRegionBodyInto(bodyBlock, prevElement);
          }
 
          cir::YieldOp::create(b, loc);
        });
  };
 
  if (partialDtorBlock) {
    cir::CleanupScopeOp::create(
        builder, loc, cir::CleanupKind::EH,
        /*bodyBuilder=*/
        [&](mlir::OpBuilder &b, mlir::Location loc) {
          emitCtorDtorLoop();
          cir::YieldOp::create(b, loc);
        },
        /*cleanupBuilder=*/
        [&](mlir::OpBuilder &b, mlir::Location loc) {
          auto cur = cir::LoadOp::create(b, loc, eltTy, tmpAddr);
          auto cmp =
              cir::CmpOp::create(builder, loc, cir::CmpOpKind::ne, cur, begin);
          cir::IfOp::create(
              builder, loc, cmp, /*withElseRegion=*/false,
              [&](mlir::OpBuilder &b, mlir::Location loc) {
                builder.createDoWhile(
                    loc,
                    /*condBuilder=*/
                    [&](mlir::OpBuilder &b, mlir::Location loc) {
                      auto el = cir::LoadOp::create(b, loc, eltTy, tmpAddr);
                      auto neq = cir::CmpOp::create(
                          builder, loc, cir::CmpOpKind::ne, el, begin);
                      builder.createCondition(neq);
                    },
                    /*bodyBuilder=*/
                    [&](mlir::OpBuilder &b, mlir::Location loc) {
                      auto el = cir::LoadOp::create(b, loc, eltTy, tmpAddr);
                      mlir::Value negOne =
                          builder.getSignedInt(loc, -1, sizeTypeSize);
                      auto prev = cir::PtrStrideOp::create(builder, loc, eltTy,
                                                           el, negOne);
                      builder.createStore(loc, prev, tmpAddr);
                      cloneRegionBodyInto(partialDtorBlock, prev);
                      builder.createYield(loc);
                    });
                cir::YieldOp::create(builder, loc);
              });
          cir::YieldOp::create(b, loc);
        });
  } else {
    emitCtorDtorLoop();
  }
 
  if (ifOp)
    cir::YieldOp::create(builder, loc);
 
  op->erase();
}
 
void LoweringPreparePass::lowerArrayDtor(cir::ArrayDtor op) {
  CIRBaseBuilderTy builder(getContext());
  builder.setInsertionPointAfter(op.getOperation());
 
  mlir::Type eltTy = op->getRegion(0).getArgument(0).getType();
 
  if (op.getNumElements()) {
    lowerArrayDtorCtorIntoLoop(builder, getTargetInfo(), op, eltTy,
                               op.getAddr(), op.getNumElements(),
                               /*arrayLen=*/0,
                               /*isCtor=*/false);
    return;
  }
 
  auto arrayLen =
      mlir::cast<cir::ArrayType>(op.getAddr().getType().getPointee()).getSize();
  lowerArrayDtorCtorIntoLoop(builder, getTargetInfo(), op, eltTy, op.getAddr(),
                             /*numElements=*/nullptr, arrayLen,
                             /*isCtor=*/false);
}
 
void LoweringPreparePass::lowerArrayCtor(cir::ArrayCtor op) {
  cir::CIRBaseBuilderTy builder(getContext());
  builder.setInsertionPointAfter(op.getOperation());
 
  mlir::Type eltTy = op->getRegion(0).getArgument(0).getType();
 
  if (op.getNumElements()) {
    lowerArrayDtorCtorIntoLoop(builder, getTargetInfo(), op, eltTy,
                               op.getAddr(), op.getNumElements(),
                               /*arrayLen=*/0,
                               /*isCtor=*/true);
    return;
  }
 
  auto arrayLen =
      mlir::cast<cir::ArrayType>(op.getAddr().getType().getPointee()).getSize();
  lowerArrayDtorCtorIntoLoop(builder, getTargetInfo(), op, eltTy, op.getAddr(),
                             /*numElements=*/nullptr, arrayLen,
                             /*isCtor=*/true);
}
 
cir::FuncOp LoweringPreparePass::getCalledFunction(cir::CallOp callOp) {
  mlir::SymbolRefAttr sym = llvm::dyn_cast_if_present<mlir::SymbolRefAttr>(
      callOp.getCallableForCallee());
  if (!sym)
    return nullptr;
  return symbolTables.lookupNearestSymbolFrom<cir::FuncOp>(callOp, sym);
}
 
void LoweringPreparePass::lowerTrivialCopyCall(cir::CallOp op) {
  cir::FuncOp funcOp = getCalledFunction(op);
  if (!funcOp)
    return;
 
  std::optional<cir::CtorKind> ctorKind = funcOp.getCxxConstructorKind();
  if (ctorKind && *ctorKind == cir::CtorKind::Copy &&
      funcOp.isCxxTrivialMemberFunction()) {
    // Replace the trivial copy constructor call with a `CopyOp`
    CIRBaseBuilderTy builder(getContext());
    mlir::ValueRange operands = op.getOperands();
    mlir::Value dest = operands[0];
    mlir::Value src = operands[1];
    builder.setInsertionPoint(op);
    builder.createCopy(dest, src);
    op.erase();
  }
}
 
cir::GlobalOp LoweringPreparePass::getOrCreateConstAggregateGlobal(
    CIRBaseBuilderTy &builder, mlir::Location loc, llvm::StringRef baseName,
    mlir::Type ty, mlir::TypedAttr constant, uint64_t alignment) {
  // Look up (and lazily populate) the per-base-name cache.
  llvm::SmallVector<cir::GlobalOp, 1> &versions =
      constAggregateGlobals[baseName];
 
  // First, check globals we've already discovered for this base name.
  for (cir::GlobalOp gv : versions) {
    if (gv.getSymType() == ty && gv.getInitialValue() == constant &&
        gv.getAlignment() == alignment)
      return gv;
  }
 
  // No cached match. Scan the module's symbol table starting from the next
  // unscanned version. In practice this should usually exit on the first
  // iteration, but it's possible that some other pass or a previous
  // invocation of this pass created globals using this same logic.
  llvm::SmallString<128> name(baseName);
  size_t baseLen = name.size();
  unsigned version = versions.size();
  while (true) {
    name.resize(baseLen);
    if (version != 0) {
      name.push_back('.');
      llvm::Twine(version).toVector(name);
    }
    auto existingGv = symbolTables.lookupSymbolIn<cir::GlobalOp>(
        mlirModule, mlir::StringAttr::get(&getContext(), name));
    if (!existingGv)
      break;
    versions.push_back(existingGv);
    if (existingGv.getSymType() == ty &&
        existingGv.getInitialValue() == constant &&
        existingGv.getAlignment() == alignment)
      return existingGv;
    ++version;
  }
 
  // No match found, create a new global. The loop above found an unused name.
  mlir::OpBuilder::InsertionGuard guard(builder);
  builder.setInsertionPointToStart(mlirModule.getBody());
  auto gv =
      cir::GlobalOp::create(builder, loc, name, ty,
                            /*isConstant=*/true,
                            cir::LangAddressSpaceAttr::get(
                                &getContext(), cir::LangAddressSpace::Default),
                            cir::GlobalLinkageKind::PrivateLinkage);
  mlir::SymbolTable::setSymbolVisibility(
      gv, mlir::SymbolTable::Visibility::Private);
  gv.setInitialValueAttr(constant);
  gv.setAlignment(alignment);
 
  // Keep the cached symbol table in sync with the new global so subsequent
  // lookups for other base names find it.
  symbolTables.getSymbolTable(mlirModule).insert(gv);
 
  versions.push_back(gv);
  return gv;
}
 
void LoweringPreparePass::lowerStoreOfConstAggregate(cir::StoreOp op) {
  // Check if the value operand is a cir.const with aggregate type.
  auto constOp = op.getValue().getDefiningOp<cir::ConstantOp>();
  if (!constOp)
    return;
 
  mlir::Type ty = constOp.getType();
  if (!mlir::isa<cir::ArrayType, cir::RecordType>(ty))
    return;
 
  // Only transform stores to local variables (backed by cir.alloca).
  // Stores to other addresses (e.g. base_class_addr) should not be
  // transformed as they may be partial initializations.
  auto alloca = op.getAddr().getDefiningOp<cir::AllocaOp>();
  if (!alloca)
    return;
 
  mlir::TypedAttr constant = constOp.getValue();
 
  // OG implements several optimization tiers for constant aggregate
  // initialization. For now we always create a global constant + memcpy
  // (shouldCreateMemCpyFromGlobal). Future work can add the intermediate
  // tiers.
  assert(!cir::MissingFeatures::shouldUseBZeroPlusStoresToInitialize());
  assert(!cir::MissingFeatures::shouldUseMemSetToInitialize());
  assert(!cir::MissingFeatures::shouldSplitConstantStore());
 
  // Get function name from parent cir.func.
  auto func = op->getParentOfType<cir::FuncOp>();
  if (!func)
    return;
  llvm::StringRef funcName = func.getSymName();
 
  // Get variable name from the alloca.
  llvm::StringRef varName = alloca.getName();
 
  // Build base name: __const.<func>.<var>
  std::string baseName = ("__const." + funcName + "." + varName).str();
  CIRBaseBuilderTy builder(getContext());
 
  // Check for existing globals and create a new global with a unique name
  // if no match is found.
  cir::GlobalOp gv = getOrCreateConstAggregateGlobal(
      builder, op.getLoc(), baseName, ty, constant, alloca.getAlignment());
 
  // Now replace the store with get_global + copy.
  builder.setInsertionPoint(op);
 
  auto ptrTy = cir::PointerType::get(ty);
  mlir::Value globalPtr =
      cir::GetGlobalOp::create(builder, op.getLoc(), ptrTy, gv.getSymName());
 
  // Replace store with copy.
  cir::CopyOp copyOp = builder.createCopy(op.getAddr(), globalPtr);
 
  cir::CIRDataLayout dataLayout(mlirModule);
  uint64_t naturalAlign = dataLayout.getABITypeAlign(ty).value();
  if (alloca.getAlignment() != naturalAlign)
    copyOp.setDstAlignment(alloca.getAlignment());
  uint64_t srcAlign = gv.getAlignment().value_or(naturalAlign);
  if (srcAlign != naturalAlign)
    copyOp.setSrcAlignment(srcAlign);
 
  // Erase the original store.
  op.erase();
 
  // Erase the cir.const if it has no remaining users.
  if (constOp.use_empty())
    constOp.erase();
}
 
// Every raised operation carries the original callee, the operands, and the
// attributes of the call, so this one function lowers any of them back to an
// equivalent plain call.
void LoweringPreparePass::lowerStdOp(cir::StdOpInterface typedOp) {
  mlir::Operation *op = typedOp.getOperation();
  cir::CIRBaseBuilderTy builder(getContext());
  builder.setInsertionPointAfter(op);
  mlir::Type resultType;
  if (op->getNumResults())
    resultType = op->getResult(0).getType();
  cir::CallOp call = builder.createCallOp(
      op->getLoc(), typedOp.getOriginalFnAttr(), resultType, op->getOperands());
 
  // IdiomRecognizer stores both the inherent and discardable attributes of the
  // original call as discardable attributes on the raised operation because
  // the cir.std operations do not share CallOp's property schema. Reconstruct
  // the original storage class from the destination CallOp schema here.
  //
  // This intentionally uses getInherentAttr as a schema query, not as a test
  // that `call` currently has the attribute: generated property accessors
  // return an engaged optional with a null attribute for a recognized but unset
  // optional property. An empty optional means the name is not inherent to
  // CallOp and must remain discardable.
  // TODO: Replace this transport encoding with shared call properties on the
  // raised operations.
  for (mlir::NamedAttribute attr : op->getDiscardableAttrs()) {
    if (call->getInherentAttr(attr.getName()).has_value())
      call->setInherentAttr(attr.getName(), attr.getValue());
    else
      call->setDiscardableAttr(attr.getName(), attr.getValue());
  }
 
  op->replaceAllUsesWith(call);
  op->erase();
}
 
void LoweringPreparePass::runOnOp(mlir::Operation *op) {
  if (auto arrayCtor = dyn_cast<cir::ArrayCtor>(op)) {
    lowerArrayCtor(arrayCtor);
  } else if (auto arrayDtor = dyn_cast<cir::ArrayDtor>(op)) {
    lowerArrayDtor(arrayDtor);
  } else if (auto registerOp = dyn_cast<cir::RegisterExitDtorOp>(op)) {
    lowerRegisterExitDtorOp(registerOp);
  } else if (auto stdOp = mlir::dyn_cast<cir::StdOpInterface>(op)) {
    lowerStdOp(stdOp);
  } else if (auto cast = mlir::dyn_cast<cir::CastOp>(op)) {
    lowerCastOp(cast);
  } else if (auto complexConj = mlir::dyn_cast<cir::ComplexConjOp>(op)) {
    lowerComplexConjOp(complexConj);
  } else if (auto complexDiv = mlir::dyn_cast<cir::ComplexFDivOp>(op)) {
    lowerComplexFDivOp(complexDiv);
  } else if (auto complexMul = mlir::dyn_cast<cir::ComplexFMulOp>(op)) {
    lowerComplexFMulOp(complexMul);
  } else if (auto complexDiv = mlir::dyn_cast<cir::ComplexDivOp>(op)) {
    lowerComplexDivOp(complexDiv);
  } else if (auto complexMul = mlir::dyn_cast<cir::ComplexMulOp>(op)) {
    lowerComplexMulOp(complexMul);
  } else if (auto glob = mlir::dyn_cast<cir::GlobalOp>(op)) {
    lowerGlobalOp(glob);
    if (auto regAttr = glob->getAttrOfType<CUDAVarRegistrationInfoAttr>(
            CUDAVarRegistrationInfoAttr::getMnemonic()))
      cudaDeviceVars.emplace_back(glob, regAttr);
  } else if (auto getGlob = mlir::dyn_cast<cir::GetGlobalOp>(op)) {
    lowerGetGlobalOp(getGlob);
  } else if (auto callOp = dyn_cast<cir::CallOp>(op)) {
    lowerTrivialCopyCall(callOp);
  } else if (auto storeOp = dyn_cast<cir::StoreOp>(op)) {
    lowerStoreOfConstAggregate(storeOp);
  } else if (auto fnOp = dyn_cast<cir::FuncOp>(op)) {
    if (auto globalCtor = fnOp.getGlobalCtorPriority())
      globalCtorList.emplace_back(fnOp.getName(), globalCtor.value());
    else if (auto globalDtor = fnOp.getGlobalDtorPriority())
      globalDtorList.emplace_back(fnOp.getName(), globalDtor.value());
 
    if (mlir::Attribute attr =
            fnOp->getAttr(cir::CUDAKernelNameAttr::getMnemonic());
        attr && !fnOp.isDeclaration()) {
      auto kernelNameAttr = dyn_cast<CUDAKernelNameAttr>(attr);
      llvm::StringRef kernelName = kernelNameAttr.getKernelName();
      cudaKernelMap[kernelName] = fnOp;
    }
  } else if (auto threeWayCmp = dyn_cast<cir::CmpThreeWayOp>(op)) {
    lowerThreeWayCmpOp(threeWayCmp);
  } else if (auto initOp = dyn_cast<cir::LocalInitOp>(op)) {
    lowerLocalInitOp(initOp);
  }
}
 
static llvm::StringRef getCUDAPrefix(const clang::LangOptions &langOpts) {
  if (langOpts.HIP)
    return "hip";
  return "cuda";
}
 
static std::string addUnderscoredPrefix(llvm::StringRef prefix,
                                        llvm::StringRef name) {
  return ("__" + prefix + name).str();
}
 
/// Creates a global constructor function for the module:
///
/// For CUDA:
/// \code
/// void __cuda_module_ctor() {
///     Handle = __cudaRegisterFatBinary(GpuBinaryBlob);
///     __cuda_register_globals(Handle);
/// }
/// \endcode
///
/// For HIP:
/// \code
/// void __hip_module_ctor() {
///     if (__hip_gpubin_handle == 0) {
///         __hip_gpubin_handle  = __hipRegisterFatBinary(GpuBinaryBlob);
///         __hip_register_globals(__hip_gpubin_handle);
///     }
/// }
/// \endcode
void LoweringPreparePass::buildCUDAModuleCtor() {
  bool isHIP = getLangOpts().HIP;
 
  if (getLangOpts().GPURelocatableDeviceCode)
    llvm_unreachable("GPU RDC NYI");
 
  // For CUDA without -fgpu-rdc, it's safe to stop generating ctor
  // if there's nothing to register.
  if (cudaKernelMap.empty() && cudaDeviceVars.empty())
    return;
 
  // There's no device-side binary, so no need to proceed for CUDA.
  // HIP creates an external symbol in this case.
  auto deviceBinaryAttr = mlirModule->getAttrOfType<mlir::StringAttr>(
      CIRDialect::getCUDADeviceBinaryAttrName());
  if (!deviceBinaryAttr && !isHIP)
    return;
 
  // Set up common types and builder.
  llvm::StringRef cudaPrefix = getCUDAPrefix(getLangOpts());
  mlir::Location loc = mlirModule->getLoc();
  CIRBaseBuilderTy builder(getContext());
  builder.setInsertionPointToStart(mlirModule.getBody());
 
  Type voidTy = builder.getVoidTy();
  PointerType voidPtrTy = builder.getVoidPtrTy();
  PointerType voidPtrPtrTy = builder.getPointerTo(voidPtrTy);
  IntType intTy = builder.getSIntNTy(32);
 
  // --- Create fatbin globals ---
 
  // The section names are different for MAC OS X.
  llvm::StringRef fatbinConstName =
      getLangOpts().HIP ? ".hip_fatbin" : ".nv_fatbin";
 
  llvm::StringRef fatbinSectionName =
      getLangOpts().HIP ? ".hipFatBinSegment" : ".nvFatBinSegment";
 
  GlobalOp fatbinStr;
  if (deviceBinaryAttr) {
    // Create the fatbin string constant with GPU binary contents.
    // The dialect verifier guarantees the attribute is typed as the array.
    auto fatbinType = mlir::cast<ArrayType>(deviceBinaryAttr.getType());
    std::string fatbinStrName = addUnderscoredPrefix(cudaPrefix, "_fatbin_str");
    fatbinStr = GlobalOp::create(builder, loc, fatbinStrName, fatbinType,
                                 /*isConstant=*/true, {},
                                 GlobalLinkageKind::PrivateLinkage);
    constexpr unsigned HIPCodeObjectAlign = 4096;
    fatbinStr.setAlignment(isHIP ? HIPCodeObjectAlign : 8);
 
    fatbinStr.setInitialValueAttr(
        cir::ConstArrayAttr::get(fatbinType, deviceBinaryAttr));
  } else {
    // If fatbin is not available, create an external symbol __hip_fatbin in
    // section .hip_fatbin. It is populated elsewhere, e.g. by lld through a
    // linker script.
    fatbinStr = GlobalOp::create(
        builder, loc, addCUIDSuffix("__hip_fatbin"), builder.getUIntNTy(8),
        /*isConstant=*/true, {}, GlobalLinkageKind::ExternalLinkage);
  }
  fatbinStr.setSection(fatbinConstName);
  fatbinStr.setPrivate();
 
  // Create the fatbin wrapper struct:
  //    struct { int magic; int version; void *fatbin; void *unused; };
  mlir::Type fatbinWrapperMembers[] = {intTy, intTy, voidPtrTy, voidPtrTy};
  auto fatbinWrapperType = cir::StructType::get(
      &getContext(), fatbinWrapperMembers, /*packed=*/false, /*is_class=*/false,
      cir::RecordType::getAllDataKinds(fatbinWrapperMembers));
  std::string fatbinWrapperName =
      addUnderscoredPrefix(cudaPrefix, "_fatbin_wrapper");
  GlobalOp fatbinWrapper = GlobalOp::create(
      builder, loc, fatbinWrapperName, fatbinWrapperType,
      /*isConstant=*/true, {}, GlobalLinkageKind::PrivateLinkage);
  fatbinWrapper.setSection(fatbinSectionName);
 
  constexpr unsigned cudaFatMagic = 0x466243b1;
  constexpr unsigned hipFatMagic = 0x48495046;
  unsigned fatMagic = isHIP ? hipFatMagic : cudaFatMagic;
 
  auto magicInit = IntAttr::get(intTy, fatMagic);
  auto versionInit = IntAttr::get(intTy, 1);
  auto fatbinStrSymbol =
      mlir::FlatSymbolRefAttr::get(fatbinStr.getSymNameAttr());
  auto fatbinInit = GlobalViewAttr::get(voidPtrTy, fatbinStrSymbol);
  mlir::TypedAttr unusedInit = builder.getConstNullPtrAttr(voidPtrTy);
  fatbinWrapper.setInitialValueAttr(cir::ConstRecordAttr::get(
      fatbinWrapperType,
      mlir::ArrayAttr::get(&getContext(),
                           {magicInit, versionInit, fatbinInit, unusedInit})));
 
  // Create the GPU binary handle global variable.
  std::string gpubinHandleName =
      addUnderscoredPrefix(cudaPrefix, "_gpubin_handle");
  if (isHIP)
    gpubinHandleName = addCUIDSuffix(gpubinHandleName);
 
  GlobalOp gpuBinHandle = GlobalOp::create(
      builder, loc, gpubinHandleName, voidPtrPtrTy,
      /*isConstant=*/false, {}, cir::GlobalLinkageKind::InternalLinkage);
  gpuBinHandle.setInitialValueAttr(builder.getConstNullPtrAttr(voidPtrPtrTy));
  gpuBinHandle.setPrivate();
 
  // Declare this function:
  //    void **__{cuda|hip}RegisterFatBinary(void *);
 
  std::string regFuncName =
      addUnderscoredPrefix(cudaPrefix, "RegisterFatBinary");
  FuncType regFuncType = FuncType::get({voidPtrTy}, voidPtrPtrTy);
  cir::FuncOp regFunc =
      buildRuntimeFunction(builder, regFuncName, loc, regFuncType);
 
  std::string moduleCtorName = addUnderscoredPrefix(cudaPrefix, "_module_ctor");
  cir::FuncOp moduleCtor = buildRuntimeFunction(
      builder, moduleCtorName, loc, FuncType::get({}, voidTy),
      GlobalLinkageKind::InternalLinkage);
 
  globalCtorList.emplace_back(moduleCtorName,
                              cir::GlobalCtorAttr::getDefaultPriority());
  builder.setInsertionPointToStart(moduleCtor.addEntryBlock());
  assert(!cir::MissingFeatures::opGlobalCtorPriority());
  if (isHIP) {
    // --- Create HIP CTOR ---
    //   if (__hip_gpubin_handle == nullptr)
    //     __hip_gpubin_handle = __hipRegisterFatBinary(&fatbinWrapper);
    //   __hip_register_globals(__hip_gpubin_handle);
    //   atexit(__hip_module_dtor);
    mlir::Block *entryBlock = builder.getInsertionBlock();
    mlir::Region *parent = entryBlock->getParent();
    mlir::Block *ifBlock = builder.createBlock(parent);
    mlir::Block *exitBlock = builder.createBlock(parent);
    {
      mlir::OpBuilder::InsertionGuard guard(builder);
      builder.setInsertionPointToEnd(entryBlock);
      mlir::Value handle =
          builder.createLoad(loc, builder.createGetGlobal(gpuBinHandle));
      auto handlePtrTy = mlir::cast<cir::PointerType>(handle.getType());
      mlir::Value nullPtr = builder.getNullPtr(handlePtrTy, loc);
      mlir::Value isNull =
          builder.createCompare(loc, cir::CmpOpKind::eq, handle, nullPtr);
      cir::BrCondOp::create(builder, loc, isNull, ifBlock, exitBlock);
    }
    {
      // Handle is null: load the fatbin and register it.
      mlir::OpBuilder::InsertionGuard guard(builder);
      builder.setInsertionPointToStart(ifBlock);
      mlir::Value wrapper = builder.createGetGlobal(fatbinWrapper);
      mlir::Value fatbinVoidPtr = builder.createBitcast(wrapper, voidPtrTy);
      cir::CallOp gpuBinaryHandleCall =
          builder.createCallOp(loc, regFunc, fatbinVoidPtr);
      mlir::Value gpuBinaryHandle = gpuBinaryHandleCall.getResult();
      // Store the value back to the global `__hip_gpubin_handle`.
      mlir::Value gpuBinaryHandleGlobal = builder.createGetGlobal(gpuBinHandle);
      builder.createStore(loc, gpuBinaryHandle, gpuBinaryHandleGlobal);
      cir::BrOp::create(builder, loc, exitBlock);
    }
    {
      // Exit block: load the (possibly newly-registered) handle, call
      // __hip_register_globals, and register the module dtor with atexit().
      mlir::OpBuilder::InsertionGuard guard(builder);
      builder.setInsertionPointToStart(exitBlock);
      mlir::Value gHandle =
          builder.createLoad(loc, builder.createGetGlobal(gpuBinHandle));
 
      if (std::optional<FuncOp> regGlobal = buildCUDARegisterGlobals())
        builder.createCallOp(loc, *regGlobal, gHandle);
 
      if (std::optional<FuncOp> dtor = buildHIPModuleDtor()) {
        cir::CIRBaseBuilderTy globalBuilder(getContext());
        globalBuilder.setInsertionPointToStart(mlirModule.getBody());
        FuncOp atexit = buildRuntimeFunction(
            globalBuilder, "atexit", loc,
            FuncType::get(PointerType::get(dtor->getFunctionType()), intTy));
        mlir::Value dtorFunc = GetGlobalOp::create(
            builder, loc, PointerType::get(dtor->getFunctionType()),
            mlir::FlatSymbolRefAttr::get(dtor->getSymNameAttr()));
        builder.createCallOp(loc, atexit, dtorFunc);
      }
      cir::ReturnOp::create(builder, loc);
    }
    return;
  }
  if (!getLangOpts().GPURelocatableDeviceCode) {
 
    // --- Create CUDA CTOR-DTOR ---
    // Register binary with CUDA runtime. This is substantially different in
    // default mode vs. separate compilation.
    // Corresponding code:
    //     gpuBinaryHandle = __cudaRegisterFatBinary(&fatbinWrapper);
    mlir::Value wrapper = builder.createGetGlobal(fatbinWrapper);
    mlir::Value fatbinVoidPtr = builder.createBitcast(wrapper, voidPtrTy);
    cir::CallOp gpuBinaryHandleCall =
        builder.createCallOp(loc, regFunc, fatbinVoidPtr);
    mlir::Value gpuBinaryHandle = gpuBinaryHandleCall.getResult();
    // Store the value back to the global `__cuda_gpubin_handle`.
    mlir::Value gpuBinaryHandleGlobal = builder.createGetGlobal(gpuBinHandle);
    builder.createStore(loc, gpuBinaryHandle, gpuBinaryHandleGlobal);
 
    // --- Generate __cuda_register_globals and call it ---
    if (std::optional<FuncOp> regGlobal = buildCUDARegisterGlobals()) {
      builder.createCallOp(loc, *regGlobal, gpuBinaryHandle);
    }
 
    // From CUDA 10.1 onwards, we must call this function to end registration:
    //      void __cudaRegisterFatBinaryEnd(void **fatbinHandle);
    // This is CUDA-specific, so no need to use `addUnderscoredPrefix`.
    std::optional<llvm::VersionTuple> sdkVersion = getSDKVersion();
    if (!sdkVersion)
      return;
    if (clang::CudaFeatureEnabled(
            *sdkVersion, clang::CudaFeature::CUDA_USES_FATBIN_REGISTER_END)) {
      cir::CIRBaseBuilderTy globalBuilder(getContext());
      globalBuilder.setInsertionPointToStart(mlirModule.getBody());
      FuncOp endFunc =
          buildRuntimeFunction(globalBuilder, "__cudaRegisterFatBinaryEnd", loc,
                               FuncType::get({voidPtrPtrTy}, voidTy));
      builder.createCallOp(loc, endFunc, gpuBinaryHandle);
    }
  } else
    llvm_unreachable("GPU RDC NYI");
 
  // Create destructor and register it with atexit() the way NVCC does it. Doing
  // it during regular destructor phase worked in CUDA before 9.2 but results in
  // double-free in 9.2.
  if (std::optional<FuncOp> dtor = buildCUDAModuleDtor()) {
 
    // extern "C" int atexit(void (*f)(void));
    cir::CIRBaseBuilderTy globalBuilder(getContext());
    globalBuilder.setInsertionPointToStart(mlirModule.getBody());
    FuncOp atexit = buildRuntimeFunction(
        globalBuilder, "atexit", loc,
        FuncType::get(PointerType::get(dtor->getFunctionType()), intTy));
    mlir::Value dtorFunc = GetGlobalOp::create(
        builder, loc, PointerType::get(dtor->getFunctionType()),
        mlir::FlatSymbolRefAttr::get(dtor->getSymNameAttr()));
    builder.createCallOp(loc, atexit, dtorFunc);
  }
  cir::ReturnOp::create(builder, loc);
}
 
std::optional<FuncOp> LoweringPreparePass::buildCUDAModuleDtor() {
  if (!mlirModule->getAttr(CIRDialect::getCUDADeviceBinaryAttrName()))
    return {};
 
  llvm::StringRef prefix = getCUDAPrefix(getLangOpts());
 
  VoidType voidTy = VoidType::get(&getContext());
  PointerType voidPtrPtrTy = PointerType::get(PointerType::get(voidTy));
 
  mlir::Location loc = mlirModule.getLoc();
 
  cir::CIRBaseBuilderTy builder(getContext());
  builder.setInsertionPointToStart(mlirModule.getBody());
 
  // define: void __cudaUnregisterFatBinary(void ** handle);
  std::string unregisterFuncName =
      addUnderscoredPrefix(prefix, "UnregisterFatBinary");
  FuncOp unregisterFunc = buildRuntimeFunction(
      builder, unregisterFuncName, loc, FuncType::get({voidPtrPtrTy}, voidTy));
 
  // void __cuda_module_dtor();
  // Despite the name, OG doesn't treat it as a destructor, so it shouldn't be
  // put into globalDtorList. If it were a real dtor, then it would cause
  // double free above CUDA 9.2. The way to use it is to manually call
  // atexit() at end of module ctor.
  std::string dtorName = addUnderscoredPrefix(prefix, "_module_dtor");
  FuncOp dtor =
      buildRuntimeFunction(builder, dtorName, loc, FuncType::get({}, voidTy),
                           GlobalLinkageKind::InternalLinkage);
 
  builder.setInsertionPointToStart(dtor.addEntryBlock());
 
  // For dtor, we only need to call:
  //    __cudaUnregisterFatBinary(__cuda_gpubin_handle);
 
  std::string gpubinName = addUnderscoredPrefix(prefix, "_gpubin_handle");
  GlobalOp gpubinGlobal = cast<GlobalOp>(mlirModule.lookupSymbol(gpubinName));
  mlir::Value gpubinAddress = builder.createGetGlobal(gpubinGlobal);
  mlir::Value gpubin = builder.createLoad(loc, gpubinAddress);
  builder.createCallOp(loc, unregisterFunc, gpubin);
  ReturnOp::create(builder, loc);
 
  return dtor;
}
 
/// Build the HIP module dtor:
///
///     void __hip_module_dtor() {
///       if (__hip_gpubin_handle != nullptr) {
///         __hipUnregisterFatBinary(__hip_gpubin_handle);
///         __hip_gpubin_handle = nullptr;
///       }
///     }
///
/// Despite the name, OG doesn't treat this as a real destructor: putting it on
/// the dtor list would cause a double-free. It is meant to be registered via
/// atexit() at the end of the module ctor.
std::optional<FuncOp> LoweringPreparePass::buildHIPModuleDtor() {
  llvm::StringRef prefix = getCUDAPrefix(getLangOpts());
 
  VoidType voidTy = VoidType::get(&getContext());
  PointerType voidPtrPtrTy = PointerType::get(PointerType::get(voidTy));
 
  mlir::Location loc = mlirModule.getLoc();
 
  cir::CIRBaseBuilderTy builder(getContext());
  builder.setInsertionPointToStart(mlirModule.getBody());
 
  // void __hipUnregisterFatBinary(void ** handle);
  std::string unregisterFuncName =
      addUnderscoredPrefix(prefix, "UnregisterFatBinary");
  FuncOp unregisterFunc = buildRuntimeFunction(
      builder, unregisterFuncName, loc, FuncType::get({voidPtrPtrTy}, voidTy));
 
  std::string dtorName = addUnderscoredPrefix(prefix, "_module_dtor");
  FuncOp dtor =
      buildRuntimeFunction(builder, dtorName, loc, FuncType::get({}, voidTy),
                           GlobalLinkageKind::InternalLinkage);
 
  std::string gpubinName =
      addCUIDSuffix(addUnderscoredPrefix(prefix, "_gpubin_handle"));
  GlobalOp gpuBinGlobal = cast<GlobalOp>(mlirModule.lookupSymbol(gpubinName));
 
  mlir::Block *entryBlock = dtor.addEntryBlock();
  mlir::Block *ifBlock = builder.createBlock(&dtor.getBody());
  mlir::Block *exitBlock = builder.createBlock(&dtor.getBody());
 
  mlir::OpBuilder::InsertionGuard guard(builder);
  builder.setInsertionPointToEnd(entryBlock);
  mlir::Value handle =
      builder.createLoad(loc, builder.createGetGlobal(gpuBinGlobal));
  auto handlePtrTy = mlir::cast<cir::PointerType>(handle.getType());
  mlir::Value nullPtr = builder.getNullPtr(handlePtrTy, loc);
  mlir::Value isNotNull =
      builder.createCompare(loc, cir::CmpOpKind::ne, handle, nullPtr);
  cir::BrCondOp::create(builder, loc, isNotNull, ifBlock, exitBlock);
 
  {
    // Handle is non-null: unregister and clear it.
    mlir::OpBuilder::InsertionGuard ifGuard(builder);
    builder.setInsertionPointToStart(ifBlock);
    builder.createCallOp(loc, unregisterFunc, handle);
    builder.createStore(loc, nullPtr, builder.createGetGlobal(gpuBinGlobal));
    cir::BrOp::create(builder, loc, exitBlock);
  }
  {
    mlir::OpBuilder::InsertionGuard exitGuard(builder);
    builder.setInsertionPointToStart(exitBlock);
    cir::ReturnOp::create(builder, loc);
  }
 
  return dtor;
}
 
std::optional<FuncOp> LoweringPreparePass::buildCUDARegisterGlobals() {
  if (cudaKernelMap.empty() && cudaDeviceVars.empty())
    return {};
 
  cir::CIRBaseBuilderTy builder(getContext());
  builder.setInsertionPointToStart(mlirModule.getBody());
 
  mlir::Location loc = mlirModule.getLoc();
  llvm::StringRef cudaPrefix = getCUDAPrefix(getLangOpts());
 
  auto voidTy = VoidType::get(&getContext());
  auto voidPtrTy = PointerType::get(voidTy);
  auto voidPtrPtrTy = PointerType::get(voidPtrTy);
 
  // Create the function:
  //      void __cuda_register_globals(void **fatbinHandle)
  std::string regGlobalFuncName =
      addUnderscoredPrefix(cudaPrefix, "_register_globals");
  auto regGlobalFuncTy = FuncType::get({voidPtrPtrTy}, voidTy);
  FuncOp regGlobalFunc =
      buildRuntimeFunction(builder, regGlobalFuncName, loc, regGlobalFuncTy,
                           /*linkage=*/GlobalLinkageKind::InternalLinkage);
  builder.setInsertionPointToStart(regGlobalFunc.addEntryBlock());
 
  buildCUDARegisterGlobalFunctions(builder, regGlobalFunc);
  buildCUDARegisterVars(builder, regGlobalFunc);
 
  ReturnOp::create(builder, loc);
  return regGlobalFunc;
}
 
void LoweringPreparePass::buildCUDARegisterGlobalFunctions(
    cir::CIRBaseBuilderTy &builder, FuncOp regGlobalFunc) {
  mlir::Location loc = mlirModule.getLoc();
  llvm::StringRef cudaPrefix = getCUDAPrefix(getLangOpts());
  cir::CIRDataLayout dataLayout(mlirModule);
 
  auto voidTy = VoidType::get(&getContext());
  auto voidPtrTy = PointerType::get(voidTy);
  auto voidPtrPtrTy = PointerType::get(voidPtrTy);
  IntType intTy = builder.getSIntNTy(32);
  IntType charTy =
      cir::IntType::get(&getContext(), getTargetInfo().getCharWidth(),
                        /*isSigned=*/false);
 
  // Extract the GPU binary handle argument.
  mlir::Value fatbinHandle = *regGlobalFunc.args_begin();
 
  cir::CIRBaseBuilderTy globalBuilder(getContext());
  globalBuilder.setInsertionPointToStart(mlirModule.getBody());
 
  // Declare CUDA internal functions:
  // int __cudaRegisterFunction(
  //   void **fatbinHandle,
  //   const char *hostFunc,
  //   char *deviceFunc,
  //   const char *deviceName,
  //   int threadLimit,
  //   uint3 *tid, uint3 *bid, dim3 *bDim, dim3 *gDim,
  //   int *wsize
  // )
  // OG doesn't care about the types at all. They're treated as void*.
 
  FuncOp cudaRegisterFunction = buildRuntimeFunction(
      globalBuilder, addUnderscoredPrefix(cudaPrefix, "RegisterFunction"), loc,
      FuncType::get({voidPtrPtrTy, voidPtrTy, voidPtrTy, voidPtrTy, intTy,
                     voidPtrTy, voidPtrTy, voidPtrTy, voidPtrTy, voidPtrTy},
                    intTy));
 
  auto makeConstantString = [&](llvm::StringRef str) -> GlobalOp {
    auto strType = ArrayType::get(&getContext(), charTy, 1 + str.size());
    auto tmpString = cir::GlobalOp::create(
        globalBuilder, loc, (".str" + str).str(), strType,
        /*isConstant=*/true, {},
        /*linkage=*/cir::GlobalLinkageKind::PrivateLinkage);
 
    // We must make the string zero-terminated.
    tmpString.setInitialValueAttr(
        ConstArrayAttr::get(strType, StringAttr::get(str + "\0", strType)));
    tmpString.setPrivate();
    return tmpString;
  };
 
  cir::ConstantOp cirNullPtr = builder.getNullPtr(voidPtrTy, loc);
  bool isHIP = getLangOpts().HIP;
  for (auto kernelName : cudaKernelMap.keys()) {
    FuncOp deviceStub = cudaKernelMap[kernelName];
    GlobalOp deviceFuncStr = makeConstantString(kernelName);
    mlir::Value deviceFunc = builder.createBitcast(
        builder.createGetGlobal(deviceFuncStr), voidPtrTy);
 
    mlir::Value hostFunc;
    if (isHIP) {
      // Under HIP, the kernel-handle is a GlobalOp shadow created by CIR
      // codegen and named with the kernel-reference mangled name (e.g.
      // `@_Z2fnv` pointing at the device-stub function
      // `_Z17__device_stub__fnv`). The CUDAKernelNameAttr on the device-stub
      // uses the same name, so we can resolve the shadow by symbol lookup.
      auto funcHandle = cast<GlobalOp>(mlirModule.lookupSymbol(kernelName));
      hostFunc =
          builder.createBitcast(builder.createGetGlobal(funcHandle), voidPtrTy);
    } else {
      hostFunc = builder.createBitcast(
          GetGlobalOp::create(
              builder, loc, PointerType::get(deviceStub.getFunctionType()),
              mlir::FlatSymbolRefAttr::get(deviceStub.getSymNameAttr())),
          voidPtrTy);
    }
    builder.createCallOp(
        loc, cudaRegisterFunction,
        {fatbinHandle, hostFunc, deviceFunc, deviceFunc,
         ConstantOp::create(builder, loc, IntAttr::get(intTy, -1)), cirNullPtr,
         cirNullPtr, cirNullPtr, cirNullPtr, cirNullPtr});
  }
}
 
// Emit `__{cuda|hip}RegisterVar` calls inside `__{cuda|hip}_register_globals`
// for every device-side shadow that carries a `cu.var_registration` attribute
// (attached by `CIRGenNVCUDARuntime::handleVarRegistration`).
void LoweringPreparePass::buildCUDARegisterVars(cir::CIRBaseBuilderTy &builder,
                                                FuncOp regGlobalFunc) {
  mlir::Location loc = mlirModule.getLoc();
  llvm::StringRef cudaPrefix = getCUDAPrefix(getLangOpts());
  cir::CIRDataLayout dataLayout(mlirModule);
 
  PointerType voidPtrTy = builder.getVoidPtrTy();
  PointerType voidPtrPtrTy = builder.getPointerTo(voidPtrTy);
  IntType intTy = builder.getSIntNTy(32);
  IntType sizeTy = builder.getUIntNTy(getTargetInfo().getMaxPointerWidth());
  IntType charTy =
      cir::IntType::get(&getContext(), getTargetInfo().getCharWidth(),
                        /*isSigned=*/false);
 
  if (cudaDeviceVars.empty())
    return;
 
  cir::CIRBaseBuilderTy globalBuilder(getContext());
  globalBuilder.setInsertionPointToStart(mlirModule.getBody());
 
  cir::VoidType voidTy = builder.getVoidTy();
  FuncOp cudaRegisterVar;
  FuncOp cudaRegisterManagedVar;
 
  // void __cudaRegisterSurface(void **fatbinHandle,
  //                            const struct surfaceReference *hostVar,
  //                            const void **deviceAddress,
  //                            const char *deviceName,
  //                            int type, int ext);
  FuncOp cudaRegisterSurface = buildRuntimeFunction(
      globalBuilder, addUnderscoredPrefix(cudaPrefix, "RegisterSurface"), loc,
      FuncType::get(
          {voidPtrPtrTy, voidPtrTy, voidPtrTy, voidPtrTy, intTy, intTy},
          voidTy));
 
  auto makeConstantString = [&](llvm::StringRef str) -> GlobalOp {
    auto strType = ArrayType::get(&getContext(), charTy, 1 + str.size());
    auto tmpString = cir::GlobalOp::create(
        globalBuilder, loc, (".str" + str).str(), strType,
        /*isConstant=*/true, {},
        /*linkage=*/cir::GlobalLinkageKind::PrivateLinkage);
    tmpString.setInitialValueAttr(
        ConstArrayAttr::get(strType, StringAttr::get(str + "\0", strType)));
    tmpString.setPrivate();
    return tmpString;
  };
 
  mlir::Value fatbinHandle = *regGlobalFunc.args_begin();
 
  for (auto &[global, regAttr] : cudaDeviceVars) {
    // External managed variables are registered by their defining TU.
    if (regAttr.getIsManaged() && global.isDeclaration())
      continue;
 
    GlobalOp deviceNameStr = makeConstantString(regAttr.getDeviceSideName());
    mlir::Value deviceName = builder.createBitcast(
        builder.createGetGlobal(deviceNameStr), voidPtrTy);
    mlir::Value hostVar =
        builder.createBitcast(builder.createGetGlobal(global), voidPtrTy);
 
    llvm::TypeSize size = dataLayout.getTypeAllocSize(global.getSymType());
 
    if (regAttr.getIsManaged()) {
      // void __hipRegisterManagedVar(void **fatbinHandle, char *managedVar,
      //                              char *var, const char *deviceName,
      //                              size_t size, unsigned align);
      if (!cudaRegisterManagedVar)
        cudaRegisterManagedVar = buildRuntimeFunction(
            globalBuilder,
            addUnderscoredPrefix(cudaPrefix, "RegisterManagedVar"), loc,
            FuncType::get(
                {voidPtrPtrTy, voidPtrTy, voidPtrTy, voidPtrTy, sizeTy, intTy},
                voidTy));
 
      llvm::StringRef name = global.getSymName();
      assert(name.ends_with(".managed") &&
             "HIP managed variables not transformed");
      auto managedVar = mlirModule.lookupSymbol<GlobalOp>(
          name.drop_back(llvm::StringRef(".managed").size()));
      mlir::Value managedPtr =
          builder.createBitcast(builder.createGetGlobal(managedVar), voidPtrTy);
      auto varSize = ConstantOp::create(
          builder, loc, IntAttr::get(sizeTy, size.getFixedValue()));
      auto align = ConstantOp::create(
          builder, loc, IntAttr::get(intTy, global.getAlignment().value_or(1)));
      builder.createCallOp(
          loc, cudaRegisterManagedVar,
          {fatbinHandle, managedPtr, hostVar, deviceName, varSize, align});
    } else {
      auto isExtern = ConstantOp::create(
          builder, loc, IntAttr::get(intTy, regAttr.getIsExtern() ? 1 : 0));
 
      switch (regAttr.getKind()) {
      case cir::CUDADeviceVarKind::Variable: {
        // void __{cuda|hip}RegisterVar(void **fatbinHandle,
        //                              char *hostVar, char *deviceAddress,
        //                              const char *deviceName, int ext,
        //                              size_t size, int constant,
        //                              int normalized);
        // OG ignores parameter types, treating pointers as void*.
        if (!cudaRegisterVar)
          cudaRegisterVar = buildRuntimeFunction(
              globalBuilder, addUnderscoredPrefix(cudaPrefix, "RegisterVar"),
              loc,
              FuncType::get({voidPtrPtrTy, voidPtrTy, voidPtrTy, voidPtrTy,
                             intTy, sizeTy, intTy, intTy},
                            voidTy));
 
        auto varSize = ConstantOp::create(
            builder, loc, IntAttr::get(sizeTy, size.getFixedValue()));
        auto isConstant = ConstantOp::create(
            builder, loc, IntAttr::get(intTy, regAttr.getIsConstant() ? 1 : 0));
        auto normalized =
            ConstantOp::create(builder, loc, IntAttr::get(intTy, 0));
 
        builder.createCallOp(loc, cudaRegisterVar,
                             {fatbinHandle, hostVar, deviceName, deviceName,
                              isExtern, varSize, isConstant, normalized});
        break;
      }
 
      case cir::CUDADeviceVarKind::Surface: {
        cir::ConstantOp surfaceType = ConstantOp::create(
            builder, loc, IntAttr::get(intTy, regAttr.getSurfTexType()));
        builder.createCallOp(loc, cudaRegisterSurface,
                             {fatbinHandle, hostVar, deviceName, deviceName,
                              surfaceType, isExtern});
        break;
      }
 
      case cir::CUDADeviceVarKind::Texture:
        llvm_unreachable("Texture registration NYI");
      }
    }
  }
}
 
void LoweringPreparePass::runOnOperation() {
  mlirModule = getOperation();
 
  lowerModule = cir::createLowerModule(mlirModule);
  if (!lowerModule) {
    mlirModule.emitError() << getArgument()
                           << " requires a module with a triple";
    return signalPassFailure();
  }
 
  llvm::SmallVector<mlir::Operation *> opsToTransform;
 
  mlirModule->walk([&](mlir::Operation *op) {
    if (mlir::isa<cir::ArrayCtor, cir::ArrayDtor, cir::CastOp,
                  cir::ComplexConjOp, cir::ComplexMulOp, cir::ComplexDivOp,
                  cir::ComplexFMulOp, cir::ComplexFDivOp, cir::DynamicCastOp,
                  cir::FuncOp, cir::CallOp, cir::GetGlobalOp, cir::GlobalOp,
                  cir::StoreOp, cir::CmpThreeWayOp, cir::LocalInitOp,
                  cir::RegisterExitDtorOp, cir::StdOpInterface>(op))
      opsToTransform.push_back(op);
  });
 
  for (mlir::Operation *o : opsToTransform)
    runOnOp(o);
 
  // Every TLS wrapper, init alias and wrapper call has been built from
  // 'tls_refs' by now. Nothing after this pass reads it, so drop it to keep a
  // second run of this pass from building the wrappers again.
  for (auto globalOp : mlirModule.getOps<cir::GlobalOp>())
    globalOp.removeTlsRefsAttr();
 
  buildCXXGlobalInitFunc();
  // The C++20 module initializer has been built from these. Nothing after this
  // pass reads them, so drop them to keep a second run of this pass from
  // building it again.
  mlirModule->removeAttr(CIRDialect::getCXXModuleInitFnNameAttrName());
  mlirModule->removeAttr(CIRDialect::getCXXModuleImportedInitsAttrName());
  buildCXXGlobalTlsFunc();
  if (getLangOpts().CUDA && !getLangOpts().CUDAIsDevice) {
    buildCUDAModuleCtor();
    // The fatbin global now references the same attribute; drop the module's
    // reference so an emitted .cir doesn't print the bytes twice. This has to
    // happen out here because the ctor and the CUDA dtor builder test the
    // attribute to decide whether a device-side binary exists at all.
    mlirModule->removeAttr(CIRDialect::getCUDADeviceBinaryAttrName());
    // The kernels and device variables have been registered from these.
    // Nothing after this pass reads them, so drop them to keep a second run
    // of this pass from registering them again.
    for (auto &entry : cudaKernelMap)
      entry.getValue()->removeAttr(CUDAKernelNameAttr::getMnemonic());
    for (auto &deviceVar : cudaDeviceVars)
      deviceVar.first->removeAttr(CUDAVarRegistrationInfoAttr::getMnemonic());
  }
 
  buildGlobalCtorDtorList();
}
 
std::unique_ptr<Pass> mlir::createLoweringPreparePass() {
  return std::make_unique<LoweringPreparePass>();
}