CIRGenAtomic.cpp

Go to the documentation of this file.

//===--- CIRGenAtomic.cpp - Emit CIR for atomic operations ----------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file contains the code for emitting atomic operations.
//
//===----------------------------------------------------------------------===//
 
#include "CIRGenFunction.h"
#include "clang/CIR/MissingFeatures.h"
 
using namespace clang;
using namespace clang::CIRGen;
using namespace cir;
 
namespace {
class AtomicInfo {
  CIRGenFunction &cgf;
  QualType atomicTy;
  QualType valueTy;
  uint64_t atomicSizeInBits = 0;
  uint64_t valueSizeInBits = 0;
  CharUnits atomicAlign;
  CharUnits valueAlign;
  TypeEvaluationKind evaluationKind = cir::TEK_Scalar;
  bool useLibCall = true;
  LValue lvalue;
  mlir::Location loc;
 
public:
  AtomicInfo(CIRGenFunction &cgf, LValue &lvalue, mlir::Location loc)
      : cgf(cgf), loc(loc) {
    assert(!lvalue.isGlobalReg());
    ASTContext &ctx = cgf.getContext();
    if (lvalue.isSimple()) {
      atomicTy = lvalue.getType();
      if (auto *ty = atomicTy->getAs<AtomicType>())
        valueTy = ty->getValueType();
      else
        valueTy = atomicTy;
      evaluationKind = cgf.getEvaluationKind(valueTy);
 
      TypeInfo valueTypeInfo = ctx.getTypeInfo(valueTy);
      TypeInfo atomicTypeInfo = ctx.getTypeInfo(atomicTy);
      uint64_t valueAlignInBits = valueTypeInfo.Align;
      uint64_t atomicAlignInBits = atomicTypeInfo.Align;
      valueSizeInBits = valueTypeInfo.Width;
      atomicSizeInBits = atomicTypeInfo.Width;
      assert(valueSizeInBits <= atomicSizeInBits);
      assert(valueAlignInBits <= atomicAlignInBits);
 
      atomicAlign = ctx.toCharUnitsFromBits(atomicAlignInBits);
      valueAlign = ctx.toCharUnitsFromBits(valueAlignInBits);
      if (lvalue.getAlignment().isZero())
        lvalue.setAlignment(atomicAlign);
 
      this->lvalue = lvalue;
    } else {
      assert(!cir::MissingFeatures::atomicInfo());
      cgf.cgm.errorNYI(loc, "AtomicInfo: non-simple lvalue");
    }
    useLibCall = !ctx.getTargetInfo().hasBuiltinAtomic(
        atomicSizeInBits, ctx.toBits(lvalue.getAlignment()));
  }
 
  QualType getValueType() const { return valueTy; }
  QualType getAtomicType() const { return atomicTy; }
  CharUnits getAtomicAlignment() const { return atomicAlign; }
  TypeEvaluationKind getEvaluationKind() const { return evaluationKind; }
  mlir::Value getAtomicPointer() const {
    if (lvalue.isSimple())
      return lvalue.getPointer();
    assert(!cir::MissingFeatures::atomicInfoGetAtomicPointer());
    return nullptr;
  }
  bool shouldUseLibCall() const { return useLibCall; }
  const LValue &getAtomicLValue() const { return lvalue; }
  Address getAtomicAddress() const {
    mlir::Type elemTy;
    if (lvalue.isSimple()) {
      elemTy = lvalue.getAddress().getElementType();
    } else {
      assert(!cir::MissingFeatures::atomicInfoGetAtomicAddress());
      cgf.cgm.errorNYI(loc, "AtomicInfo::getAtomicAddress: non-simple lvalue");
    }
    return Address(getAtomicPointer(), elemTy, getAtomicAlignment());
  }
 
  /// Is the atomic size larger than the underlying value type?
  ///
  /// Note that the absence of padding does not mean that atomic
  /// objects are completely interchangeable with non-atomic
  /// objects: we might have promoted the alignment of a type
  /// without making it bigger.
  bool hasPadding() const { return (valueSizeInBits != atomicSizeInBits); }
 
  bool emitMemSetZeroIfNecessary() const;
 
  mlir::Value getScalarRValValueOrNull(RValue rvalue) const;
 
  /// Cast the given pointer to an integer pointer suitable for atomic
  /// operations on the source.
  Address castToAtomicIntPointer(Address addr) const;
 
  /// If addr is compatible with the iN that will be used for an atomic
  /// operation, bitcast it. Otherwise, create a temporary that is suitable and
  /// copy the value across.
  Address convertToAtomicIntPointer(Address addr, mlir::Location loc) const;
 
  /// Turn an atomic-layout object into an r-value.
  RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
                                   SourceLocation loc, bool asValue) const;
 
  /// Converts a rvalue to integer value.
  mlir::Value convertRValueToInt(RValue rvalue, mlir::Location loc,
                                 bool cmpxchg = false) const;
 
  RValue convertToValueOrAtomic(mlir::Value intVal, AggValueSlot resultSlot,
                                SourceLocation loc, bool asValue,
                                bool cmpxchg = false) const;
 
  /// Copy an atomic r-value into atomic-layout memory.
  void emitCopyIntoMemory(RValue rvalue) const;
 
  /// Project an l-value down to the value field.
  LValue projectValue() const {
    assert(lvalue.isSimple());
    Address addr = getAtomicAddress();
    if (hasPadding())
      addr = cgf.getBuilder().createGetMember(loc, addr, /*name=*/"value",
                                              /*index=*/0,
                                              cgf.cgm.getDataLayout());
 
    assert(!cir::MissingFeatures::opTBAA());
    return LValue::makeAddr(addr, getValueType(), lvalue.getBaseInfo());
  }
 
  /// Emits atomic load.
  /// \returns Loaded value.
  RValue emitAtomicLoad(AggValueSlot resultSlot, SourceLocation loc,
                        bool asValue, cir::MemOrder order, bool isVolatile);
 
  /// Materialize an atomic r-value in atomic-layout memory.
  Address materializeRValue(RValue rvalue, mlir::Location loc) const;
 
  /// Creates temp alloca for intermediate operations on atomic value.
  Address createTempAlloca() const;
 
private:
  bool requiresMemSetZero(mlir::Type ty) const;
 
  /// Emits atomic load as a CIR operation.
  mlir::Value emitAtomicLoadOp(cir::MemOrder order, bool isVolatile,
                               bool cmpxchg = false);
};
} // namespace
 
/// Attach the AMDGPU atomic metadata markers that the current atomic options
/// call for. The "no.X" metadata is emitted when the corresponding option is
/// off, since it asserts the absence of that memory kind. The clang::atomic
/// attribute is what turns the options on and off.
static void setAMDGPUAtomicMetadata(CIRGenFunction &cgf, mlir::Operation *op,
                                    const AtomicExpr *expr) {
  // TODO: AMDGPUTargetCodeGenInfo::setTargetAtomicMetadata also calls
  // CGF.AddAMDGPUAvailableVisibleMMRA on every atomic instruction; this is
  // tied to the AMDGPUAvailableVisible statement attribute, which
  // CIRGenStmt.cpp's emitAttributedStmt currently rejects via errorNYI.
  assert(!cir::MissingFeatures::atomicAMDGPUAvailableVisibleMMRA());
 
  const llvm::Triple &triple = cgf.cgm.getTriple();
  bool isAMDGCN = triple.isAMDGCN();
  bool isAMDSPIRV = triple.isSPIRV() && triple.getVendor() == llvm::Triple::AMD;
  if (!isAMDGCN && !isAMDSPIRV)
    return;
 
  // Lowering keeps this only on a flat pointer.
  mlir::UnitAttr unit = cgf.getBuilder().getUnitAttr();
  if (isAMDGCN && expr->threadPrivateMemoryAtomicsAreUndefined())
    op->setAttr(cir::CIRDialect::getAMDGPUNoPrivateAddrSpaceAttrName(), unit);
 
  // Only a read-modify-write instruction carries these; a plain load, store or
  // cmpxchg does not.
  if (!mlir::isa<cir::AtomicFetchOp, cir::AtomicXchgOp>(op))
    return;
 
  clang::AtomicOptions atomicOpts = cgf.cgm.getAtomicOpts();
  if (!atomicOpts.getOption(clang::AtomicOptionKind::FineGrainedMemory))
    op->setAttr(cir::CIRDialect::getAMDGPUNoFineGrainedMemoryAttrName(), unit);
  if (!atomicOpts.getOption(clang::AtomicOptionKind::RemoteMemory))
    op->setAttr(cir::CIRDialect::getAMDGPUNoRemoteMemoryAttrName(), unit);
  // Denormal flushing only matters for a float add.
  auto fetchOp = mlir::dyn_cast<cir::AtomicFetchOp>(op);
  if (fetchOp &&
      atomicOpts.getOption(clang::AtomicOptionKind::IgnoreDenormalMode) &&
      fetchOp.getBinop() == cir::AtomicFetchKind::Add &&
      mlir::isa<cir::SingleType>(fetchOp.getVal().getType()))
    op->setAttr(cir::CIRDialect::getAMDGPUIgnoreDenormalModeAttrName(), unit);
}
 
// This function emits any expression (scalar, complex, or aggregate)
// into a temporary alloca.
static Address emitValToTemp(CIRGenFunction &cgf, Expr *e) {
  Address declPtr = cgf.createMemTemp(
      e->getType(), cgf.getLoc(e->getSourceRange()), ".atomictmp");
  cgf.emitAnyExprToMem(e, declPtr, e->getType().getQualifiers(),
                       /*Init*/ true);
  return declPtr;
}
 
/// Does a store of the given IR type modify the full expected width?
static bool isFullSizeType(CIRGenModule &cgm, mlir::Type ty,
                           uint64_t expectedSize) {
  return cgm.getDataLayout().getTypeStoreSize(ty) * 8 == expectedSize;
}
 
/// Does the atomic type require memsetting to zero before initialization?
///
/// The IR type is provided as a way of making certain queries faster.
bool AtomicInfo::requiresMemSetZero(mlir::Type ty) const {
  // If the atomic type has size padding, we definitely need a memset.
  if (hasPadding())
    return true;
 
  // Otherwise, do some simple heuristics to try to avoid it:
  switch (getEvaluationKind()) {
  // For scalars and complexes, check whether the store size of the
  // type uses the full size.
  case cir::TEK_Scalar:
    return !isFullSizeType(cgf.cgm, ty, atomicSizeInBits);
  case cir::TEK_Complex:
    return !isFullSizeType(cgf.cgm,
                           mlir::cast<cir::ComplexType>(ty).getElementType(),
                           atomicSizeInBits / 2);
  // Padding in structs has an undefined bit pattern.  User beware.
  case cir::TEK_Aggregate:
    return false;
  }
  llvm_unreachable("bad evaluation kind");
}
 
Address AtomicInfo::convertToAtomicIntPointer(Address addr,
                                              mlir::Location loc) const {
  mlir::Type ty = addr.getElementType();
  uint64_t sourceSizeInBits = cgf.cgm.getDataLayout().getTypeSizeInBits(ty);
  if (sourceSizeInBits != atomicSizeInBits) {
    CIRGenBuilderTy &builder = cgf.getBuilder();
 
    Address tmp = createTempAlloca();
    mlir::Value zero = builder.getConstInt(loc, cgf.cgm.uInt8Ty, 0);
    unsigned size =
        cgf.getContext().toCharUnitsFromBits(atomicSizeInBits).getQuantity();
    mlir::Value memSetSize = builder.getConstInt(loc, cgf.cgm.uInt64Ty, size);
    addr = addr.withElementType(builder, cgf.cgm.voidTy);
    tmp = tmp.withElementType(builder, cgf.cgm.voidTy);
    builder.createMemSet(loc, tmp, zero, memSetSize);
 
    builder.createMemCpy(
        loc, tmp, addr,
        builder.getConstInt(loc, cgf.cgm.uInt64Ty,
                            std::min(atomicSizeInBits, sourceSizeInBits) / 8));
    addr = tmp;
  }
 
  return castToAtomicIntPointer(addr);
}
 
RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
                                             AggValueSlot resultSlot,
                                             SourceLocation loc,
                                             bool asValue) const {
  if (lvalue.isSimple()) {
    if (evaluationKind == TEK_Aggregate)
      return resultSlot.asRValue();
 
    // Drill into the padding structure if we have one.
    if (hasPadding()) {
      cgf.cgm.errorNYI(loc,
                       "AtomicInfo::convertAtomicTempToRValue: hasPadding");
      return RValue::get(nullptr);
    }
 
    // Otherwise, just convert the temporary to an r-value using the
    // normal conversion routine.
    return cgf.convertTempToRValue(addr, getValueType(), loc);
  }
 
  cgf.cgm.errorNYI(
      loc, "AtomicInfo::convertAtomicTempToRValue: lvalue is not simple");
  return RValue::get(nullptr);
}
 
RValue AtomicInfo::emitAtomicLoad(AggValueSlot resultSlot, SourceLocation loc,
                                  bool asValue, cir::MemOrder order,
                                  bool isVolatile) {
  // Check whether we should use a library call.
  if (shouldUseLibCall()) {
    assert(!cir::MissingFeatures::atomicUseLibCall());
    cgf.cgm.errorNYI(loc, "emitAtomicLoad: emit atomic lib call");
    return RValue::get(nullptr);
  }
 
  // Okay, we're doing this natively.
  mlir::Value loadOp = emitAtomicLoadOp(order, isVolatile);
 
  // If we're ignoring an aggregate return, don't do anything.
  if (getEvaluationKind() == TEK_Aggregate && resultSlot.isIgnored())
    return RValue::getAggregate(Address::invalid(), false);
 
  // Okay, turn that back into the original value or atomic (for non-simple
  // lvalues) type.
  return convertToValueOrAtomic(loadOp, resultSlot, loc, asValue);
}
 
Address AtomicInfo::createTempAlloca() const {
  // Remove addrspace info from the atomic pointer element when making the
  // alloca pointer element.
  QualType tmpTy = (lvalue.isBitField() && valueSizeInBits > atomicSizeInBits)
                       ? valueTy
                       : atomicTy.getUnqualifiedType();
  Address tempAlloca =
      cgf.createMemTemp(tmpTy, getAtomicAlignment(), loc, "atomic-temp");
 
  // Cast to pointer to value type for bitfields.
  if (lvalue.isBitField()) {
    cgf.cgm.errorNYI(loc, "AtomicInfo::createTempAlloca: bitfield lvalue");
  }
 
  return tempAlloca;
}
 
mlir::Value AtomicInfo::getScalarRValValueOrNull(RValue rvalue) const {
  if (rvalue.isScalar() && (!hasPadding() || !lvalue.isSimple()))
    return rvalue.getValue();
  return nullptr;
}
 
Address AtomicInfo::castToAtomicIntPointer(Address addr) const {
  auto intTy = mlir::dyn_cast<cir::IntType>(addr.getElementType());
  // Don't bother with int casts if the integer size is the same.
  if (intTy && intTy.getWidth() == atomicSizeInBits)
    return addr;
  auto ty = cgf.getBuilder().getUIntNTy(atomicSizeInBits);
  return addr.withElementType(cgf.getBuilder(), ty);
}
 
bool AtomicInfo::emitMemSetZeroIfNecessary() const {
  assert(lvalue.isSimple());
  Address addr = lvalue.getAddress();
  if (!requiresMemSetZero(addr.getElementType()))
    return false;
 
  addr = addr.withElementType(cgf.getBuilder(), cgf.cgm.voidTy);
  mlir::Value zero = cgf.getBuilder().getConstInt(loc, cgf.cgm.uInt8Ty, 0);
  mlir::Value memSetSize = cgf.getBuilder().getConstInt(
      loc, cgf.cgm.uInt64Ty,
      cgf.getContext().toCharUnitsFromBits(atomicSizeInBits).getQuantity());
 
  cgf.getBuilder().createMemSet(loc, addr, zero, memSetSize);
  return true;
}
 
/// Return true if \param valueTy is a type that should be casted to integer
/// around the atomic memory operation. If \param cmpxchg is true, then the
/// cast of a floating point type is made as that instruction can not have
/// floating point operands.  TODO: Allow compare-and-exchange and FP - see
/// comment in CIRGenAtomicExpandPass.cpp.
static bool shouldCastToInt(mlir::Type valueTy, bool cmpxchg) {
  if (cir::isAnyFloatingPointType(valueTy))
    return isa<cir::FP80Type>(valueTy) || cmpxchg;
  return !isa<cir::IntType>(valueTy) && !isa<cir::PointerType>(valueTy);
}
 
mlir::Value AtomicInfo::emitAtomicLoadOp(cir::MemOrder order, bool isVolatile,
                                         bool cmpxchg) {
  Address addr = getAtomicAddress();
  if (shouldCastToInt(addr.getElementType(), cmpxchg))
    addr = castToAtomicIntPointer(addr);
 
  cir::LoadOp op =
      cgf.getBuilder().createLoad(loc, addr, /*isVolatile=*/isVolatile);
  op.setMemOrder(order);
 
  assert(!cir::MissingFeatures::opTBAA());
  return op;
}
 
mlir::Value AtomicInfo::convertRValueToInt(RValue rvalue, mlir::Location loc,
                                           bool cmpxchg) const {
  // If we've got a scalar value of the right size, try to avoid going
  // through memory. Floats get casted if needed by AtomicExpandPass.
  if (mlir::Value value = getScalarRValValueOrNull(rvalue)) {
    if (!shouldCastToInt(value.getType(), cmpxchg))
      return cgf.emitToMemory(value, valueTy);
 
    cgf.cgm.errorNYI(
        loc, "AtomicInfo::convertRValueToInt: cast scalar rvalue to int");
    return nullptr;
  }
 
  // Otherwise, we need to go through memory.
  // Put the r-value in memory.
  Address addr = materializeRValue(rvalue, loc);
 
  // Cast the temporary to the atomic int type and pull a value out.
  addr = castToAtomicIntPointer(addr);
 
  return cgf.getBuilder().createLoad(loc, addr);
}
 
RValue AtomicInfo::convertToValueOrAtomic(mlir::Value intVal,
                                          AggValueSlot resultSlot,
                                          SourceLocation loc, bool asValue,
                                          bool cmpxchg) const {
  // Try not to in some easy cases.
  assert((mlir::isa<cir::IntType, cir::PointerType, cir::FPTypeInterface>(
             intVal.getType())) &&
         "Expected integer, pointer or floating point value when converting "
         "result.");
  bool isWholeValue =
      !lvalue.isBitField() || lvalue.getBitFieldInfo().size == valueSizeInBits;
  if (getEvaluationKind() == TEK_Scalar &&
      ((isWholeValue && !hasPadding()) || !asValue)) {
    mlir::Type valTy = asValue ? cgf.convertTypeForMem(valueTy)
                               : getAtomicAddress().getElementType();
    if (!shouldCastToInt(valTy, cmpxchg)) {
      assert((!mlir::isa<cir::IntType>(valTy) || intVal.getType() == valTy) &&
             "Different integer types.");
      return RValue::get(cgf.emitFromMemory(intVal, valueTy));
    }
 
    cgf.cgm.errorNYI("convertToValueOrAtomic: convert through bitcast");
    return RValue::get(nullptr);
  }
 
  // Create a temporary.  This needs to be big enough to hold the
  // atomic integer.
  Address temp = Address::invalid();
  bool tempIsVolatile = false;
  if (asValue && getEvaluationKind() == TEK_Aggregate) {
    assert(!resultSlot.isIgnored());
    temp = resultSlot.getAddress();
    tempIsVolatile = resultSlot.isVolatile();
  } else {
    temp = createTempAlloca();
  }
 
  // Slam the integer into the temporary.
  Address castTemp = castToAtomicIntPointer(temp);
  cgf.getBuilder().createStore(cgf.getLoc(loc), intVal, castTemp,
                               tempIsVolatile);
  return convertAtomicTempToRValue(temp, resultSlot, loc, asValue);
}
 
/// Copy an r-value into memory as part of storing to an atomic type.
/// This needs to create a bit-pattern suitable for atomic operations.
void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
  assert(lvalue.isSimple());
 
  // If we have an r-value, the rvalue should be of the atomic type,
  // which means that the caller is responsible for having zeroed
  // any padding.  Just do an aggregate copy of that type.
  if (rvalue.isAggregate()) {
    cgf.cgm.errorNYI("copying aggregate into atomic lvalue");
    return;
  }
 
  // Okay, otherwise we're copying stuff.
 
  // Zero out the buffer if necessary.
  emitMemSetZeroIfNecessary();
 
  // Drill past the padding if present.
  LValue tempLValue = projectValue();
 
  // Okay, store the rvalue in.
  if (rvalue.isScalar()) {
    cgf.emitStoreOfScalar(rvalue.getValue(), tempLValue, /*isInit=*/true);
  } else {
    cgf.emitStoreOfComplex(loc, rvalue.getComplexValue(), tempLValue,
                           /*isInit=*/true);
  }
}
 
/// Materialize an r-value into memory for the purposes of storing it
/// to an atomic type.
Address AtomicInfo::materializeRValue(RValue rvalue, mlir::Location loc) const {
  // Aggregate r-values are already in memory, and EmitAtomicStore
  // requires them to be values of the atomic type.
  if (rvalue.isAggregate())
    return rvalue.getAggregateAddress();
 
  // Otherwise, make a temporary and materialize into it.
  LValue tempLV = cgf.makeAddrLValue(createTempAlloca(), getAtomicType());
  AtomicInfo atomics(cgf, tempLV, loc);
 
  atomics.emitCopyIntoMemory(rvalue);
  return tempLV.getAddress();
}
 
static void emitDefaultCaseLabel(CIRGenBuilderTy &builder, mlir::Location loc) {
  mlir::ArrayAttr valuesAttr = builder.getArrayAttr({});
  mlir::OpBuilder::InsertPoint insertPoint;
  cir::CaseOp::create(builder, loc, valuesAttr, cir::CaseOpKind::Default,
                      insertPoint);
  builder.restoreInsertionPoint(insertPoint);
}
 
// Create a "case" operation with the given list of orders as its values. Also
// create the region that will hold the body of the switch-case label.
static void emitMemOrderCaseLabel(CIRGenBuilderTy &builder, mlir::Location loc,
                                  mlir::Type orderType,
                                  llvm::ArrayRef<cir::MemOrder> orders) {
  llvm::SmallVector<mlir::Attribute, 2> orderAttrs;
  for (cir::MemOrder order : orders)
    orderAttrs.push_back(cir::IntAttr::get(orderType, static_cast<int>(order)));
  mlir::ArrayAttr ordersAttr = builder.getArrayAttr(orderAttrs);
 
  mlir::OpBuilder::InsertPoint insertPoint;
  cir::CaseOp::create(builder, loc, ordersAttr, cir::CaseOpKind::Anyof,
                      insertPoint);
  builder.restoreInsertionPoint(insertPoint);
}
 
static void emitAtomicCmpXchg(CIRGenFunction &cgf, AtomicExpr *e, bool isWeak,
                              Address dest, Address ptr, Address val1,
                              Address val2, Address expectedResult,
                              uint64_t size, cir::MemOrder successOrder,
                              cir::MemOrder failureOrder,
                              cir::SyncScopeKind scope) {
  mlir::Location loc = cgf.getLoc(e->getSourceRange());
 
  CIRGenBuilderTy &builder = cgf.getBuilder();
  mlir::Value expected = builder.createLoad(loc, val1);
  mlir::Value desired = builder.createLoad(loc, val2);
 
  auto cmpxchg = cir::AtomicCmpXchgOp::create(
      builder, loc, expected.getType(), builder.getBoolTy(), ptr.getPointer(),
      expected, desired,
      cir::MemOrderAttr::get(&cgf.getMLIRContext(), successOrder),
      cir::MemOrderAttr::get(&cgf.getMLIRContext(), failureOrder),
      cir::SyncScopeKindAttr::get(&cgf.getMLIRContext(), scope),
      builder.getI64IntegerAttr(ptr.getAlignment().getAsAlign().value()));
 
  cmpxchg.setIsVolatile(e->isVolatile());
  cmpxchg.setWeak(isWeak);
  setAMDGPUAtomicMetadata(cgf, cmpxchg, e);
 
  mlir::Value failed = builder.createNot(cmpxchg.getSuccess());
  cir::IfOp::create(
      builder, loc, failed, /*withElseRegion=*/false,
      [&](mlir::OpBuilder &, mlir::Location) {
        uint64_t expectedSizeInBytes = cgf.cgm.getDataLayout().getTypeStoreSize(
            expectedResult.getElementType());
 
        if (expectedSizeInBytes == size) {
          Address storeAddr = expectedResult.withElementType(
              builder, cmpxchg.getOld().getType());
          builder.createStore(loc, cmpxchg.getOld(), storeAddr);
          assert(!MissingFeatures::generateDebugInfo());
        } else {
          Address oldTmp = cgf.createTempAlloca(
              cmpxchg.getOld().getType(), ptr.getAlignment(), loc, "old.tmp");
          builder.createStore(loc, cmpxchg.getOld(), oldTmp);
          assert(!MissingFeatures::generateDebugInfo());
 
          Address oldTmpVoid = oldTmp.withElementType(builder, cgf.cgm.voidTy);
          Address expectedVoid =
              expectedResult.withElementType(builder, cgf.cgm.voidTy);
          builder.createMemCpy(
              loc, expectedVoid, oldTmpVoid,
              builder.getConstInt(loc, cgf.cgm.uInt64Ty, expectedSizeInBytes));
        }
        builder.createYield(loc);
      });
 
  // Update the memory at Dest with Success's value.
  cgf.emitStoreOfScalar(cmpxchg.getSuccess(),
                        cgf.makeAddrLValue(dest, e->getType()),
                        /*isInit=*/false);
}
 
static void emitAtomicCmpXchgFailureSet(
    CIRGenFunction &cgf, AtomicExpr *e, bool isWeak, Address dest, Address ptr,
    Address val1, Address val2, Address expectedResult, Expr *failureOrderExpr,
    uint64_t size, cir::MemOrder successOrder, cir::SyncScopeKind scope) {
  Expr::EvalResult failureOrderEval;
  if (failureOrderExpr->EvaluateAsInt(failureOrderEval, cgf.getContext())) {
    uint64_t failureOrderInt = failureOrderEval.Val.getInt().getZExtValue();
 
    cir::MemOrder failureOrder;
    if (!cir::isValidCIRAtomicOrderingCABI(failureOrderInt)) {
      failureOrder = cir::MemOrder::Relaxed;
    } else {
      switch ((cir::MemOrder)failureOrderInt) {
      case cir::MemOrder::Relaxed:
        // 31.7.2.18: "The failure argument shall not be memory_order_release
        // nor memory_order_acq_rel". Fallback to monotonic.
      case cir::MemOrder::Release:
      case cir::MemOrder::AcquireRelease:
        failureOrder = cir::MemOrder::Relaxed;
        break;
      case cir::MemOrder::Consume:
      case cir::MemOrder::Acquire:
        failureOrder = cir::MemOrder::Acquire;
        break;
      case cir::MemOrder::SequentiallyConsistent:
        failureOrder = cir::MemOrder::SequentiallyConsistent;
        break;
      }
    }
 
    // Prior to c++17, "the failure argument shall be no stronger than the
    // success argument". This condition has been lifted and the only
    // precondition is 31.7.2.18. Effectively treat this as a DR and skip
    // language version checks.
    emitAtomicCmpXchg(cgf, e, isWeak, dest, ptr, val1, val2, expectedResult,
                      size, successOrder, failureOrder, scope);
    return;
  }
 
  // The failure memory order is not a compile time constant. The CIR atomic ops
  // require a constant value, so that memory order is known at compile time. In
  // this case, we can switch based on the memory order and call each variant
  // individually.
  mlir::Value failureOrderVal = cgf.emitScalarExpr(failureOrderExpr);
  mlir::Location atomicLoc = cgf.getLoc(e->getSourceRange());
  cir::SwitchOp::create(
      cgf.getBuilder(), atomicLoc, failureOrderVal,
      [&](mlir::OpBuilder &b, mlir::Location loc, mlir::OperationState &os) {
        mlir::Block *switchBlock = cgf.getBuilder().getBlock();
 
        // case cir::MemOrder::Relaxed:
        //   // 31.7.2.18: "The failure argument shall not be
        //   memory_order_release
        //   // nor memory_order_acq_rel". Fallback to monotonic.
        // case cir::MemOrder::Release:
        // case cir::MemOrder::AcquireRelease:
        //  Note: Since there are 3 options, this makes sense to just emit as a
        //  'default', which prevents user code from 'falling off' of this,
        //  which seems reasonable.  Also, 'relaxed' being the default behavior
        //  is also probably the least harmful.
        emitDefaultCaseLabel(cgf.getBuilder(), atomicLoc);
        emitAtomicCmpXchg(cgf, e, isWeak, dest, ptr, val1, val2, expectedResult,
                          size, successOrder, cir::MemOrder::Relaxed, scope);
        cgf.getBuilder().createBreak(atomicLoc);
        cgf.getBuilder().setInsertionPointToEnd(switchBlock);
 
        // case cir::MemOrder::Consume:
        // case cir::MemOrder::Acquire:
        emitMemOrderCaseLabel(cgf.getBuilder(), loc, failureOrderVal.getType(),
                              {cir::MemOrder::Consume, cir::MemOrder::Acquire});
        emitAtomicCmpXchg(cgf, e, isWeak, dest, ptr, val1, val2, expectedResult,
                          size, successOrder, cir::MemOrder::Acquire, scope);
        cgf.getBuilder().createBreak(atomicLoc);
        cgf.getBuilder().setInsertionPointToEnd(switchBlock);
 
        // case cir::MemOrder::SequentiallyConsistent:
        emitMemOrderCaseLabel(cgf.getBuilder(), loc, failureOrderVal.getType(),
                              {cir::MemOrder::SequentiallyConsistent});
        emitAtomicCmpXchg(cgf, e, isWeak, dest, ptr, val1, val2, expectedResult,
                          size, successOrder,
                          cir::MemOrder::SequentiallyConsistent, scope);
        cgf.getBuilder().createBreak(atomicLoc);
        cgf.getBuilder().setInsertionPointToEnd(switchBlock);
 
        cgf.getBuilder().createYield(atomicLoc);
      });
}
 
// A version of the emitAtomicCmpXchgFailureSet function that ALSO checks
// whether it is 'weak' or not (by adding an 'if' around it, and calling
// emitAtomicCmpXchgFailureSet 2x).
static void emitAtomicCmpXchgFailureSetCheckWeak(
    CIRGenFunction &cgf, AtomicExpr *e, Expr *isWeakExpr, Address dest,
    Address ptr, Address val1, Address val2, Address expectedResult,
    Expr *failureOrderExpr, uint64_t size, cir::MemOrder successOrder,
    cir::SyncScopeKind scope) {
  mlir::Value isWeakVal = cgf.emitScalarExpr(isWeakExpr);
  // The AST seems to be inserting a 'bool' cast (even in C mode) here, so we'll
  // just emit it like a scalar.
  assert(isWeakVal.getType() == cgf.getBuilder().getBoolTy());
  mlir::Location atomicLoc = cgf.getLoc(e->getSourceRange());
 
  // Unlike classic compiler, we use an 'if' here instead of a switch, simply to
  // make this more readable/logical, plus we don't allow switch over a bool in
  // CIR.
  cir::IfOp::create(
      cgf.getBuilder(), atomicLoc, isWeakVal, /*elseRegion=*/true,
      [&](mlir::OpBuilder &b, mlir::Location loc) {
        emitAtomicCmpXchgFailureSet(cgf, e, /*isWeak=*/true, dest, ptr, val1,
                                    val2, expectedResult, failureOrderExpr,
                                    size, successOrder, scope);
        cgf.getBuilder().createYield(atomicLoc);
      },
      [&](mlir::OpBuilder &b, mlir::Location loc) {
        emitAtomicCmpXchgFailureSet(cgf, e, /*isWeak=*/false, dest, ptr, val1,
                                    val2, expectedResult, failureOrderExpr,
                                    size, successOrder, scope);
        cgf.getBuilder().createYield(atomicLoc);
      });
}
 
static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest,
                         Address ptr, Address val1, Address val2,
                         Address expectedResult, Expr *isWeakExpr,
                         Expr *failureOrderExpr, int64_t size,
                         cir::MemOrder order, cir::SyncScopeKind scope) {
  assert(!cir::MissingFeatures::atomicSyncScopeID());
  llvm::StringRef opName;
 
  CIRGenBuilderTy &builder = cgf.getBuilder();
  mlir::Location loc = cgf.getLoc(expr->getSourceRange());
  auto orderAttr = cir::MemOrderAttr::get(builder.getContext(), order);
  auto scopeAttr = cir::SyncScopeKindAttr::get(builder.getContext(), scope);
  cir::AtomicFetchKindAttr fetchAttr;
  bool fetchFirst = true;
 
  auto handleFetchOp = [&](cir::AtomicFetchKind kind) {
    opName = cir::AtomicFetchOp::getOperationName();
    fetchAttr = cir::AtomicFetchKindAttr::get(builder.getContext(), kind);
  };
 
  switch (expr->getOp()) {
  case AtomicExpr::AO__c11_atomic_init:
    llvm_unreachable("already handled!");
 
  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
  case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
  case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
    emitAtomicCmpXchgFailureSet(cgf, expr, /*isWeak=*/false, dest, ptr, val1,
                                val2, expectedResult, failureOrderExpr, size,
                                order, scope);
    return;
 
  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
  case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
  case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
    emitAtomicCmpXchgFailureSet(cgf, expr, /*isWeak=*/true, dest, ptr, val1,
                                val2, expectedResult, failureOrderExpr, size,
                                order, scope);
    return;
 
  case AtomicExpr::AO__atomic_compare_exchange:
  case AtomicExpr::AO__atomic_compare_exchange_n:
  case AtomicExpr::AO__scoped_atomic_compare_exchange:
  case AtomicExpr::AO__scoped_atomic_compare_exchange_n: {
    bool isWeak = false;
    if (isWeakExpr->EvaluateAsBooleanCondition(isWeak, cgf.getContext())) {
      emitAtomicCmpXchgFailureSet(cgf, expr, isWeak, dest, ptr, val1, val2,
                                  expectedResult, failureOrderExpr, size, order,
                                  scope);
    } else {
      emitAtomicCmpXchgFailureSetCheckWeak(
          cgf, expr, isWeakExpr, dest, ptr, val1, val2, expectedResult,
          failureOrderExpr, size, order, scope);
    }
    return;
  }
 
  case AtomicExpr::AO__c11_atomic_load:
  case AtomicExpr::AO__atomic_load_n:
  case AtomicExpr::AO__atomic_load:
  case AtomicExpr::AO__scoped_atomic_load_n:
  case AtomicExpr::AO__scoped_atomic_load:
  case AtomicExpr::AO__hip_atomic_load:
  case AtomicExpr::AO__opencl_atomic_load: {
    cir::LoadOp load =
        builder.createLoad(loc, ptr, /*isVolatile=*/expr->isVolatile());
 
    load->setAttr("mem_order", orderAttr);
    load->setAttr("sync_scope", scopeAttr);
 
    builder.createStore(loc, load->getResult(0), dest);
    return;
  }
 
  case AtomicExpr::AO__c11_atomic_store:
  case AtomicExpr::AO__atomic_store_n:
  case AtomicExpr::AO__atomic_store:
  case AtomicExpr::AO__scoped_atomic_store:
  case AtomicExpr::AO__scoped_atomic_store_n:
  case AtomicExpr::AO__hip_atomic_store:
  case AtomicExpr::AO__opencl_atomic_store: {
    cir::LoadOp loadVal1 = builder.createLoad(loc, val1);
 
    assert(!cir::MissingFeatures::atomicSyncScopeID());
 
    builder.createStore(loc, loadVal1, ptr, expr->isVolatile(),
                        /*isNontemporal=*/false,
                        /*align=*/mlir::IntegerAttr{}, scopeAttr, orderAttr);
    return;
  }
 
  case AtomicExpr::AO__c11_atomic_exchange:
  case AtomicExpr::AO__atomic_exchange_n:
  case AtomicExpr::AO__atomic_exchange:
  case AtomicExpr::AO__scoped_atomic_exchange_n:
  case AtomicExpr::AO__scoped_atomic_exchange:
  case AtomicExpr::AO__hip_atomic_exchange:
  case AtomicExpr::AO__opencl_atomic_exchange:
    opName = cir::AtomicXchgOp::getOperationName();
    break;
 
  case AtomicExpr::AO__atomic_add_fetch:
  case AtomicExpr::AO__scoped_atomic_add_fetch:
    fetchFirst = false;
    [[fallthrough]];
  case AtomicExpr::AO__c11_atomic_fetch_add:
  case AtomicExpr::AO__atomic_fetch_add:
  case AtomicExpr::AO__scoped_atomic_fetch_add:
  case AtomicExpr::AO__hip_atomic_fetch_add:
  case AtomicExpr::AO__opencl_atomic_fetch_add:
    handleFetchOp(cir::AtomicFetchKind::Add);
    break;
 
  case AtomicExpr::AO__atomic_sub_fetch:
  case AtomicExpr::AO__scoped_atomic_sub_fetch:
    fetchFirst = false;
    [[fallthrough]];
  case AtomicExpr::AO__c11_atomic_fetch_sub:
  case AtomicExpr::AO__atomic_fetch_sub:
  case AtomicExpr::AO__scoped_atomic_fetch_sub:
  case AtomicExpr::AO__hip_atomic_fetch_sub:
  case AtomicExpr::AO__opencl_atomic_fetch_sub:
    handleFetchOp(cir::AtomicFetchKind::Sub);
    break;
 
  case AtomicExpr::AO__atomic_min_fetch:
  case AtomicExpr::AO__scoped_atomic_min_fetch:
    fetchFirst = false;
    [[fallthrough]];
  case AtomicExpr::AO__c11_atomic_fetch_min:
  case AtomicExpr::AO__atomic_fetch_min:
  case AtomicExpr::AO__scoped_atomic_fetch_min:
  case AtomicExpr::AO__hip_atomic_fetch_min:
  case AtomicExpr::AO__opencl_atomic_fetch_min:
    handleFetchOp(cir::AtomicFetchKind::Min);
    break;
 
  case AtomicExpr::AO__atomic_fetch_fminimum:
  case AtomicExpr::AO__scoped_atomic_fetch_fminimum:
    assert(expr->getValueType()->isFloatingType() &&
           "fminimum operations only support floating-point types");
    handleFetchOp(cir::AtomicFetchKind::Minimum);
    break;
 
  case AtomicExpr::AO__atomic_fetch_fminimum_num:
  case AtomicExpr::AO__scoped_atomic_fetch_fminimum_num:
    assert(expr->getValueType()->isFloatingType() &&
           "fminimum_num operations only support floating-point types");
    handleFetchOp(cir::AtomicFetchKind::MinimumNum);
    break;
 
  case AtomicExpr::AO__atomic_max_fetch:
  case AtomicExpr::AO__scoped_atomic_max_fetch:
    fetchFirst = false;
    [[fallthrough]];
  case AtomicExpr::AO__c11_atomic_fetch_max:
  case AtomicExpr::AO__atomic_fetch_max:
  case AtomicExpr::AO__scoped_atomic_fetch_max:
  case AtomicExpr::AO__hip_atomic_fetch_max:
  case AtomicExpr::AO__opencl_atomic_fetch_max:
    handleFetchOp(cir::AtomicFetchKind::Max);
    break;
 
  case AtomicExpr::AO__atomic_fetch_fmaximum:
  case AtomicExpr::AO__scoped_atomic_fetch_fmaximum:
    assert(expr->getValueType()->isFloatingType() &&
           "fmaximum operations only support floating-point types");
    handleFetchOp(cir::AtomicFetchKind::Maximum);
    break;
 
  case AtomicExpr::AO__atomic_fetch_fmaximum_num:
  case AtomicExpr::AO__scoped_atomic_fetch_fmaximum_num:
    assert(expr->getValueType()->isFloatingType() &&
           "fmaximum_num operations only support floating-point types");
    handleFetchOp(cir::AtomicFetchKind::MaximumNum);
    break;
 
  case AtomicExpr::AO__atomic_and_fetch:
  case AtomicExpr::AO__scoped_atomic_and_fetch:
    fetchFirst = false;
    [[fallthrough]];
  case AtomicExpr::AO__c11_atomic_fetch_and:
  case AtomicExpr::AO__atomic_fetch_and:
  case AtomicExpr::AO__scoped_atomic_fetch_and:
  case AtomicExpr::AO__hip_atomic_fetch_and:
  case AtomicExpr::AO__opencl_atomic_fetch_and:
    handleFetchOp(cir::AtomicFetchKind::And);
    break;
 
  case AtomicExpr::AO__atomic_or_fetch:
  case AtomicExpr::AO__scoped_atomic_or_fetch:
    fetchFirst = false;
    [[fallthrough]];
  case AtomicExpr::AO__c11_atomic_fetch_or:
  case AtomicExpr::AO__atomic_fetch_or:
  case AtomicExpr::AO__scoped_atomic_fetch_or:
  case AtomicExpr::AO__hip_atomic_fetch_or:
  case AtomicExpr::AO__opencl_atomic_fetch_or:
    handleFetchOp(cir::AtomicFetchKind::Or);
    break;
 
  case AtomicExpr::AO__atomic_xor_fetch:
  case AtomicExpr::AO__scoped_atomic_xor_fetch:
    fetchFirst = false;
    [[fallthrough]];
  case AtomicExpr::AO__c11_atomic_fetch_xor:
  case AtomicExpr::AO__atomic_fetch_xor:
  case AtomicExpr::AO__scoped_atomic_fetch_xor:
  case AtomicExpr::AO__hip_atomic_fetch_xor:
  case AtomicExpr::AO__opencl_atomic_fetch_xor:
    handleFetchOp(cir::AtomicFetchKind::Xor);
    break;
 
  case AtomicExpr::AO__atomic_nand_fetch:
  case AtomicExpr::AO__scoped_atomic_nand_fetch:
    fetchFirst = false;
    [[fallthrough]];
  case AtomicExpr::AO__c11_atomic_fetch_nand:
  case AtomicExpr::AO__atomic_fetch_nand:
  case AtomicExpr::AO__scoped_atomic_fetch_nand:
    handleFetchOp(cir::AtomicFetchKind::Nand);
    break;
 
  case AtomicExpr::AO__atomic_test_and_set: {
    auto op = cir::AtomicTestAndSetOp::create(
        builder, loc, ptr.getPointer(), order,
        builder.getI64IntegerAttr(ptr.getAlignment().getQuantity()),
        expr->isVolatile());
    builder.createStore(loc, op, dest);
    return;
  }
 
  case AtomicExpr::AO__atomic_clear: {
    cir::AtomicClearOp::create(
        builder, loc, ptr.getPointer(), order,
        builder.getI64IntegerAttr(ptr.getAlignment().getQuantity()),
        expr->isVolatile());
    return;
  }
 
  case AtomicExpr::AO__atomic_fetch_uinc:
  case AtomicExpr::AO__scoped_atomic_fetch_uinc:
    handleFetchOp(cir::AtomicFetchKind::UIncWrap);
    break;
 
  case AtomicExpr::AO__atomic_fetch_udec:
  case AtomicExpr::AO__scoped_atomic_fetch_udec:
    handleFetchOp(cir::AtomicFetchKind::UDecWrap);
    break;
 
  case AtomicExpr::AO__opencl_atomic_init:
    cgf.cgm.errorNYI(expr->getSourceRange(), "emitAtomicOp: expr op NYI");
    return;
  }
 
  assert(!opName.empty() && "expected operation name to build");
  mlir::Value loadVal1 = builder.createLoad(loc, val1);
 
  SmallVector<mlir::Value> atomicOperands = {ptr.getPointer(), loadVal1};
  SmallVector<mlir::Type> atomicResTys = {loadVal1.getType()};
  mlir::Operation *rmwOp = builder.create(loc, builder.getStringAttr(opName),
                                          atomicOperands, atomicResTys);
 
  if (fetchAttr)
    rmwOp->setAttr("binop", fetchAttr);
  rmwOp->setAttr("mem_order", orderAttr);
  rmwOp->setAttr("sync_scope", scopeAttr);
  if (expr->isVolatile())
    rmwOp->setAttr("is_volatile", builder.getUnitAttr());
  if (fetchFirst && opName == cir::AtomicFetchOp::getOperationName())
    rmwOp->setAttr("fetch_first", builder.getUnitAttr());
 
  setAMDGPUAtomicMetadata(cgf, rmwOp, expr);
 
  mlir::Value result = rmwOp->getResult(0);
 
  builder.createStore(loc, result, dest);
}
 
// Map clang sync scope to CIR sync scope.
static cir::SyncScopeKind convertSyncScopeToCIR(CIRGenFunction &cgf,
                                                SourceRange range,
                                                clang::SyncScope scope) {
  switch (scope) {
  case clang::SyncScope::SingleScope:
    return cir::SyncScopeKind::SingleThread;
  case clang::SyncScope::SystemScope:
    return cir::SyncScopeKind::System;
  case clang::SyncScope::DeviceScope:
    return cir::SyncScopeKind::Device;
  case clang::SyncScope::WorkgroupScope:
    return cir::SyncScopeKind::Workgroup;
  case clang::SyncScope::WavefrontScope:
    return cir::SyncScopeKind::Wavefront;
  case clang::SyncScope::ClusterScope:
    return cir::SyncScopeKind::Cluster;
 
  case clang::SyncScope::HIPSingleThread:
    return cir::SyncScopeKind::HIPSingleThread;
  case clang::SyncScope::HIPSystem:
    return cir::SyncScopeKind::HIPSystem;
  case clang::SyncScope::HIPAgent:
    return cir::SyncScopeKind::HIPAgent;
  case clang::SyncScope::HIPWorkgroup:
    return cir::SyncScopeKind::HIPWorkgroup;
  case clang::SyncScope::HIPWavefront:
    return cir::SyncScopeKind::HIPWavefront;
  case clang::SyncScope::HIPCluster:
    return cir::SyncScopeKind::HIPCluster;
 
  case clang::SyncScope::OpenCLWorkGroup:
    return cir::SyncScopeKind::OpenCLWorkGroup;
  case clang::SyncScope::OpenCLDevice:
    return cir::SyncScopeKind::OpenCLDevice;
  case clang::SyncScope::OpenCLAllSVMDevices:
    return cir::SyncScopeKind::OpenCLAllSVMDevices;
  case clang::SyncScope::OpenCLSubGroup:
    return cir::SyncScopeKind::OpenCLSubGroup;
  }
 
  llvm_unreachable("unhandled sync scope");
}
 
static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest,
                         Address ptr, Address val1, Address val2,
                         Address expectedResult, Expr *isWeakExpr,
                         Expr *failureOrderExpr, int64_t size,
                         cir::MemOrder order,
                         const std::optional<Expr::EvalResult> &scopeConst,
                         mlir::Value scopeValue) {
  std::unique_ptr<AtomicScopeModel> scopeModel = expr->getScopeModel();
 
  if (!scopeModel) {
    emitAtomicOp(cgf, expr, dest, ptr, val1, val2, expectedResult, isWeakExpr,
                 failureOrderExpr, size, order, cir::SyncScopeKind::System);
    return;
  }
 
  if (scopeConst.has_value()) {
    cir::SyncScopeKind mappedScope = convertSyncScopeToCIR(
        cgf, expr->getScope()->getSourceRange(),
        scopeModel->map(scopeConst->Val.getInt().getZExtValue()));
    emitAtomicOp(cgf, expr, dest, ptr, val1, val2, expectedResult, isWeakExpr,
                 failureOrderExpr, size, order, mappedScope);
    return;
  }
 
  // The sync scope is not a compile-time constant. Emit a switch statement to
  // handle each possible value of the sync scope.
  CIRGenBuilderTy &builder = cgf.getBuilder();
  mlir::Location loc = cgf.getLoc(expr->getSourceRange());
  llvm::ArrayRef<unsigned> allScopes = scopeModel->getRuntimeValues();
  unsigned fallback = scopeModel->getFallBackValue();
 
  cir::SwitchOp::create(
      builder, loc, scopeValue,
      [&](mlir::OpBuilder &, mlir::Location loc, mlir::OperationState &) {
        mlir::Block *switchBlock = builder.getBlock();
 
        // Default case -- use fallback scope
        cir::SyncScopeKind fallbackScope = convertSyncScopeToCIR(
            cgf, expr->getScope()->getSourceRange(), scopeModel->map(fallback));
        emitDefaultCaseLabel(builder, loc);
        emitAtomicOp(cgf, expr, dest, ptr, val1, val2, expectedResult,
                     isWeakExpr, failureOrderExpr, size, order, fallbackScope);
        builder.createBreak(loc);
        builder.setInsertionPointToEnd(switchBlock);
 
        // Emit a switch case for each non-fallback runtime scope value
        for (unsigned scope : allScopes) {
          if (scope == fallback)
            continue;
 
          cir::SyncScopeKind cirScope = convertSyncScopeToCIR(
              cgf, expr->getScope()->getSourceRange(), scopeModel->map(scope));
 
          mlir::ArrayAttr casesAttr = builder.getArrayAttr(
              {cir::IntAttr::get(scopeValue.getType(), scope)});
          mlir::OpBuilder::InsertPoint insertPoint;
          cir::CaseOp::create(builder, loc, casesAttr, cir::CaseOpKind::Equal,
                              insertPoint);
 
          builder.restoreInsertionPoint(insertPoint);
          emitAtomicOp(cgf, expr, dest, ptr, val1, val2, expectedResult,
                       isWeakExpr, failureOrderExpr, size, order, cirScope);
          builder.createBreak(loc);
          builder.setInsertionPointToEnd(switchBlock);
        }
 
        builder.createYield(loc);
      });
}
 
static std::optional<cir::MemOrder>
getEffectiveAtomicMemOrder(cir::MemOrder oriOrder, bool isStore, bool isLoad,
                           bool isFence) {
  // Some memory orders are not supported by partial atomic operation:
  // {memory_order_releaxed} is not valid for fence operations.
  // {memory_order_consume, memory_order_acquire} are not valid for write-only
  // operations.
  // {memory_order_release} is not valid for read-only operations.
  // {memory_order_acq_rel} is only valid for read-write operations.
  if (isStore) {
    if (oriOrder == cir::MemOrder::Consume ||
        oriOrder == cir::MemOrder::Acquire ||
        oriOrder == cir::MemOrder::AcquireRelease)
      return std::nullopt;
  } else if (isLoad) {
    if (oriOrder == cir::MemOrder::Release ||
        oriOrder == cir::MemOrder::AcquireRelease)
      return std::nullopt;
  } else if (isFence) {
    if (oriOrder == cir::MemOrder::Relaxed)
      return std::nullopt;
  }
  // memory_order_consume is not implemented, it is always treated like
  // memory_order_acquire
  if (oriOrder == cir::MemOrder::Consume)
    return cir::MemOrder::Acquire;
  return oriOrder;
}
 
static void emitAtomicExprWithDynamicMemOrder(
    CIRGenFunction &cgf, mlir::Value order, bool isStore, bool isLoad,
    bool isFence, llvm::function_ref<void(cir::MemOrder)> emitAtomicOpFn) {
  if (!order)
    return;
  // The memory order is not known at compile-time.  The atomic operations
  // can't handle runtime memory orders; the memory order must be hard coded.
  // Generate a "switch" statement that converts a runtime value into a
  // compile-time value.
  CIRGenBuilderTy &builder = cgf.getBuilder();
  cir::SwitchOp::create(
      builder, order.getLoc(), order,
      [&](mlir::OpBuilder &, mlir::Location loc, mlir::OperationState &) {
        mlir::Block *switchBlock = builder.getBlock();
 
        auto emitMemOrderCase = [&](llvm::ArrayRef<cir::MemOrder> caseOrders) {
          // Checking there are same effective memory order for each case.
          for (int i = 1, e = caseOrders.size(); i < e; i++)
            assert((getEffectiveAtomicMemOrder(caseOrders[i - 1], isStore,
                                               isLoad, isFence) ==
                    getEffectiveAtomicMemOrder(caseOrders[i], isStore, isLoad,
                                               isFence)) &&
                   "Effective memory order must be same!");
          // Emit case label and atomic opeartion if neccessary.
          if (caseOrders.empty()) {
            emitDefaultCaseLabel(builder, loc);
            // There is no good way to report an unsupported memory order at
            // runtime, hence the fallback to memory_order_relaxed.
            if (!isFence)
              emitAtomicOpFn(cir::MemOrder::Relaxed);
          } else if (std::optional<cir::MemOrder> actualOrder =
                         getEffectiveAtomicMemOrder(caseOrders[0], isStore,
                                                    isLoad, isFence)) {
            // Included in default case.
            if (!isFence && actualOrder == cir::MemOrder::Relaxed)
              return;
            // Creating case operation for effective memory order. If there are
            // multiple cases in `caseOrders`, the actual order of each case
            // must be same, this needs to be guaranteed by the caller.
            emitMemOrderCaseLabel(builder, loc, order.getType(), caseOrders);
            emitAtomicOpFn(actualOrder.value());
          } else {
            // Do nothing if (!caseOrders.empty() && !actualOrder)
            return;
          }
          builder.createBreak(loc);
          builder.setInsertionPointToEnd(switchBlock);
        };
 
        emitMemOrderCase(/*default:*/ {});
        emitMemOrderCase({cir::MemOrder::Relaxed});
        emitMemOrderCase({cir::MemOrder::Consume, cir::MemOrder::Acquire});
        emitMemOrderCase({cir::MemOrder::Release});
        emitMemOrderCase({cir::MemOrder::AcquireRelease});
        emitMemOrderCase({cir::MemOrder::SequentiallyConsistent});
 
        builder.createYield(loc);
      });
}
 
void CIRGenFunction::emitAtomicExprWithMemOrder(
    const Expr *memOrder, bool isStore, bool isLoad, bool isFence,
    llvm::function_ref<void(cir::MemOrder)> emitAtomicOpFn) {
  // Emit the memory order operand, and try to evaluate it as a constant.
  Expr::EvalResult eval;
  if (memOrder->EvaluateAsInt(eval, getContext())) {
    uint64_t constOrder = eval.Val.getInt().getZExtValue();
    // We should not ever get to a case where the ordering isn't a valid CABI
    // value, but it's hard to enforce that in general.
    if (!cir::isValidCIRAtomicOrderingCABI(constOrder))
      return;
    cir::MemOrder oriOrder = static_cast<cir::MemOrder>(constOrder);
    if (std::optional<cir::MemOrder> actualOrder =
            getEffectiveAtomicMemOrder(oriOrder, isStore, isLoad, isFence))
      emitAtomicOpFn(actualOrder.value());
    return;
  }
 
  // Otherwise, handle variable memory ordering. Emit `SwitchOp` to convert
  // dynamic value to static value.
  mlir::Value dynOrder = emitScalarExpr(memOrder);
  emitAtomicExprWithDynamicMemOrder(*this, dynOrder, isStore, isLoad, isFence,
                                    emitAtomicOpFn);
}
 
static RValue emitAtomicLibCall(CIRGenFunction &cgf, llvm::StringRef funcName,
                                QualType resultType, CallArgList &args) {
  const CIRGenFunctionInfo &fnInfo =
      cgf.cgm.getTypes().arrangeBuiltinFunctionCall(resultType, args);
  cir::FuncType fnTy = cgf.cgm.getTypes().getFunctionType(fnInfo);
 
  mlir::NamedAttrList fnAttrs;
  assert(!cir::MissingFeatures::opFuncExtraAttrs());
 
  cir::FuncOp fn = cgf.cgm.createRuntimeFunction(fnTy, funcName, fnAttrs);
  auto callee = CIRGenCallee::forDirect(fn);
  return cgf.emitCall(fnInfo, callee, ReturnValueSlot(), args,
                      /*isMustTail=*/false);
}
 
static RValue emitLibCallForAtomicExpr(CIRGenFunction &cgf, AtomicExpr *e,
                                       Address atomicPtr, Address dest,
                                       Address val1, Address val2,
                                       uint64_t atomicTySize,
                                       QualType resultTy) {
  mlir::Location loc = cgf.getLoc(e->getSourceRange());
 
  CallArgList args;
  // For non-optimized library calls, the size is the first parameter.
  args.add(
      RValue::get(cgf.getBuilder().getConstInt(loc, cgf.sizeTy, atomicTySize)),
      cgf.getContext().getSizeType());
 
  // The atomic address is the second parameter.
  // The OpenCL atomic library functions only accept pointer arguments to
  // generic address space.
  auto castToGenericAddrSpace = [&](mlir::Value v, QualType pt) {
    if (!e->isOpenCL())
      return cgf.getBuilder().createPtrBitcast(v, cgf.voidTy);
 
    assert(!cir::MissingFeatures::openCL());
    cgf.cgm.errorNYI(loc, "emitLibCallForAtomicExpr: openCL");
    return cgf.getBuilder().createPtrBitcast(v, cgf.voidTy);
  };
  args.add(RValue::get(castToGenericAddrSpace(atomicPtr.emitRawPointer(),
                                              e->getPtr()->getType())),
           cgf.getContext().VoidPtrTy);
 
  mlir::Value order = cgf.emitScalarExpr(e->getOrder());
 
  // The next 1-3 parameters are op-dependent.
  llvm::StringRef calleeName;
  QualType retTy;
  bool hasRetTy = false;
  switch (e->getOp()) {
  case AtomicExpr::AO__c11_atomic_init:
  case AtomicExpr::AO__opencl_atomic_init:
    llvm_unreachable("Already handled!");
 
  // There is only one libcall for compare an exchange, because there is no
  // optimisation benefit possible from a libcall version of a weak compare
  // and exchange.
  // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
  //                                void *desired, int success, int failure)
  case AtomicExpr::AO__atomic_compare_exchange:
  case AtomicExpr::AO__atomic_compare_exchange_n:
  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
  case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
  case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
  case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
  case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
  case AtomicExpr::AO__scoped_atomic_compare_exchange:
  case AtomicExpr::AO__scoped_atomic_compare_exchange_n: {
    calleeName = "__atomic_compare_exchange";
    retTy = cgf.getContext().BoolTy;
    hasRetTy = true;
    mlir::Value orderFail = cgf.emitScalarExpr(e->getOrderFail());
    args.add(RValue::get(castToGenericAddrSpace(val1.emitRawPointer(),
                                                e->getVal1()->getType())),
             cgf.getContext().VoidPtrTy);
    args.add(RValue::get(castToGenericAddrSpace(val2.emitRawPointer(),
                                                e->getVal2()->getType())),
             cgf.getContext().VoidPtrTy);
    args.add(RValue::get(order), cgf.getContext().IntTy);
    order = orderFail;
    break;
  }
 
  // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
  //                        int order)
  case AtomicExpr::AO__atomic_exchange:
  case AtomicExpr::AO__atomic_exchange_n:
  case AtomicExpr::AO__c11_atomic_exchange:
  case AtomicExpr::AO__hip_atomic_exchange:
  case AtomicExpr::AO__opencl_atomic_exchange:
  case AtomicExpr::AO__scoped_atomic_exchange:
  case AtomicExpr::AO__scoped_atomic_exchange_n:
    calleeName = "__atomic_exchange";
    args.add(RValue::get(castToGenericAddrSpace(val1.emitRawPointer(),
                                                e->getVal1()->getType())),
             cgf.getContext().VoidPtrTy);
    break;
 
  // void __atomic_store(size_t size, void *mem, void *val, int order)
  case AtomicExpr::AO__atomic_store:
  case AtomicExpr::AO__atomic_store_n:
  case AtomicExpr::AO__c11_atomic_store:
  case AtomicExpr::AO__hip_atomic_store:
  case AtomicExpr::AO__opencl_atomic_store:
  case AtomicExpr::AO__scoped_atomic_store:
  case AtomicExpr::AO__scoped_atomic_store_n:
    calleeName = "__atomic_store";
    retTy = cgf.getContext().VoidTy;
    hasRetTy = true;
    args.add(RValue::get(castToGenericAddrSpace(val1.emitRawPointer(),
                                                e->getVal1()->getType())),
             cgf.getContext().VoidPtrTy);
    break;
 
  // void __atomic_load(size_t size, void *mem, void *return, int order)
  case AtomicExpr::AO__atomic_load:
  case AtomicExpr::AO__atomic_load_n:
  case AtomicExpr::AO__c11_atomic_load:
  case AtomicExpr::AO__hip_atomic_load:
  case AtomicExpr::AO__opencl_atomic_load:
  case AtomicExpr::AO__scoped_atomic_load:
  case AtomicExpr::AO__scoped_atomic_load_n:
    calleeName = "__atomic_load";
    break;
 
  case AtomicExpr::AO__atomic_add_fetch:
  case AtomicExpr::AO__scoped_atomic_add_fetch:
  case AtomicExpr::AO__atomic_fetch_add:
  case AtomicExpr::AO__c11_atomic_fetch_add:
  case AtomicExpr::AO__hip_atomic_fetch_add:
  case AtomicExpr::AO__opencl_atomic_fetch_add:
  case AtomicExpr::AO__scoped_atomic_fetch_add:
  case AtomicExpr::AO__atomic_and_fetch:
  case AtomicExpr::AO__scoped_atomic_and_fetch:
  case AtomicExpr::AO__atomic_fetch_and:
  case AtomicExpr::AO__c11_atomic_fetch_and:
  case AtomicExpr::AO__hip_atomic_fetch_and:
  case AtomicExpr::AO__opencl_atomic_fetch_and:
  case AtomicExpr::AO__scoped_atomic_fetch_and:
  case AtomicExpr::AO__atomic_or_fetch:
  case AtomicExpr::AO__scoped_atomic_or_fetch:
  case AtomicExpr::AO__atomic_fetch_or:
  case AtomicExpr::AO__c11_atomic_fetch_or:
  case AtomicExpr::AO__hip_atomic_fetch_or:
  case AtomicExpr::AO__opencl_atomic_fetch_or:
  case AtomicExpr::AO__scoped_atomic_fetch_or:
  case AtomicExpr::AO__atomic_sub_fetch:
  case AtomicExpr::AO__scoped_atomic_sub_fetch:
  case AtomicExpr::AO__atomic_fetch_sub:
  case AtomicExpr::AO__c11_atomic_fetch_sub:
  case AtomicExpr::AO__hip_atomic_fetch_sub:
  case AtomicExpr::AO__opencl_atomic_fetch_sub:
  case AtomicExpr::AO__scoped_atomic_fetch_sub:
  case AtomicExpr::AO__atomic_xor_fetch:
  case AtomicExpr::AO__scoped_atomic_xor_fetch:
  case AtomicExpr::AO__atomic_fetch_xor:
  case AtomicExpr::AO__c11_atomic_fetch_xor:
  case AtomicExpr::AO__hip_atomic_fetch_xor:
  case AtomicExpr::AO__opencl_atomic_fetch_xor:
  case AtomicExpr::AO__scoped_atomic_fetch_xor:
  case AtomicExpr::AO__atomic_nand_fetch:
  case AtomicExpr::AO__atomic_fetch_nand:
  case AtomicExpr::AO__c11_atomic_fetch_nand:
  case AtomicExpr::AO__scoped_atomic_fetch_nand:
  case AtomicExpr::AO__scoped_atomic_nand_fetch:
  case AtomicExpr::AO__atomic_min_fetch:
  case AtomicExpr::AO__atomic_fetch_min:
  case AtomicExpr::AO__c11_atomic_fetch_min:
  case AtomicExpr::AO__hip_atomic_fetch_min:
  case AtomicExpr::AO__opencl_atomic_fetch_min:
  case AtomicExpr::AO__scoped_atomic_fetch_min:
  case AtomicExpr::AO__scoped_atomic_min_fetch:
  case AtomicExpr::AO__atomic_max_fetch:
  case AtomicExpr::AO__atomic_fetch_max:
  case AtomicExpr::AO__c11_atomic_fetch_max:
  case AtomicExpr::AO__hip_atomic_fetch_max:
  case AtomicExpr::AO__opencl_atomic_fetch_max:
  case AtomicExpr::AO__scoped_atomic_fetch_max:
  case AtomicExpr::AO__scoped_atomic_max_fetch:
  case AtomicExpr::AO__scoped_atomic_fetch_uinc:
  case AtomicExpr::AO__scoped_atomic_fetch_udec:
  case AtomicExpr::AO__atomic_fetch_fmaximum:
  case AtomicExpr::AO__atomic_fetch_fmaximum_num:
  case AtomicExpr::AO__atomic_fetch_fminimum:
  case AtomicExpr::AO__atomic_fetch_fminimum_num:
  case AtomicExpr::AO__scoped_atomic_fetch_fmaximum:
  case AtomicExpr::AO__scoped_atomic_fetch_fmaximum_num:
  case AtomicExpr::AO__scoped_atomic_fetch_fminimum:
  case AtomicExpr::AO__scoped_atomic_fetch_fminimum_num:
  case AtomicExpr::AO__atomic_test_and_set:
  case AtomicExpr::AO__atomic_clear:
  case AtomicExpr::AO__atomic_fetch_uinc:
  case AtomicExpr::AO__atomic_fetch_udec:
    llvm_unreachable("Integral atomic operations always become atomicrmw!");
  }
 
  if (e->isOpenCL()) {
    assert(!cir::MissingFeatures::openCL());
    cgf.cgm.errorNYI(loc, "emitLibCallForAtomicExpr: openCL");
    return RValue::get(nullptr);
  }
 
  // By default, assume we return a value of the atomic type.
  if (!hasRetTy) {
    // Value is returned through parameter before the order.
    retTy = cgf.getContext().VoidTy;
    args.add(RValue::get(castToGenericAddrSpace(dest.emitRawPointer(), retTy)),
             cgf.getContext().VoidPtrTy);
  }
 
  // Order is always the last parameter.
  args.add(RValue::get(order), cgf.getContext().IntTy);
  if (e->isOpenCL()) {
    assert(!cir::MissingFeatures::openCL());
    cgf.cgm.errorNYI(loc, "emitLibCallForAtomicExpr: openCL");
    return RValue::get(nullptr);
  }
 
  RValue res = emitAtomicLibCall(cgf, calleeName, retTy, args);
 
  // The value is returned directly from the libcall.
  if (e->isCmpXChg())
    return res;
 
  if (resultTy->isVoidType())
    return RValue::get(nullptr);
 
  return cgf.convertTempToRValue(
      dest.withElementType(cgf.getBuilder(), cgf.convertTypeForMem(resultTy)),
      resultTy, e->getExprLoc());
}
 
RValue CIRGenFunction::emitAtomicExpr(AtomicExpr *e) {
  QualType atomicTy = e->getPtr()->getType()->getPointeeType();
  QualType memTy = atomicTy;
  if (const auto *ty = atomicTy->getAs<AtomicType>())
    memTy = ty->getValueType();
 
  Expr *isWeakExpr = nullptr;
  Expr *orderFailExpr = nullptr;
 
  Address val1 = Address::invalid();
  Address val2 = Address::invalid();
  Address dest = Address::invalid();
  Address ptr = emitPointerWithAlignment(e->getPtr());
 
  assert(!cir::MissingFeatures::openCL());
  if (e->getOp() == AtomicExpr::AO__c11_atomic_init) {
    LValue lvalue = makeAddrLValue(ptr, atomicTy);
    emitAtomicInit(e->getVal1(), lvalue);
    return RValue::get(nullptr);
  }
 
  TypeInfoChars typeInfo = getContext().getTypeInfoInChars(atomicTy);
  uint64_t size = typeInfo.Width.getQuantity();
 
  // Emit the sync scope operand, and try to evaluate it as a constant.
  mlir::Value scope =
      e->getScopeModel() ? emitScalarExpr(e->getScope()) : nullptr;
  std::optional<Expr::EvalResult> scopeConst;
  if (Expr::EvalResult eval;
      e->getScopeModel() && e->getScope()->EvaluateAsInt(eval, getContext()))
    scopeConst.emplace(std::move(eval));
 
  switch (e->getOp()) {
  default:
    cgm.errorNYI(e->getSourceRange(), "atomic op NYI");
    return RValue::get(nullptr);
 
  case AtomicExpr::AO__c11_atomic_init:
    llvm_unreachable("already handled above with emitAtomicInit");
 
  case AtomicExpr::AO__atomic_load_n:
  case AtomicExpr::AO__scoped_atomic_load_n:
  case AtomicExpr::AO__c11_atomic_load:
  case AtomicExpr::AO__opencl_atomic_load:
  case AtomicExpr::AO__hip_atomic_load:
  case AtomicExpr::AO__atomic_test_and_set:
  case AtomicExpr::AO__atomic_clear:
    break;
 
  case AtomicExpr::AO__atomic_load:
  case AtomicExpr::AO__scoped_atomic_load:
    dest = emitPointerWithAlignment(e->getVal1());
    break;
 
  case AtomicExpr::AO__atomic_store:
  case AtomicExpr::AO__scoped_atomic_store:
    val1 = emitPointerWithAlignment(e->getVal1());
    break;
 
  case AtomicExpr::AO__atomic_exchange:
  case AtomicExpr::AO__scoped_atomic_exchange:
    val1 = emitPointerWithAlignment(e->getVal1());
    dest = emitPointerWithAlignment(e->getVal2());
    break;
 
  case AtomicExpr::AO__atomic_compare_exchange:
  case AtomicExpr::AO__atomic_compare_exchange_n:
  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
  case AtomicExpr::AO__scoped_atomic_compare_exchange:
  case AtomicExpr::AO__scoped_atomic_compare_exchange_n:
  case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
  case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
  case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
  case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
    val1 = emitPointerWithAlignment(e->getVal1());
    if (e->getOp() == AtomicExpr::AO__atomic_compare_exchange ||
        e->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange)
      val2 = emitPointerWithAlignment(e->getVal2());
    else
      val2 = emitValToTemp(*this, e->getVal2());
    orderFailExpr = e->getOrderFail();
    if (e->getOp() == AtomicExpr::AO__atomic_compare_exchange_n ||
        e->getOp() == AtomicExpr::AO__atomic_compare_exchange ||
        e->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange_n ||
        e->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange)
      isWeakExpr = e->getWeak();
    break;
 
  case AtomicExpr::AO__c11_atomic_fetch_add:
  case AtomicExpr::AO__c11_atomic_fetch_sub:
    if (memTy->isPointerType()) {
      // For pointer arithmetic, we're required to do a bit of math:
      // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
      // ... but only for the C11 builtins. The GNU builtins expect the
      // user to multiply by sizeof(T).
      QualType val1Ty = e->getVal1()->getType();
      mlir::Location loc = getLoc(e->getSourceRange());
      mlir::Value val1Scalar = emitScalarExpr(e->getVal1());
      CharUnits pointeeIncAmt =
          getContext().getTypeSizeInChars(memTy->getPointeeType());
      mlir::Value scale = builder.getConstInt(loc, val1Scalar.getType(),
                                              pointeeIncAmt.getQuantity());
      val1Scalar = builder.createMul(loc, val1Scalar, scale);
      val1 = createMemTemp(val1Ty, loc, ".atomictmp");
      emitStoreOfScalar(val1Scalar, makeAddrLValue(val1, val1Ty),
                        /*isInit=*/true);
    }
    [[fallthrough]];
  case AtomicExpr::AO__atomic_fetch_add:
  case AtomicExpr::AO__atomic_fetch_sub:
  case AtomicExpr::AO__atomic_add_fetch:
  case AtomicExpr::AO__atomic_sub_fetch:
    if (memTy->isPointerType()) {
      // Fetch-and-update atomic operation on pointers should treat the pointer
      // value as uintptr_t values
      if (!val1.isValid())
        val1 = emitValToTemp(*this, e->getVal1());
      ptr = ptr.withElementType(builder, val1.getElementType());
      break;
    }
    [[fallthrough]];
  case AtomicExpr::AO__atomic_fetch_max:
  case AtomicExpr::AO__atomic_fetch_min:
  case AtomicExpr::AO__atomic_max_fetch:
  case AtomicExpr::AO__atomic_min_fetch:
  case AtomicExpr::AO__c11_atomic_fetch_max:
  case AtomicExpr::AO__c11_atomic_fetch_min:
  case AtomicExpr::AO__scoped_atomic_fetch_add:
  case AtomicExpr::AO__scoped_atomic_fetch_max:
  case AtomicExpr::AO__scoped_atomic_fetch_min:
  case AtomicExpr::AO__scoped_atomic_fetch_sub:
  case AtomicExpr::AO__scoped_atomic_fetch_fminimum:
  case AtomicExpr::AO__scoped_atomic_fetch_fmaximum:
  case AtomicExpr::AO__scoped_atomic_fetch_fminimum_num:
  case AtomicExpr::AO__scoped_atomic_fetch_fmaximum_num:
  case AtomicExpr::AO__scoped_atomic_add_fetch:
  case AtomicExpr::AO__scoped_atomic_max_fetch:
  case AtomicExpr::AO__scoped_atomic_min_fetch:
  case AtomicExpr::AO__scoped_atomic_sub_fetch:
    [[fallthrough]];
 
  case AtomicExpr::AO__atomic_fetch_and:
  case AtomicExpr::AO__atomic_fetch_nand:
  case AtomicExpr::AO__atomic_fetch_or:
  case AtomicExpr::AO__atomic_fetch_xor:
  case AtomicExpr::AO__atomic_and_fetch:
  case AtomicExpr::AO__atomic_nand_fetch:
  case AtomicExpr::AO__atomic_or_fetch:
  case AtomicExpr::AO__atomic_xor_fetch:
  case AtomicExpr::AO__atomic_exchange_n:
  case AtomicExpr::AO__atomic_store_n:
  case AtomicExpr::AO__c11_atomic_fetch_and:
  case AtomicExpr::AO__c11_atomic_fetch_nand:
  case AtomicExpr::AO__c11_atomic_fetch_or:
  case AtomicExpr::AO__c11_atomic_fetch_xor:
  case AtomicExpr::AO__c11_atomic_exchange:
  case AtomicExpr::AO__c11_atomic_store:
  case AtomicExpr::AO__scoped_atomic_fetch_and:
  case AtomicExpr::AO__scoped_atomic_fetch_nand:
  case AtomicExpr::AO__scoped_atomic_fetch_or:
  case AtomicExpr::AO__scoped_atomic_fetch_xor:
  case AtomicExpr::AO__scoped_atomic_and_fetch:
  case AtomicExpr::AO__scoped_atomic_nand_fetch:
  case AtomicExpr::AO__scoped_atomic_or_fetch:
  case AtomicExpr::AO__scoped_atomic_xor_fetch:
  case AtomicExpr::AO__scoped_atomic_store_n:
  case AtomicExpr::AO__scoped_atomic_exchange_n:
  case AtomicExpr::AO__atomic_fetch_uinc:
  case AtomicExpr::AO__atomic_fetch_udec:
  case AtomicExpr::AO__scoped_atomic_fetch_uinc:
  case AtomicExpr::AO__scoped_atomic_fetch_udec:
  case AtomicExpr::AO__atomic_fetch_fminimum:
  case AtomicExpr::AO__atomic_fetch_fmaximum:
  case AtomicExpr::AO__atomic_fetch_fminimum_num:
  case AtomicExpr::AO__atomic_fetch_fmaximum_num:
  case AtomicExpr::AO__hip_atomic_exchange:
  case AtomicExpr::AO__hip_atomic_store:
  case AtomicExpr::AO__hip_atomic_fetch_add:
  case AtomicExpr::AO__hip_atomic_fetch_sub:
  case AtomicExpr::AO__hip_atomic_fetch_min:
  case AtomicExpr::AO__hip_atomic_fetch_max:
  case AtomicExpr::AO__hip_atomic_fetch_and:
  case AtomicExpr::AO__hip_atomic_fetch_or:
  case AtomicExpr::AO__hip_atomic_fetch_xor:
  case AtomicExpr::AO__opencl_atomic_exchange:
  case AtomicExpr::AO__opencl_atomic_store:
  case AtomicExpr::AO__opencl_atomic_fetch_add:
  case AtomicExpr::AO__opencl_atomic_fetch_sub:
  case AtomicExpr::AO__opencl_atomic_fetch_min:
  case AtomicExpr::AO__opencl_atomic_fetch_max:
  case AtomicExpr::AO__opencl_atomic_fetch_and:
  case AtomicExpr::AO__opencl_atomic_fetch_or:
  case AtomicExpr::AO__opencl_atomic_fetch_xor:
    val1 = emitValToTemp(*this, e->getVal1());
    break;
  }
 
  QualType resultTy = e->getType().getUnqualifiedType();
 
  bool shouldCastToIntPtrTy =
      shouldCastToInt(convertTypeForMem(memTy), e->isCmpXChg());
 
  // The inlined atomics only function on iN types, where N is a power of 2. We
  // need to make sure (via temporaries if necessary) that all incoming values
  // are compatible.
  mlir::Location loc = getLoc(e->getSourceRange());
  LValue atomicValue = makeAddrLValue(ptr, atomicTy);
  AtomicInfo atomics(*this, atomicValue, loc);
 
  // Save val1's address before it is (possibly) converted to a temporary
  // sized to the full atomic width below.
  Address originalVal1 = val1;
 
  if (shouldCastToIntPtrTy) {
    ptr = atomics.castToAtomicIntPointer(ptr);
    if (val1.isValid())
      val1 = atomics.convertToAtomicIntPointer(val1, loc);
    if (val2.isValid())
      val2 = atomics.convertToAtomicIntPointer(val2, loc);
  }
  if (dest.isValid()) {
    if (shouldCastToIntPtrTy)
      dest = atomics.castToAtomicIntPointer(dest);
  } else if (e->isCmpXChg()) {
    dest = createMemTemp(resultTy, loc, "cmpxchg.bool");
  } else if (e->getOp() == AtomicExpr::AO__atomic_test_and_set) {
    dest = createMemTemp(resultTy, loc, "test_and_set.bool");
  } else if (!resultTy->isVoidType()) {
    dest = atomics.createTempAlloca();
    if (shouldCastToIntPtrTy)
      dest = atomics.castToAtomicIntPointer(dest);
  }
 
  bool powerOf2Size = (size & (size - 1)) == 0;
  bool useLibCall = !powerOf2Size || (size > 16);
 
  // For atomics larger than 16 bytes, emit a libcall from the frontend. This
  // avoids the overhead of dealing with excessively-large value types in IR.
  // Non-power-of-2 values also lower to libcall here, as they are not currently
  // permitted in IR instructions (although that constraint could be relaxed in
  // the future). For other cases where a libcall is required on a given
  // platform, we let the backend handle it (this includes handling for all of
  // the size-optimized libcall variants, which are only valid up to 16 bytes.)
  //
  // See: https://llvm.org/docs/Atomics.html#libcalls-atomic
  if (useLibCall)
    return emitLibCallForAtomicExpr(*this, e, ptr, dest, val1, val2, size,
                                    resultTy);
 
  bool isStore = e->getOp() == AtomicExpr::AO__c11_atomic_store ||
                 e->getOp() == AtomicExpr::AO__opencl_atomic_store ||
                 e->getOp() == AtomicExpr::AO__hip_atomic_store ||
                 e->getOp() == AtomicExpr::AO__atomic_store ||
                 e->getOp() == AtomicExpr::AO__atomic_store_n ||
                 e->getOp() == AtomicExpr::AO__scoped_atomic_store ||
                 e->getOp() == AtomicExpr::AO__scoped_atomic_store_n ||
                 e->getOp() == AtomicExpr::AO__atomic_clear;
  bool isLoad = e->getOp() == AtomicExpr::AO__c11_atomic_load ||
                e->getOp() == AtomicExpr::AO__opencl_atomic_load ||
                e->getOp() == AtomicExpr::AO__hip_atomic_load ||
                e->getOp() == AtomicExpr::AO__atomic_load ||
                e->getOp() == AtomicExpr::AO__atomic_load_n ||
                e->getOp() == AtomicExpr::AO__scoped_atomic_load ||
                e->getOp() == AtomicExpr::AO__scoped_atomic_load_n;
 
  auto emitAtomicOpCallBackFn = [&](cir::MemOrder memOrder) {
    emitAtomicOp(*this, e, dest, ptr, val1, val2, originalVal1, isWeakExpr,
                 orderFailExpr, size, memOrder, scopeConst, scope);
  };
  emitAtomicExprWithMemOrder(e->getOrder(), isStore, isLoad, /*isFence*/ false,
                             emitAtomicOpCallBackFn);
 
  if (resultTy->isVoidType())
    return RValue::get(nullptr);
 
  return convertTempToRValue(
      dest.withElementType(builder, convertTypeForMem(resultTy)), resultTy,
      e->getExprLoc());
}
 
RValue CIRGenFunction::emitAtomicLoad(LValue lvalue, SourceLocation loc,
                                      AggValueSlot slot) {
  if (lvalue.getType()->isAtomicType())
    return emitAtomicLoad(lvalue, loc, cir::MemOrder::SequentiallyConsistent,
                          /*isVolatile=*/lvalue.isVolatileQualified(), slot);
  return emitAtomicLoad(lvalue, loc, cir::MemOrder::Acquire,
                        /*isVolatile=*/true, slot);
}
 
RValue CIRGenFunction::emitAtomicLoad(LValue lvalue, SourceLocation loc,
                                      cir::MemOrder order, bool isVolatile,
                                      AggValueSlot slot) {
  AtomicInfo info(*this, lvalue, getLoc(loc));
  return info.emitAtomicLoad(slot, loc, /*asValue=*/true, order, isVolatile);
}
 
void CIRGenFunction::emitAtomicStore(RValue rvalue, LValue dest, bool isInit) {
  bool isVolatile = dest.isVolatileQualified();
  auto order = cir::MemOrder::SequentiallyConsistent;
  if (!dest.getType()->isAtomicType()) {
    assert(!cir::MissingFeatures::atomicMicrosoftVolatile());
  }
  return emitAtomicStore(rvalue, dest, order, isVolatile, isInit);
}
 
/// Emit a store to an l-value of atomic type.
///
/// Note that the r-value is expected to be an r-value of the atomic type; this
/// means that for aggregate r-values, it should include storage for any padding
/// that was necessary.
void CIRGenFunction::emitAtomicStore(RValue rvalue, LValue dest,
                                     cir::MemOrder order, bool isVolatile,
                                     bool isInit) {
  // If this is an aggregate r-value, it should agree in type except
  // maybe for address-space qualification.
  mlir::Location loc = dest.getPointer().getLoc();
  assert(!rvalue.isAggregate() ||
         rvalue.getAggregateAddress().getElementType() ==
             dest.getAddress().getElementType());
 
  AtomicInfo atomics(*this, dest, loc);
  LValue lvalue = atomics.getAtomicLValue();
 
  if (lvalue.isSimple()) {
    // If this is an initialization, just put the value there normally.
    if (isInit) {
      atomics.emitCopyIntoMemory(rvalue);
      return;
    }
 
    // Check whether we should use a library call.
    if (atomics.shouldUseLibCall()) {
      assert(!cir::MissingFeatures::atomicUseLibCall());
      cgm.errorNYI(loc, "emitAtomicStore: atomic store with library call");
      return;
    }
 
    // Okay, we're doing this natively.
    mlir::Value valueToStore = atomics.convertRValueToInt(rvalue, loc);
 
    // Do the atomic store.
    Address addr = atomics.getAtomicAddress();
    if (mlir::Value value = atomics.getScalarRValValueOrNull(rvalue)) {
      if (shouldCastToInt(value.getType(), /*CmpXchg=*/false)) {
        addr = atomics.castToAtomicIntPointer(addr);
        valueToStore =
            builder.createIntCast(valueToStore, addr.getElementType());
      }
    }
    cir::StoreOp store = builder.createStore(loc, valueToStore, addr);
 
    // Initializations don't need to be atomic.
    if (!isInit) {
      assert(!cir::MissingFeatures::atomicOpenMP());
      store.setMemOrder(order);
    }
 
    // Other decoration.
    if (isVolatile)
      store.setIsVolatile(true);
 
    assert(!cir::MissingFeatures::opLoadStoreTbaa());
    return;
  }
 
  cgm.errorNYI(loc, "emitAtomicStore: non-simple atomic lvalue");
  assert(!cir::MissingFeatures::opLoadStoreAtomic());
}
 
void CIRGenFunction::emitAtomicInit(Expr *init, LValue dest) {
  AtomicInfo atomics(*this, dest, getLoc(init->getSourceRange()));
 
  switch (atomics.getEvaluationKind()) {
  case cir::TEK_Scalar: {
    mlir::Value value = emitScalarExpr(init);
    atomics.emitCopyIntoMemory(RValue::get(value));
    return;
  }
 
  case cir::TEK_Complex: {
    mlir::Value value = emitComplexExpr(init);
    atomics.emitCopyIntoMemory(RValue::get(value));
    return;
  }
 
  case cir::TEK_Aggregate: {
    // Fix up the destination if the initializer isn't an expression
    // of atomic type.
    bool zeroed = false;
    if (!init->getType()->isAtomicType()) {
      zeroed = atomics.emitMemSetZeroIfNecessary();
      dest = atomics.projectValue();
    }
 
    // Evaluate the expression directly into the destination.
    assert(!cir::MissingFeatures::aggValueSlotGC());
    AggValueSlot slot = AggValueSlot::forLValue(
        dest, AggValueSlot::IsNotDestructed, AggValueSlot::IsNotAliased,
        AggValueSlot::DoesNotOverlap,
        zeroed ? AggValueSlot::IsZeroed : AggValueSlot::IsNotZeroed);
 
    emitAggExpr(init, slot);
    return;
  }
  }
 
  llvm_unreachable("bad evaluation kind");
}