35 : cgf(cgf), loc(loc) {
41 valueTy = ty->getValueType();
50 valueSizeInBits = valueTypeInfo.
Width;
51 atomicSizeInBits = atomicTypeInfo.
Width;
52 assert(valueSizeInBits <= atomicSizeInBits);
53 assert(valueAlignInBits <= atomicAlignInBits);
60 this->lvalue = lvalue;
63 cgf.
cgm.
errorNYI(loc,
"AtomicInfo: non-simple lvalue");
69 QualType getValueType()
const {
return valueTy; }
70 CharUnits getAtomicAlignment()
const {
return atomicAlign; }
72 mlir::Value getAtomicPointer()
const {
78 Address getAtomicAddress()
const {
84 cgf.
cgm.
errorNYI(loc,
"AtomicInfo::getAtomicAddress: non-simple lvalue");
86 return Address(getAtomicPointer(), elemTy, getAtomicAlignment());
95 bool hasPadding()
const {
return (valueSizeInBits != atomicSizeInBits); }
97 bool emitMemSetZeroIfNecessary()
const;
109 void emitCopyIntoMemory(
RValue rvalue)
const;
112 LValue projectValue()
const {
114 Address addr = getAtomicAddress();
116 cgf.
cgm.
errorNYI(loc,
"AtomicInfo::projectValue: padding");
120 return LValue::makeAddr(addr, getValueType(), lvalue.
getBaseInfo());
124 Address createTempAlloca()
const;
127 bool requiresMemSetZero(mlir::Type ty)
const;
143 uint64_t expectedSize) {
150bool AtomicInfo::requiresMemSetZero(mlir::Type ty)
const {
156 switch (getEvaluationKind()) {
162 cgf.
cgm.
errorNYI(loc,
"AtomicInfo::requiresMemSetZero: complex type");
169 llvm_unreachable(
"bad evaluation kind");
172Address AtomicInfo::convertToAtomicIntPointer(
Address addr)
const {
175 if (sourceSizeInBits != atomicSizeInBits) {
178 "AtomicInfo::convertToAtomicIntPointer: convert through temp alloca");
181 return castToAtomicIntPointer(addr);
184Address AtomicInfo::createTempAlloca()
const {
186 (lvalue.
isBitField() && valueSizeInBits > atomicSizeInBits) ? valueTy
188 getAtomicAlignment(), loc,
"atomic-temp");
192 cgf.
cgm.
errorNYI(loc,
"AtomicInfo::createTempAlloca: bitfield lvalue");
201 if (intTy && intTy.getWidth() == atomicSizeInBits)
207bool AtomicInfo::emitMemSetZeroIfNecessary()
const {
214 "AtomicInfo::emitMemSetZeroIfNecessary: emit memset zero");
220void AtomicInfo::emitCopyIntoMemory(
RValue rvalue)
const {
227 cgf.
cgm.
errorNYI(
"copying aggregate into atomic lvalue");
234 emitMemSetZeroIfNecessary();
237 LValue tempLValue = projectValue();
243 cgf.
cgm.
errorNYI(
"copying complex into atomic lvalue");
249 cir::MemOrder order) {
250 std::unique_ptr<AtomicScopeModel> scopeModel =
expr->getScopeModel();
253 cgf.
cgm.
errorNYI(
expr->getSourceRange(),
"emitAtomicOp: atomic scope");
260 mlir::Location loc = cgf.
getLoc(
expr->getSourceRange());
261 auto orderAttr = cir::MemOrderAttr::get(builder.getContext(), order);
263 switch (
expr->getOp()) {
264 case AtomicExpr::AO__c11_atomic_init:
265 llvm_unreachable(
"already handled!");
267 case AtomicExpr::AO__c11_atomic_load:
268 case AtomicExpr::AO__atomic_load_n:
269 case AtomicExpr::AO__atomic_load: {
275 load->setAttr(
"mem_order", orderAttr);
277 builder.
createStore(loc, load->getResult(0), dest);
281 case AtomicExpr::AO__c11_atomic_store:
282 case AtomicExpr::AO__atomic_store_n:
283 case AtomicExpr::AO__atomic_store: {
284 cir::LoadOp loadVal1 = builder.
createLoad(loc, val1);
289 mlir::IntegerAttr{}, orderAttr);
293 case AtomicExpr::AO__opencl_atomic_init:
295 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
296 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
297 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
299 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
300 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
301 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
303 case AtomicExpr::AO__atomic_compare_exchange:
304 case AtomicExpr::AO__atomic_compare_exchange_n:
305 case AtomicExpr::AO__scoped_atomic_compare_exchange:
306 case AtomicExpr::AO__scoped_atomic_compare_exchange_n:
308 case AtomicExpr::AO__opencl_atomic_load:
309 case AtomicExpr::AO__hip_atomic_load:
310 case AtomicExpr::AO__scoped_atomic_load_n:
311 case AtomicExpr::AO__scoped_atomic_load:
313 case AtomicExpr::AO__opencl_atomic_store:
314 case AtomicExpr::AO__hip_atomic_store:
315 case AtomicExpr::AO__scoped_atomic_store:
316 case AtomicExpr::AO__scoped_atomic_store_n:
318 case AtomicExpr::AO__c11_atomic_exchange:
319 case AtomicExpr::AO__hip_atomic_exchange:
320 case AtomicExpr::AO__opencl_atomic_exchange:
321 case AtomicExpr::AO__atomic_exchange_n:
322 case AtomicExpr::AO__atomic_exchange:
323 case AtomicExpr::AO__scoped_atomic_exchange_n:
324 case AtomicExpr::AO__scoped_atomic_exchange:
326 case AtomicExpr::AO__atomic_add_fetch:
327 case AtomicExpr::AO__scoped_atomic_add_fetch:
329 case AtomicExpr::AO__c11_atomic_fetch_add:
330 case AtomicExpr::AO__hip_atomic_fetch_add:
331 case AtomicExpr::AO__opencl_atomic_fetch_add:
332 case AtomicExpr::AO__atomic_fetch_add:
333 case AtomicExpr::AO__scoped_atomic_fetch_add:
335 case AtomicExpr::AO__atomic_sub_fetch:
336 case AtomicExpr::AO__scoped_atomic_sub_fetch:
338 case AtomicExpr::AO__c11_atomic_fetch_sub:
339 case AtomicExpr::AO__hip_atomic_fetch_sub:
340 case AtomicExpr::AO__opencl_atomic_fetch_sub:
341 case AtomicExpr::AO__atomic_fetch_sub:
342 case AtomicExpr::AO__scoped_atomic_fetch_sub:
344 case AtomicExpr::AO__atomic_min_fetch:
345 case AtomicExpr::AO__scoped_atomic_min_fetch:
347 case AtomicExpr::AO__c11_atomic_fetch_min:
348 case AtomicExpr::AO__hip_atomic_fetch_min:
349 case AtomicExpr::AO__opencl_atomic_fetch_min:
350 case AtomicExpr::AO__atomic_fetch_min:
351 case AtomicExpr::AO__scoped_atomic_fetch_min:
353 case AtomicExpr::AO__atomic_max_fetch:
354 case AtomicExpr::AO__scoped_atomic_max_fetch:
356 case AtomicExpr::AO__c11_atomic_fetch_max:
357 case AtomicExpr::AO__hip_atomic_fetch_max:
358 case AtomicExpr::AO__opencl_atomic_fetch_max:
359 case AtomicExpr::AO__atomic_fetch_max:
360 case AtomicExpr::AO__scoped_atomic_fetch_max:
362 case AtomicExpr::AO__atomic_and_fetch:
363 case AtomicExpr::AO__scoped_atomic_and_fetch:
365 case AtomicExpr::AO__c11_atomic_fetch_and:
366 case AtomicExpr::AO__hip_atomic_fetch_and:
367 case AtomicExpr::AO__opencl_atomic_fetch_and:
368 case AtomicExpr::AO__atomic_fetch_and:
369 case AtomicExpr::AO__scoped_atomic_fetch_and:
371 case AtomicExpr::AO__atomic_or_fetch:
372 case AtomicExpr::AO__scoped_atomic_or_fetch:
374 case AtomicExpr::AO__c11_atomic_fetch_or:
375 case AtomicExpr::AO__hip_atomic_fetch_or:
376 case AtomicExpr::AO__opencl_atomic_fetch_or:
377 case AtomicExpr::AO__atomic_fetch_or:
378 case AtomicExpr::AO__scoped_atomic_fetch_or:
380 case AtomicExpr::AO__atomic_xor_fetch:
381 case AtomicExpr::AO__scoped_atomic_xor_fetch:
383 case AtomicExpr::AO__c11_atomic_fetch_xor:
384 case AtomicExpr::AO__hip_atomic_fetch_xor:
385 case AtomicExpr::AO__opencl_atomic_fetch_xor:
386 case AtomicExpr::AO__atomic_fetch_xor:
387 case AtomicExpr::AO__scoped_atomic_fetch_xor:
389 case AtomicExpr::AO__atomic_nand_fetch:
390 case AtomicExpr::AO__scoped_atomic_nand_fetch:
392 case AtomicExpr::AO__c11_atomic_fetch_nand:
393 case AtomicExpr::AO__atomic_fetch_nand:
394 case AtomicExpr::AO__scoped_atomic_fetch_nand:
396 case AtomicExpr::AO__atomic_test_and_set:
398 case AtomicExpr::AO__atomic_clear:
407 auto memOrder =
static_cast<cir::MemOrder
>(order);
409 return memOrder != cir::MemOrder::Consume &&
410 memOrder != cir::MemOrder::Acquire &&
411 memOrder != cir::MemOrder::AcquireRelease;
413 return memOrder != cir::MemOrder::Release &&
414 memOrder != cir::MemOrder::AcquireRelease;
422 memTy = ty->getValueType();
429 if (e->
getOp() == AtomicExpr::AO__c11_atomic_init) {
443 bool shouldCastToIntPtrTy =
true;
445 switch (e->
getOp()) {
450 case AtomicExpr::AO__c11_atomic_init:
451 llvm_unreachable(
"already handled above with emitAtomicInit");
453 case AtomicExpr::AO__atomic_load_n:
454 case AtomicExpr::AO__c11_atomic_load:
457 case AtomicExpr::AO__atomic_load:
461 case AtomicExpr::AO__atomic_store:
465 case AtomicExpr::AO__atomic_store_n:
466 case AtomicExpr::AO__c11_atomic_store:
479 if (shouldCastToIntPtrTy) {
480 ptr = atomics.castToAtomicIntPointer(ptr);
482 val1 = atomics.convertToAtomicIntPointer(val1);
485 if (shouldCastToIntPtrTy)
486 dest = atomics.castToAtomicIntPointer(dest);
488 dest = atomics.createTempAlloca();
489 if (shouldCastToIntPtrTy)
490 dest = atomics.castToAtomicIntPointer(dest);
493 bool powerOf2Size = (size & (size - 1)) == 0;
494 bool useLibCall = !powerOf2Size || (size > 16);
511 bool isStore = e->
getOp() == AtomicExpr::AO__c11_atomic_store ||
512 e->
getOp() == AtomicExpr::AO__opencl_atomic_store ||
513 e->
getOp() == AtomicExpr::AO__hip_atomic_store ||
514 e->
getOp() == AtomicExpr::AO__atomic_store ||
515 e->
getOp() == AtomicExpr::AO__atomic_store_n ||
516 e->
getOp() == AtomicExpr::AO__scoped_atomic_store ||
517 e->
getOp() == AtomicExpr::AO__scoped_atomic_store_n ||
518 e->
getOp() == AtomicExpr::AO__atomic_clear;
519 bool isLoad = e->
getOp() == AtomicExpr::AO__c11_atomic_load ||
520 e->
getOp() == AtomicExpr::AO__opencl_atomic_load ||
521 e->
getOp() == AtomicExpr::AO__hip_atomic_load ||
522 e->
getOp() == AtomicExpr::AO__atomic_load ||
523 e->
getOp() == AtomicExpr::AO__atomic_load_n ||
524 e->
getOp() == AtomicExpr::AO__scoped_atomic_load ||
525 e->
getOp() == AtomicExpr::AO__scoped_atomic_load_n;
531 uint64_t ord = orderConst.
Val.
getInt().getZExtValue();
534 static_cast<cir::MemOrder
>(ord));
552 switch (atomics.getEvaluationKind()) {
568 llvm_unreachable(
"bad evaluation kind");
static Address emitValToTemp(CIRGenFunction &cgf, Expr *e)
static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest, Address ptr, Address val1, uint64_t size, cir::MemOrder order)
static bool isMemOrderValid(uint64_t order, bool isStore, bool isLoad)
static bool isFullSizeType(CIRGenModule &cgm, mlir::Type ty, uint64_t expectedSize)
Does a store of the given IR type modify the full expected width?
llvm::TypeSize getTypeSizeInBits(mlir::Type ty) const
llvm::TypeSize getTypeStoreSize(mlir::Type ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type.
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
TypeInfo getTypeInfo(const Type *T) const
Get the size and alignment of the specified complete type in bits.
TypeInfoChars getTypeInfoInChars(const Type *T) const
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
AtomicExpr - Variadic atomic builtins: __atomic_exchange, __atomic_fetch_*, __atomic_load,...
mlir::Type getElementType() const
Address withElementType(CIRGenBuilderTy &builder, mlir::Type ElemTy) const
Return address with different element type, a bitcast pointer, and the same alignment.
cir::LoadOp createLoad(mlir::Location loc, Address addr, bool isVolatile=false)
cir::IntType getUIntNTy(int n)
cir::StoreOp createStore(mlir::Location loc, mlir::Value val, Address dst, bool isVolatile=false, mlir::IntegerAttr align={}, cir::MemOrderAttr order={})
static cir::TypeEvaluationKind getEvaluationKind(clang::QualType type)
Return the cir::TypeEvaluationKind of QualType type.
RValue convertTempToRValue(Address addr, clang::QualType type, clang::SourceLocation loc)
Given the address of a temporary variable, produce an r-value of its type.
Address emitPointerWithAlignment(const clang::Expr *expr, LValueBaseInfo *baseInfo=nullptr)
Given an expression with a pointer type, emit the value and compute our best estimate of the alignmen...
void emitStoreOfScalar(mlir::Value value, Address addr, bool isVolatile, clang::QualType ty, bool isInit=false, bool isNontemporal=false)
mlir::Location getLoc(clang::SourceLocation srcLoc)
Helpers to convert Clang's SourceLocation to a MLIR Location.
void emitAnyExprToMem(const Expr *e, Address location, Qualifiers quals, bool isInitializer)
Emits the code necessary to evaluate an arbitrary expression into the given memory location.
RValue emitAtomicExpr(AtomicExpr *e)
mlir::Type convertTypeForMem(QualType t)
mlir::Value emitScalarExpr(const clang::Expr *e)
Emit the computation of the specified expression of scalar type.
CIRGenBuilderTy & getBuilder()
void emitAtomicInit(Expr *init, LValue dest)
LValue makeAddrLValue(Address addr, QualType ty, AlignmentSource source=AlignmentSource::Type)
clang::ASTContext & getContext() const
Address createMemTemp(QualType t, mlir::Location loc, const Twine &name="tmp", Address *alloca=nullptr, mlir::OpBuilder::InsertPoint ip={})
Create a temporary memory object of the given type, with appropriate alignmen and cast it to the defa...
This class organizes the cross-function state that is used while generating CIR code.
DiagnosticBuilder errorNYI(SourceLocation, llvm::StringRef)
Helpers to emit "not yet implemented" error diagnostics.
const cir::CIRDataLayout getDataLayout() const
Address getAddress() const
clang::QualType getType() const
mlir::Value getPointer() const
void setAlignment(clang::CharUnits a)
clang::CharUnits getAlignment() const
LValueBaseInfo getBaseInfo() const
This trivial value class is used to represent the result of an expression that is evaluated.
static RValue get(mlir::Value v)
mlir::Value getValue() const
Return the value of this scalar value.
CharUnits - This is an opaque type for sizes expressed in character units.
bool isZero() const
isZero - Test whether the quantity equals zero.
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
This represents one expression.
bool EvaluateAsInt(EvalResult &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer,...
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
A (possibly-)qualified type.
Qualifiers getQualifiers() const
Retrieve the set of qualifiers applied to this type.
QualType getUnqualifiedType() const
Retrieve the unqualified variant of the given type, removing as little sugar as possible.
SourceRange getSourceRange() const LLVM_READONLY
SourceLocation tokens are not useful in isolation - they are low level value objects created/interpre...
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
const T * getAs() const
Member-template getAs<specific type>'.
bool isValidCIRAtomicOrderingCABI(Int value)
const internal::VariadicDynCastAllOfMatcher< Stmt, Expr > expr
Matches expressions.
The JSON file list parser is used to communicate input to InstallAPI.
static bool atomicInfoGetAtomicPointer()
static bool atomicScope()
static bool atomicUseLibCall()
static bool atomicSyncScopeID()
static bool atomicInfoGetAtomicAddress()
EvalResult is a struct with detailed info about an evaluated expression.
APValue Val
Val - This is the value the expression can be folded to.