clang 22.0.0git
CGAtomic.cpp
Go to the documentation of this file.
1//===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the code for emitting atomic operations.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGCall.h"
14#include "CGRecordLayout.h"
15#include "CodeGenFunction.h"
16#include "CodeGenModule.h"
17#include "TargetInfo.h"
21#include "llvm/ADT/DenseMap.h"
22#include "llvm/IR/DataLayout.h"
23#include "llvm/IR/Intrinsics.h"
24
25using namespace clang;
26using namespace CodeGen;
27
28namespace {
29 class AtomicInfo {
30 CodeGenFunction &CGF;
31 QualType AtomicTy;
32 QualType ValueTy;
33 uint64_t AtomicSizeInBits;
34 uint64_t ValueSizeInBits;
35 CharUnits AtomicAlign;
36 CharUnits ValueAlign;
37 TypeEvaluationKind EvaluationKind;
38 bool UseLibcall;
39 LValue LVal;
40 CGBitFieldInfo BFI;
41 public:
42 AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
43 : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
44 EvaluationKind(TEK_Scalar), UseLibcall(true) {
45 assert(!lvalue.isGlobalReg());
46 ASTContext &C = CGF.getContext();
47 if (lvalue.isSimple()) {
48 AtomicTy = lvalue.getType();
49 if (auto *ATy = AtomicTy->getAs<AtomicType>())
50 ValueTy = ATy->getValueType();
51 else
52 ValueTy = AtomicTy;
53 EvaluationKind = CGF.getEvaluationKind(ValueTy);
54
55 uint64_t ValueAlignInBits;
56 uint64_t AtomicAlignInBits;
57 TypeInfo ValueTI = C.getTypeInfo(ValueTy);
58 ValueSizeInBits = ValueTI.Width;
59 ValueAlignInBits = ValueTI.Align;
60
61 TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
62 AtomicSizeInBits = AtomicTI.Width;
63 AtomicAlignInBits = AtomicTI.Align;
64
65 assert(ValueSizeInBits <= AtomicSizeInBits);
66 assert(ValueAlignInBits <= AtomicAlignInBits);
67
68 AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
69 ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
70 if (lvalue.getAlignment().isZero())
71 lvalue.setAlignment(AtomicAlign);
72
73 LVal = lvalue;
74 } else if (lvalue.isBitField()) {
75 ValueTy = lvalue.getType();
76 ValueSizeInBits = C.getTypeSize(ValueTy);
77 auto &OrigBFI = lvalue.getBitFieldInfo();
78 auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
79 AtomicSizeInBits = C.toBits(
80 C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
81 .alignTo(lvalue.getAlignment()));
82 llvm::Value *BitFieldPtr = lvalue.getRawBitFieldPointer(CGF);
83 auto OffsetInChars =
84 (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
85 lvalue.getAlignment();
86 llvm::Value *StoragePtr = CGF.Builder.CreateConstGEP1_64(
87 CGF.Int8Ty, BitFieldPtr, OffsetInChars.getQuantity());
88 StoragePtr = CGF.Builder.CreateAddrSpaceCast(
89 StoragePtr, CGF.UnqualPtrTy, "atomic_bitfield_base");
90 BFI = OrigBFI;
91 BFI.Offset = Offset;
92 BFI.StorageSize = AtomicSizeInBits;
93 BFI.StorageOffset += OffsetInChars;
94 llvm::Type *StorageTy = CGF.Builder.getIntNTy(AtomicSizeInBits);
95 LVal = LValue::MakeBitfield(
96 Address(StoragePtr, StorageTy, lvalue.getAlignment()), BFI,
97 lvalue.getType(), lvalue.getBaseInfo(), lvalue.getTBAAInfo());
98 AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
99 if (AtomicTy.isNull()) {
100 llvm::APInt Size(
101 /*numBits=*/32,
102 C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
103 AtomicTy = C.getConstantArrayType(C.CharTy, Size, nullptr,
104 ArraySizeModifier::Normal,
105 /*IndexTypeQuals=*/0);
106 }
107 AtomicAlign = ValueAlign = lvalue.getAlignment();
108 } else if (lvalue.isVectorElt()) {
109 ValueTy = lvalue.getType()->castAs<VectorType>()->getElementType();
110 ValueSizeInBits = C.getTypeSize(ValueTy);
111 AtomicTy = lvalue.getType();
112 AtomicSizeInBits = C.getTypeSize(AtomicTy);
113 AtomicAlign = ValueAlign = lvalue.getAlignment();
114 LVal = lvalue;
115 } else {
116 assert(lvalue.isExtVectorElt());
117 ValueTy = lvalue.getType();
118 ValueSizeInBits = C.getTypeSize(ValueTy);
119 AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
120 lvalue.getType(), cast<llvm::FixedVectorType>(
121 lvalue.getExtVectorAddress().getElementType())
122 ->getNumElements());
123 AtomicSizeInBits = C.getTypeSize(AtomicTy);
124 AtomicAlign = ValueAlign = lvalue.getAlignment();
125 LVal = lvalue;
126 }
127 UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
128 AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
129 }
130
131 QualType getAtomicType() const { return AtomicTy; }
132 QualType getValueType() const { return ValueTy; }
133 CharUnits getAtomicAlignment() const { return AtomicAlign; }
134 uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
135 uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
136 TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
137 bool shouldUseLibcall() const { return UseLibcall; }
138 const LValue &getAtomicLValue() const { return LVal; }
139 llvm::Value *getAtomicPointer() const {
140 if (LVal.isSimple())
141 return LVal.emitRawPointer(CGF);
142 else if (LVal.isBitField())
143 return LVal.getRawBitFieldPointer(CGF);
144 else if (LVal.isVectorElt())
145 return LVal.getRawVectorPointer(CGF);
146 assert(LVal.isExtVectorElt());
147 return LVal.getRawExtVectorPointer(CGF);
148 }
149 Address getAtomicAddress() const {
150 llvm::Type *ElTy;
151 if (LVal.isSimple())
152 ElTy = LVal.getAddress().getElementType();
153 else if (LVal.isBitField())
154 ElTy = LVal.getBitFieldAddress().getElementType();
155 else if (LVal.isVectorElt())
156 ElTy = LVal.getVectorAddress().getElementType();
157 else
158 ElTy = LVal.getExtVectorAddress().getElementType();
159 return Address(getAtomicPointer(), ElTy, getAtomicAlignment());
160 }
161
162 Address getAtomicAddressAsAtomicIntPointer() const {
163 return castToAtomicIntPointer(getAtomicAddress());
164 }
165
166 /// Is the atomic size larger than the underlying value type?
167 ///
168 /// Note that the absence of padding does not mean that atomic
169 /// objects are completely interchangeable with non-atomic
170 /// objects: we might have promoted the alignment of a type
171 /// without making it bigger.
172 bool hasPadding() const {
173 return (ValueSizeInBits != AtomicSizeInBits);
174 }
175
176 bool emitMemSetZeroIfNecessary() const;
177
178 llvm::Value *getAtomicSizeValue() const {
179 CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
180 return CGF.CGM.getSize(size);
181 }
182
183 /// Cast the given pointer to an integer pointer suitable for atomic
184 /// operations if the source.
185 Address castToAtomicIntPointer(Address Addr) const;
186
187 /// If Addr is compatible with the iN that will be used for an atomic
188 /// operation, bitcast it. Otherwise, create a temporary that is suitable
189 /// and copy the value across.
190 Address convertToAtomicIntPointer(Address Addr) const;
191
192 /// Turn an atomic-layout object into an r-value.
193 RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
194 SourceLocation loc, bool AsValue) const;
195
196 llvm::Value *getScalarRValValueOrNull(RValue RVal) const;
197
198 /// Converts an rvalue to integer value if needed.
199 llvm::Value *convertRValueToInt(RValue RVal, bool CmpXchg = false) const;
200
201 RValue ConvertToValueOrAtomic(llvm::Value *IntVal, AggValueSlot ResultSlot,
202 SourceLocation Loc, bool AsValue,
203 bool CmpXchg = false) const;
204
205 /// Copy an atomic r-value into atomic-layout memory.
206 void emitCopyIntoMemory(RValue rvalue) const;
207
208 /// Project an l-value down to the value field.
209 LValue projectValue() const {
210 assert(LVal.isSimple());
211 Address addr = getAtomicAddress();
212 if (hasPadding())
213 addr = CGF.Builder.CreateStructGEP(addr, 0);
214
215 return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
216 LVal.getBaseInfo(), LVal.getTBAAInfo());
217 }
218
219 /// Emits atomic load.
220 /// \returns Loaded value.
221 RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
222 bool AsValue, llvm::AtomicOrdering AO,
223 bool IsVolatile);
224
225 /// Emits atomic compare-and-exchange sequence.
226 /// \param Expected Expected value.
227 /// \param Desired Desired value.
228 /// \param Success Atomic ordering for success operation.
229 /// \param Failure Atomic ordering for failed operation.
230 /// \param IsWeak true if atomic operation is weak, false otherwise.
231 /// \returns Pair of values: previous value from storage (value type) and
232 /// boolean flag (i1 type) with true if success and false otherwise.
233 std::pair<RValue, llvm::Value *>
234 EmitAtomicCompareExchange(RValue Expected, RValue Desired,
235 llvm::AtomicOrdering Success =
236 llvm::AtomicOrdering::SequentiallyConsistent,
237 llvm::AtomicOrdering Failure =
238 llvm::AtomicOrdering::SequentiallyConsistent,
239 bool IsWeak = false);
240
241 /// Emits atomic update.
242 /// \param AO Atomic ordering.
243 /// \param UpdateOp Update operation for the current lvalue.
244 void EmitAtomicUpdate(llvm::AtomicOrdering AO,
245 const llvm::function_ref<RValue(RValue)> &UpdateOp,
246 bool IsVolatile);
247 /// Emits atomic update.
248 /// \param AO Atomic ordering.
249 void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
250 bool IsVolatile);
251
252 /// Materialize an atomic r-value in atomic-layout memory.
253 Address materializeRValue(RValue rvalue) const;
254
255 /// Creates temp alloca for intermediate operations on atomic value.
256 Address CreateTempAlloca() const;
257 private:
258 bool requiresMemSetZero(llvm::Type *type) const;
259
260
261 /// Emits atomic load as a libcall.
262 void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
263 llvm::AtomicOrdering AO, bool IsVolatile);
264 /// Emits atomic load as LLVM instruction.
265 llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile,
266 bool CmpXchg = false);
267 /// Emits atomic compare-and-exchange op as a libcall.
268 llvm::Value *EmitAtomicCompareExchangeLibcall(
269 llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
270 llvm::AtomicOrdering Success =
271 llvm::AtomicOrdering::SequentiallyConsistent,
272 llvm::AtomicOrdering Failure =
273 llvm::AtomicOrdering::SequentiallyConsistent);
274 /// Emits atomic compare-and-exchange op as LLVM instruction.
275 std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
276 llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
277 llvm::AtomicOrdering Success =
278 llvm::AtomicOrdering::SequentiallyConsistent,
279 llvm::AtomicOrdering Failure =
280 llvm::AtomicOrdering::SequentiallyConsistent,
281 bool IsWeak = false);
282 /// Emit atomic update as libcalls.
283 void
284 EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
285 const llvm::function_ref<RValue(RValue)> &UpdateOp,
286 bool IsVolatile);
287 /// Emit atomic update as LLVM instructions.
288 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
289 const llvm::function_ref<RValue(RValue)> &UpdateOp,
290 bool IsVolatile);
291 /// Emit atomic update as libcalls.
292 void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
293 bool IsVolatile);
294 /// Emit atomic update as LLVM instructions.
295 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
296 bool IsVolatile);
297 };
298}
299
300Address AtomicInfo::CreateTempAlloca() const {
301 Address TempAlloca = CGF.CreateMemTemp(
302 (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
303 : AtomicTy,
304 getAtomicAlignment(),
305 "atomic-temp");
306 // Cast to pointer to value type for bitfields.
307 if (LVal.isBitField())
309 TempAlloca, getAtomicAddress().getType(),
310 getAtomicAddress().getElementType());
311 return TempAlloca;
312}
313
315 StringRef fnName,
316 QualType resultType,
317 CallArgList &args) {
318 const CGFunctionInfo &fnInfo =
319 CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args);
320 llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
321 llvm::AttrBuilder fnAttrB(CGF.getLLVMContext());
322 fnAttrB.addAttribute(llvm::Attribute::NoUnwind);
323 fnAttrB.addAttribute(llvm::Attribute::WillReturn);
324 llvm::AttributeList fnAttrs = llvm::AttributeList::get(
325 CGF.getLLVMContext(), llvm::AttributeList::FunctionIndex, fnAttrB);
326
327 llvm::FunctionCallee fn =
328 CGF.CGM.CreateRuntimeFunction(fnTy, fnName, fnAttrs);
329 auto callee = CGCallee::forDirect(fn);
330 return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args);
331}
332
333/// Does a store of the given IR type modify the full expected width?
334static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
335 uint64_t expectedSize) {
336 return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
337}
338
339/// Does the atomic type require memsetting to zero before initialization?
340///
341/// The IR type is provided as a way of making certain queries faster.
342bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
343 // If the atomic type has size padding, we definitely need a memset.
344 if (hasPadding()) return true;
345
346 // Otherwise, do some simple heuristics to try to avoid it:
347 switch (getEvaluationKind()) {
348 // For scalars and complexes, check whether the store size of the
349 // type uses the full size.
350 case TEK_Scalar:
351 return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
352 case TEK_Complex:
353 return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
354 AtomicSizeInBits / 2);
355
356 // Padding in structs has an undefined bit pattern. User beware.
357 case TEK_Aggregate:
358 return false;
359 }
360 llvm_unreachable("bad evaluation kind");
361}
362
363bool AtomicInfo::emitMemSetZeroIfNecessary() const {
364 assert(LVal.isSimple());
365 Address addr = LVal.getAddress();
366 if (!requiresMemSetZero(addr.getElementType()))
367 return false;
368
370 addr.emitRawPointer(CGF), llvm::ConstantInt::get(CGF.Int8Ty, 0),
371 CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
372 LVal.getAlignment().getAsAlign());
373 return true;
374}
375
376static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
377 Address Dest, Address Ptr,
378 Address Val1, Address Val2,
379 uint64_t Size,
380 llvm::AtomicOrdering SuccessOrder,
381 llvm::AtomicOrdering FailureOrder,
382 llvm::SyncScope::ID Scope) {
383 // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
384 llvm::Value *Expected = CGF.Builder.CreateLoad(Val1);
385 llvm::Value *Desired = CGF.Builder.CreateLoad(Val2);
386
387 llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
388 Ptr, Expected, Desired, SuccessOrder, FailureOrder, Scope);
389 Pair->setVolatile(E->isVolatile());
390 Pair->setWeak(IsWeak);
391 CGF.getTargetHooks().setTargetAtomicMetadata(CGF, *Pair, E);
392
393 // Cmp holds the result of the compare-exchange operation: true on success,
394 // false on failure.
395 llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
396 llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
397
398 // This basic block is used to hold the store instruction if the operation
399 // failed.
400 llvm::BasicBlock *StoreExpectedBB =
401 CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
402
403 // This basic block is the exit point of the operation, we should end up
404 // here regardless of whether or not the operation succeeded.
405 llvm::BasicBlock *ContinueBB =
406 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
407
408 // Update Expected if Expected isn't equal to Old, otherwise branch to the
409 // exit point.
410 CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
411
412 CGF.Builder.SetInsertPoint(StoreExpectedBB);
413 // Update the memory at Expected with Old's value.
414 auto *I = CGF.Builder.CreateStore(Old, Val1);
415 CGF.addInstToCurrentSourceAtom(I, Old);
416
417 // Finally, branch to the exit point.
418 CGF.Builder.CreateBr(ContinueBB);
419
420 CGF.Builder.SetInsertPoint(ContinueBB);
421 // Update the memory at Dest with Cmp's value.
422 CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
423}
424
425/// Given an ordering required on success, emit all possible cmpxchg
426/// instructions to cope with the provided (but possibly only dynamically known)
427/// FailureOrder.
429 bool IsWeak, Address Dest, Address Ptr,
430 Address Val1, Address Val2,
431 llvm::Value *FailureOrderVal,
432 uint64_t Size,
433 llvm::AtomicOrdering SuccessOrder,
434 llvm::SyncScope::ID Scope) {
435 llvm::AtomicOrdering FailureOrder;
436 if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
437 auto FOS = FO->getSExtValue();
438 if (!llvm::isValidAtomicOrderingCABI(FOS))
439 FailureOrder = llvm::AtomicOrdering::Monotonic;
440 else
441 switch ((llvm::AtomicOrderingCABI)FOS) {
442 case llvm::AtomicOrderingCABI::relaxed:
443 // 31.7.2.18: "The failure argument shall not be memory_order_release
444 // nor memory_order_acq_rel". Fallback to monotonic.
445 case llvm::AtomicOrderingCABI::release:
446 case llvm::AtomicOrderingCABI::acq_rel:
447 FailureOrder = llvm::AtomicOrdering::Monotonic;
448 break;
449 case llvm::AtomicOrderingCABI::consume:
450 case llvm::AtomicOrderingCABI::acquire:
451 FailureOrder = llvm::AtomicOrdering::Acquire;
452 break;
453 case llvm::AtomicOrderingCABI::seq_cst:
454 FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent;
455 break;
456 }
457 // Prior to c++17, "the failure argument shall be no stronger than the
458 // success argument". This condition has been lifted and the only
459 // precondition is 31.7.2.18. Effectively treat this as a DR and skip
460 // language version checks.
461 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
462 FailureOrder, Scope);
463 return;
464 }
465
466 // Create all the relevant BB's
467 auto *MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
468 auto *AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
469 auto *SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
470 auto *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
471
472 // MonotonicBB is arbitrarily chosen as the default case; in practice, this
473 // doesn't matter unless someone is crazy enough to use something that
474 // doesn't fold to a constant for the ordering.
475 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
476 // Implemented as acquire, since it's the closest in LLVM.
477 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
478 AcquireBB);
479 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
480 AcquireBB);
481 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
482 SeqCstBB);
483
484 // Emit all the different atomics
485 CGF.Builder.SetInsertPoint(MonotonicBB);
486 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
487 Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope);
488 CGF.Builder.CreateBr(ContBB);
489
490 CGF.Builder.SetInsertPoint(AcquireBB);
491 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
492 llvm::AtomicOrdering::Acquire, Scope);
493 CGF.Builder.CreateBr(ContBB);
494
495 CGF.Builder.SetInsertPoint(SeqCstBB);
496 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
497 llvm::AtomicOrdering::SequentiallyConsistent, Scope);
498 CGF.Builder.CreateBr(ContBB);
499
500 CGF.Builder.SetInsertPoint(ContBB);
501}
502
503/// Duplicate the atomic min/max operation in conventional IR for the builtin
504/// variants that return the new rather than the original value.
505static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder,
507 bool IsSigned,
508 llvm::Value *OldVal,
509 llvm::Value *RHS) {
510 const bool IsFP = OldVal->getType()->isFloatingPointTy();
511
512 if (IsFP) {
513 llvm::Intrinsic::ID IID = (Op == AtomicExpr::AO__atomic_max_fetch ||
514 Op == AtomicExpr::AO__scoped_atomic_max_fetch)
515 ? llvm::Intrinsic::maxnum
516 : llvm::Intrinsic::minnum;
517
518 return Builder.CreateBinaryIntrinsic(IID, OldVal, RHS, llvm::FMFSource(),
519 "newval");
520 }
521
522 llvm::CmpInst::Predicate Pred;
523 switch (Op) {
524 default:
525 llvm_unreachable("Unexpected min/max operation");
526 case AtomicExpr::AO__atomic_max_fetch:
527 case AtomicExpr::AO__scoped_atomic_max_fetch:
528 Pred = IsSigned ? llvm::CmpInst::ICMP_SGT : llvm::CmpInst::ICMP_UGT;
529 break;
530 case AtomicExpr::AO__atomic_min_fetch:
531 case AtomicExpr::AO__scoped_atomic_min_fetch:
532 Pred = IsSigned ? llvm::CmpInst::ICMP_SLT : llvm::CmpInst::ICMP_ULT;
533 break;
534 }
535 llvm::Value *Cmp = Builder.CreateICmp(Pred, OldVal, RHS, "tst");
536 return Builder.CreateSelect(Cmp, OldVal, RHS, "newval");
537}
538
540 Address Ptr, Address Val1, Address Val2,
541 llvm::Value *IsWeak, llvm::Value *FailureOrder,
542 uint64_t Size, llvm::AtomicOrdering Order,
543 llvm::SyncScope::ID Scope) {
544 llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
545 bool PostOpMinMax = false;
546 unsigned PostOp = 0;
547
548 switch (E->getOp()) {
549 case AtomicExpr::AO__c11_atomic_init:
550 case AtomicExpr::AO__opencl_atomic_init:
551 llvm_unreachable("Already handled!");
552
553 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
554 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
555 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
556 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
557 FailureOrder, Size, Order, Scope);
558 return;
559 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
560 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
561 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
562 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
563 FailureOrder, Size, Order, Scope);
564 return;
565 case AtomicExpr::AO__atomic_compare_exchange:
566 case AtomicExpr::AO__atomic_compare_exchange_n:
567 case AtomicExpr::AO__scoped_atomic_compare_exchange:
568 case AtomicExpr::AO__scoped_atomic_compare_exchange_n: {
569 if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
570 emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
571 Val1, Val2, FailureOrder, Size, Order, Scope);
572 } else {
573 // Create all the relevant BB's
574 llvm::BasicBlock *StrongBB =
575 CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
576 llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
577 llvm::BasicBlock *ContBB =
578 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
579
580 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
581 SI->addCase(CGF.Builder.getInt1(false), StrongBB);
582
583 CGF.Builder.SetInsertPoint(StrongBB);
584 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
585 FailureOrder, Size, Order, Scope);
586 CGF.Builder.CreateBr(ContBB);
587
588 CGF.Builder.SetInsertPoint(WeakBB);
589 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
590 FailureOrder, Size, Order, Scope);
591 CGF.Builder.CreateBr(ContBB);
592
593 CGF.Builder.SetInsertPoint(ContBB);
594 }
595 return;
596 }
597 case AtomicExpr::AO__c11_atomic_load:
598 case AtomicExpr::AO__opencl_atomic_load:
599 case AtomicExpr::AO__hip_atomic_load:
600 case AtomicExpr::AO__atomic_load_n:
601 case AtomicExpr::AO__atomic_load:
602 case AtomicExpr::AO__scoped_atomic_load_n:
603 case AtomicExpr::AO__scoped_atomic_load: {
604 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
605 Load->setAtomic(Order, Scope);
606 Load->setVolatile(E->isVolatile());
607 CGF.maybeAttachRangeForLoad(Load, E->getValueType(), E->getExprLoc());
608 auto *I = CGF.Builder.CreateStore(Load, Dest);
609 CGF.addInstToCurrentSourceAtom(I, Load);
610 return;
611 }
612
613 case AtomicExpr::AO__c11_atomic_store:
614 case AtomicExpr::AO__opencl_atomic_store:
615 case AtomicExpr::AO__hip_atomic_store:
616 case AtomicExpr::AO__atomic_store:
617 case AtomicExpr::AO__atomic_store_n:
618 case AtomicExpr::AO__scoped_atomic_store:
619 case AtomicExpr::AO__scoped_atomic_store_n: {
620 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
621 llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
622 Store->setAtomic(Order, Scope);
623 Store->setVolatile(E->isVolatile());
624 CGF.addInstToCurrentSourceAtom(Store, LoadVal1);
625 return;
626 }
627
628 case AtomicExpr::AO__c11_atomic_exchange:
629 case AtomicExpr::AO__hip_atomic_exchange:
630 case AtomicExpr::AO__opencl_atomic_exchange:
631 case AtomicExpr::AO__atomic_exchange_n:
632 case AtomicExpr::AO__atomic_exchange:
633 case AtomicExpr::AO__scoped_atomic_exchange_n:
634 case AtomicExpr::AO__scoped_atomic_exchange:
635 Op = llvm::AtomicRMWInst::Xchg;
636 break;
637
638 case AtomicExpr::AO__atomic_add_fetch:
639 case AtomicExpr::AO__scoped_atomic_add_fetch:
640 PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FAdd
641 : llvm::Instruction::Add;
642 [[fallthrough]];
643 case AtomicExpr::AO__c11_atomic_fetch_add:
644 case AtomicExpr::AO__hip_atomic_fetch_add:
645 case AtomicExpr::AO__opencl_atomic_fetch_add:
646 case AtomicExpr::AO__atomic_fetch_add:
647 case AtomicExpr::AO__scoped_atomic_fetch_add:
648 Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FAdd
649 : llvm::AtomicRMWInst::Add;
650 break;
651
652 case AtomicExpr::AO__atomic_sub_fetch:
653 case AtomicExpr::AO__scoped_atomic_sub_fetch:
654 PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FSub
655 : llvm::Instruction::Sub;
656 [[fallthrough]];
657 case AtomicExpr::AO__c11_atomic_fetch_sub:
658 case AtomicExpr::AO__hip_atomic_fetch_sub:
659 case AtomicExpr::AO__opencl_atomic_fetch_sub:
660 case AtomicExpr::AO__atomic_fetch_sub:
661 case AtomicExpr::AO__scoped_atomic_fetch_sub:
662 Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FSub
663 : llvm::AtomicRMWInst::Sub;
664 break;
665
666 case AtomicExpr::AO__atomic_min_fetch:
667 case AtomicExpr::AO__scoped_atomic_min_fetch:
668 PostOpMinMax = true;
669 [[fallthrough]];
670 case AtomicExpr::AO__c11_atomic_fetch_min:
671 case AtomicExpr::AO__hip_atomic_fetch_min:
672 case AtomicExpr::AO__opencl_atomic_fetch_min:
673 case AtomicExpr::AO__atomic_fetch_min:
674 case AtomicExpr::AO__scoped_atomic_fetch_min:
675 Op = E->getValueType()->isFloatingType()
676 ? llvm::AtomicRMWInst::FMin
677 : (E->getValueType()->isSignedIntegerType()
678 ? llvm::AtomicRMWInst::Min
679 : llvm::AtomicRMWInst::UMin);
680 break;
681
682 case AtomicExpr::AO__atomic_max_fetch:
683 case AtomicExpr::AO__scoped_atomic_max_fetch:
684 PostOpMinMax = true;
685 [[fallthrough]];
686 case AtomicExpr::AO__c11_atomic_fetch_max:
687 case AtomicExpr::AO__hip_atomic_fetch_max:
688 case AtomicExpr::AO__opencl_atomic_fetch_max:
689 case AtomicExpr::AO__atomic_fetch_max:
690 case AtomicExpr::AO__scoped_atomic_fetch_max:
691 Op = E->getValueType()->isFloatingType()
692 ? llvm::AtomicRMWInst::FMax
693 : (E->getValueType()->isSignedIntegerType()
694 ? llvm::AtomicRMWInst::Max
695 : llvm::AtomicRMWInst::UMax);
696 break;
697
698 case AtomicExpr::AO__atomic_and_fetch:
699 case AtomicExpr::AO__scoped_atomic_and_fetch:
700 PostOp = llvm::Instruction::And;
701 [[fallthrough]];
702 case AtomicExpr::AO__c11_atomic_fetch_and:
703 case AtomicExpr::AO__hip_atomic_fetch_and:
704 case AtomicExpr::AO__opencl_atomic_fetch_and:
705 case AtomicExpr::AO__atomic_fetch_and:
706 case AtomicExpr::AO__scoped_atomic_fetch_and:
707 Op = llvm::AtomicRMWInst::And;
708 break;
709
710 case AtomicExpr::AO__atomic_or_fetch:
711 case AtomicExpr::AO__scoped_atomic_or_fetch:
712 PostOp = llvm::Instruction::Or;
713 [[fallthrough]];
714 case AtomicExpr::AO__c11_atomic_fetch_or:
715 case AtomicExpr::AO__hip_atomic_fetch_or:
716 case AtomicExpr::AO__opencl_atomic_fetch_or:
717 case AtomicExpr::AO__atomic_fetch_or:
718 case AtomicExpr::AO__scoped_atomic_fetch_or:
719 Op = llvm::AtomicRMWInst::Or;
720 break;
721
722 case AtomicExpr::AO__atomic_xor_fetch:
723 case AtomicExpr::AO__scoped_atomic_xor_fetch:
724 PostOp = llvm::Instruction::Xor;
725 [[fallthrough]];
726 case AtomicExpr::AO__c11_atomic_fetch_xor:
727 case AtomicExpr::AO__hip_atomic_fetch_xor:
728 case AtomicExpr::AO__opencl_atomic_fetch_xor:
729 case AtomicExpr::AO__atomic_fetch_xor:
730 case AtomicExpr::AO__scoped_atomic_fetch_xor:
731 Op = llvm::AtomicRMWInst::Xor;
732 break;
733
734 case AtomicExpr::AO__atomic_nand_fetch:
735 case AtomicExpr::AO__scoped_atomic_nand_fetch:
736 PostOp = llvm::Instruction::And; // the NOT is special cased below
737 [[fallthrough]];
738 case AtomicExpr::AO__c11_atomic_fetch_nand:
739 case AtomicExpr::AO__atomic_fetch_nand:
740 case AtomicExpr::AO__scoped_atomic_fetch_nand:
741 Op = llvm::AtomicRMWInst::Nand;
742 break;
743
744 case AtomicExpr::AO__atomic_test_and_set: {
745 llvm::AtomicRMWInst *RMWI =
746 CGF.emitAtomicRMWInst(llvm::AtomicRMWInst::Xchg, Ptr,
747 CGF.Builder.getInt8(1), Order, Scope, E);
748 RMWI->setVolatile(E->isVolatile());
749 llvm::Value *Result = CGF.EmitToMemory(
750 CGF.Builder.CreateIsNotNull(RMWI, "tobool"), E->getType());
751 auto *I = CGF.Builder.CreateStore(Result, Dest);
752 CGF.addInstToCurrentSourceAtom(I, Result);
753 return;
754 }
755
756 case AtomicExpr::AO__atomic_clear: {
757 llvm::StoreInst *Store =
758 CGF.Builder.CreateStore(CGF.Builder.getInt8(0), Ptr);
759 Store->setAtomic(Order, Scope);
760 Store->setVolatile(E->isVolatile());
761 CGF.addInstToCurrentSourceAtom(Store, nullptr);
762 return;
763 }
764 }
765
766 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
767 llvm::AtomicRMWInst *RMWI =
768 CGF.emitAtomicRMWInst(Op, Ptr, LoadVal1, Order, Scope, E);
769 RMWI->setVolatile(E->isVolatile());
770
771 // For __atomic_*_fetch operations, perform the operation again to
772 // determine the value which was written.
773 llvm::Value *Result = RMWI;
774 if (PostOpMinMax)
775 Result = EmitPostAtomicMinMax(CGF.Builder, E->getOp(),
777 RMWI, LoadVal1);
778 else if (PostOp)
779 Result = CGF.Builder.CreateBinOp((llvm::Instruction::BinaryOps)PostOp, RMWI,
780 LoadVal1);
781 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch ||
782 E->getOp() == AtomicExpr::AO__scoped_atomic_nand_fetch)
783 Result = CGF.Builder.CreateNot(Result);
784 auto *I = CGF.Builder.CreateStore(Result, Dest);
785 CGF.addInstToCurrentSourceAtom(I, Result);
786}
787
788// This function emits any expression (scalar, complex, or aggregate)
789// into a temporary alloca.
790static Address
792 Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
793 CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
794 /*Init*/ true);
795 return DeclPtr;
796}
797
799 Address Ptr, Address Val1, Address Val2,
800 llvm::Value *IsWeak, llvm::Value *FailureOrder,
801 uint64_t Size, llvm::AtomicOrdering Order,
802 llvm::Value *Scope) {
803 auto ScopeModel = Expr->getScopeModel();
804
805 // LLVM atomic instructions always have sync scope. If clang atomic
806 // expression has no scope operand, use default LLVM sync scope.
807 if (!ScopeModel) {
808 llvm::SyncScope::ID SS;
809 if (CGF.getLangOpts().OpenCL)
810 // OpenCL approach is: "The functions that do not have memory_scope
811 // argument have the same semantics as the corresponding functions with
812 // the memory_scope argument set to memory_scope_device." See ref.:
813 // https://registry.khronos.org/OpenCL/specs/3.0-unified/html/OpenCL_C.html#atomic-functions
816 Order, CGF.getLLVMContext());
817 else
818 SS = llvm::SyncScope::System;
819 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
820 Order, SS);
821 return;
822 }
823
824 // Handle constant scope.
825 if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) {
826 auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID(
827 CGF.CGM.getLangOpts(), ScopeModel->map(SC->getZExtValue()),
828 Order, CGF.CGM.getLLVMContext());
829 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
830 Order, SCID);
831 return;
832 }
833
834 // Handle non-constant scope.
835 auto &Builder = CGF.Builder;
836 auto Scopes = ScopeModel->getRuntimeValues();
837 llvm::DenseMap<unsigned, llvm::BasicBlock *> BB;
838 for (auto S : Scopes)
839 BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn);
840
841 llvm::BasicBlock *ContBB =
842 CGF.createBasicBlock("atomic.scope.continue", CGF.CurFn);
843
844 auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false);
845 // If unsupported sync scope is encountered at run time, assume a fallback
846 // sync scope value.
847 auto FallBack = ScopeModel->getFallBackValue();
848 llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]);
849 for (auto S : Scopes) {
850 auto *B = BB[S];
851 if (S != FallBack)
852 SI->addCase(Builder.getInt32(S), B);
853
854 Builder.SetInsertPoint(B);
855 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
856 Order,
858 ScopeModel->map(S),
859 Order,
860 CGF.getLLVMContext()));
861 Builder.CreateBr(ContBB);
862 }
863
864 Builder.SetInsertPoint(ContBB);
865}
866
869
870 QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
871 QualType MemTy = AtomicTy;
872 if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
873 MemTy = AT->getValueType();
874 llvm::Value *IsWeak = nullptr, *OrderFail = nullptr;
875
876 Address Val1 = Address::invalid();
877 Address Val2 = Address::invalid();
878 Address Dest = Address::invalid();
880
881 if (E->getOp() == AtomicExpr::AO__c11_atomic_init ||
882 E->getOp() == AtomicExpr::AO__opencl_atomic_init) {
883 LValue lvalue = MakeAddrLValue(Ptr, AtomicTy);
884 EmitAtomicInit(E->getVal1(), lvalue);
885 return RValue::get(nullptr);
886 }
887
888 auto TInfo = getContext().getTypeInfoInChars(AtomicTy);
889 uint64_t Size = TInfo.Width.getQuantity();
890 unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();
891
892 CharUnits MaxInlineWidth =
893 getContext().toCharUnitsFromBits(MaxInlineWidthInBits);
894 DiagnosticsEngine &Diags = CGM.getDiags();
895 bool Misaligned = !Ptr.getAlignment().isMultipleOf(TInfo.Width);
896 bool Oversized = getContext().toBits(TInfo.Width) > MaxInlineWidthInBits;
897 if (Misaligned) {
898 Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned)
899 << (int)TInfo.Width.getQuantity()
900 << (int)Ptr.getAlignment().getQuantity();
901 }
902 if (Oversized) {
903 Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_oversized)
904 << (int)TInfo.Width.getQuantity() << (int)MaxInlineWidth.getQuantity();
905 }
906
907 llvm::Value *Order = EmitScalarExpr(E->getOrder());
908 llvm::Value *Scope =
909 E->getScopeModel() ? EmitScalarExpr(E->getScope()) : nullptr;
910 bool ShouldCastToIntPtrTy = true;
911
912 switch (E->getOp()) {
913 case AtomicExpr::AO__c11_atomic_init:
914 case AtomicExpr::AO__opencl_atomic_init:
915 llvm_unreachable("Already handled above with EmitAtomicInit!");
916
917 case AtomicExpr::AO__atomic_load_n:
918 case AtomicExpr::AO__scoped_atomic_load_n:
919 case AtomicExpr::AO__c11_atomic_load:
920 case AtomicExpr::AO__opencl_atomic_load:
921 case AtomicExpr::AO__hip_atomic_load:
922 case AtomicExpr::AO__atomic_test_and_set:
923 case AtomicExpr::AO__atomic_clear:
924 break;
925
926 case AtomicExpr::AO__atomic_load:
927 case AtomicExpr::AO__scoped_atomic_load:
929 break;
930
931 case AtomicExpr::AO__atomic_store:
932 case AtomicExpr::AO__scoped_atomic_store:
934 break;
935
936 case AtomicExpr::AO__atomic_exchange:
937 case AtomicExpr::AO__scoped_atomic_exchange:
940 break;
941
942 case AtomicExpr::AO__atomic_compare_exchange:
943 case AtomicExpr::AO__atomic_compare_exchange_n:
944 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
945 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
946 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
947 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
948 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
949 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
950 case AtomicExpr::AO__scoped_atomic_compare_exchange:
951 case AtomicExpr::AO__scoped_atomic_compare_exchange_n:
953 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange ||
954 E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange)
956 else
957 Val2 = EmitValToTemp(*this, E->getVal2());
958 OrderFail = EmitScalarExpr(E->getOrderFail());
959 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n ||
960 E->getOp() == AtomicExpr::AO__atomic_compare_exchange ||
961 E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange_n ||
962 E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange)
963 IsWeak = EmitScalarExpr(E->getWeak());
964 break;
965
966 case AtomicExpr::AO__c11_atomic_fetch_add:
967 case AtomicExpr::AO__c11_atomic_fetch_sub:
968 case AtomicExpr::AO__hip_atomic_fetch_add:
969 case AtomicExpr::AO__hip_atomic_fetch_sub:
970 case AtomicExpr::AO__opencl_atomic_fetch_add:
971 case AtomicExpr::AO__opencl_atomic_fetch_sub:
972 if (MemTy->isPointerType()) {
973 // For pointer arithmetic, we're required to do a bit of math:
974 // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
975 // ... but only for the C11 builtins. The GNU builtins expect the
976 // user to multiply by sizeof(T).
977 QualType Val1Ty = E->getVal1()->getType();
978 llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
979 CharUnits PointeeIncAmt =
980 getContext().getTypeSizeInChars(MemTy->getPointeeType());
981 Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
982 auto Temp = CreateMemTemp(Val1Ty, ".atomictmp");
983 Val1 = Temp;
984 EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty));
985 break;
986 }
987 [[fallthrough]];
988 case AtomicExpr::AO__atomic_fetch_add:
989 case AtomicExpr::AO__atomic_fetch_max:
990 case AtomicExpr::AO__atomic_fetch_min:
991 case AtomicExpr::AO__atomic_fetch_sub:
992 case AtomicExpr::AO__atomic_add_fetch:
993 case AtomicExpr::AO__atomic_max_fetch:
994 case AtomicExpr::AO__atomic_min_fetch:
995 case AtomicExpr::AO__atomic_sub_fetch:
996 case AtomicExpr::AO__c11_atomic_fetch_max:
997 case AtomicExpr::AO__c11_atomic_fetch_min:
998 case AtomicExpr::AO__opencl_atomic_fetch_max:
999 case AtomicExpr::AO__opencl_atomic_fetch_min:
1000 case AtomicExpr::AO__hip_atomic_fetch_max:
1001 case AtomicExpr::AO__hip_atomic_fetch_min:
1002 case AtomicExpr::AO__scoped_atomic_fetch_add:
1003 case AtomicExpr::AO__scoped_atomic_fetch_max:
1004 case AtomicExpr::AO__scoped_atomic_fetch_min:
1005 case AtomicExpr::AO__scoped_atomic_fetch_sub:
1006 case AtomicExpr::AO__scoped_atomic_add_fetch:
1007 case AtomicExpr::AO__scoped_atomic_max_fetch:
1008 case AtomicExpr::AO__scoped_atomic_min_fetch:
1009 case AtomicExpr::AO__scoped_atomic_sub_fetch:
1010 ShouldCastToIntPtrTy = !MemTy->isFloatingType();
1011 [[fallthrough]];
1012
1013 case AtomicExpr::AO__atomic_fetch_and:
1014 case AtomicExpr::AO__atomic_fetch_nand:
1015 case AtomicExpr::AO__atomic_fetch_or:
1016 case AtomicExpr::AO__atomic_fetch_xor:
1017 case AtomicExpr::AO__atomic_and_fetch:
1018 case AtomicExpr::AO__atomic_nand_fetch:
1019 case AtomicExpr::AO__atomic_or_fetch:
1020 case AtomicExpr::AO__atomic_xor_fetch:
1021 case AtomicExpr::AO__atomic_store_n:
1022 case AtomicExpr::AO__atomic_exchange_n:
1023 case AtomicExpr::AO__c11_atomic_fetch_and:
1024 case AtomicExpr::AO__c11_atomic_fetch_nand:
1025 case AtomicExpr::AO__c11_atomic_fetch_or:
1026 case AtomicExpr::AO__c11_atomic_fetch_xor:
1027 case AtomicExpr::AO__c11_atomic_store:
1028 case AtomicExpr::AO__c11_atomic_exchange:
1029 case AtomicExpr::AO__hip_atomic_fetch_and:
1030 case AtomicExpr::AO__hip_atomic_fetch_or:
1031 case AtomicExpr::AO__hip_atomic_fetch_xor:
1032 case AtomicExpr::AO__hip_atomic_store:
1033 case AtomicExpr::AO__hip_atomic_exchange:
1034 case AtomicExpr::AO__opencl_atomic_fetch_and:
1035 case AtomicExpr::AO__opencl_atomic_fetch_or:
1036 case AtomicExpr::AO__opencl_atomic_fetch_xor:
1037 case AtomicExpr::AO__opencl_atomic_store:
1038 case AtomicExpr::AO__opencl_atomic_exchange:
1039 case AtomicExpr::AO__scoped_atomic_fetch_and:
1040 case AtomicExpr::AO__scoped_atomic_fetch_nand:
1041 case AtomicExpr::AO__scoped_atomic_fetch_or:
1042 case AtomicExpr::AO__scoped_atomic_fetch_xor:
1043 case AtomicExpr::AO__scoped_atomic_and_fetch:
1044 case AtomicExpr::AO__scoped_atomic_nand_fetch:
1045 case AtomicExpr::AO__scoped_atomic_or_fetch:
1046 case AtomicExpr::AO__scoped_atomic_xor_fetch:
1047 case AtomicExpr::AO__scoped_atomic_store_n:
1048 case AtomicExpr::AO__scoped_atomic_exchange_n:
1049 Val1 = EmitValToTemp(*this, E->getVal1());
1050 break;
1051 }
1052
1053 QualType RValTy = E->getType().getUnqualifiedType();
1054
1055 // The inlined atomics only function on iN types, where N is a power of 2. We
1056 // need to make sure (via temporaries if necessary) that all incoming values
1057 // are compatible.
1058 LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy);
1059 AtomicInfo Atomics(*this, AtomicVal);
1060
1061 if (ShouldCastToIntPtrTy) {
1062 Ptr = Atomics.castToAtomicIntPointer(Ptr);
1063 if (Val1.isValid())
1064 Val1 = Atomics.convertToAtomicIntPointer(Val1);
1065 if (Val2.isValid())
1066 Val2 = Atomics.convertToAtomicIntPointer(Val2);
1067 }
1068 if (Dest.isValid()) {
1069 if (ShouldCastToIntPtrTy)
1070 Dest = Atomics.castToAtomicIntPointer(Dest);
1071 } else if (E->isCmpXChg())
1072 Dest = CreateMemTemp(RValTy, "cmpxchg.bool");
1073 else if (!RValTy->isVoidType()) {
1074 Dest = Atomics.CreateTempAlloca();
1075 if (ShouldCastToIntPtrTy)
1076 Dest = Atomics.castToAtomicIntPointer(Dest);
1077 }
1078
1079 bool PowerOf2Size = (Size & (Size - 1)) == 0;
1080 bool UseLibcall = !PowerOf2Size || (Size > 16);
1081
1082 // For atomics larger than 16 bytes, emit a libcall from the frontend. This
1083 // avoids the overhead of dealing with excessively-large value types in IR.
1084 // Non-power-of-2 values also lower to libcall here, as they are not currently
1085 // permitted in IR instructions (although that constraint could be relaxed in
1086 // the future). For other cases where a libcall is required on a given
1087 // platform, we let the backend handle it (this includes handling for all of
1088 // the size-optimized libcall variants, which are only valid up to 16 bytes.)
1089 //
1090 // See: https://llvm.org/docs/Atomics.html#libcalls-atomic
1091 if (UseLibcall) {
1092 CallArgList Args;
1093 // For non-optimized library calls, the size is the first parameter.
1094 Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
1095 getContext().getSizeType());
1096
1097 // The atomic address is the second parameter.
1098 // The OpenCL atomic library functions only accept pointer arguments to
1099 // generic address space.
1100 auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) {
1101 if (!E->isOpenCL())
1102 return V;
1103 auto AS = PT->castAs<PointerType>()->getPointeeType().getAddressSpace();
1104 if (AS == LangAS::opencl_generic)
1105 return V;
1106 auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic);
1107 auto *DestType = llvm::PointerType::get(getLLVMContext(), DestAS);
1108
1109 return getTargetHooks().performAddrSpaceCast(*this, V, AS, DestType,
1110 false);
1111 };
1112
1113 Args.add(RValue::get(CastToGenericAddrSpace(Ptr.emitRawPointer(*this),
1114 E->getPtr()->getType())),
1116
1117 // The next 1-3 parameters are op-dependent.
1118 std::string LibCallName;
1119 QualType RetTy;
1120 bool HaveRetTy = false;
1121 switch (E->getOp()) {
1122 case AtomicExpr::AO__c11_atomic_init:
1123 case AtomicExpr::AO__opencl_atomic_init:
1124 llvm_unreachable("Already handled!");
1125
1126 // There is only one libcall for compare an exchange, because there is no
1127 // optimisation benefit possible from a libcall version of a weak compare
1128 // and exchange.
1129 // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
1130 // void *desired, int success, int failure)
1131 case AtomicExpr::AO__atomic_compare_exchange:
1132 case AtomicExpr::AO__atomic_compare_exchange_n:
1133 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1134 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1135 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
1136 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
1137 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1138 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1139 case AtomicExpr::AO__scoped_atomic_compare_exchange:
1140 case AtomicExpr::AO__scoped_atomic_compare_exchange_n:
1141 LibCallName = "__atomic_compare_exchange";
1142 RetTy = getContext().BoolTy;
1143 HaveRetTy = true;
1144 Args.add(RValue::get(CastToGenericAddrSpace(Val1.emitRawPointer(*this),
1145 E->getVal1()->getType())),
1147 Args.add(RValue::get(CastToGenericAddrSpace(Val2.emitRawPointer(*this),
1148 E->getVal2()->getType())),
1150 Args.add(RValue::get(Order), getContext().IntTy);
1151 Order = OrderFail;
1152 break;
1153 // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
1154 // int order)
1155 case AtomicExpr::AO__atomic_exchange:
1156 case AtomicExpr::AO__atomic_exchange_n:
1157 case AtomicExpr::AO__c11_atomic_exchange:
1158 case AtomicExpr::AO__hip_atomic_exchange:
1159 case AtomicExpr::AO__opencl_atomic_exchange:
1160 case AtomicExpr::AO__scoped_atomic_exchange:
1161 case AtomicExpr::AO__scoped_atomic_exchange_n:
1162 LibCallName = "__atomic_exchange";
1163 Args.add(RValue::get(CastToGenericAddrSpace(Val1.emitRawPointer(*this),
1164 E->getVal1()->getType())),
1166 break;
1167 // void __atomic_store(size_t size, void *mem, void *val, int order)
1168 case AtomicExpr::AO__atomic_store:
1169 case AtomicExpr::AO__atomic_store_n:
1170 case AtomicExpr::AO__c11_atomic_store:
1171 case AtomicExpr::AO__hip_atomic_store:
1172 case AtomicExpr::AO__opencl_atomic_store:
1173 case AtomicExpr::AO__scoped_atomic_store:
1174 case AtomicExpr::AO__scoped_atomic_store_n:
1175 LibCallName = "__atomic_store";
1176 RetTy = getContext().VoidTy;
1177 HaveRetTy = true;
1178 Args.add(RValue::get(CastToGenericAddrSpace(Val1.emitRawPointer(*this),
1179 E->getVal1()->getType())),
1181 break;
1182 // void __atomic_load(size_t size, void *mem, void *return, int order)
1183 case AtomicExpr::AO__atomic_load:
1184 case AtomicExpr::AO__atomic_load_n:
1185 case AtomicExpr::AO__c11_atomic_load:
1186 case AtomicExpr::AO__hip_atomic_load:
1187 case AtomicExpr::AO__opencl_atomic_load:
1188 case AtomicExpr::AO__scoped_atomic_load:
1189 case AtomicExpr::AO__scoped_atomic_load_n:
1190 LibCallName = "__atomic_load";
1191 break;
1192 case AtomicExpr::AO__atomic_add_fetch:
1193 case AtomicExpr::AO__scoped_atomic_add_fetch:
1194 case AtomicExpr::AO__atomic_fetch_add:
1195 case AtomicExpr::AO__c11_atomic_fetch_add:
1196 case AtomicExpr::AO__hip_atomic_fetch_add:
1197 case AtomicExpr::AO__opencl_atomic_fetch_add:
1198 case AtomicExpr::AO__scoped_atomic_fetch_add:
1199 case AtomicExpr::AO__atomic_and_fetch:
1200 case AtomicExpr::AO__scoped_atomic_and_fetch:
1201 case AtomicExpr::AO__atomic_fetch_and:
1202 case AtomicExpr::AO__c11_atomic_fetch_and:
1203 case AtomicExpr::AO__hip_atomic_fetch_and:
1204 case AtomicExpr::AO__opencl_atomic_fetch_and:
1205 case AtomicExpr::AO__scoped_atomic_fetch_and:
1206 case AtomicExpr::AO__atomic_or_fetch:
1207 case AtomicExpr::AO__scoped_atomic_or_fetch:
1208 case AtomicExpr::AO__atomic_fetch_or:
1209 case AtomicExpr::AO__c11_atomic_fetch_or:
1210 case AtomicExpr::AO__hip_atomic_fetch_or:
1211 case AtomicExpr::AO__opencl_atomic_fetch_or:
1212 case AtomicExpr::AO__scoped_atomic_fetch_or:
1213 case AtomicExpr::AO__atomic_sub_fetch:
1214 case AtomicExpr::AO__scoped_atomic_sub_fetch:
1215 case AtomicExpr::AO__atomic_fetch_sub:
1216 case AtomicExpr::AO__c11_atomic_fetch_sub:
1217 case AtomicExpr::AO__hip_atomic_fetch_sub:
1218 case AtomicExpr::AO__opencl_atomic_fetch_sub:
1219 case AtomicExpr::AO__scoped_atomic_fetch_sub:
1220 case AtomicExpr::AO__atomic_xor_fetch:
1221 case AtomicExpr::AO__scoped_atomic_xor_fetch:
1222 case AtomicExpr::AO__atomic_fetch_xor:
1223 case AtomicExpr::AO__c11_atomic_fetch_xor:
1224 case AtomicExpr::AO__hip_atomic_fetch_xor:
1225 case AtomicExpr::AO__opencl_atomic_fetch_xor:
1226 case AtomicExpr::AO__scoped_atomic_fetch_xor:
1227 case AtomicExpr::AO__atomic_nand_fetch:
1228 case AtomicExpr::AO__atomic_fetch_nand:
1229 case AtomicExpr::AO__c11_atomic_fetch_nand:
1230 case AtomicExpr::AO__scoped_atomic_fetch_nand:
1231 case AtomicExpr::AO__scoped_atomic_nand_fetch:
1232 case AtomicExpr::AO__atomic_min_fetch:
1233 case AtomicExpr::AO__atomic_fetch_min:
1234 case AtomicExpr::AO__c11_atomic_fetch_min:
1235 case AtomicExpr::AO__hip_atomic_fetch_min:
1236 case AtomicExpr::AO__opencl_atomic_fetch_min:
1237 case AtomicExpr::AO__scoped_atomic_fetch_min:
1238 case AtomicExpr::AO__scoped_atomic_min_fetch:
1239 case AtomicExpr::AO__atomic_max_fetch:
1240 case AtomicExpr::AO__atomic_fetch_max:
1241 case AtomicExpr::AO__c11_atomic_fetch_max:
1242 case AtomicExpr::AO__hip_atomic_fetch_max:
1243 case AtomicExpr::AO__opencl_atomic_fetch_max:
1244 case AtomicExpr::AO__scoped_atomic_fetch_max:
1245 case AtomicExpr::AO__scoped_atomic_max_fetch:
1246 case AtomicExpr::AO__atomic_test_and_set:
1247 case AtomicExpr::AO__atomic_clear:
1248 llvm_unreachable("Integral atomic operations always become atomicrmw!");
1249 }
1250
1251 if (E->isOpenCL()) {
1252 LibCallName =
1253 std::string("__opencl") + StringRef(LibCallName).drop_front(1).str();
1254 }
1255 // By default, assume we return a value of the atomic type.
1256 if (!HaveRetTy) {
1257 // Value is returned through parameter before the order.
1258 RetTy = getContext().VoidTy;
1259 Args.add(RValue::get(
1260 CastToGenericAddrSpace(Dest.emitRawPointer(*this), RetTy)),
1262 }
1263 // Order is always the last parameter.
1264 Args.add(RValue::get(Order),
1265 getContext().IntTy);
1266 if (E->isOpenCL())
1268
1269 RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
1270 // The value is returned directly from the libcall.
1271 if (E->isCmpXChg())
1272 return Res;
1273
1274 if (RValTy->isVoidType())
1275 return RValue::get(nullptr);
1276
1278 RValTy, E->getExprLoc());
1279 }
1280
1281 bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
1282 E->getOp() == AtomicExpr::AO__opencl_atomic_store ||
1283 E->getOp() == AtomicExpr::AO__hip_atomic_store ||
1284 E->getOp() == AtomicExpr::AO__atomic_store ||
1285 E->getOp() == AtomicExpr::AO__atomic_store_n ||
1286 E->getOp() == AtomicExpr::AO__scoped_atomic_store ||
1287 E->getOp() == AtomicExpr::AO__scoped_atomic_store_n ||
1288 E->getOp() == AtomicExpr::AO__atomic_clear;
1289 bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
1290 E->getOp() == AtomicExpr::AO__opencl_atomic_load ||
1291 E->getOp() == AtomicExpr::AO__hip_atomic_load ||
1292 E->getOp() == AtomicExpr::AO__atomic_load ||
1293 E->getOp() == AtomicExpr::AO__atomic_load_n ||
1294 E->getOp() == AtomicExpr::AO__scoped_atomic_load ||
1295 E->getOp() == AtomicExpr::AO__scoped_atomic_load_n;
1296
1297 if (isa<llvm::ConstantInt>(Order)) {
1298 auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1299 // We should not ever get to a case where the ordering isn't a valid C ABI
1300 // value, but it's hard to enforce that in general.
1301 if (llvm::isValidAtomicOrderingCABI(ord))
1302 switch ((llvm::AtomicOrderingCABI)ord) {
1303 case llvm::AtomicOrderingCABI::relaxed:
1304 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1305 llvm::AtomicOrdering::Monotonic, Scope);
1306 break;
1307 case llvm::AtomicOrderingCABI::consume:
1308 case llvm::AtomicOrderingCABI::acquire:
1309 if (IsStore)
1310 break; // Avoid crashing on code with undefined behavior
1311 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1312 llvm::AtomicOrdering::Acquire, Scope);
1313 break;
1314 case llvm::AtomicOrderingCABI::release:
1315 if (IsLoad)
1316 break; // Avoid crashing on code with undefined behavior
1317 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1318 llvm::AtomicOrdering::Release, Scope);
1319 break;
1320 case llvm::AtomicOrderingCABI::acq_rel:
1321 if (IsLoad || IsStore)
1322 break; // Avoid crashing on code with undefined behavior
1323 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1324 llvm::AtomicOrdering::AcquireRelease, Scope);
1325 break;
1326 case llvm::AtomicOrderingCABI::seq_cst:
1327 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1328 llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1329 break;
1330 }
1331 if (RValTy->isVoidType())
1332 return RValue::get(nullptr);
1333
1335 RValTy, E->getExprLoc());
1336 }
1337
1338 // Long case, when Order isn't obviously constant.
1339
1340 // Create all the relevant BB's
1341 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
1342 *ReleaseBB = nullptr, *AcqRelBB = nullptr,
1343 *SeqCstBB = nullptr;
1344 MonotonicBB = createBasicBlock("monotonic", CurFn);
1345 if (!IsStore)
1346 AcquireBB = createBasicBlock("acquire", CurFn);
1347 if (!IsLoad)
1348 ReleaseBB = createBasicBlock("release", CurFn);
1349 if (!IsLoad && !IsStore)
1350 AcqRelBB = createBasicBlock("acqrel", CurFn);
1351 SeqCstBB = createBasicBlock("seqcst", CurFn);
1352 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1353
1354 // Create the switch for the split
1355 // MonotonicBB is arbitrarily chosen as the default case; in practice, this
1356 // doesn't matter unless someone is crazy enough to use something that
1357 // doesn't fold to a constant for the ordering.
1358 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1359 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
1360
1361 // Emit all the different atomics
1362 Builder.SetInsertPoint(MonotonicBB);
1363 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1364 llvm::AtomicOrdering::Monotonic, Scope);
1365 Builder.CreateBr(ContBB);
1366 if (!IsStore) {
1367 Builder.SetInsertPoint(AcquireBB);
1368 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1369 llvm::AtomicOrdering::Acquire, Scope);
1370 Builder.CreateBr(ContBB);
1371 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
1372 AcquireBB);
1373 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
1374 AcquireBB);
1375 }
1376 if (!IsLoad) {
1377 Builder.SetInsertPoint(ReleaseBB);
1378 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1379 llvm::AtomicOrdering::Release, Scope);
1380 Builder.CreateBr(ContBB);
1381 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release),
1382 ReleaseBB);
1383 }
1384 if (!IsLoad && !IsStore) {
1385 Builder.SetInsertPoint(AcqRelBB);
1386 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1387 llvm::AtomicOrdering::AcquireRelease, Scope);
1388 Builder.CreateBr(ContBB);
1389 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel),
1390 AcqRelBB);
1391 }
1392 Builder.SetInsertPoint(SeqCstBB);
1393 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1394 llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1395 Builder.CreateBr(ContBB);
1396 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
1397 SeqCstBB);
1398
1399 // Cleanup and return
1400 Builder.SetInsertPoint(ContBB);
1401 if (RValTy->isVoidType())
1402 return RValue::get(nullptr);
1403
1404 assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
1406 RValTy, E->getExprLoc());
1407}
1408
1409Address AtomicInfo::castToAtomicIntPointer(Address addr) const {
1410 llvm::IntegerType *ty =
1411 llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
1412 return addr.withElementType(ty);
1413}
1414
1415Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {
1416 llvm::Type *Ty = Addr.getElementType();
1417 uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
1418 if (SourceSizeInBits != AtomicSizeInBits) {
1419 Address Tmp = CreateTempAlloca();
1420 CGF.Builder.CreateMemCpy(Tmp, Addr,
1421 std::min(AtomicSizeInBits, SourceSizeInBits) / 8);
1422 Addr = Tmp;
1423 }
1424
1425 return castToAtomicIntPointer(Addr);
1426}
1427
1428RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
1429 AggValueSlot resultSlot,
1430 SourceLocation loc,
1431 bool asValue) const {
1432 if (LVal.isSimple()) {
1433 if (EvaluationKind == TEK_Aggregate)
1434 return resultSlot.asRValue();
1435
1436 // Drill into the padding structure if we have one.
1437 if (hasPadding())
1438 addr = CGF.Builder.CreateStructGEP(addr, 0);
1439
1440 // Otherwise, just convert the temporary to an r-value using the
1441 // normal conversion routine.
1442 return CGF.convertTempToRValue(addr, getValueType(), loc);
1443 }
1444 if (!asValue)
1445 // Get RValue from temp memory as atomic for non-simple lvalues
1446 return RValue::get(CGF.Builder.CreateLoad(addr));
1447 if (LVal.isBitField())
1448 return CGF.EmitLoadOfBitfieldLValue(
1449 LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(),
1450 LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1451 if (LVal.isVectorElt())
1452 return CGF.EmitLoadOfLValue(
1453 LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(),
1454 LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1455 assert(LVal.isExtVectorElt());
1456 return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt(
1457 addr, LVal.getExtVectorElts(), LVal.getType(),
1458 LVal.getBaseInfo(), TBAAAccessInfo()));
1459}
1460
1461/// Return true if \param ValTy is a type that should be casted to integer
1462/// around the atomic memory operation. If \param CmpXchg is true, then the
1463/// cast of a floating point type is made as that instruction can not have
1464/// floating point operands. TODO: Allow compare-and-exchange and FP - see
1465/// comment in AtomicExpandPass.cpp.
1466static bool shouldCastToInt(llvm::Type *ValTy, bool CmpXchg) {
1467 if (ValTy->isFloatingPointTy())
1468 return ValTy->isX86_FP80Ty() || CmpXchg;
1469 return !ValTy->isIntegerTy() && !ValTy->isPointerTy();
1470}
1471
1472RValue AtomicInfo::ConvertToValueOrAtomic(llvm::Value *Val,
1473 AggValueSlot ResultSlot,
1474 SourceLocation Loc, bool AsValue,
1475 bool CmpXchg) const {
1476 // Try not to in some easy cases.
1477 assert((Val->getType()->isIntegerTy() || Val->getType()->isPointerTy() ||
1478 Val->getType()->isIEEELikeFPTy()) &&
1479 "Expected integer, pointer or floating point value when converting "
1480 "result.");
1481 if (getEvaluationKind() == TEK_Scalar &&
1482 (((!LVal.isBitField() ||
1483 LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
1484 !hasPadding()) ||
1485 !AsValue)) {
1486 auto *ValTy = AsValue
1487 ? CGF.ConvertTypeForMem(ValueTy)
1488 : getAtomicAddress().getElementType();
1489 if (!shouldCastToInt(ValTy, CmpXchg)) {
1490 assert((!ValTy->isIntegerTy() || Val->getType() == ValTy) &&
1491 "Different integer types.");
1492 return RValue::get(CGF.EmitFromMemory(Val, ValueTy));
1493 }
1494 if (llvm::CastInst::isBitCastable(Val->getType(), ValTy))
1495 return RValue::get(CGF.Builder.CreateBitCast(Val, ValTy));
1496 }
1497
1498 // Create a temporary. This needs to be big enough to hold the
1499 // atomic integer.
1500 Address Temp = Address::invalid();
1501 bool TempIsVolatile = false;
1502 if (AsValue && getEvaluationKind() == TEK_Aggregate) {
1503 assert(!ResultSlot.isIgnored());
1504 Temp = ResultSlot.getAddress();
1505 TempIsVolatile = ResultSlot.isVolatile();
1506 } else {
1507 Temp = CreateTempAlloca();
1508 }
1509
1510 // Slam the integer into the temporary.
1511 Address CastTemp = castToAtomicIntPointer(Temp);
1512 CGF.Builder.CreateStore(Val, CastTemp)->setVolatile(TempIsVolatile);
1513
1514 return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue);
1515}
1516
1517void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1518 llvm::AtomicOrdering AO, bool) {
1519 // void __atomic_load(size_t size, void *mem, void *return, int order);
1520 CallArgList Args;
1521 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1522 Args.add(RValue::get(getAtomicPointer()), CGF.getContext().VoidPtrTy);
1523 Args.add(RValue::get(AddForLoaded), CGF.getContext().VoidPtrTy);
1524 Args.add(
1525 RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))),
1526 CGF.getContext().IntTy);
1527 emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1528}
1529
1530llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1531 bool IsVolatile, bool CmpXchg) {
1532 // Okay, we're doing this natively.
1533 Address Addr = getAtomicAddress();
1534 if (shouldCastToInt(Addr.getElementType(), CmpXchg))
1535 Addr = castToAtomicIntPointer(Addr);
1536 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
1537 Load->setAtomic(AO);
1538
1539 // Other decoration.
1540 if (IsVolatile)
1541 Load->setVolatile(true);
1542 CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo());
1543 return Load;
1544}
1545
1546/// An LValue is a candidate for having its loads and stores be made atomic if
1547/// we are operating under /volatile:ms *and* the LValue itself is volatile and
1548/// performing such an operation can be performed without a libcall.
1550 if (!CGM.getLangOpts().MSVolatile) return false;
1551 AtomicInfo AI(*this, LV);
1552 bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType());
1553 // An atomic is inline if we don't need to use a libcall.
1554 bool AtomicIsInline = !AI.shouldUseLibcall();
1555 // MSVC doesn't seem to do this for types wider than a pointer.
1556 if (getContext().getTypeSize(LV.getType()) >
1557 getContext().getTypeSize(getContext().getIntPtrType()))
1558 return false;
1559 return IsVolatile && AtomicIsInline;
1560}
1561
1563 AggValueSlot Slot) {
1564 llvm::AtomicOrdering AO;
1565 bool IsVolatile = LV.isVolatileQualified();
1566 if (LV.getType()->isAtomicType()) {
1567 AO = llvm::AtomicOrdering::SequentiallyConsistent;
1568 } else {
1569 AO = llvm::AtomicOrdering::Acquire;
1570 IsVolatile = true;
1571 }
1572 return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
1573}
1574
1575RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1576 bool AsValue, llvm::AtomicOrdering AO,
1577 bool IsVolatile) {
1578 // Check whether we should use a library call.
1579 if (shouldUseLibcall()) {
1580 Address TempAddr = Address::invalid();
1581 if (LVal.isSimple() && !ResultSlot.isIgnored()) {
1582 assert(getEvaluationKind() == TEK_Aggregate);
1583 TempAddr = ResultSlot.getAddress();
1584 } else
1585 TempAddr = CreateTempAlloca();
1586
1587 EmitAtomicLoadLibcall(TempAddr.emitRawPointer(CGF), AO, IsVolatile);
1588
1589 // Okay, turn that back into the original value or whole atomic (for
1590 // non-simple lvalues) type.
1591 return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
1592 }
1593
1594 // Okay, we're doing this natively.
1595 auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1596
1597 // If we're ignoring an aggregate return, don't do anything.
1598 if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
1599 return RValue::getAggregate(Address::invalid(), false);
1600
1601 // Okay, turn that back into the original value or atomic (for non-simple
1602 // lvalues) type.
1603 return ConvertToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
1604}
1605
1606/// Emit a load from an l-value of atomic type. Note that the r-value
1607/// we produce is an r-value of the atomic *value* type.
1609 llvm::AtomicOrdering AO, bool IsVolatile,
1610 AggValueSlot resultSlot) {
1611 AtomicInfo Atomics(*this, src);
1612 return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
1613 IsVolatile);
1614}
1615
1616/// Copy an r-value into memory as part of storing to an atomic type.
1617/// This needs to create a bit-pattern suitable for atomic operations.
1618void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1619 assert(LVal.isSimple());
1620 // If we have an r-value, the rvalue should be of the atomic type,
1621 // which means that the caller is responsible for having zeroed
1622 // any padding. Just do an aggregate copy of that type.
1623 if (rvalue.isAggregate()) {
1624 LValue Dest = CGF.MakeAddrLValue(getAtomicAddress(), getAtomicType());
1625 LValue Src = CGF.MakeAddrLValue(rvalue.getAggregateAddress(),
1626 getAtomicType());
1627 bool IsVolatile = rvalue.isVolatileQualified() ||
1628 LVal.isVolatileQualified();
1629 CGF.EmitAggregateCopy(Dest, Src, getAtomicType(),
1630 AggValueSlot::DoesNotOverlap, IsVolatile);
1631 return;
1632 }
1633
1634 // Okay, otherwise we're copying stuff.
1635
1636 // Zero out the buffer if necessary.
1637 emitMemSetZeroIfNecessary();
1638
1639 // Drill past the padding if present.
1640 LValue TempLVal = projectValue();
1641
1642 // Okay, store the rvalue in.
1643 if (rvalue.isScalar()) {
1644 CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
1645 } else {
1646 CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
1647 }
1648}
1649
1650
1651/// Materialize an r-value into memory for the purposes of storing it
1652/// to an atomic type.
1653Address AtomicInfo::materializeRValue(RValue rvalue) const {
1654 // Aggregate r-values are already in memory, and EmitAtomicStore
1655 // requires them to be values of the atomic type.
1656 if (rvalue.isAggregate())
1657 return rvalue.getAggregateAddress();
1658
1659 // Otherwise, make a temporary and materialize into it.
1660 LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType());
1661 AtomicInfo Atomics(CGF, TempLV);
1662 Atomics.emitCopyIntoMemory(rvalue);
1663 return TempLV.getAddress();
1664}
1665
1666llvm::Value *AtomicInfo::getScalarRValValueOrNull(RValue RVal) const {
1667 if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple()))
1668 return RVal.getScalarVal();
1669 return nullptr;
1670}
1671
1672llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal, bool CmpXchg) const {
1673 // If we've got a scalar value of the right size, try to avoid going
1674 // through memory. Floats get casted if needed by AtomicExpandPass.
1675 if (llvm::Value *Value = getScalarRValValueOrNull(RVal)) {
1676 if (!shouldCastToInt(Value->getType(), CmpXchg))
1677 return CGF.EmitToMemory(Value, ValueTy);
1678 else {
1679 llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1680 CGF.getLLVMContext(),
1681 LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1682 if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
1683 return CGF.Builder.CreateBitCast(Value, InputIntTy);
1684 }
1685 }
1686 // Otherwise, we need to go through memory.
1687 // Put the r-value in memory.
1688 Address Addr = materializeRValue(RVal);
1689
1690 // Cast the temporary to the atomic int type and pull a value out.
1691 Addr = castToAtomicIntPointer(Addr);
1692 return CGF.Builder.CreateLoad(Addr);
1693}
1694
1695std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1696 llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1697 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1698 // Do the atomic store.
1699 Address Addr = getAtomicAddressAsAtomicIntPointer();
1700 auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr, ExpectedVal, DesiredVal,
1701 Success, Failure);
1702 // Other decoration.
1703 Inst->setVolatile(LVal.isVolatileQualified());
1704 Inst->setWeak(IsWeak);
1705
1706 // Okay, turn that back into the original value type.
1707 auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
1708 auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
1709 return std::make_pair(PreviousVal, SuccessFailureVal);
1710}
1711
1712llvm::Value *
1713AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1714 llvm::Value *DesiredAddr,
1715 llvm::AtomicOrdering Success,
1716 llvm::AtomicOrdering Failure) {
1717 // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1718 // void *desired, int success, int failure);
1719 CallArgList Args;
1720 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1721 Args.add(RValue::get(getAtomicPointer()), CGF.getContext().VoidPtrTy);
1722 Args.add(RValue::get(ExpectedAddr), CGF.getContext().VoidPtrTy);
1723 Args.add(RValue::get(DesiredAddr), CGF.getContext().VoidPtrTy);
1724 Args.add(RValue::get(
1725 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))),
1726 CGF.getContext().IntTy);
1727 Args.add(RValue::get(
1728 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))),
1729 CGF.getContext().IntTy);
1730 auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1731 CGF.getContext().BoolTy, Args);
1732
1733 return SuccessFailureRVal.getScalarVal();
1734}
1735
1736std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1737 RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1738 llvm::AtomicOrdering Failure, bool IsWeak) {
1739 // Check whether we should use a library call.
1740 if (shouldUseLibcall()) {
1741 // Produce a source address.
1742 Address ExpectedAddr = materializeRValue(Expected);
1743 llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF);
1744 llvm::Value *DesiredPtr = materializeRValue(Desired).emitRawPointer(CGF);
1745 auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr,
1746 Success, Failure);
1747 return std::make_pair(
1748 convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1749 SourceLocation(), /*AsValue=*/false),
1750 Res);
1751 }
1752
1753 // If we've got a scalar value of the right size, try to avoid going
1754 // through memory.
1755 auto *ExpectedVal = convertRValueToInt(Expected, /*CmpXchg=*/true);
1756 auto *DesiredVal = convertRValueToInt(Desired, /*CmpXchg=*/true);
1757 auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1758 Failure, IsWeak);
1759 return std::make_pair(
1760 ConvertToValueOrAtomic(Res.first, AggValueSlot::ignored(),
1761 SourceLocation(), /*AsValue=*/false,
1762 /*CmpXchg=*/true),
1763 Res.second);
1764}
1765
1766static void
1767EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1768 const llvm::function_ref<RValue(RValue)> &UpdateOp,
1769 Address DesiredAddr) {
1770 RValue UpRVal;
1771 LValue AtomicLVal = Atomics.getAtomicLValue();
1772 LValue DesiredLVal;
1773 if (AtomicLVal.isSimple()) {
1774 UpRVal = OldRVal;
1775 DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType());
1776 } else {
1777 // Build new lvalue for temp address.
1778 Address Ptr = Atomics.materializeRValue(OldRVal);
1779 LValue UpdateLVal;
1780 if (AtomicLVal.isBitField()) {
1781 UpdateLVal =
1782 LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
1783 AtomicLVal.getType(),
1784 AtomicLVal.getBaseInfo(),
1785 AtomicLVal.getTBAAInfo());
1786 DesiredLVal =
1787 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1788 AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1789 AtomicLVal.getTBAAInfo());
1790 } else if (AtomicLVal.isVectorElt()) {
1791 UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
1792 AtomicLVal.getType(),
1793 AtomicLVal.getBaseInfo(),
1794 AtomicLVal.getTBAAInfo());
1795 DesiredLVal = LValue::MakeVectorElt(
1796 DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(),
1797 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1798 } else {
1799 assert(AtomicLVal.isExtVectorElt());
1800 UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
1801 AtomicLVal.getType(),
1802 AtomicLVal.getBaseInfo(),
1803 AtomicLVal.getTBAAInfo());
1804 DesiredLVal = LValue::MakeExtVectorElt(
1805 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1806 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1807 }
1808 UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());
1809 }
1810 // Store new value in the corresponding memory area.
1811 RValue NewRVal = UpdateOp(UpRVal);
1812 if (NewRVal.isScalar()) {
1813 CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);
1814 } else {
1815 assert(NewRVal.isComplex());
1816 CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,
1817 /*isInit=*/false);
1818 }
1819}
1820
1821void AtomicInfo::EmitAtomicUpdateLibcall(
1822 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1823 bool IsVolatile) {
1824 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1825
1826 Address ExpectedAddr = CreateTempAlloca();
1827
1828 EmitAtomicLoadLibcall(ExpectedAddr.emitRawPointer(CGF), AO, IsVolatile);
1829 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1830 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1831 CGF.EmitBlock(ContBB);
1832 Address DesiredAddr = CreateTempAlloca();
1833 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1834 requiresMemSetZero(getAtomicAddress().getElementType())) {
1835 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1836 CGF.Builder.CreateStore(OldVal, DesiredAddr);
1837 }
1838 auto OldRVal = convertAtomicTempToRValue(ExpectedAddr,
1840 SourceLocation(), /*AsValue=*/false);
1841 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
1842 llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF);
1843 llvm::Value *DesiredPtr = DesiredAddr.emitRawPointer(CGF);
1844 auto *Res =
1845 EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr, AO, Failure);
1846 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1847 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1848}
1849
1850void AtomicInfo::EmitAtomicUpdateOp(
1851 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1852 bool IsVolatile) {
1853 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1854
1855 // Do the atomic load.
1856 auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile, /*CmpXchg=*/true);
1857 // For non-simple lvalues perform compare-and-swap procedure.
1858 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1859 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1860 auto *CurBB = CGF.Builder.GetInsertBlock();
1861 CGF.EmitBlock(ContBB);
1862 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1863 /*NumReservedValues=*/2);
1864 PHI->addIncoming(OldVal, CurBB);
1865 Address NewAtomicAddr = CreateTempAlloca();
1866 Address NewAtomicIntAddr =
1867 shouldCastToInt(NewAtomicAddr.getElementType(), /*CmpXchg=*/true)
1868 ? castToAtomicIntPointer(NewAtomicAddr)
1869 : NewAtomicAddr;
1870
1871 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1872 requiresMemSetZero(getAtomicAddress().getElementType())) {
1873 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1874 }
1875 auto OldRVal = ConvertToValueOrAtomic(PHI, AggValueSlot::ignored(),
1876 SourceLocation(), /*AsValue=*/false,
1877 /*CmpXchg=*/true);
1878 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
1879 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1880 // Try to write new value using cmpxchg operation.
1881 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1882 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1883 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1884 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1885}
1886
1887static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1888 RValue UpdateRVal, Address DesiredAddr) {
1889 LValue AtomicLVal = Atomics.getAtomicLValue();
1890 LValue DesiredLVal;
1891 // Build new lvalue for temp address.
1892 if (AtomicLVal.isBitField()) {
1893 DesiredLVal =
1894 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1895 AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1896 AtomicLVal.getTBAAInfo());
1897 } else if (AtomicLVal.isVectorElt()) {
1898 DesiredLVal =
1899 LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(),
1900 AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1901 AtomicLVal.getTBAAInfo());
1902 } else {
1903 assert(AtomicLVal.isExtVectorElt());
1904 DesiredLVal = LValue::MakeExtVectorElt(
1905 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1906 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1907 }
1908 // Store new value in the corresponding memory area.
1909 assert(UpdateRVal.isScalar());
1910 CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);
1911}
1912
1913void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1914 RValue UpdateRVal, bool IsVolatile) {
1915 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1916
1917 Address ExpectedAddr = CreateTempAlloca();
1918
1919 EmitAtomicLoadLibcall(ExpectedAddr.emitRawPointer(CGF), AO, IsVolatile);
1920 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1921 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1922 CGF.EmitBlock(ContBB);
1923 Address DesiredAddr = CreateTempAlloca();
1924 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1925 requiresMemSetZero(getAtomicAddress().getElementType())) {
1926 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1927 CGF.Builder.CreateStore(OldVal, DesiredAddr);
1928 }
1929 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
1930 llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF);
1931 llvm::Value *DesiredPtr = DesiredAddr.emitRawPointer(CGF);
1932 auto *Res =
1933 EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr, AO, Failure);
1934 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1935 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1936}
1937
1938void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
1939 bool IsVolatile) {
1940 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1941
1942 // Do the atomic load.
1943 auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile, /*CmpXchg=*/true);
1944 // For non-simple lvalues perform compare-and-swap procedure.
1945 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1946 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1947 auto *CurBB = CGF.Builder.GetInsertBlock();
1948 CGF.EmitBlock(ContBB);
1949 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1950 /*NumReservedValues=*/2);
1951 PHI->addIncoming(OldVal, CurBB);
1952 Address NewAtomicAddr = CreateTempAlloca();
1953 Address NewAtomicIntAddr = castToAtomicIntPointer(NewAtomicAddr);
1954 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1955 requiresMemSetZero(getAtomicAddress().getElementType())) {
1956 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1957 }
1958 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
1959 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1960 // Try to write new value using cmpxchg operation.
1961 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1962 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1963 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1964 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1965}
1966
1967void AtomicInfo::EmitAtomicUpdate(
1968 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1969 bool IsVolatile) {
1970 if (shouldUseLibcall()) {
1971 EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
1972 } else {
1973 EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
1974 }
1975}
1976
1977void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
1978 bool IsVolatile) {
1979 if (shouldUseLibcall()) {
1980 EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
1981 } else {
1982 EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
1983 }
1984}
1985
1987 bool isInit) {
1988 bool IsVolatile = lvalue.isVolatileQualified();
1989 llvm::AtomicOrdering AO;
1990 if (lvalue.getType()->isAtomicType()) {
1991 AO = llvm::AtomicOrdering::SequentiallyConsistent;
1992 } else {
1993 AO = llvm::AtomicOrdering::Release;
1994 IsVolatile = true;
1995 }
1996 return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
1997}
1998
1999/// Emit a store to an l-value of atomic type.
2000///
2001/// Note that the r-value is expected to be an r-value *of the atomic
2002/// type*; this means that for aggregate r-values, it should include
2003/// storage for any padding that was necessary.
2005 llvm::AtomicOrdering AO, bool IsVolatile,
2006 bool isInit) {
2007 // If this is an aggregate r-value, it should agree in type except
2008 // maybe for address-space qualification.
2009 assert(!rvalue.isAggregate() ||
2011 dest.getAddress().getElementType());
2012
2013 AtomicInfo atomics(*this, dest);
2014 LValue LVal = atomics.getAtomicLValue();
2015
2016 // If this is an initialization, just put the value there normally.
2017 if (LVal.isSimple()) {
2018 if (isInit) {
2019 atomics.emitCopyIntoMemory(rvalue);
2020 return;
2021 }
2022
2023 // Check whether we should use a library call.
2024 if (atomics.shouldUseLibcall()) {
2025 // Produce a source address.
2026 Address srcAddr = atomics.materializeRValue(rvalue);
2027
2028 // void __atomic_store(size_t size, void *mem, void *val, int order)
2029 CallArgList args;
2030 args.add(RValue::get(atomics.getAtomicSizeValue()),
2031 getContext().getSizeType());
2032 args.add(RValue::get(atomics.getAtomicPointer()), getContext().VoidPtrTy);
2033 args.add(RValue::get(srcAddr.emitRawPointer(*this)),
2035 args.add(
2036 RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))),
2037 getContext().IntTy);
2038 emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
2039 return;
2040 }
2041
2042 // Okay, we're doing this natively.
2043 llvm::Value *ValToStore = atomics.convertRValueToInt(rvalue);
2044
2045 // Do the atomic store.
2046 Address Addr = atomics.getAtomicAddress();
2047 if (llvm::Value *Value = atomics.getScalarRValValueOrNull(rvalue))
2048 if (shouldCastToInt(Value->getType(), /*CmpXchg=*/false)) {
2049 Addr = atomics.castToAtomicIntPointer(Addr);
2050 ValToStore = Builder.CreateIntCast(ValToStore, Addr.getElementType(),
2051 /*isSigned=*/false);
2052 }
2053 llvm::StoreInst *store = Builder.CreateStore(ValToStore, Addr);
2054
2055 if (AO == llvm::AtomicOrdering::Acquire)
2056 AO = llvm::AtomicOrdering::Monotonic;
2057 else if (AO == llvm::AtomicOrdering::AcquireRelease)
2058 AO = llvm::AtomicOrdering::Release;
2059 // Initializations don't need to be atomic.
2060 if (!isInit)
2061 store->setAtomic(AO);
2062
2063 // Other decoration.
2064 if (IsVolatile)
2065 store->setVolatile(true);
2066 CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo());
2067 return;
2068 }
2069
2070 // Emit simple atomic update operation.
2071 atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
2072}
2073
2074/// Emit a compare-and-exchange op for atomic type.
2075///
2076std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
2077 LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
2078 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
2079 AggValueSlot Slot) {
2080 // If this is an aggregate r-value, it should agree in type except
2081 // maybe for address-space qualification.
2082 assert(!Expected.isAggregate() ||
2083 Expected.getAggregateAddress().getElementType() ==
2084 Obj.getAddress().getElementType());
2085 assert(!Desired.isAggregate() ||
2086 Desired.getAggregateAddress().getElementType() ==
2087 Obj.getAddress().getElementType());
2088 AtomicInfo Atomics(*this, Obj);
2089
2090 return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
2091 IsWeak);
2092}
2093
2094llvm::AtomicRMWInst *
2095CodeGenFunction::emitAtomicRMWInst(llvm::AtomicRMWInst::BinOp Op, Address Addr,
2096 llvm::Value *Val, llvm::AtomicOrdering Order,
2097 llvm::SyncScope::ID SSID,
2098 const AtomicExpr *AE) {
2099 llvm::AtomicRMWInst *RMW =
2100 Builder.CreateAtomicRMW(Op, Addr, Val, Order, SSID);
2101 getTargetHooks().setTargetAtomicMetadata(*this, *RMW, AE);
2102 return RMW;
2103}
2104
2106 LValue LVal, llvm::AtomicOrdering AO,
2107 const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
2108 AtomicInfo Atomics(*this, LVal);
2109 Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
2110}
2111
2113 AtomicInfo atomics(*this, dest);
2114
2115 switch (atomics.getEvaluationKind()) {
2116 case TEK_Scalar: {
2117 llvm::Value *value = EmitScalarExpr(init);
2118 atomics.emitCopyIntoMemory(RValue::get(value));
2119 return;
2120 }
2121
2122 case TEK_Complex: {
2123 ComplexPairTy value = EmitComplexExpr(init);
2124 atomics.emitCopyIntoMemory(RValue::getComplex(value));
2125 return;
2126 }
2127
2128 case TEK_Aggregate: {
2129 // Fix up the destination if the initializer isn't an expression
2130 // of atomic type.
2131 bool Zeroed = false;
2132 if (!init->getType()->isAtomicType()) {
2133 Zeroed = atomics.emitMemSetZeroIfNecessary();
2134 dest = atomics.projectValue();
2135 }
2136
2137 // Evaluate the expression directly into the destination.
2143
2144 EmitAggExpr(init, slot);
2145 return;
2146 }
2147 }
2148 llvm_unreachable("bad evaluation kind");
2149}
Defines the clang::ASTContext interface.
#define V(N, I)
static llvm::Value * EmitPostAtomicMinMax(CGBuilderTy &Builder, AtomicExpr::AtomicOp Op, bool IsSigned, llvm::Value *OldVal, llvm::Value *RHS)
Duplicate the atomic min/max operation in conventional IR for the builtin variants that return the ne...
Definition CGAtomic.cpp:505
static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal, const llvm::function_ref< RValue(RValue)> &UpdateOp, Address DesiredAddr)
static Address EmitValToTemp(CodeGenFunction &CGF, Expr *E)
Definition CGAtomic.cpp:791
static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, Address Ptr, Address Val1, Address Val2, llvm::Value *IsWeak, llvm::Value *FailureOrder, uint64_t Size, llvm::AtomicOrdering Order, llvm::SyncScope::ID Scope)
Definition CGAtomic.cpp:539
static RValue emitAtomicLibcall(CodeGenFunction &CGF, StringRef fnName, QualType resultType, CallArgList &args)
Definition CGAtomic.cpp:314
static bool shouldCastToInt(llvm::Type *ValTy, bool CmpXchg)
Return true if.
static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo, TBAAAccessInfo *TBAAInfo, KnownNonNull_t IsKnownNonNull, CodeGenFunction &CGF)
Definition CGExpr.cpp:1567
static void emitAtomicCmpXchg(CIRGenFunction &cgf, AtomicExpr *e, bool isWeak, Address dest, Address ptr, Address val1, Address val2, uint64_t size, cir::MemOrder successOrder, cir::MemOrder failureOrder)
static void emitAtomicCmpXchgFailureSet(CIRGenFunction &cgf, AtomicExpr *e, bool isWeak, Address dest, Address ptr, Address val1, Address val2, Expr *failureOrderExpr, uint64_t size, cir::MemOrder successOrder)
static bool isFullSizeType(CIRGenModule &cgm, mlir::Type ty, uint64_t expectedSize)
Does a store of the given IR type modify the full expected width?
TokenType getType() const
Returns the token's type, e.g.
static QualType getPointeeType(const MemRegion *R)
CanQualType VoidPtrTy
CanQualType BoolTy
CanQualType IntTy
CanQualType VoidTy
QualType getSizeType() const
Return the unique type for "size_t" (C99 7.17), defined in <stddef.h>.
QualType getExtVectorType(QualType VectorType, unsigned NumElts) const
Return the unique reference to an extended vector type of the specified element type and size.
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
AtomicExpr - Variadic atomic builtins: __atomic_exchange, __atomic_fetch_*, __atomic_load,...
Definition Expr.h:6814
static std::unique_ptr< AtomicScopeModel > getScopeModel(AtomicOp Op)
Get atomic scope model for the atomic op code.
Definition Expr.h:6963
Expr * getVal2() const
Definition Expr.h:6865
Expr * getOrder() const
Definition Expr.h:6848
QualType getValueType() const
Definition Expr.cpp:5256
Expr * getScope() const
Definition Expr.h:6851
bool isCmpXChg() const
Definition Expr.h:6898
AtomicOp getOp() const
Definition Expr.h:6877
bool isOpenCL() const
Definition Expr.h:6926
Expr * getVal1() const
Definition Expr.h:6855
Expr * getPtr() const
Definition Expr.h:6845
Expr * getWeak() const
Definition Expr.h:6871
SourceLocation getBeginLoc() const LLVM_READONLY
Definition Expr.h:6945
Expr * getOrderFail() const
Definition Expr.h:6861
bool isVolatile() const
Definition Expr.h:6894
CharUnits - This is an opaque type for sizes expressed in character units.
Definition CharUnits.h:38
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
Definition CharUnits.h:185
bool isMultipleOf(CharUnits N) const
Test whether this is a multiple of the other value.
Definition CharUnits.h:143
Like RawAddress, an abstract representation of an aligned address, but the pointer contained in this ...
Definition Address.h:128
static Address invalid()
Definition Address.h:176
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
Return the pointer contained in this class after authenticating it and adding offset to it if necessa...
Definition Address.h:253
CharUnits getAlignment() const
Definition Address.h:194
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Definition Address.h:209
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
Definition Address.h:276
bool isValid() const
Definition Address.h:177
An aggregate value slot.
Definition CGValue.h:504
static AggValueSlot ignored()
ignored - Returns an aggregate value slot indicating that the aggregate value is being ignored.
Definition CGValue.h:572
Address getAddress() const
Definition CGValue.h:644
static AggValueSlot forLValue(const LValue &LV, IsDestructed_t isDestructed, NeedsGCBarriers_t needsGC, IsAliased_t isAliased, Overlap_t mayOverlap, IsZeroed_t isZeroed=IsNotZeroed, IsSanitizerChecked_t isChecked=IsNotSanitizerChecked)
Definition CGValue.h:602
RValue asRValue() const
Definition CGValue.h:666
A scoped helper to set the current source atom group for CGDebugInfo::addInstToCurrentSourceAtom.
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
Definition CGBuilder.h:140
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
Definition CGBuilder.h:207
llvm::CallInst * CreateMemSet(Address Dest, llvm::Value *Value, llvm::Value *Size, bool IsVolatile=false)
Definition CGBuilder.h:402
Address CreateStructGEP(Address Addr, unsigned Index, const llvm::Twine &Name="")
Definition CGBuilder.h:223
llvm::AtomicCmpXchgInst * CreateAtomicCmpXchg(Address Addr, llvm::Value *Cmp, llvm::Value *New, llvm::AtomicOrdering SuccessOrdering, llvm::AtomicOrdering FailureOrdering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
Definition CGBuilder.h:173
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
Definition CGBuilder.h:112
llvm::CallInst * CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
Definition CGBuilder.h:369
Address CreateAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
Definition CGBuilder.h:193
static CGCallee forDirect(llvm::Constant *functionPtr, const CGCalleeInfo &abstractInfo=CGCalleeInfo())
Definition CGCall.h:137
CGFunctionInfo - Class to encapsulate the information about a function definition.
CallArgList - Type for representing both the value and type of arguments in a call.
Definition CGCall.h:274
void add(RValue rvalue, QualType type)
Definition CGCall.h:302
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
void EmitAtomicInit(Expr *E, LValue lvalue)
RValue convertTempToRValue(Address addr, QualType type, SourceLocation Loc)
Given the address of a temporary variable, produce an r-value of its type.
Definition CGExpr.cpp:6833
bool hasVolatileMember(QualType T)
hasVolatileMember - returns true if aggregate type has a volatile member.
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
void addInstToCurrentSourceAtom(llvm::Instruction *KeyInstruction, llvm::Value *Backup)
See CGDebugInfo::addInstToCurrentSourceAtom.
const LangOptions & getLangOpts() const
void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO, const llvm::function_ref< RValue(RValue)> &UpdateOp, bool IsVolatile)
std::pair< RValue, llvm::Value * > EmitAtomicCompareExchange(LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, llvm::AtomicOrdering Success=llvm::AtomicOrdering::SequentiallyConsistent, llvm::AtomicOrdering Failure=llvm::AtomicOrdering::SequentiallyConsistent, bool IsWeak=false, AggValueSlot Slot=AggValueSlot::ignored())
Emit a compare-and-exchange op for atomic type.
void maybeAttachRangeForLoad(llvm::LoadInst *Load, QualType Ty, SourceLocation Loc)
Definition CGExpr.cpp:2203
void EmitAggregateCopy(LValue Dest, LValue Src, QualType EltTy, AggValueSlot::Overlap_t MayOverlap, bool isVolatile=false)
EmitAggregateCopy - Emit an aggregate copy.
const TargetInfo & getTarget() const
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
Definition CGExpr.cpp:2527
RValue EmitAtomicLoad(LValue LV, SourceLocation SL, AggValueSlot Slot=AggValueSlot::ignored())
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
llvm::Value * EmitToMemory(llvm::Value *Value, QualType Ty)
EmitToMemory - Change a scalar value from its value representation to its in-memory representation.
Definition CGExpr.cpp:2344
ComplexPairTy EmitComplexExpr(const Expr *E, bool IgnoreReal=false, bool IgnoreImag=false)
EmitComplexExpr - Emit the computation of the specified expression of complex type,...
RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee, ReturnValueSlot ReturnValue, const CallArgList &Args, llvm::CallBase **CallOrInvoke, bool IsMustTail, SourceLocation Loc, bool IsVirtualFunctionPointerThunk=false)
EmitCall - Generate a call of the given function, expecting the given result type,...
Definition CGCall.cpp:5233
const TargetCodeGenInfo & getTargetHooks() const
void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit)
EmitStoreOfComplex - Store a complex number into the specified l-value.
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
Definition CGExpr.cpp:2724
llvm::AtomicRMWInst * emitAtomicRMWInst(llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val, llvm::AtomicOrdering Order=llvm::AtomicOrdering::SequentiallyConsistent, llvm::SyncScope::ID SSID=llvm::SyncScope::System, const AtomicExpr *AE=nullptr)
Emit an atomicrmw instruction, and applying relevant metadata when applicable.
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
Definition CGExpr.cpp:294
llvm::Type * ConvertTypeForMem(QualType T)
RValue EmitLoadOfBitfieldLValue(LValue LV, SourceLocation Loc)
Definition CGExpr.cpp:2598
RValue EmitAtomicExpr(AtomicExpr *E)
Definition CGAtomic.cpp:867
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
bool LValueIsSuitableForInlineAtomic(LValue Src)
An LValue is a candidate for having its loads and stores be made atomic if we are operating under /vo...
RawAddress CreateMemTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
Definition CGExpr.cpp:187
void EmitAggExpr(const Expr *E, AggValueSlot AS)
EmitAggExpr - Emit the computation of the specified expression of aggregate type.
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
RValue EmitLoadOfExtVectorElementLValue(LValue V)
Definition CGExpr.cpp:2635
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
llvm::Value * EmitFromMemory(llvm::Value *Value, QualType Ty)
EmitFromMemory - Change a scalar value from its memory representation to its value representation.
Definition CGExpr.cpp:2374
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
llvm::LLVMContext & getLLVMContext()
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
Definition CGStmt.cpp:655
This class organizes the cross-function state that is used while generating LLVM code.
llvm::FunctionCallee CreateRuntimeFunction(llvm::FunctionType *Ty, StringRef Name, llvm::AttributeList ExtraAttrs=llvm::AttributeList(), bool Local=false, bool AssumeConvergent=false)
Create or return a runtime function declaration with the specified type and name.
const LangOptions & getLangOpts() const
const llvm::DataLayout & getDataLayout() const
void DecorateInstructionWithTBAA(llvm::Instruction *Inst, TBAAAccessInfo TBAAInfo)
DecorateInstructionWithTBAA - Decorate the instruction with a TBAA tag.
llvm::LLVMContext & getLLVMContext()
llvm::ConstantInt * getSize(CharUnits numChars)
Emit the given number of characters as a value of type size_t.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
Definition CGCall.cpp:1701
const CGFunctionInfo & arrangeBuiltinFunctionCall(QualType resultType, const CallArgList &args)
Definition CGCall.cpp:728
LValue - This represents an lvalue references.
Definition CGValue.h:182
bool isSimple() const
Definition CGValue.h:278
bool isVolatileQualified() const
Definition CGValue.h:285
bool isVolatile() const
Definition CGValue.h:328
Address getAddress() const
Definition CGValue.h:361
QualType getType() const
Definition CGValue.h:291
TBAAAccessInfo getTBAAInfo() const
Definition CGValue.h:335
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
Definition CGValue.h:42
bool isScalar() const
Definition CGValue.h:64
static RValue get(llvm::Value *V)
Definition CGValue.h:98
static RValue getAggregate(Address addr, bool isVolatile=false)
Convert an Address to an RValue.
Definition CGValue.h:125
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
Definition CGValue.h:108
bool isAggregate() const
Definition CGValue.h:66
Address getAggregateAddress() const
getAggregateAddr() - Return the Value* of the address of the aggregate.
Definition CGValue.h:83
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
Definition CGValue.h:71
bool isComplex() const
Definition CGValue.h:65
bool isVolatileQualified() const
Definition CGValue.h:68
std::pair< llvm::Value *, llvm::Value * > getComplexVal() const
getComplexVal - Return the real/imag components of this complex value.
Definition CGValue.h:78
ReturnValueSlot - Contains the address where the return value of a function can be stored,...
Definition CGCall.h:379
virtual llvm::SyncScope::ID getLLVMSyncScopeID(const LangOptions &LangOpts, SyncScope Scope, llvm::AtomicOrdering Ordering, llvm::LLVMContext &Ctx) const
Get the syncscope used in LLVM IR.
virtual void setTargetAtomicMetadata(CodeGenFunction &CGF, llvm::Instruction &AtomicInst, const AtomicExpr *Expr=nullptr) const
Allow the target to apply other metadata to an atomic instruction.
Definition TargetInfo.h:359
Concrete class used by the front-end to report problems and issues.
Definition Diagnostic.h:232
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
This represents one expression.
Definition Expr.h:112
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
Definition Expr.cpp:273
QualType getType() const
Definition Expr.h:144
PointerType - C99 6.7.5.1 - Pointer Declarators.
Definition TypeBase.h:3328
A (possibly-)qualified type.
Definition TypeBase.h:937
bool isNull() const
Return true if this QualType doesn't point to a type yet.
Definition TypeBase.h:1004
LangAS getAddressSpace() const
Return the address space of this type.
Definition TypeBase.h:8404
Qualifiers getQualifiers() const
Retrieve the set of qualifiers applied to this type.
Definition TypeBase.h:8318
QualType getUnqualifiedType() const
Retrieve the unqualified variant of the given type, removing as little sugar as possible.
Definition TypeBase.h:8372
Scope - A scope is a transient data structure that is used while parsing the program.
Definition Scope.h:41
Encodes a location in the source.
bool isVoidType() const
Definition TypeBase.h:8871
bool isSignedIntegerType() const
Return true if this is an integer type that is signed, according to C99 6.2.5p4 [char,...
Definition Type.cpp:2205
bool isPointerType() const
Definition TypeBase.h:8515
const T * castAs() const
Member-template castAs<specific type>.
Definition TypeBase.h:9158
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
Definition Type.cpp:752
bool isAtomicType() const
Definition TypeBase.h:8697
bool isFloatingType() const
Definition Type.cpp:2304
const T * getAs() const
Member-template getAs<specific type>'.
Definition TypeBase.h:9091
QualType getType() const
Definition Value.cpp:237
const internal::VariadicAllOfMatcher< Type > type
Matches Types in the clang AST.
bool Load(InterpState &S, CodePtr OpPC)
Definition Interp.h:1938
The JSON file list parser is used to communicate input to InstallAPI.
bool isa(CodeGen::Address addr)
Definition Address.h:330
@ Success
Annotation was successful.
Definition Parser.h:65
llvm::StringRef getAsString(SyncScope S)
Definition SyncScope.h:60
U cast(CodeGen::Address addr)
Definition Address.h:327
unsigned long uint64_t
#define true
Definition stdbool.h:25
CharUnits StorageOffset
The offset of the bitfield storage from the start of the struct.
unsigned Offset
The offset within a contiguous run of bitfields that are represented as a single "field" within the L...
unsigned Size
The total size of the bit-field, in bits.
unsigned StorageSize
The storage size in bits which should be used when accessing this bitfield.
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64