clang 17.0.0git
CGAtomic.cpp
Go to the documentation of this file.
1//===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the code for emitting atomic operations.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGCall.h"
14#include "CGRecordLayout.h"
15#include "CodeGenFunction.h"
16#include "CodeGenModule.h"
17#include "TargetInfo.h"
21#include "llvm/ADT/DenseMap.h"
22#include "llvm/IR/DataLayout.h"
23#include "llvm/IR/Intrinsics.h"
24#include "llvm/IR/Operator.h"
25
26using namespace clang;
27using namespace CodeGen;
28
29namespace {
30 class AtomicInfo {
31 CodeGenFunction &CGF;
32 QualType AtomicTy;
33 QualType ValueTy;
34 uint64_t AtomicSizeInBits;
35 uint64_t ValueSizeInBits;
36 CharUnits AtomicAlign;
37 CharUnits ValueAlign;
38 TypeEvaluationKind EvaluationKind;
39 bool UseLibcall;
40 LValue LVal;
42 public:
43 AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
44 : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
45 EvaluationKind(TEK_Scalar), UseLibcall(true) {
46 assert(!lvalue.isGlobalReg());
47 ASTContext &C = CGF.getContext();
48 if (lvalue.isSimple()) {
49 AtomicTy = lvalue.getType();
50 if (auto *ATy = AtomicTy->getAs<AtomicType>())
51 ValueTy = ATy->getValueType();
52 else
53 ValueTy = AtomicTy;
54 EvaluationKind = CGF.getEvaluationKind(ValueTy);
55
56 uint64_t ValueAlignInBits;
57 uint64_t AtomicAlignInBits;
58 TypeInfo ValueTI = C.getTypeInfo(ValueTy);
59 ValueSizeInBits = ValueTI.Width;
60 ValueAlignInBits = ValueTI.Align;
61
62 TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
63 AtomicSizeInBits = AtomicTI.Width;
64 AtomicAlignInBits = AtomicTI.Align;
65
66 assert(ValueSizeInBits <= AtomicSizeInBits);
67 assert(ValueAlignInBits <= AtomicAlignInBits);
68
69 AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
70 ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
71 if (lvalue.getAlignment().isZero())
72 lvalue.setAlignment(AtomicAlign);
73
74 LVal = lvalue;
75 } else if (lvalue.isBitField()) {
76 ValueTy = lvalue.getType();
77 ValueSizeInBits = C.getTypeSize(ValueTy);
78 auto &OrigBFI = lvalue.getBitFieldInfo();
79 auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
80 AtomicSizeInBits = C.toBits(
81 C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
82 .alignTo(lvalue.getAlignment()));
83 auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer());
84 auto OffsetInChars =
85 (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
86 lvalue.getAlignment();
87 VoidPtrAddr = CGF.Builder.CreateConstGEP1_64(
88 CGF.Int8Ty, VoidPtrAddr, OffsetInChars.getQuantity());
89 llvm::Type *IntTy = CGF.Builder.getIntNTy(AtomicSizeInBits);
91 VoidPtrAddr, IntTy->getPointerTo(), "atomic_bitfield_base");
92 BFI = OrigBFI;
93 BFI.Offset = Offset;
94 BFI.StorageSize = AtomicSizeInBits;
95 BFI.StorageOffset += OffsetInChars;
96 LVal = LValue::MakeBitfield(Address(Addr, IntTy, lvalue.getAlignment()),
97 BFI, lvalue.getType(), lvalue.getBaseInfo(),
98 lvalue.getTBAAInfo());
99 AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
100 if (AtomicTy.isNull()) {
101 llvm::APInt Size(
102 /*numBits=*/32,
103 C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
104 AtomicTy =
105 C.getConstantArrayType(C.CharTy, Size, nullptr, ArrayType::Normal,
106 /*IndexTypeQuals=*/0);
107 }
108 AtomicAlign = ValueAlign = lvalue.getAlignment();
109 } else if (lvalue.isVectorElt()) {
110 ValueTy = lvalue.getType()->castAs<VectorType>()->getElementType();
111 ValueSizeInBits = C.getTypeSize(ValueTy);
112 AtomicTy = lvalue.getType();
113 AtomicSizeInBits = C.getTypeSize(AtomicTy);
114 AtomicAlign = ValueAlign = lvalue.getAlignment();
115 LVal = lvalue;
116 } else {
117 assert(lvalue.isExtVectorElt());
118 ValueTy = lvalue.getType();
119 ValueSizeInBits = C.getTypeSize(ValueTy);
120 AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
121 lvalue.getType(), cast<llvm::FixedVectorType>(
123 ->getNumElements());
124 AtomicSizeInBits = C.getTypeSize(AtomicTy);
125 AtomicAlign = ValueAlign = lvalue.getAlignment();
126 LVal = lvalue;
127 }
128 UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
129 AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
130 }
131
132 QualType getAtomicType() const { return AtomicTy; }
133 QualType getValueType() const { return ValueTy; }
134 CharUnits getAtomicAlignment() const { return AtomicAlign; }
135 uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
136 uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
137 TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
138 bool shouldUseLibcall() const { return UseLibcall; }
139 const LValue &getAtomicLValue() const { return LVal; }
140 llvm::Value *getAtomicPointer() const {
141 if (LVal.isSimple())
142 return LVal.getPointer(CGF);
143 else if (LVal.isBitField())
144 return LVal.getBitFieldPointer();
145 else if (LVal.isVectorElt())
146 return LVal.getVectorPointer();
147 assert(LVal.isExtVectorElt());
148 return LVal.getExtVectorPointer();
149 }
150 Address getAtomicAddress() const {
151 llvm::Type *ElTy;
152 if (LVal.isSimple())
153 ElTy = LVal.getAddress(CGF).getElementType();
154 else if (LVal.isBitField())
155 ElTy = LVal.getBitFieldAddress().getElementType();
156 else if (LVal.isVectorElt())
157 ElTy = LVal.getVectorAddress().getElementType();
158 else
159 ElTy = LVal.getExtVectorAddress().getElementType();
160 return Address(getAtomicPointer(), ElTy, getAtomicAlignment());
161 }
162
163 Address getAtomicAddressAsAtomicIntPointer() const {
164 return emitCastToAtomicIntPointer(getAtomicAddress());
165 }
166
167 /// Is the atomic size larger than the underlying value type?
168 ///
169 /// Note that the absence of padding does not mean that atomic
170 /// objects are completely interchangeable with non-atomic
171 /// objects: we might have promoted the alignment of a type
172 /// without making it bigger.
173 bool hasPadding() const {
174 return (ValueSizeInBits != AtomicSizeInBits);
175 }
176
177 bool emitMemSetZeroIfNecessary() const;
178
179 llvm::Value *getAtomicSizeValue() const {
180 CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
181 return CGF.CGM.getSize(size);
182 }
183
184 /// Cast the given pointer to an integer pointer suitable for atomic
185 /// operations if the source.
186 Address emitCastToAtomicIntPointer(Address Addr) const;
187
188 /// If Addr is compatible with the iN that will be used for an atomic
189 /// operation, bitcast it. Otherwise, create a temporary that is suitable
190 /// and copy the value across.
191 Address convertToAtomicIntPointer(Address Addr) const;
192
193 /// Turn an atomic-layout object into an r-value.
194 RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
195 SourceLocation loc, bool AsValue) const;
196
197 /// Converts a rvalue to integer value.
198 llvm::Value *convertRValueToInt(RValue RVal) const;
199
200 RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
201 AggValueSlot ResultSlot,
202 SourceLocation Loc, bool AsValue) const;
203
204 /// Copy an atomic r-value into atomic-layout memory.
205 void emitCopyIntoMemory(RValue rvalue) const;
206
207 /// Project an l-value down to the value field.
208 LValue projectValue() const {
209 assert(LVal.isSimple());
210 Address addr = getAtomicAddress();
211 if (hasPadding())
212 addr = CGF.Builder.CreateStructGEP(addr, 0);
213
214 return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
215 LVal.getBaseInfo(), LVal.getTBAAInfo());
216 }
217
218 /// Emits atomic load.
219 /// \returns Loaded value.
220 RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
221 bool AsValue, llvm::AtomicOrdering AO,
222 bool IsVolatile);
223
224 /// Emits atomic compare-and-exchange sequence.
225 /// \param Expected Expected value.
226 /// \param Desired Desired value.
227 /// \param Success Atomic ordering for success operation.
228 /// \param Failure Atomic ordering for failed operation.
229 /// \param IsWeak true if atomic operation is weak, false otherwise.
230 /// \returns Pair of values: previous value from storage (value type) and
231 /// boolean flag (i1 type) with true if success and false otherwise.
232 std::pair<RValue, llvm::Value *>
233 EmitAtomicCompareExchange(RValue Expected, RValue Desired,
234 llvm::AtomicOrdering Success =
235 llvm::AtomicOrdering::SequentiallyConsistent,
236 llvm::AtomicOrdering Failure =
237 llvm::AtomicOrdering::SequentiallyConsistent,
238 bool IsWeak = false);
239
240 /// Emits atomic update.
241 /// \param AO Atomic ordering.
242 /// \param UpdateOp Update operation for the current lvalue.
243 void EmitAtomicUpdate(llvm::AtomicOrdering AO,
244 const llvm::function_ref<RValue(RValue)> &UpdateOp,
245 bool IsVolatile);
246 /// Emits atomic update.
247 /// \param AO Atomic ordering.
248 void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
249 bool IsVolatile);
250
251 /// Materialize an atomic r-value in atomic-layout memory.
252 Address materializeRValue(RValue rvalue) const;
253
254 /// Creates temp alloca for intermediate operations on atomic value.
255 Address CreateTempAlloca() const;
256 private:
257 bool requiresMemSetZero(llvm::Type *type) const;
258
259
260 /// Emits atomic load as a libcall.
261 void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
262 llvm::AtomicOrdering AO, bool IsVolatile);
263 /// Emits atomic load as LLVM instruction.
264 llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
265 /// Emits atomic compare-and-exchange op as a libcall.
266 llvm::Value *EmitAtomicCompareExchangeLibcall(
267 llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
268 llvm::AtomicOrdering Success =
269 llvm::AtomicOrdering::SequentiallyConsistent,
270 llvm::AtomicOrdering Failure =
271 llvm::AtomicOrdering::SequentiallyConsistent);
272 /// Emits atomic compare-and-exchange op as LLVM instruction.
273 std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
274 llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
275 llvm::AtomicOrdering Success =
276 llvm::AtomicOrdering::SequentiallyConsistent,
277 llvm::AtomicOrdering Failure =
278 llvm::AtomicOrdering::SequentiallyConsistent,
279 bool IsWeak = false);
280 /// Emit atomic update as libcalls.
281 void
282 EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
283 const llvm::function_ref<RValue(RValue)> &UpdateOp,
284 bool IsVolatile);
285 /// Emit atomic update as LLVM instructions.
286 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
287 const llvm::function_ref<RValue(RValue)> &UpdateOp,
288 bool IsVolatile);
289 /// Emit atomic update as libcalls.
290 void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
291 bool IsVolatile);
292 /// Emit atomic update as LLVM instructions.
293 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
294 bool IsVolatile);
295 };
296}
297
298Address AtomicInfo::CreateTempAlloca() const {
299 Address TempAlloca = CGF.CreateMemTemp(
300 (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
301 : AtomicTy,
302 getAtomicAlignment(),
303 "atomic-temp");
304 // Cast to pointer to value type for bitfields.
305 if (LVal.isBitField())
307 TempAlloca, getAtomicAddress().getType(),
308 getAtomicAddress().getElementType());
309 return TempAlloca;
310}
311
313 StringRef fnName,
314 QualType resultType,
315 CallArgList &args) {
316 const CGFunctionInfo &fnInfo =
317 CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args);
318 llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
319 llvm::AttrBuilder fnAttrB(CGF.getLLVMContext());
320 fnAttrB.addAttribute(llvm::Attribute::NoUnwind);
321 fnAttrB.addAttribute(llvm::Attribute::WillReturn);
322 llvm::AttributeList fnAttrs = llvm::AttributeList::get(
323 CGF.getLLVMContext(), llvm::AttributeList::FunctionIndex, fnAttrB);
324
325 llvm::FunctionCallee fn =
326 CGF.CGM.CreateRuntimeFunction(fnTy, fnName, fnAttrs);
327 auto callee = CGCallee::forDirect(fn);
328 return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args);
329}
330
331/// Does a store of the given IR type modify the full expected width?
332static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
333 uint64_t expectedSize) {
334 return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
335}
336
337/// Does the atomic type require memsetting to zero before initialization?
338///
339/// The IR type is provided as a way of making certain queries faster.
340bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
341 // If the atomic type has size padding, we definitely need a memset.
342 if (hasPadding()) return true;
343
344 // Otherwise, do some simple heuristics to try to avoid it:
345 switch (getEvaluationKind()) {
346 // For scalars and complexes, check whether the store size of the
347 // type uses the full size.
348 case TEK_Scalar:
349 return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
350 case TEK_Complex:
351 return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
352 AtomicSizeInBits / 2);
353
354 // Padding in structs has an undefined bit pattern. User beware.
355 case TEK_Aggregate:
356 return false;
357 }
358 llvm_unreachable("bad evaluation kind");
359}
360
361bool AtomicInfo::emitMemSetZeroIfNecessary() const {
362 assert(LVal.isSimple());
363 Address addr = LVal.getAddress(CGF);
364 if (!requiresMemSetZero(addr.getElementType()))
365 return false;
366
368 addr.getPointer(), llvm::ConstantInt::get(CGF.Int8Ty, 0),
369 CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
370 LVal.getAlignment().getAsAlign());
371 return true;
372}
373
374static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
375 Address Dest, Address Ptr,
376 Address Val1, Address Val2,
377 uint64_t Size,
378 llvm::AtomicOrdering SuccessOrder,
379 llvm::AtomicOrdering FailureOrder,
380 llvm::SyncScope::ID Scope) {
381 // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
382 llvm::Value *Expected = CGF.Builder.CreateLoad(Val1);
383 llvm::Value *Desired = CGF.Builder.CreateLoad(Val2);
384
385 llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
386 Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder,
387 Scope);
388 Pair->setVolatile(E->isVolatile());
389 Pair->setWeak(IsWeak);
390
391 // Cmp holds the result of the compare-exchange operation: true on success,
392 // false on failure.
393 llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
394 llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
395
396 // This basic block is used to hold the store instruction if the operation
397 // failed.
398 llvm::BasicBlock *StoreExpectedBB =
399 CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
400
401 // This basic block is the exit point of the operation, we should end up
402 // here regardless of whether or not the operation succeeded.
403 llvm::BasicBlock *ContinueBB =
404 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
405
406 // Update Expected if Expected isn't equal to Old, otherwise branch to the
407 // exit point.
408 CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
409
410 CGF.Builder.SetInsertPoint(StoreExpectedBB);
411 // Update the memory at Expected with Old's value.
412 CGF.Builder.CreateStore(Old, Val1);
413 // Finally, branch to the exit point.
414 CGF.Builder.CreateBr(ContinueBB);
415
416 CGF.Builder.SetInsertPoint(ContinueBB);
417 // Update the memory at Dest with Cmp's value.
418 CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
419}
420
421/// Given an ordering required on success, emit all possible cmpxchg
422/// instructions to cope with the provided (but possibly only dynamically known)
423/// FailureOrder.
425 bool IsWeak, Address Dest, Address Ptr,
426 Address Val1, Address Val2,
427 llvm::Value *FailureOrderVal,
428 uint64_t Size,
429 llvm::AtomicOrdering SuccessOrder,
430 llvm::SyncScope::ID Scope) {
431 llvm::AtomicOrdering FailureOrder;
432 if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
433 auto FOS = FO->getSExtValue();
434 if (!llvm::isValidAtomicOrderingCABI(FOS))
435 FailureOrder = llvm::AtomicOrdering::Monotonic;
436 else
437 switch ((llvm::AtomicOrderingCABI)FOS) {
438 case llvm::AtomicOrderingCABI::relaxed:
439 // 31.7.2.18: "The failure argument shall not be memory_order_release
440 // nor memory_order_acq_rel". Fallback to monotonic.
441 case llvm::AtomicOrderingCABI::release:
442 case llvm::AtomicOrderingCABI::acq_rel:
443 FailureOrder = llvm::AtomicOrdering::Monotonic;
444 break;
445 case llvm::AtomicOrderingCABI::consume:
446 case llvm::AtomicOrderingCABI::acquire:
447 FailureOrder = llvm::AtomicOrdering::Acquire;
448 break;
449 case llvm::AtomicOrderingCABI::seq_cst:
450 FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent;
451 break;
452 }
453 // Prior to c++17, "the failure argument shall be no stronger than the
454 // success argument". This condition has been lifted and the only
455 // precondition is 31.7.2.18. Effectively treat this as a DR and skip
456 // language version checks.
457 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
458 FailureOrder, Scope);
459 return;
460 }
461
462 // Create all the relevant BB's
463 auto *MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
464 auto *AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
465 auto *SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
466 auto *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
467
468 // MonotonicBB is arbitrarily chosen as the default case; in practice, this
469 // doesn't matter unless someone is crazy enough to use something that
470 // doesn't fold to a constant for the ordering.
471 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
472 // Implemented as acquire, since it's the closest in LLVM.
473 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
474 AcquireBB);
475 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
476 AcquireBB);
477 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
478 SeqCstBB);
479
480 // Emit all the different atomics
481 CGF.Builder.SetInsertPoint(MonotonicBB);
482 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
483 Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope);
484 CGF.Builder.CreateBr(ContBB);
485
486 CGF.Builder.SetInsertPoint(AcquireBB);
487 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
488 llvm::AtomicOrdering::Acquire, Scope);
489 CGF.Builder.CreateBr(ContBB);
490
491 CGF.Builder.SetInsertPoint(SeqCstBB);
492 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
493 llvm::AtomicOrdering::SequentiallyConsistent, Scope);
494 CGF.Builder.CreateBr(ContBB);
495
496 CGF.Builder.SetInsertPoint(ContBB);
497}
498
499/// Duplicate the atomic min/max operation in conventional IR for the builtin
500/// variants that return the new rather than the original value.
501static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder,
503 bool IsSigned,
504 llvm::Value *OldVal,
505 llvm::Value *RHS) {
506 llvm::CmpInst::Predicate Pred;
507 switch (Op) {
508 default:
509 llvm_unreachable("Unexpected min/max operation");
510 case AtomicExpr::AO__atomic_max_fetch:
511 Pred = IsSigned ? llvm::CmpInst::ICMP_SGT : llvm::CmpInst::ICMP_UGT;
512 break;
513 case AtomicExpr::AO__atomic_min_fetch:
514 Pred = IsSigned ? llvm::CmpInst::ICMP_SLT : llvm::CmpInst::ICMP_ULT;
515 break;
516 }
517 llvm::Value *Cmp = Builder.CreateICmp(Pred, OldVal, RHS, "tst");
518 return Builder.CreateSelect(Cmp, OldVal, RHS, "newval");
519}
520
522 Address Ptr, Address Val1, Address Val2,
523 llvm::Value *IsWeak, llvm::Value *FailureOrder,
524 uint64_t Size, llvm::AtomicOrdering Order,
525 llvm::SyncScope::ID Scope) {
526 llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
527 bool PostOpMinMax = false;
528 unsigned PostOp = 0;
529
530 switch (E->getOp()) {
531 case AtomicExpr::AO__c11_atomic_init:
532 case AtomicExpr::AO__opencl_atomic_init:
533 llvm_unreachable("Already handled!");
534
535 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
536 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
537 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
538 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
539 FailureOrder, Size, Order, Scope);
540 return;
541 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
542 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
543 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
544 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
545 FailureOrder, Size, Order, Scope);
546 return;
547 case AtomicExpr::AO__atomic_compare_exchange:
548 case AtomicExpr::AO__atomic_compare_exchange_n: {
549 if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
550 emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
551 Val1, Val2, FailureOrder, Size, Order, Scope);
552 } else {
553 // Create all the relevant BB's
554 llvm::BasicBlock *StrongBB =
555 CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
556 llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
557 llvm::BasicBlock *ContBB =
558 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
559
560 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
561 SI->addCase(CGF.Builder.getInt1(false), StrongBB);
562
563 CGF.Builder.SetInsertPoint(StrongBB);
564 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
565 FailureOrder, Size, Order, Scope);
566 CGF.Builder.CreateBr(ContBB);
567
568 CGF.Builder.SetInsertPoint(WeakBB);
569 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
570 FailureOrder, Size, Order, Scope);
571 CGF.Builder.CreateBr(ContBB);
572
573 CGF.Builder.SetInsertPoint(ContBB);
574 }
575 return;
576 }
577 case AtomicExpr::AO__c11_atomic_load:
578 case AtomicExpr::AO__opencl_atomic_load:
579 case AtomicExpr::AO__hip_atomic_load:
580 case AtomicExpr::AO__atomic_load_n:
581 case AtomicExpr::AO__atomic_load: {
582 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
583 Load->setAtomic(Order, Scope);
584 Load->setVolatile(E->isVolatile());
585 CGF.Builder.CreateStore(Load, Dest);
586 return;
587 }
588
589 case AtomicExpr::AO__c11_atomic_store:
590 case AtomicExpr::AO__opencl_atomic_store:
591 case AtomicExpr::AO__hip_atomic_store:
592 case AtomicExpr::AO__atomic_store:
593 case AtomicExpr::AO__atomic_store_n: {
594 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
595 llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
596 Store->setAtomic(Order, Scope);
597 Store->setVolatile(E->isVolatile());
598 return;
599 }
600
601 case AtomicExpr::AO__c11_atomic_exchange:
602 case AtomicExpr::AO__hip_atomic_exchange:
603 case AtomicExpr::AO__opencl_atomic_exchange:
604 case AtomicExpr::AO__atomic_exchange_n:
605 case AtomicExpr::AO__atomic_exchange:
606 Op = llvm::AtomicRMWInst::Xchg;
607 break;
608
609 case AtomicExpr::AO__atomic_add_fetch:
610 PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FAdd
611 : llvm::Instruction::Add;
612 [[fallthrough]];
613 case AtomicExpr::AO__c11_atomic_fetch_add:
614 case AtomicExpr::AO__hip_atomic_fetch_add:
615 case AtomicExpr::AO__opencl_atomic_fetch_add:
616 case AtomicExpr::AO__atomic_fetch_add:
617 Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FAdd
618 : llvm::AtomicRMWInst::Add;
619 break;
620
621 case AtomicExpr::AO__atomic_sub_fetch:
622 PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FSub
623 : llvm::Instruction::Sub;
624 [[fallthrough]];
625 case AtomicExpr::AO__c11_atomic_fetch_sub:
626 case AtomicExpr::AO__opencl_atomic_fetch_sub:
627 case AtomicExpr::AO__atomic_fetch_sub:
628 Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FSub
629 : llvm::AtomicRMWInst::Sub;
630 break;
631
632 case AtomicExpr::AO__atomic_min_fetch:
633 PostOpMinMax = true;
634 [[fallthrough]];
635 case AtomicExpr::AO__c11_atomic_fetch_min:
636 case AtomicExpr::AO__hip_atomic_fetch_min:
637 case AtomicExpr::AO__opencl_atomic_fetch_min:
638 case AtomicExpr::AO__atomic_fetch_min:
639 Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Min
640 : llvm::AtomicRMWInst::UMin;
641 break;
642
643 case AtomicExpr::AO__atomic_max_fetch:
644 PostOpMinMax = true;
645 [[fallthrough]];
646 case AtomicExpr::AO__c11_atomic_fetch_max:
647 case AtomicExpr::AO__hip_atomic_fetch_max:
648 case AtomicExpr::AO__opencl_atomic_fetch_max:
649 case AtomicExpr::AO__atomic_fetch_max:
650 Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Max
651 : llvm::AtomicRMWInst::UMax;
652 break;
653
654 case AtomicExpr::AO__atomic_and_fetch:
655 PostOp = llvm::Instruction::And;
656 [[fallthrough]];
657 case AtomicExpr::AO__c11_atomic_fetch_and:
658 case AtomicExpr::AO__hip_atomic_fetch_and:
659 case AtomicExpr::AO__opencl_atomic_fetch_and:
660 case AtomicExpr::AO__atomic_fetch_and:
661 Op = llvm::AtomicRMWInst::And;
662 break;
663
664 case AtomicExpr::AO__atomic_or_fetch:
665 PostOp = llvm::Instruction::Or;
666 [[fallthrough]];
667 case AtomicExpr::AO__c11_atomic_fetch_or:
668 case AtomicExpr::AO__hip_atomic_fetch_or:
669 case AtomicExpr::AO__opencl_atomic_fetch_or:
670 case AtomicExpr::AO__atomic_fetch_or:
671 Op = llvm::AtomicRMWInst::Or;
672 break;
673
674 case AtomicExpr::AO__atomic_xor_fetch:
675 PostOp = llvm::Instruction::Xor;
676 [[fallthrough]];
677 case AtomicExpr::AO__c11_atomic_fetch_xor:
678 case AtomicExpr::AO__hip_atomic_fetch_xor:
679 case AtomicExpr::AO__opencl_atomic_fetch_xor:
680 case AtomicExpr::AO__atomic_fetch_xor:
681 Op = llvm::AtomicRMWInst::Xor;
682 break;
683
684 case AtomicExpr::AO__atomic_nand_fetch:
685 PostOp = llvm::Instruction::And; // the NOT is special cased below
686 [[fallthrough]];
687 case AtomicExpr::AO__c11_atomic_fetch_nand:
688 case AtomicExpr::AO__atomic_fetch_nand:
689 Op = llvm::AtomicRMWInst::Nand;
690 break;
691 }
692
693 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
694 llvm::AtomicRMWInst *RMWI =
695 CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order, Scope);
696 RMWI->setVolatile(E->isVolatile());
697
698 // For __atomic_*_fetch operations, perform the operation again to
699 // determine the value which was written.
700 llvm::Value *Result = RMWI;
701 if (PostOpMinMax)
702 Result = EmitPostAtomicMinMax(CGF.Builder, E->getOp(),
704 RMWI, LoadVal1);
705 else if (PostOp)
706 Result = CGF.Builder.CreateBinOp((llvm::Instruction::BinaryOps)PostOp, RMWI,
707 LoadVal1);
708 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
709 Result = CGF.Builder.CreateNot(Result);
710 CGF.Builder.CreateStore(Result, Dest);
711}
712
713// This function emits any expression (scalar, complex, or aggregate)
714// into a temporary alloca.
715static Address
717 Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
718 CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
719 /*Init*/ true);
720 return DeclPtr;
721}
722
724 Address Ptr, Address Val1, Address Val2,
725 llvm::Value *IsWeak, llvm::Value *FailureOrder,
726 uint64_t Size, llvm::AtomicOrdering Order,
727 llvm::Value *Scope) {
728 auto ScopeModel = Expr->getScopeModel();
729
730 // LLVM atomic instructions always have synch scope. If clang atomic
731 // expression has no scope operand, use default LLVM synch scope.
732 if (!ScopeModel) {
733 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
734 Order, CGF.CGM.getLLVMContext().getOrInsertSyncScopeID(""));
735 return;
736 }
737
738 // Handle constant scope.
739 if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) {
740 auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID(
741 CGF.CGM.getLangOpts(), ScopeModel->map(SC->getZExtValue()),
742 Order, CGF.CGM.getLLVMContext());
743 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
744 Order, SCID);
745 return;
746 }
747
748 // Handle non-constant scope.
749 auto &Builder = CGF.Builder;
750 auto Scopes = ScopeModel->getRuntimeValues();
751 llvm::DenseMap<unsigned, llvm::BasicBlock *> BB;
752 for (auto S : Scopes)
753 BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn);
754
755 llvm::BasicBlock *ContBB =
756 CGF.createBasicBlock("atomic.scope.continue", CGF.CurFn);
757
758 auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false);
759 // If unsupported synch scope is encountered at run time, assume a fallback
760 // synch scope value.
761 auto FallBack = ScopeModel->getFallBackValue();
762 llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]);
763 for (auto S : Scopes) {
764 auto *B = BB[S];
765 if (S != FallBack)
766 SI->addCase(Builder.getInt32(S), B);
767
768 Builder.SetInsertPoint(B);
769 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
770 Order,
772 ScopeModel->map(S),
773 Order,
774 CGF.getLLVMContext()));
775 Builder.CreateBr(ContBB);
776 }
777
778 Builder.SetInsertPoint(ContBB);
779}
780
781static void
783 bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,
784 SourceLocation Loc, CharUnits SizeInChars) {
785 if (UseOptimizedLibcall) {
786 // Load value and pass it to the function directly.
787 CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy);
788 int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
789 ValTy =
790 CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
791 llvm::Type *ITy = llvm::IntegerType::get(CGF.getLLVMContext(), SizeInBits);
792 Address Ptr = Address(CGF.Builder.CreateBitCast(Val, ITy->getPointerTo()),
793 ITy, Align);
794 Val = CGF.EmitLoadOfScalar(Ptr, false,
795 CGF.getContext().getPointerType(ValTy),
796 Loc);
797 // Coerce the value into an appropriately sized integer type.
798 Args.add(RValue::get(Val), ValTy);
799 } else {
800 // Non-optimized functions always take a reference.
801 Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)),
802 CGF.getContext().VoidPtrTy);
803 }
804}
805
806RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
807 QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
808 QualType MemTy = AtomicTy;
809 if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
810 MemTy = AT->getValueType();
811 llvm::Value *IsWeak = nullptr, *OrderFail = nullptr;
812
813 Address Val1 = Address::invalid();
814 Address Val2 = Address::invalid();
815 Address Dest = Address::invalid();
817
818 if (E->getOp() == AtomicExpr::AO__c11_atomic_init ||
819 E->getOp() == AtomicExpr::AO__opencl_atomic_init) {
820 LValue lvalue = MakeAddrLValue(Ptr, AtomicTy);
821 EmitAtomicInit(E->getVal1(), lvalue);
822 return RValue::get(nullptr);
823 }
824
825 auto TInfo = getContext().getTypeInfoInChars(AtomicTy);
826 uint64_t Size = TInfo.Width.getQuantity();
827 unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();
828
829 bool Oversized = getContext().toBits(TInfo.Width) > MaxInlineWidthInBits;
830 bool Misaligned = (Ptr.getAlignment() % TInfo.Width) != 0;
831 bool UseLibcall = Misaligned | Oversized;
832 bool ShouldCastToIntPtrTy = true;
833
834 CharUnits MaxInlineWidth =
835 getContext().toCharUnitsFromBits(MaxInlineWidthInBits);
836
837 DiagnosticsEngine &Diags = CGM.getDiags();
838
839 if (Misaligned) {
840 Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned)
841 << (int)TInfo.Width.getQuantity()
842 << (int)Ptr.getAlignment().getQuantity();
843 }
844
845 if (Oversized) {
846 Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_oversized)
847 << (int)TInfo.Width.getQuantity() << (int)MaxInlineWidth.getQuantity();
848 }
849
850 llvm::Value *Order = EmitScalarExpr(E->getOrder());
851 llvm::Value *Scope =
852 E->getScopeModel() ? EmitScalarExpr(E->getScope()) : nullptr;
853
854 switch (E->getOp()) {
855 case AtomicExpr::AO__c11_atomic_init:
856 case AtomicExpr::AO__opencl_atomic_init:
857 llvm_unreachable("Already handled above with EmitAtomicInit!");
858
859 case AtomicExpr::AO__c11_atomic_load:
860 case AtomicExpr::AO__opencl_atomic_load:
861 case AtomicExpr::AO__hip_atomic_load:
862 case AtomicExpr::AO__atomic_load_n:
863 break;
864
865 case AtomicExpr::AO__atomic_load:
867 break;
868
869 case AtomicExpr::AO__atomic_store:
871 break;
872
873 case AtomicExpr::AO__atomic_exchange:
876 break;
877
878 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
879 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
880 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
881 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
882 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
883 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
884 case AtomicExpr::AO__atomic_compare_exchange_n:
885 case AtomicExpr::AO__atomic_compare_exchange:
887 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
889 else
890 Val2 = EmitValToTemp(*this, E->getVal2());
891 OrderFail = EmitScalarExpr(E->getOrderFail());
892 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n ||
893 E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
894 IsWeak = EmitScalarExpr(E->getWeak());
895 break;
896
897 case AtomicExpr::AO__c11_atomic_fetch_add:
898 case AtomicExpr::AO__c11_atomic_fetch_sub:
899 case AtomicExpr::AO__hip_atomic_fetch_add:
900 case AtomicExpr::AO__opencl_atomic_fetch_add:
901 case AtomicExpr::AO__opencl_atomic_fetch_sub:
902 if (MemTy->isPointerType()) {
903 // For pointer arithmetic, we're required to do a bit of math:
904 // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
905 // ... but only for the C11 builtins. The GNU builtins expect the
906 // user to multiply by sizeof(T).
907 QualType Val1Ty = E->getVal1()->getType();
908 llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
909 CharUnits PointeeIncAmt =
911 Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
912 auto Temp = CreateMemTemp(Val1Ty, ".atomictmp");
913 Val1 = Temp;
914 EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty));
915 break;
916 }
917 [[fallthrough]];
918 case AtomicExpr::AO__atomic_fetch_add:
919 case AtomicExpr::AO__atomic_fetch_sub:
920 case AtomicExpr::AO__atomic_add_fetch:
921 case AtomicExpr::AO__atomic_sub_fetch:
922 ShouldCastToIntPtrTy = !MemTy->isFloatingType();
923 [[fallthrough]];
924
925 case AtomicExpr::AO__c11_atomic_store:
926 case AtomicExpr::AO__c11_atomic_exchange:
927 case AtomicExpr::AO__opencl_atomic_store:
928 case AtomicExpr::AO__hip_atomic_store:
929 case AtomicExpr::AO__opencl_atomic_exchange:
930 case AtomicExpr::AO__hip_atomic_exchange:
931 case AtomicExpr::AO__atomic_store_n:
932 case AtomicExpr::AO__atomic_exchange_n:
933 case AtomicExpr::AO__c11_atomic_fetch_and:
934 case AtomicExpr::AO__c11_atomic_fetch_or:
935 case AtomicExpr::AO__c11_atomic_fetch_xor:
936 case AtomicExpr::AO__c11_atomic_fetch_nand:
937 case AtomicExpr::AO__c11_atomic_fetch_max:
938 case AtomicExpr::AO__c11_atomic_fetch_min:
939 case AtomicExpr::AO__opencl_atomic_fetch_and:
940 case AtomicExpr::AO__opencl_atomic_fetch_or:
941 case AtomicExpr::AO__opencl_atomic_fetch_xor:
942 case AtomicExpr::AO__opencl_atomic_fetch_min:
943 case AtomicExpr::AO__opencl_atomic_fetch_max:
944 case AtomicExpr::AO__atomic_fetch_and:
945 case AtomicExpr::AO__hip_atomic_fetch_and:
946 case AtomicExpr::AO__atomic_fetch_or:
947 case AtomicExpr::AO__hip_atomic_fetch_or:
948 case AtomicExpr::AO__atomic_fetch_xor:
949 case AtomicExpr::AO__hip_atomic_fetch_xor:
950 case AtomicExpr::AO__atomic_fetch_nand:
951 case AtomicExpr::AO__atomic_and_fetch:
952 case AtomicExpr::AO__atomic_or_fetch:
953 case AtomicExpr::AO__atomic_xor_fetch:
954 case AtomicExpr::AO__atomic_nand_fetch:
955 case AtomicExpr::AO__atomic_max_fetch:
956 case AtomicExpr::AO__atomic_min_fetch:
957 case AtomicExpr::AO__atomic_fetch_max:
958 case AtomicExpr::AO__hip_atomic_fetch_max:
959 case AtomicExpr::AO__atomic_fetch_min:
960 case AtomicExpr::AO__hip_atomic_fetch_min:
961 Val1 = EmitValToTemp(*this, E->getVal1());
962 break;
963 }
964
965 QualType RValTy = E->getType().getUnqualifiedType();
966
967 // The inlined atomics only function on iN types, where N is a power of 2. We
968 // need to make sure (via temporaries if necessary) that all incoming values
969 // are compatible.
970 LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy);
971 AtomicInfo Atomics(*this, AtomicVal);
972
973 if (ShouldCastToIntPtrTy) {
974 Ptr = Atomics.emitCastToAtomicIntPointer(Ptr);
975 if (Val1.isValid())
976 Val1 = Atomics.convertToAtomicIntPointer(Val1);
977 if (Val2.isValid())
978 Val2 = Atomics.convertToAtomicIntPointer(Val2);
979 }
980 if (Dest.isValid()) {
981 if (ShouldCastToIntPtrTy)
982 Dest = Atomics.emitCastToAtomicIntPointer(Dest);
983 } else if (E->isCmpXChg())
984 Dest = CreateMemTemp(RValTy, "cmpxchg.bool");
985 else if (!RValTy->isVoidType()) {
986 Dest = Atomics.CreateTempAlloca();
987 if (ShouldCastToIntPtrTy)
988 Dest = Atomics.emitCastToAtomicIntPointer(Dest);
989 }
990
991 // Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
992 if (UseLibcall) {
993 bool UseOptimizedLibcall = false;
994 switch (E->getOp()) {
995 case AtomicExpr::AO__c11_atomic_init:
996 case AtomicExpr::AO__opencl_atomic_init:
997 llvm_unreachable("Already handled above with EmitAtomicInit!");
998
999 case AtomicExpr::AO__c11_atomic_fetch_add:
1000 case AtomicExpr::AO__opencl_atomic_fetch_add:
1001 case AtomicExpr::AO__atomic_fetch_add:
1002 case AtomicExpr::AO__hip_atomic_fetch_add:
1003 case AtomicExpr::AO__c11_atomic_fetch_and:
1004 case AtomicExpr::AO__opencl_atomic_fetch_and:
1005 case AtomicExpr::AO__hip_atomic_fetch_and:
1006 case AtomicExpr::AO__atomic_fetch_and:
1007 case AtomicExpr::AO__c11_atomic_fetch_or:
1008 case AtomicExpr::AO__opencl_atomic_fetch_or:
1009 case AtomicExpr::AO__hip_atomic_fetch_or:
1010 case AtomicExpr::AO__atomic_fetch_or:
1011 case AtomicExpr::AO__c11_atomic_fetch_nand:
1012 case AtomicExpr::AO__atomic_fetch_nand:
1013 case AtomicExpr::AO__c11_atomic_fetch_sub:
1014 case AtomicExpr::AO__opencl_atomic_fetch_sub:
1015 case AtomicExpr::AO__atomic_fetch_sub:
1016 case AtomicExpr::AO__c11_atomic_fetch_xor:
1017 case AtomicExpr::AO__opencl_atomic_fetch_xor:
1018 case AtomicExpr::AO__opencl_atomic_fetch_min:
1019 case AtomicExpr::AO__opencl_atomic_fetch_max:
1020 case AtomicExpr::AO__atomic_fetch_xor:
1021 case AtomicExpr::AO__hip_atomic_fetch_xor:
1022 case AtomicExpr::AO__c11_atomic_fetch_max:
1023 case AtomicExpr::AO__c11_atomic_fetch_min:
1024 case AtomicExpr::AO__atomic_add_fetch:
1025 case AtomicExpr::AO__atomic_and_fetch:
1026 case AtomicExpr::AO__atomic_nand_fetch:
1027 case AtomicExpr::AO__atomic_or_fetch:
1028 case AtomicExpr::AO__atomic_sub_fetch:
1029 case AtomicExpr::AO__atomic_xor_fetch:
1030 case AtomicExpr::AO__atomic_fetch_max:
1031 case AtomicExpr::AO__hip_atomic_fetch_max:
1032 case AtomicExpr::AO__atomic_fetch_min:
1033 case AtomicExpr::AO__hip_atomic_fetch_min:
1034 case AtomicExpr::AO__atomic_max_fetch:
1035 case AtomicExpr::AO__atomic_min_fetch:
1036 // For these, only library calls for certain sizes exist.
1037 UseOptimizedLibcall = true;
1038 break;
1039
1040 case AtomicExpr::AO__atomic_load:
1041 case AtomicExpr::AO__atomic_store:
1042 case AtomicExpr::AO__atomic_exchange:
1043 case AtomicExpr::AO__atomic_compare_exchange:
1044 // Use the generic version if we don't know that the operand will be
1045 // suitably aligned for the optimized version.
1046 if (Misaligned)
1047 break;
1048 [[fallthrough]];
1049 case AtomicExpr::AO__c11_atomic_load:
1050 case AtomicExpr::AO__c11_atomic_store:
1051 case AtomicExpr::AO__c11_atomic_exchange:
1052 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1053 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1054 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
1055 case AtomicExpr::AO__opencl_atomic_load:
1056 case AtomicExpr::AO__hip_atomic_load:
1057 case AtomicExpr::AO__opencl_atomic_store:
1058 case AtomicExpr::AO__hip_atomic_store:
1059 case AtomicExpr::AO__opencl_atomic_exchange:
1060 case AtomicExpr::AO__hip_atomic_exchange:
1061 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1062 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
1063 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1064 case AtomicExpr::AO__atomic_load_n:
1065 case AtomicExpr::AO__atomic_store_n:
1066 case AtomicExpr::AO__atomic_exchange_n:
1067 case AtomicExpr::AO__atomic_compare_exchange_n:
1068 // Only use optimized library calls for sizes for which they exist.
1069 // FIXME: Size == 16 optimized library functions exist too.
1070 if (Size == 1 || Size == 2 || Size == 4 || Size == 8)
1071 UseOptimizedLibcall = true;
1072 break;
1073 }
1074
1075 CallArgList Args;
1076 if (!UseOptimizedLibcall) {
1077 // For non-optimized library calls, the size is the first parameter
1078 Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
1079 getContext().getSizeType());
1080 }
1081 // Atomic address is the first or second parameter
1082 // The OpenCL atomic library functions only accept pointer arguments to
1083 // generic address space.
1084 auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) {
1085 if (!E->isOpenCL())
1086 return V;
1087 auto AS = PT->castAs<PointerType>()->getPointeeType().getAddressSpace();
1088 if (AS == LangAS::opencl_generic)
1089 return V;
1091 auto T = llvm::cast<llvm::PointerType>(V->getType());
1092 auto *DestType = llvm::PointerType::getWithSamePointeeType(T, DestAS);
1093
1095 *this, V, AS, LangAS::opencl_generic, DestType, false);
1096 };
1097
1098 Args.add(RValue::get(CastToGenericAddrSpace(
1099 EmitCastToVoidPtr(Ptr.getPointer()), E->getPtr()->getType())),
1101
1102 std::string LibCallName;
1103 QualType LoweredMemTy =
1104 MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy;
1105 QualType RetTy;
1106 bool HaveRetTy = false;
1107 llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
1108 bool PostOpMinMax = false;
1109 switch (E->getOp()) {
1110 case AtomicExpr::AO__c11_atomic_init:
1111 case AtomicExpr::AO__opencl_atomic_init:
1112 llvm_unreachable("Already handled!");
1113
1114 // There is only one libcall for compare an exchange, because there is no
1115 // optimisation benefit possible from a libcall version of a weak compare
1116 // and exchange.
1117 // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
1118 // void *desired, int success, int failure)
1119 // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired,
1120 // int success, int failure)
1121 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1122 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1123 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1124 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
1125 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1126 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
1127 case AtomicExpr::AO__atomic_compare_exchange:
1128 case AtomicExpr::AO__atomic_compare_exchange_n:
1129 LibCallName = "__atomic_compare_exchange";
1130 RetTy = getContext().BoolTy;
1131 HaveRetTy = true;
1132 Args.add(
1133 RValue::get(CastToGenericAddrSpace(
1134 EmitCastToVoidPtr(Val1.getPointer()), E->getVal1()->getType())),
1136 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(),
1137 MemTy, E->getExprLoc(), TInfo.Width);
1138 Args.add(RValue::get(Order), getContext().IntTy);
1139 Order = OrderFail;
1140 break;
1141 // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
1142 // int order)
1143 // T __atomic_exchange_N(T *mem, T val, int order)
1144 case AtomicExpr::AO__c11_atomic_exchange:
1145 case AtomicExpr::AO__opencl_atomic_exchange:
1146 case AtomicExpr::AO__atomic_exchange_n:
1147 case AtomicExpr::AO__atomic_exchange:
1148 case AtomicExpr::AO__hip_atomic_exchange:
1149 LibCallName = "__atomic_exchange";
1150 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1151 MemTy, E->getExprLoc(), TInfo.Width);
1152 break;
1153 // void __atomic_store(size_t size, void *mem, void *val, int order)
1154 // void __atomic_store_N(T *mem, T val, int order)
1155 case AtomicExpr::AO__c11_atomic_store:
1156 case AtomicExpr::AO__opencl_atomic_store:
1157 case AtomicExpr::AO__hip_atomic_store:
1158 case AtomicExpr::AO__atomic_store:
1159 case AtomicExpr::AO__atomic_store_n:
1160 LibCallName = "__atomic_store";
1161 RetTy = getContext().VoidTy;
1162 HaveRetTy = true;
1163 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1164 MemTy, E->getExprLoc(), TInfo.Width);
1165 break;
1166 // void __atomic_load(size_t size, void *mem, void *return, int order)
1167 // T __atomic_load_N(T *mem, int order)
1168 case AtomicExpr::AO__c11_atomic_load:
1169 case AtomicExpr::AO__opencl_atomic_load:
1170 case AtomicExpr::AO__hip_atomic_load:
1171 case AtomicExpr::AO__atomic_load:
1172 case AtomicExpr::AO__atomic_load_n:
1173 LibCallName = "__atomic_load";
1174 break;
1175 // T __atomic_add_fetch_N(T *mem, T val, int order)
1176 // T __atomic_fetch_add_N(T *mem, T val, int order)
1177 case AtomicExpr::AO__atomic_add_fetch:
1178 PostOp = llvm::Instruction::Add;
1179 [[fallthrough]];
1180 case AtomicExpr::AO__c11_atomic_fetch_add:
1181 case AtomicExpr::AO__opencl_atomic_fetch_add:
1182 case AtomicExpr::AO__atomic_fetch_add:
1183 case AtomicExpr::AO__hip_atomic_fetch_add:
1184 LibCallName = "__atomic_fetch_add";
1185 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1186 LoweredMemTy, E->getExprLoc(), TInfo.Width);
1187 break;
1188 // T __atomic_and_fetch_N(T *mem, T val, int order)
1189 // T __atomic_fetch_and_N(T *mem, T val, int order)
1190 case AtomicExpr::AO__atomic_and_fetch:
1191 PostOp = llvm::Instruction::And;
1192 [[fallthrough]];
1193 case AtomicExpr::AO__c11_atomic_fetch_and:
1194 case AtomicExpr::AO__opencl_atomic_fetch_and:
1195 case AtomicExpr::AO__hip_atomic_fetch_and:
1196 case AtomicExpr::AO__atomic_fetch_and:
1197 LibCallName = "__atomic_fetch_and";
1198 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1199 MemTy, E->getExprLoc(), TInfo.Width);
1200 break;
1201 // T __atomic_or_fetch_N(T *mem, T val, int order)
1202 // T __atomic_fetch_or_N(T *mem, T val, int order)
1203 case AtomicExpr::AO__atomic_or_fetch:
1204 PostOp = llvm::Instruction::Or;
1205 [[fallthrough]];
1206 case AtomicExpr::AO__c11_atomic_fetch_or:
1207 case AtomicExpr::AO__opencl_atomic_fetch_or:
1208 case AtomicExpr::AO__hip_atomic_fetch_or:
1209 case AtomicExpr::AO__atomic_fetch_or:
1210 LibCallName = "__atomic_fetch_or";
1211 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1212 MemTy, E->getExprLoc(), TInfo.Width);
1213 break;
1214 // T __atomic_sub_fetch_N(T *mem, T val, int order)
1215 // T __atomic_fetch_sub_N(T *mem, T val, int order)
1216 case AtomicExpr::AO__atomic_sub_fetch:
1217 PostOp = llvm::Instruction::Sub;
1218 [[fallthrough]];
1219 case AtomicExpr::AO__c11_atomic_fetch_sub:
1220 case AtomicExpr::AO__opencl_atomic_fetch_sub:
1221 case AtomicExpr::AO__atomic_fetch_sub:
1222 LibCallName = "__atomic_fetch_sub";
1223 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1224 LoweredMemTy, E->getExprLoc(), TInfo.Width);
1225 break;
1226 // T __atomic_xor_fetch_N(T *mem, T val, int order)
1227 // T __atomic_fetch_xor_N(T *mem, T val, int order)
1228 case AtomicExpr::AO__atomic_xor_fetch:
1229 PostOp = llvm::Instruction::Xor;
1230 [[fallthrough]];
1231 case AtomicExpr::AO__c11_atomic_fetch_xor:
1232 case AtomicExpr::AO__opencl_atomic_fetch_xor:
1233 case AtomicExpr::AO__hip_atomic_fetch_xor:
1234 case AtomicExpr::AO__atomic_fetch_xor:
1235 LibCallName = "__atomic_fetch_xor";
1236 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1237 MemTy, E->getExprLoc(), TInfo.Width);
1238 break;
1239 case AtomicExpr::AO__atomic_min_fetch:
1240 PostOpMinMax = true;
1241 [[fallthrough]];
1242 case AtomicExpr::AO__c11_atomic_fetch_min:
1243 case AtomicExpr::AO__atomic_fetch_min:
1244 case AtomicExpr::AO__hip_atomic_fetch_min:
1245 case AtomicExpr::AO__opencl_atomic_fetch_min:
1246 LibCallName = E->getValueType()->isSignedIntegerType()
1247 ? "__atomic_fetch_min"
1248 : "__atomic_fetch_umin";
1249 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1250 LoweredMemTy, E->getExprLoc(), TInfo.Width);
1251 break;
1252 case AtomicExpr::AO__atomic_max_fetch:
1253 PostOpMinMax = true;
1254 [[fallthrough]];
1255 case AtomicExpr::AO__c11_atomic_fetch_max:
1256 case AtomicExpr::AO__atomic_fetch_max:
1257 case AtomicExpr::AO__hip_atomic_fetch_max:
1258 case AtomicExpr::AO__opencl_atomic_fetch_max:
1259 LibCallName = E->getValueType()->isSignedIntegerType()
1260 ? "__atomic_fetch_max"
1261 : "__atomic_fetch_umax";
1262 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1263 LoweredMemTy, E->getExprLoc(), TInfo.Width);
1264 break;
1265 // T __atomic_nand_fetch_N(T *mem, T val, int order)
1266 // T __atomic_fetch_nand_N(T *mem, T val, int order)
1267 case AtomicExpr::AO__atomic_nand_fetch:
1268 PostOp = llvm::Instruction::And; // the NOT is special cased below
1269 [[fallthrough]];
1270 case AtomicExpr::AO__c11_atomic_fetch_nand:
1271 case AtomicExpr::AO__atomic_fetch_nand:
1272 LibCallName = "__atomic_fetch_nand";
1273 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1274 MemTy, E->getExprLoc(), TInfo.Width);
1275 break;
1276 }
1277
1278 if (E->isOpenCL()) {
1279 LibCallName = std::string("__opencl") +
1280 StringRef(LibCallName).drop_front(1).str();
1281
1282 }
1283 // Optimized functions have the size in their name.
1284 if (UseOptimizedLibcall)
1285 LibCallName += "_" + llvm::utostr(Size);
1286 // By default, assume we return a value of the atomic type.
1287 if (!HaveRetTy) {
1288 if (UseOptimizedLibcall) {
1289 // Value is returned directly.
1290 // The function returns an appropriately sized integer type.
1292 getContext().toBits(TInfo.Width), /*Signed=*/false);
1293 } else {
1294 // Value is returned through parameter before the order.
1295 RetTy = getContext().VoidTy;
1298 }
1299 }
1300 // order is always the last parameter
1301 Args.add(RValue::get(Order),
1302 getContext().IntTy);
1303 if (E->isOpenCL())
1305
1306 // PostOp is only needed for the atomic_*_fetch operations, and
1307 // thus is only needed for and implemented in the
1308 // UseOptimizedLibcall codepath.
1309 assert(UseOptimizedLibcall || (!PostOp && !PostOpMinMax));
1310
1311 RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
1312 // The value is returned directly from the libcall.
1313 if (E->isCmpXChg())
1314 return Res;
1315
1316 // The value is returned directly for optimized libcalls but the expr
1317 // provided an out-param.
1318 if (UseOptimizedLibcall && Res.getScalarVal()) {
1319 llvm::Value *ResVal = Res.getScalarVal();
1320 if (PostOpMinMax) {
1321 llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal();
1322 ResVal = EmitPostAtomicMinMax(Builder, E->getOp(),
1324 ResVal, LoadVal1);
1325 } else if (PostOp) {
1326 llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal();
1327 ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1);
1328 }
1329 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
1330 ResVal = Builder.CreateNot(ResVal);
1331
1333 ResVal, Builder.CreateElementBitCast(Dest, ResVal->getType()));
1334 }
1335
1336 if (RValTy->isVoidType())
1337 return RValue::get(nullptr);
1338
1339 return convertTempToRValue(
1341 RValTy, E->getExprLoc());
1342 }
1343
1344 bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
1345 E->getOp() == AtomicExpr::AO__opencl_atomic_store ||
1346 E->getOp() == AtomicExpr::AO__hip_atomic_store ||
1347 E->getOp() == AtomicExpr::AO__atomic_store ||
1348 E->getOp() == AtomicExpr::AO__atomic_store_n;
1349 bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
1350 E->getOp() == AtomicExpr::AO__opencl_atomic_load ||
1351 E->getOp() == AtomicExpr::AO__hip_atomic_load ||
1352 E->getOp() == AtomicExpr::AO__atomic_load ||
1353 E->getOp() == AtomicExpr::AO__atomic_load_n;
1354
1355 if (isa<llvm::ConstantInt>(Order)) {
1356 auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1357 // We should not ever get to a case where the ordering isn't a valid C ABI
1358 // value, but it's hard to enforce that in general.
1359 if (llvm::isValidAtomicOrderingCABI(ord))
1360 switch ((llvm::AtomicOrderingCABI)ord) {
1361 case llvm::AtomicOrderingCABI::relaxed:
1362 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1363 llvm::AtomicOrdering::Monotonic, Scope);
1364 break;
1365 case llvm::AtomicOrderingCABI::consume:
1366 case llvm::AtomicOrderingCABI::acquire:
1367 if (IsStore)
1368 break; // Avoid crashing on code with undefined behavior
1369 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1370 llvm::AtomicOrdering::Acquire, Scope);
1371 break;
1372 case llvm::AtomicOrderingCABI::release:
1373 if (IsLoad)
1374 break; // Avoid crashing on code with undefined behavior
1375 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1376 llvm::AtomicOrdering::Release, Scope);
1377 break;
1378 case llvm::AtomicOrderingCABI::acq_rel:
1379 if (IsLoad || IsStore)
1380 break; // Avoid crashing on code with undefined behavior
1381 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1382 llvm::AtomicOrdering::AcquireRelease, Scope);
1383 break;
1384 case llvm::AtomicOrderingCABI::seq_cst:
1385 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1386 llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1387 break;
1388 }
1389 if (RValTy->isVoidType())
1390 return RValue::get(nullptr);
1391
1392 return convertTempToRValue(
1394 RValTy, E->getExprLoc());
1395 }
1396
1397 // Long case, when Order isn't obviously constant.
1398
1399 // Create all the relevant BB's
1400 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
1401 *ReleaseBB = nullptr, *AcqRelBB = nullptr,
1402 *SeqCstBB = nullptr;
1403 MonotonicBB = createBasicBlock("monotonic", CurFn);
1404 if (!IsStore)
1405 AcquireBB = createBasicBlock("acquire", CurFn);
1406 if (!IsLoad)
1407 ReleaseBB = createBasicBlock("release", CurFn);
1408 if (!IsLoad && !IsStore)
1409 AcqRelBB = createBasicBlock("acqrel", CurFn);
1410 SeqCstBB = createBasicBlock("seqcst", CurFn);
1411 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1412
1413 // Create the switch for the split
1414 // MonotonicBB is arbitrarily chosen as the default case; in practice, this
1415 // doesn't matter unless someone is crazy enough to use something that
1416 // doesn't fold to a constant for the ordering.
1417 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1418 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
1419
1420 // Emit all the different atomics
1421 Builder.SetInsertPoint(MonotonicBB);
1422 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1423 llvm::AtomicOrdering::Monotonic, Scope);
1424 Builder.CreateBr(ContBB);
1425 if (!IsStore) {
1426 Builder.SetInsertPoint(AcquireBB);
1427 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1428 llvm::AtomicOrdering::Acquire, Scope);
1429 Builder.CreateBr(ContBB);
1430 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
1431 AcquireBB);
1432 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
1433 AcquireBB);
1434 }
1435 if (!IsLoad) {
1436 Builder.SetInsertPoint(ReleaseBB);
1437 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1438 llvm::AtomicOrdering::Release, Scope);
1439 Builder.CreateBr(ContBB);
1440 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release),
1441 ReleaseBB);
1442 }
1443 if (!IsLoad && !IsStore) {
1444 Builder.SetInsertPoint(AcqRelBB);
1445 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1446 llvm::AtomicOrdering::AcquireRelease, Scope);
1447 Builder.CreateBr(ContBB);
1448 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel),
1449 AcqRelBB);
1450 }
1451 Builder.SetInsertPoint(SeqCstBB);
1452 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1453 llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1454 Builder.CreateBr(ContBB);
1455 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
1456 SeqCstBB);
1457
1458 // Cleanup and return
1459 Builder.SetInsertPoint(ContBB);
1460 if (RValTy->isVoidType())
1461 return RValue::get(nullptr);
1462
1463 assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
1464 return convertTempToRValue(
1466 RValTy, E->getExprLoc());
1467}
1468
1469Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const {
1470 llvm::IntegerType *ty =
1471 llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
1472 return CGF.Builder.CreateElementBitCast(addr, ty);
1473}
1474
1475Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {
1476 llvm::Type *Ty = Addr.getElementType();
1477 uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
1478 if (SourceSizeInBits != AtomicSizeInBits) {
1479 Address Tmp = CreateTempAlloca();
1480 CGF.Builder.CreateMemCpy(Tmp, Addr,
1481 std::min(AtomicSizeInBits, SourceSizeInBits) / 8);
1482 Addr = Tmp;
1483 }
1484
1485 return emitCastToAtomicIntPointer(Addr);
1486}
1487
1488RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
1489 AggValueSlot resultSlot,
1490 SourceLocation loc,
1491 bool asValue) const {
1492 if (LVal.isSimple()) {
1493 if (EvaluationKind == TEK_Aggregate)
1494 return resultSlot.asRValue();
1495
1496 // Drill into the padding structure if we have one.
1497 if (hasPadding())
1498 addr = CGF.Builder.CreateStructGEP(addr, 0);
1499
1500 // Otherwise, just convert the temporary to an r-value using the
1501 // normal conversion routine.
1502 return CGF.convertTempToRValue(addr, getValueType(), loc);
1503 }
1504 if (!asValue)
1505 // Get RValue from temp memory as atomic for non-simple lvalues
1506 return RValue::get(CGF.Builder.CreateLoad(addr));
1507 if (LVal.isBitField())
1508 return CGF.EmitLoadOfBitfieldLValue(
1509 LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(),
1510 LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1511 if (LVal.isVectorElt())
1512 return CGF.EmitLoadOfLValue(
1513 LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(),
1514 LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1515 assert(LVal.isExtVectorElt());
1517 addr, LVal.getExtVectorElts(), LVal.getType(),
1518 LVal.getBaseInfo(), TBAAAccessInfo()));
1519}
1520
1521RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
1522 AggValueSlot ResultSlot,
1523 SourceLocation Loc,
1524 bool AsValue) const {
1525 // Try not to in some easy cases.
1526 assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
1527 if (getEvaluationKind() == TEK_Scalar &&
1528 (((!LVal.isBitField() ||
1529 LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
1530 !hasPadding()) ||
1531 !AsValue)) {
1532 auto *ValTy = AsValue
1533 ? CGF.ConvertTypeForMem(ValueTy)
1534 : getAtomicAddress().getElementType();
1535 if (ValTy->isIntegerTy()) {
1536 assert(IntVal->getType() == ValTy && "Different integer types.");
1537 return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
1538 } else if (ValTy->isPointerTy())
1539 return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
1540 else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
1541 return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
1542 }
1543
1544 // Create a temporary. This needs to be big enough to hold the
1545 // atomic integer.
1546 Address Temp = Address::invalid();
1547 bool TempIsVolatile = false;
1548 if (AsValue && getEvaluationKind() == TEK_Aggregate) {
1549 assert(!ResultSlot.isIgnored());
1550 Temp = ResultSlot.getAddress();
1551 TempIsVolatile = ResultSlot.isVolatile();
1552 } else {
1553 Temp = CreateTempAlloca();
1554 }
1555
1556 // Slam the integer into the temporary.
1557 Address CastTemp = emitCastToAtomicIntPointer(Temp);
1558 CGF.Builder.CreateStore(IntVal, CastTemp)
1559 ->setVolatile(TempIsVolatile);
1560
1561 return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue);
1562}
1563
1564void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1565 llvm::AtomicOrdering AO, bool) {
1566 // void __atomic_load(size_t size, void *mem, void *return, int order);
1567 CallArgList Args;
1568 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1569 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1570 CGF.getContext().VoidPtrTy);
1571 Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)),
1572 CGF.getContext().VoidPtrTy);
1573 Args.add(
1574 RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))),
1575 CGF.getContext().IntTy);
1576 emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1577}
1578
1579llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1580 bool IsVolatile) {
1581 // Okay, we're doing this natively.
1582 Address Addr = getAtomicAddressAsAtomicIntPointer();
1583 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
1584 Load->setAtomic(AO);
1585
1586 // Other decoration.
1587 if (IsVolatile)
1588 Load->setVolatile(true);
1590 return Load;
1591}
1592
1593/// An LValue is a candidate for having its loads and stores be made atomic if
1594/// we are operating under /volatile:ms *and* the LValue itself is volatile and
1595/// performing such an operation can be performed without a libcall.
1597 if (!CGM.getLangOpts().MSVolatile) return false;
1598 AtomicInfo AI(*this, LV);
1599 bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType());
1600 // An atomic is inline if we don't need to use a libcall.
1601 bool AtomicIsInline = !AI.shouldUseLibcall();
1602 // MSVC doesn't seem to do this for types wider than a pointer.
1603 if (getContext().getTypeSize(LV.getType()) >
1604 getContext().getTypeSize(getContext().getIntPtrType()))
1605 return false;
1606 return IsVolatile && AtomicIsInline;
1607}
1608
1610 AggValueSlot Slot) {
1611 llvm::AtomicOrdering AO;
1612 bool IsVolatile = LV.isVolatileQualified();
1613 if (LV.getType()->isAtomicType()) {
1614 AO = llvm::AtomicOrdering::SequentiallyConsistent;
1615 } else {
1616 AO = llvm::AtomicOrdering::Acquire;
1617 IsVolatile = true;
1618 }
1619 return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
1620}
1621
1622RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1623 bool AsValue, llvm::AtomicOrdering AO,
1624 bool IsVolatile) {
1625 // Check whether we should use a library call.
1626 if (shouldUseLibcall()) {
1627 Address TempAddr = Address::invalid();
1628 if (LVal.isSimple() && !ResultSlot.isIgnored()) {
1629 assert(getEvaluationKind() == TEK_Aggregate);
1630 TempAddr = ResultSlot.getAddress();
1631 } else
1632 TempAddr = CreateTempAlloca();
1633
1634 EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile);
1635
1636 // Okay, turn that back into the original value or whole atomic (for
1637 // non-simple lvalues) type.
1638 return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
1639 }
1640
1641 // Okay, we're doing this natively.
1642 auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1643
1644 // If we're ignoring an aggregate return, don't do anything.
1645 if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
1646 return RValue::getAggregate(Address::invalid(), false);
1647
1648 // Okay, turn that back into the original value or atomic (for non-simple
1649 // lvalues) type.
1650 return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
1651}
1652
1653/// Emit a load from an l-value of atomic type. Note that the r-value
1654/// we produce is an r-value of the atomic *value* type.
1656 llvm::AtomicOrdering AO, bool IsVolatile,
1657 AggValueSlot resultSlot) {
1658 AtomicInfo Atomics(*this, src);
1659 return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
1660 IsVolatile);
1661}
1662
1663/// Copy an r-value into memory as part of storing to an atomic type.
1664/// This needs to create a bit-pattern suitable for atomic operations.
1665void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1666 assert(LVal.isSimple());
1667 // If we have an r-value, the rvalue should be of the atomic type,
1668 // which means that the caller is responsible for having zeroed
1669 // any padding. Just do an aggregate copy of that type.
1670 if (rvalue.isAggregate()) {
1671 LValue Dest = CGF.MakeAddrLValue(getAtomicAddress(), getAtomicType());
1672 LValue Src = CGF.MakeAddrLValue(rvalue.getAggregateAddress(),
1673 getAtomicType());
1674 bool IsVolatile = rvalue.isVolatileQualified() ||
1675 LVal.isVolatileQualified();
1676 CGF.EmitAggregateCopy(Dest, Src, getAtomicType(),
1677 AggValueSlot::DoesNotOverlap, IsVolatile);
1678 return;
1679 }
1680
1681 // Okay, otherwise we're copying stuff.
1682
1683 // Zero out the buffer if necessary.
1684 emitMemSetZeroIfNecessary();
1685
1686 // Drill past the padding if present.
1687 LValue TempLVal = projectValue();
1688
1689 // Okay, store the rvalue in.
1690 if (rvalue.isScalar()) {
1691 CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
1692 } else {
1693 CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
1694 }
1695}
1696
1697
1698/// Materialize an r-value into memory for the purposes of storing it
1699/// to an atomic type.
1700Address AtomicInfo::materializeRValue(RValue rvalue) const {
1701 // Aggregate r-values are already in memory, and EmitAtomicStore
1702 // requires them to be values of the atomic type.
1703 if (rvalue.isAggregate())
1704 return rvalue.getAggregateAddress();
1705
1706 // Otherwise, make a temporary and materialize into it.
1707 LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType());
1708 AtomicInfo Atomics(CGF, TempLV);
1709 Atomics.emitCopyIntoMemory(rvalue);
1710 return TempLV.getAddress(CGF);
1711}
1712
1713llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
1714 // If we've got a scalar value of the right size, try to avoid going
1715 // through memory.
1716 if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) {
1717 llvm::Value *Value = RVal.getScalarVal();
1718 if (isa<llvm::IntegerType>(Value->getType()))
1719 return CGF.EmitToMemory(Value, ValueTy);
1720 else {
1721 llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1722 CGF.getLLVMContext(),
1723 LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1724 if (isa<llvm::PointerType>(Value->getType()))
1725 return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
1726 else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
1727 return CGF.Builder.CreateBitCast(Value, InputIntTy);
1728 }
1729 }
1730 // Otherwise, we need to go through memory.
1731 // Put the r-value in memory.
1732 Address Addr = materializeRValue(RVal);
1733
1734 // Cast the temporary to the atomic int type and pull a value out.
1735 Addr = emitCastToAtomicIntPointer(Addr);
1736 return CGF.Builder.CreateLoad(Addr);
1737}
1738
1739std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1740 llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1741 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1742 // Do the atomic store.
1743 Address Addr = getAtomicAddressAsAtomicIntPointer();
1744 auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(),
1745 ExpectedVal, DesiredVal,
1746 Success, Failure);
1747 // Other decoration.
1748 Inst->setVolatile(LVal.isVolatileQualified());
1749 Inst->setWeak(IsWeak);
1750
1751 // Okay, turn that back into the original value type.
1752 auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
1753 auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
1754 return std::make_pair(PreviousVal, SuccessFailureVal);
1755}
1756
1757llvm::Value *
1758AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1759 llvm::Value *DesiredAddr,
1760 llvm::AtomicOrdering Success,
1761 llvm::AtomicOrdering Failure) {
1762 // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1763 // void *desired, int success, int failure);
1764 CallArgList Args;
1765 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1766 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1767 CGF.getContext().VoidPtrTy);
1768 Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)),
1769 CGF.getContext().VoidPtrTy);
1770 Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)),
1771 CGF.getContext().VoidPtrTy);
1772 Args.add(RValue::get(
1773 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))),
1774 CGF.getContext().IntTy);
1775 Args.add(RValue::get(
1776 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))),
1777 CGF.getContext().IntTy);
1778 auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1779 CGF.getContext().BoolTy, Args);
1780
1781 return SuccessFailureRVal.getScalarVal();
1782}
1783
1784std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1785 RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1786 llvm::AtomicOrdering Failure, bool IsWeak) {
1787 // Check whether we should use a library call.
1788 if (shouldUseLibcall()) {
1789 // Produce a source address.
1790 Address ExpectedAddr = materializeRValue(Expected);
1791 Address DesiredAddr = materializeRValue(Desired);
1792 auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1793 DesiredAddr.getPointer(),
1794 Success, Failure);
1795 return std::make_pair(
1796 convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1797 SourceLocation(), /*AsValue=*/false),
1798 Res);
1799 }
1800
1801 // If we've got a scalar value of the right size, try to avoid going
1802 // through memory.
1803 auto *ExpectedVal = convertRValueToInt(Expected);
1804 auto *DesiredVal = convertRValueToInt(Desired);
1805 auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1806 Failure, IsWeak);
1807 return std::make_pair(
1808 ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(),
1809 SourceLocation(), /*AsValue=*/false),
1810 Res.second);
1811}
1812
1813static void
1814EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1815 const llvm::function_ref<RValue(RValue)> &UpdateOp,
1816 Address DesiredAddr) {
1817 RValue UpRVal;
1818 LValue AtomicLVal = Atomics.getAtomicLValue();
1819 LValue DesiredLVal;
1820 if (AtomicLVal.isSimple()) {
1821 UpRVal = OldRVal;
1822 DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType());
1823 } else {
1824 // Build new lvalue for temp address.
1825 Address Ptr = Atomics.materializeRValue(OldRVal);
1826 LValue UpdateLVal;
1827 if (AtomicLVal.isBitField()) {
1828 UpdateLVal =
1829 LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
1830 AtomicLVal.getType(),
1831 AtomicLVal.getBaseInfo(),
1832 AtomicLVal.getTBAAInfo());
1833 DesiredLVal =
1834 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1835 AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1836 AtomicLVal.getTBAAInfo());
1837 } else if (AtomicLVal.isVectorElt()) {
1838 UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
1839 AtomicLVal.getType(),
1840 AtomicLVal.getBaseInfo(),
1841 AtomicLVal.getTBAAInfo());
1842 DesiredLVal = LValue::MakeVectorElt(
1843 DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(),
1844 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1845 } else {
1846 assert(AtomicLVal.isExtVectorElt());
1847 UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
1848 AtomicLVal.getType(),
1849 AtomicLVal.getBaseInfo(),
1850 AtomicLVal.getTBAAInfo());
1851 DesiredLVal = LValue::MakeExtVectorElt(
1852 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1853 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1854 }
1855 UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());
1856 }
1857 // Store new value in the corresponding memory area.
1858 RValue NewRVal = UpdateOp(UpRVal);
1859 if (NewRVal.isScalar()) {
1860 CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);
1861 } else {
1862 assert(NewRVal.isComplex());
1863 CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,
1864 /*isInit=*/false);
1865 }
1866}
1867
1868void AtomicInfo::EmitAtomicUpdateLibcall(
1869 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1870 bool IsVolatile) {
1871 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1872
1873 Address ExpectedAddr = CreateTempAlloca();
1874
1875 EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1876 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1877 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1878 CGF.EmitBlock(ContBB);
1879 Address DesiredAddr = CreateTempAlloca();
1880 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1881 requiresMemSetZero(getAtomicAddress().getElementType())) {
1882 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1883 CGF.Builder.CreateStore(OldVal, DesiredAddr);
1884 }
1885 auto OldRVal = convertAtomicTempToRValue(ExpectedAddr,
1887 SourceLocation(), /*AsValue=*/false);
1888 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
1889 auto *Res =
1890 EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1891 DesiredAddr.getPointer(),
1892 AO, Failure);
1893 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1894 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1895}
1896
1897void AtomicInfo::EmitAtomicUpdateOp(
1898 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1899 bool IsVolatile) {
1900 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1901
1902 // Do the atomic load.
1903 auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile);
1904 // For non-simple lvalues perform compare-and-swap procedure.
1905 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1906 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1907 auto *CurBB = CGF.Builder.GetInsertBlock();
1908 CGF.EmitBlock(ContBB);
1909 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1910 /*NumReservedValues=*/2);
1911 PHI->addIncoming(OldVal, CurBB);
1912 Address NewAtomicAddr = CreateTempAlloca();
1913 Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1914 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1915 requiresMemSetZero(getAtomicAddress().getElementType())) {
1916 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1917 }
1918 auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(),
1919 SourceLocation(), /*AsValue=*/false);
1920 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
1921 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1922 // Try to write new value using cmpxchg operation.
1923 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1924 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1925 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1926 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1927}
1928
1929static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1930 RValue UpdateRVal, Address DesiredAddr) {
1931 LValue AtomicLVal = Atomics.getAtomicLValue();
1932 LValue DesiredLVal;
1933 // Build new lvalue for temp address.
1934 if (AtomicLVal.isBitField()) {
1935 DesiredLVal =
1936 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1937 AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1938 AtomicLVal.getTBAAInfo());
1939 } else if (AtomicLVal.isVectorElt()) {
1940 DesiredLVal =
1941 LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(),
1942 AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1943 AtomicLVal.getTBAAInfo());
1944 } else {
1945 assert(AtomicLVal.isExtVectorElt());
1946 DesiredLVal = LValue::MakeExtVectorElt(
1947 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1948 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1949 }
1950 // Store new value in the corresponding memory area.
1951 assert(UpdateRVal.isScalar());
1952 CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);
1953}
1954
1955void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1956 RValue UpdateRVal, bool IsVolatile) {
1957 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1958
1959 Address ExpectedAddr = CreateTempAlloca();
1960
1961 EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1962 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1963 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1964 CGF.EmitBlock(ContBB);
1965 Address DesiredAddr = CreateTempAlloca();
1966 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1967 requiresMemSetZero(getAtomicAddress().getElementType())) {
1968 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1969 CGF.Builder.CreateStore(OldVal, DesiredAddr);
1970 }
1971 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
1972 auto *Res =
1973 EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1974 DesiredAddr.getPointer(),
1975 AO, Failure);
1976 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1977 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1978}
1979
1980void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
1981 bool IsVolatile) {
1982 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1983
1984 // Do the atomic load.
1985 auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile);
1986 // For non-simple lvalues perform compare-and-swap procedure.
1987 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1988 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1989 auto *CurBB = CGF.Builder.GetInsertBlock();
1990 CGF.EmitBlock(ContBB);
1991 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1992 /*NumReservedValues=*/2);
1993 PHI->addIncoming(OldVal, CurBB);
1994 Address NewAtomicAddr = CreateTempAlloca();
1995 Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1996 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1997 requiresMemSetZero(getAtomicAddress().getElementType())) {
1998 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1999 }
2000 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
2001 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
2002 // Try to write new value using cmpxchg operation.
2003 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
2004 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
2005 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
2006 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
2007}
2008
2009void AtomicInfo::EmitAtomicUpdate(
2010 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
2011 bool IsVolatile) {
2012 if (shouldUseLibcall()) {
2013 EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
2014 } else {
2015 EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
2016 }
2017}
2018
2019void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
2020 bool IsVolatile) {
2021 if (shouldUseLibcall()) {
2022 EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
2023 } else {
2024 EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
2025 }
2026}
2027
2029 bool isInit) {
2030 bool IsVolatile = lvalue.isVolatileQualified();
2031 llvm::AtomicOrdering AO;
2032 if (lvalue.getType()->isAtomicType()) {
2033 AO = llvm::AtomicOrdering::SequentiallyConsistent;
2034 } else {
2035 AO = llvm::AtomicOrdering::Release;
2036 IsVolatile = true;
2037 }
2038 return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
2039}
2040
2041/// Emit a store to an l-value of atomic type.
2042///
2043/// Note that the r-value is expected to be an r-value *of the atomic
2044/// type*; this means that for aggregate r-values, it should include
2045/// storage for any padding that was necessary.
2047 llvm::AtomicOrdering AO, bool IsVolatile,
2048 bool isInit) {
2049 // If this is an aggregate r-value, it should agree in type except
2050 // maybe for address-space qualification.
2051 assert(!rvalue.isAggregate() ||
2053 dest.getAddress(*this).getElementType());
2054
2055 AtomicInfo atomics(*this, dest);
2056 LValue LVal = atomics.getAtomicLValue();
2057
2058 // If this is an initialization, just put the value there normally.
2059 if (LVal.isSimple()) {
2060 if (isInit) {
2061 atomics.emitCopyIntoMemory(rvalue);
2062 return;
2063 }
2064
2065 // Check whether we should use a library call.
2066 if (atomics.shouldUseLibcall()) {
2067 // Produce a source address.
2068 Address srcAddr = atomics.materializeRValue(rvalue);
2069
2070 // void __atomic_store(size_t size, void *mem, void *val, int order)
2071 CallArgList args;
2072 args.add(RValue::get(atomics.getAtomicSizeValue()),
2073 getContext().getSizeType());
2074 args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())),
2078 args.add(
2079 RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))),
2080 getContext().IntTy);
2081 emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
2082 return;
2083 }
2084
2085 // Okay, we're doing this natively.
2086 llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
2087
2088 // Do the atomic store.
2089 Address addr =
2090 atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress());
2091 intValue = Builder.CreateIntCast(
2092 intValue, addr.getElementType(), /*isSigned=*/false);
2093 llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
2094
2095 if (AO == llvm::AtomicOrdering::Acquire)
2096 AO = llvm::AtomicOrdering::Monotonic;
2097 else if (AO == llvm::AtomicOrdering::AcquireRelease)
2098 AO = llvm::AtomicOrdering::Release;
2099 // Initializations don't need to be atomic.
2100 if (!isInit)
2101 store->setAtomic(AO);
2102
2103 // Other decoration.
2104 if (IsVolatile)
2105 store->setVolatile(true);
2107 return;
2108 }
2109
2110 // Emit simple atomic update operation.
2111 atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
2112}
2113
2114/// Emit a compare-and-exchange op for atomic type.
2115///
2116std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
2117 LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
2118 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
2119 AggValueSlot Slot) {
2120 // If this is an aggregate r-value, it should agree in type except
2121 // maybe for address-space qualification.
2122 assert(!Expected.isAggregate() ||
2123 Expected.getAggregateAddress().getElementType() ==
2124 Obj.getAddress(*this).getElementType());
2125 assert(!Desired.isAggregate() ||
2126 Desired.getAggregateAddress().getElementType() ==
2127 Obj.getAddress(*this).getElementType());
2128 AtomicInfo Atomics(*this, Obj);
2129
2130 return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
2131 IsWeak);
2132}
2133
2135 LValue LVal, llvm::AtomicOrdering AO,
2136 const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
2137 AtomicInfo Atomics(*this, LVal);
2138 Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
2139}
2140
2142 AtomicInfo atomics(*this, dest);
2143
2144 switch (atomics.getEvaluationKind()) {
2145 case TEK_Scalar: {
2146 llvm::Value *value = EmitScalarExpr(init);
2147 atomics.emitCopyIntoMemory(RValue::get(value));
2148 return;
2149 }
2150
2151 case TEK_Complex: {
2152 ComplexPairTy value = EmitComplexExpr(init);
2153 atomics.emitCopyIntoMemory(RValue::getComplex(value));
2154 return;
2155 }
2156
2157 case TEK_Aggregate: {
2158 // Fix up the destination if the initializer isn't an expression
2159 // of atomic type.
2160 bool Zeroed = false;
2161 if (!init->getType()->isAtomicType()) {
2162 Zeroed = atomics.emitMemSetZeroIfNecessary();
2163 dest = atomics.projectValue();
2164 }
2165
2166 // Evaluate the expression directly into the destination.
2168 dest, *this, AggValueSlot::IsNotDestructed,
2172
2173 EmitAggExpr(init, slot);
2174 return;
2175 }
2176 }
2177 llvm_unreachable("bad evaluation kind");
2178}
Defines the clang::ASTContext interface.
#define V(N, I)
Definition: ASTContext.h:3217
static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type, uint64_t expectedSize)
Does a store of the given IR type modify the full expected width?
Definition: CGAtomic.cpp:332
static llvm::Value * EmitPostAtomicMinMax(CGBuilderTy &Builder, AtomicExpr::AtomicOp Op, bool IsSigned, llvm::Value *OldVal, llvm::Value *RHS)
Duplicate the atomic min/max operation in conventional IR for the builtin variants that return the ne...
Definition: CGAtomic.cpp:501
static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal, const llvm::function_ref< RValue(RValue)> &UpdateOp, Address DesiredAddr)
Definition: CGAtomic.cpp:1814
static Address EmitValToTemp(CodeGenFunction &CGF, Expr *E)
Definition: CGAtomic.cpp:716
static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, Address Ptr, Address Val1, Address Val2, llvm::Value *IsWeak, llvm::Value *FailureOrder, uint64_t Size, llvm::AtomicOrdering Order, llvm::SyncScope::ID Scope)
Definition: CGAtomic.cpp:521
static void AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args, bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy, SourceLocation Loc, CharUnits SizeInChars)
Definition: CGAtomic.cpp:782
static RValue emitAtomicLibcall(CodeGenFunction &CGF, StringRef fnName, QualType resultType, CallArgList &args)
Definition: CGAtomic.cpp:312
static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, Address Dest, Address Ptr, Address Val1, Address Val2, llvm::Value *FailureOrderVal, uint64_t Size, llvm::AtomicOrdering SuccessOrder, llvm::SyncScope::ID Scope)
Given an ordering required on success, emit all possible cmpxchg instructions to cope with the provid...
Definition: CGAtomic.cpp:424
static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, Address Dest, Address Ptr, Address Val1, Address Val2, uint64_t Size, llvm::AtomicOrdering SuccessOrder, llvm::AtomicOrdering FailureOrder, llvm::SyncScope::ID Scope)
Definition: CGAtomic.cpp:374
CodeGenFunction::ComplexPairTy ComplexPairTy
unsigned Offset
Definition: Format.cpp:2776
__device__ int
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition: ASTContext.h:182
CharUnits getTypeAlignInChars(QualType T) const
Return the ABI-specified alignment of a (complete) type T, in characters.
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
CanQualType VoidPtrTy
Definition: ASTContext.h:1105
CanQualType BoolTy
Definition: ASTContext.h:1079
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth,...
CanQualType getSizeType() const
Return the unique type for "size_t" (C99 7.17), defined in <stddef.h>.
CanQualType IntTy
Definition: ASTContext.h:1087
TypeInfoChars getTypeInfoInChars(const Type *T) const
int64_t toBits(CharUnits CharSize) const
Convert a size in characters to a size in bits.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
CanQualType VoidTy
Definition: ASTContext.h:1078
QualType getExtVectorType(QualType VectorType, unsigned NumElts) const
Return the unique reference to an extended vector type of the specified element type and size.
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
unsigned getTargetAddressSpace(LangAS AS) const
QualType getIntPtrType() const
Return a type compatible with "intptr_t" (C99 7.18.1.4), as defined by the target.
AtomicExpr - Variadic atomic builtins: __atomic_exchange, __atomic_fetch_*, __atomic_load,...
Definition: Expr.h:6239
static std::unique_ptr< AtomicScopeModel > getScopeModel(AtomicOp Op)
Get atomic scope model for the atomic op code.
Definition: Expr.h:6352
Expr * getVal2() const
Definition: Expr.h:6291
Expr * getOrder() const
Definition: Expr.h:6274
QualType getValueType() const
Definition: Expr.cpp:4887
Expr * getScope() const
Definition: Expr.h:6277
bool isCmpXChg() const
Definition: Expr.h:6315
AtomicOp getOp() const
Definition: Expr.h:6303
bool isOpenCL() const
Definition: Expr.h:6326
Expr * getVal1() const
Definition: Expr.h:6281
Expr * getPtr() const
Definition: Expr.h:6271
Expr * getWeak() const
Definition: Expr.h:6297
SourceLocation getBeginLoc() const LLVM_READONLY
Definition: Expr.h:6334
Expr * getOrderFail() const
Definition: Expr.h:6287
bool isVolatile() const
Definition: Expr.h:6311
CharUnits - This is an opaque type for sizes expressed in character units.
Definition: CharUnits.h:38
bool isZero() const
isZero - Test whether the quantity equals zero.
Definition: CharUnits.h:122
llvm::Align getAsAlign() const
getAsAlign - Returns Quantity as a valid llvm::Align, Beware llvm::Align assumes power of two 8-bit b...
Definition: CharUnits.h:189
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
Definition: CharUnits.h:185
An aligned address.
Definition: Address.h:29
static Address invalid()
Definition: Address.h:49
CharUnits getAlignment() const
Return the alignment of this pointer.
Definition: Address.h:81
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Definition: Address.h:65
llvm::Value * getPointer() const
Definition: Address.h:54
bool isValid() const
Definition: Address.h:50
An aggregate value slot.
Definition: CGValue.h:514
static AggValueSlot ignored()
ignored - Returns an aggregate value slot indicating that the aggregate value is being ignored.
Definition: CGValue.h:582
Address getAddress() const
Definition: CGValue.h:652
static AggValueSlot forLValue(const LValue &LV, CodeGenFunction &CGF, IsDestructed_t isDestructed, NeedsGCBarriers_t needsGC, IsAliased_t isAliased, Overlap_t mayOverlap, IsZeroed_t isZeroed=IsNotZeroed, IsSanitizerChecked_t isChecked=IsNotSanitizerChecked)
Definition: CGValue.h:612
RValue asRValue() const
Definition: CGValue.h:676
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
Definition: CGBuilder.h:99
Address CreateElementBitCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
Cast the element type of the given address to a different type, preserving information like the align...
Definition: CGBuilder.h:169
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
Definition: CGBuilder.h:177
llvm::AtomicRMWInst * CreateAtomicRMW(llvm::AtomicRMWInst::BinOp Op, llvm::Value *Ptr, llvm::Value *Val, llvm::AtomicOrdering Ordering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
Definition: CGBuilder.h:150
llvm::CallInst * CreateMemSet(Address Dest, llvm::Value *Value, llvm::Value *Size, bool IsVolatile=false)
Definition: CGBuilder.h:347
Address CreateStructGEP(Address Addr, unsigned Index, const llvm::Twine &Name="")
Definition: CGBuilder.h:193
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
Definition: CGBuilder.h:71
llvm::CallInst * CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
Definition: CGBuilder.h:318
llvm::AtomicCmpXchgInst * CreateAtomicCmpXchg(llvm::Value *Ptr, llvm::Value *Cmp, llvm::Value *New, llvm::AtomicOrdering SuccessOrdering, llvm::AtomicOrdering FailureOrdering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
Definition: CGBuilder.h:138
static CGCallee forDirect(llvm::Constant *functionPtr, const CGCalleeInfo &abstractInfo=CGCalleeInfo())
Definition: CGCall.h:130
CGFunctionInfo - Class to encapsulate the information about a function definition.
CallArgList - Type for representing both the value and type of arguments in a call.
Definition: CGCall.h:259
void add(RValue rvalue, QualType type)
Definition: CGCall.h:283
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
llvm::Value * EmitFromMemory(llvm::Value *Value, QualType Ty)
EmitFromMemory - Change a scalar value from its memory representation to its value representation.
std::pair< RValue, llvm::Value * > EmitAtomicCompareExchange(LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, llvm::AtomicOrdering Success=llvm::AtomicOrdering::SequentiallyConsistent, llvm::AtomicOrdering Failure=llvm::AtomicOrdering::SequentiallyConsistent, bool IsWeak=false, AggValueSlot Slot=AggValueSlot::ignored())
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
RValue EmitAtomicLoad(LValue LV, SourceLocation SL, AggValueSlot Slot=AggValueSlot::ignored())
bool hasVolatileMember(QualType T)
hasVolatileMember - returns true if aggregate type has a volatile member.
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO, const llvm::function_ref< RValue(RValue)> &UpdateOp, bool IsVolatile)
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
ComplexPairTy EmitComplexExpr(const Expr *E, bool IgnoreReal=false, bool IgnoreImag=false)
EmitComplexExpr - Emit the computation of the specified expression of complex type,...
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
RValue convertTempToRValue(Address addr, QualType type, SourceLocation Loc)
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee, ReturnValueSlot ReturnValue, const CallArgList &Args, llvm::CallBase **callOrInvoke, bool IsMustTail, SourceLocation Loc)
EmitCall - Generate a call of the given function, expecting the given result type,...
llvm::Type * ConvertTypeForMem(QualType T)
void EmitAtomicInit(Expr *E, LValue lvalue)
const TargetInfo & getTarget() const
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
Address EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitPointerWithAlignment - Given an expression with a pointer type, emit the value and compute our be...
RValue EmitLoadOfExtVectorElementLValue(LValue V)
void EmitAggregateCopy(LValue Dest, LValue Src, QualType EltTy, AggValueSlot::Overlap_t MayOverlap, bool isVolatile=false)
EmitAggregateCopy - Emit an aggregate copy.
const TargetCodeGenInfo & getTargetHooks() const
void EmitAggExpr(const Expr *E, AggValueSlot AS)
EmitAggExpr - Emit the computation of the specified expression of aggregate type.
llvm::Value * EmitToMemory(llvm::Value *Value, QualType Ty)
EmitToMemory - Change a scalar value from its value representation to its in-memory representation.
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
Address CreateMemTemp(QualType T, const Twine &Name="tmp", Address *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
llvm::Value * EmitCastToVoidPtr(llvm::Value *value)
Emit a cast to void* in the appropriate address space.
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit)
EmitStoreOfComplex - Store a complex number into the specified l-value.
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
llvm::LLVMContext & getLLVMContext()
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
bool LValueIsSuitableForInlineAtomic(LValue Src)
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
RValue EmitLoadOfBitfieldLValue(LValue LV, SourceLocation Loc)
This class organizes the cross-function state that is used while generating LLVM code.
llvm::FunctionCallee CreateRuntimeFunction(llvm::FunctionType *Ty, StringRef Name, llvm::AttributeList ExtraAttrs=llvm::AttributeList(), bool Local=false, bool AssumeConvergent=false)
Create or return a runtime function declaration with the specified type and name.
DiagnosticsEngine & getDiags() const
const LangOptions & getLangOpts() const
const llvm::DataLayout & getDataLayout() const
void DecorateInstructionWithTBAA(llvm::Instruction *Inst, TBAAAccessInfo TBAAInfo)
DecorateInstructionWithTBAA - Decorate the instruction with a TBAA tag.
llvm::LLVMContext & getLLVMContext()
llvm::ConstantInt * getSize(CharUnits numChars)
Emit the given number of characters as a value of type size_t.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
Definition: CGCall.cpp:1618
const CGFunctionInfo & arrangeBuiltinFunctionCall(QualType resultType, const CallArgList &args)
Definition: CGCall.cpp:658
LValue - This represents an lvalue references.
Definition: CGValue.h:171
bool isBitField() const
Definition: CGValue.h:270
llvm::Constant * getExtVectorElts() const
Definition: CGValue.h:400
static LValue MakeExtVectorElt(Address vecAddress, llvm::Constant *Elts, QualType type, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo)
Definition: CGValue.h:448
void setAlignment(CharUnits A)
Definition: CGValue.h:334
llvm::Value * getBitFieldPointer() const
Definition: CGValue.h:410
bool isVectorElt() const
Definition: CGValue.h:269
bool isSimple() const
Definition: CGValue.h:268
bool isVolatileQualified() const
Definition: CGValue.h:275
CharUnits getAlignment() const
Definition: CGValue.h:333
Address getAddress(CodeGenFunction &CGF) const
Definition: CGValue.h:352
bool isVolatile() const
Definition: CGValue.h:318
bool isGlobalReg() const
Definition: CGValue.h:272
llvm::Value * getVectorPointer() const
Definition: CGValue.h:369
bool isExtVectorElt() const
Definition: CGValue.h:271
llvm::Value * getVectorIdx() const
Definition: CGValue.h:373
LValueBaseInfo getBaseInfo() const
Definition: CGValue.h:336
llvm::Value * getPointer(CodeGenFunction &CGF) const
Definition: CGValue.h:348
QualType getType() const
Definition: CGValue.h:281
llvm::Value * getExtVectorPointer() const
Definition: CGValue.h:396
const CGBitFieldInfo & getBitFieldInfo() const
Definition: CGValue.h:411
TBAAAccessInfo getTBAAInfo() const
Definition: CGValue.h:325
Address getVectorAddress() const
Definition: CGValue.h:365
static LValue MakeBitfield(Address Addr, const CGBitFieldInfo &Info, QualType type, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo)
Create a new object to represent a bit-field access.
Definition: CGValue.h:468
static LValue MakeVectorElt(Address vecAddress, llvm::Value *Idx, QualType type, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo)
Definition: CGValue.h:434
Address getExtVectorAddress() const
Definition: CGValue.h:392
Address getBitFieldAddress() const
Definition: CGValue.h:406
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
Definition: CGValue.h:39
bool isScalar() const
Definition: CGValue.h:54
static RValue get(llvm::Value *V)
Definition: CGValue.h:89
static RValue getAggregate(Address addr, bool isVolatile=false)
Definition: CGValue.h:110
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
Definition: CGValue.h:96
bool isAggregate() const
Definition: CGValue.h:56
Address getAggregateAddress() const
getAggregateAddr() - Return the Value* of the address of the aggregate.
Definition: CGValue.h:73
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
Definition: CGValue.h:61
bool isComplex() const
Definition: CGValue.h:55
bool isVolatileQualified() const
Definition: CGValue.h:58
std::pair< llvm::Value *, llvm::Value * > getComplexVal() const
getComplexVal - Return the real/imag components of this complex value.
Definition: CGValue.h:68
ReturnValueSlot - Contains the address where the return value of a function can be stored,...
Definition: CGCall.h:357
virtual llvm::SyncScope::ID getLLVMSyncScopeID(const LangOptions &LangOpts, SyncScope Scope, llvm::AtomicOrdering Ordering, llvm::LLVMContext &Ctx) const
Get the syncscope used in LLVM IR.
Definition: TargetInfo.cpp:544
virtual llvm::Value * performAddrSpaceCast(CodeGen::CodeGenFunction &CGF, llvm::Value *V, LangAS SrcAddr, LangAS DestAddr, llvm::Type *DestTy, bool IsNonNull=false) const
Perform address space cast of an expression of pointer type.
Definition: TargetInfo.cpp:522
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:192
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
Definition: Diagnostic.h:1542
This represents one expression.
Definition: Expr.h:110
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
Definition: Expr.cpp:330
QualType getType() const
Definition: Expr.h:142
PointerType - C99 6.7.5.1 - Pointer Declarators.
Definition: Type.h:2788
A (possibly-)qualified type.
Definition: Type.h:736
bool isNull() const
Return true if this QualType doesn't point to a type yet.
Definition: Type.h:803
Qualifiers getQualifiers() const
Retrieve the set of qualifiers applied to this type.
Definition: Type.h:6689
QualType getUnqualifiedType() const
Retrieve the unqualified variant of the given type, removing as little sugar as possible.
Definition: Type.h:6742
Scope - A scope is a transient data structure that is used while parsing the program.
Definition: Scope.h:41
Encodes a location in the source.
unsigned getMaxAtomicInlineWidth() const
Return the maximum width lock-free atomic operation which can be inlined given the supported features...
Definition: TargetInfo.h:780
bool isVoidType() const
Definition: Type.h:7218
bool isSignedIntegerType() const
Return true if this is an integer type that is signed, according to C99 6.2.5p4 [char,...
Definition: Type.cpp:2042
bool isPointerType() const
Definition: Type.h:6910
const T * castAs() const
Member-template castAs<specific type>.
Definition: Type.h:7491
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
Definition: Type.cpp:629
bool isAtomicType() const
Definition: Type.h:7051
bool isFloatingType() const
Definition: Type.cpp:2145
const T * getAs() const
Member-template getAs<specific type>'.
Definition: Type.h:7424
Represents a GCC generic vector type.
Definition: Type.h:3377
TypeEvaluationKind
The kind of evaluation to perform on values of a particular type.
const internal::VariadicAllOfMatcher< Type > type
Matches Types in the clang AST.
bool Load(InterpState &S, CodePtr OpPC)
Definition: Interp.h:1043
@ C
Languages that the frontend can parse and compile.
llvm::StringRef getAsString(SyncScope S)
Definition: SyncScope.h:55
unsigned long uint64_t
#define true
Definition: stdbool.h:21
Structure with information about how a bitfield should be accessed.
CharUnits StorageOffset
The offset of the bitfield storage from the start of the struct.
unsigned Offset
The offset within a contiguous run of bitfields that are represented as a single "field" within the L...
unsigned Size
The total size of the bit-field, in bits.
unsigned StorageSize
The storage size in bits which should be used when accessing this bitfield.
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
llvm::IntegerType * IntTy
int
uint64_t Width
Definition: ASTContext.h:153
unsigned Align
Definition: ASTContext.h:154