clang 18.0.0git
CGAtomic.cpp
Go to the documentation of this file.
1//===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the code for emitting atomic operations.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGCall.h"
14#include "CGRecordLayout.h"
15#include "CodeGenFunction.h"
16#include "CodeGenModule.h"
17#include "TargetInfo.h"
21#include "llvm/ADT/DenseMap.h"
22#include "llvm/IR/DataLayout.h"
23#include "llvm/IR/Intrinsics.h"
24#include "llvm/IR/Operator.h"
25
26using namespace clang;
27using namespace CodeGen;
28
29namespace {
30 class AtomicInfo {
31 CodeGenFunction &CGF;
32 QualType AtomicTy;
33 QualType ValueTy;
34 uint64_t AtomicSizeInBits;
35 uint64_t ValueSizeInBits;
36 CharUnits AtomicAlign;
37 CharUnits ValueAlign;
38 TypeEvaluationKind EvaluationKind;
39 bool UseLibcall;
40 LValue LVal;
42 public:
43 AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
44 : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
45 EvaluationKind(TEK_Scalar), UseLibcall(true) {
46 assert(!lvalue.isGlobalReg());
47 ASTContext &C = CGF.getContext();
48 if (lvalue.isSimple()) {
49 AtomicTy = lvalue.getType();
50 if (auto *ATy = AtomicTy->getAs<AtomicType>())
51 ValueTy = ATy->getValueType();
52 else
53 ValueTy = AtomicTy;
54 EvaluationKind = CGF.getEvaluationKind(ValueTy);
55
56 uint64_t ValueAlignInBits;
57 uint64_t AtomicAlignInBits;
58 TypeInfo ValueTI = C.getTypeInfo(ValueTy);
59 ValueSizeInBits = ValueTI.Width;
60 ValueAlignInBits = ValueTI.Align;
61
62 TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
63 AtomicSizeInBits = AtomicTI.Width;
64 AtomicAlignInBits = AtomicTI.Align;
65
66 assert(ValueSizeInBits <= AtomicSizeInBits);
67 assert(ValueAlignInBits <= AtomicAlignInBits);
68
69 AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
70 ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
71 if (lvalue.getAlignment().isZero())
72 lvalue.setAlignment(AtomicAlign);
73
74 LVal = lvalue;
75 } else if (lvalue.isBitField()) {
76 ValueTy = lvalue.getType();
77 ValueSizeInBits = C.getTypeSize(ValueTy);
78 auto &OrigBFI = lvalue.getBitFieldInfo();
79 auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
80 AtomicSizeInBits = C.toBits(
81 C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
82 .alignTo(lvalue.getAlignment()));
83 llvm::Value *BitFieldPtr = lvalue.getBitFieldPointer();
84 auto OffsetInChars =
85 (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
86 lvalue.getAlignment();
87 llvm::Value *StoragePtr = CGF.Builder.CreateConstGEP1_64(
88 CGF.Int8Ty, BitFieldPtr, OffsetInChars.getQuantity());
89 StoragePtr = CGF.Builder.CreateAddrSpaceCast(
90 StoragePtr, CGF.UnqualPtrTy, "atomic_bitfield_base");
91 BFI = OrigBFI;
92 BFI.Offset = Offset;
93 BFI.StorageSize = AtomicSizeInBits;
94 BFI.StorageOffset += OffsetInChars;
95 llvm::Type *StorageTy = CGF.Builder.getIntNTy(AtomicSizeInBits);
96 LVal = LValue::MakeBitfield(
97 Address(StoragePtr, StorageTy, lvalue.getAlignment()), BFI,
98 lvalue.getType(), lvalue.getBaseInfo(), lvalue.getTBAAInfo());
99 AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
100 if (AtomicTy.isNull()) {
101 llvm::APInt Size(
102 /*numBits=*/32,
103 C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
104 AtomicTy =
105 C.getConstantArrayType(C.CharTy, Size, nullptr, ArrayType::Normal,
106 /*IndexTypeQuals=*/0);
107 }
108 AtomicAlign = ValueAlign = lvalue.getAlignment();
109 } else if (lvalue.isVectorElt()) {
110 ValueTy = lvalue.getType()->castAs<VectorType>()->getElementType();
111 ValueSizeInBits = C.getTypeSize(ValueTy);
112 AtomicTy = lvalue.getType();
113 AtomicSizeInBits = C.getTypeSize(AtomicTy);
114 AtomicAlign = ValueAlign = lvalue.getAlignment();
115 LVal = lvalue;
116 } else {
117 assert(lvalue.isExtVectorElt());
118 ValueTy = lvalue.getType();
119 ValueSizeInBits = C.getTypeSize(ValueTy);
120 AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
121 lvalue.getType(), cast<llvm::FixedVectorType>(
123 ->getNumElements());
124 AtomicSizeInBits = C.getTypeSize(AtomicTy);
125 AtomicAlign = ValueAlign = lvalue.getAlignment();
126 LVal = lvalue;
127 }
128 UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
129 AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
130 }
131
132 QualType getAtomicType() const { return AtomicTy; }
133 QualType getValueType() const { return ValueTy; }
134 CharUnits getAtomicAlignment() const { return AtomicAlign; }
135 uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
136 uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
137 TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
138 bool shouldUseLibcall() const { return UseLibcall; }
139 const LValue &getAtomicLValue() const { return LVal; }
140 llvm::Value *getAtomicPointer() const {
141 if (LVal.isSimple())
142 return LVal.getPointer(CGF);
143 else if (LVal.isBitField())
144 return LVal.getBitFieldPointer();
145 else if (LVal.isVectorElt())
146 return LVal.getVectorPointer();
147 assert(LVal.isExtVectorElt());
148 return LVal.getExtVectorPointer();
149 }
150 Address getAtomicAddress() const {
151 llvm::Type *ElTy;
152 if (LVal.isSimple())
153 ElTy = LVal.getAddress(CGF).getElementType();
154 else if (LVal.isBitField())
155 ElTy = LVal.getBitFieldAddress().getElementType();
156 else if (LVal.isVectorElt())
157 ElTy = LVal.getVectorAddress().getElementType();
158 else
159 ElTy = LVal.getExtVectorAddress().getElementType();
160 return Address(getAtomicPointer(), ElTy, getAtomicAlignment());
161 }
162
163 Address getAtomicAddressAsAtomicIntPointer() const {
164 return castToAtomicIntPointer(getAtomicAddress());
165 }
166
167 /// Is the atomic size larger than the underlying value type?
168 ///
169 /// Note that the absence of padding does not mean that atomic
170 /// objects are completely interchangeable with non-atomic
171 /// objects: we might have promoted the alignment of a type
172 /// without making it bigger.
173 bool hasPadding() const {
174 return (ValueSizeInBits != AtomicSizeInBits);
175 }
176
177 bool emitMemSetZeroIfNecessary() const;
178
179 llvm::Value *getAtomicSizeValue() const {
180 CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
181 return CGF.CGM.getSize(size);
182 }
183
184 /// Cast the given pointer to an integer pointer suitable for atomic
185 /// operations if the source.
186 Address castToAtomicIntPointer(Address Addr) const;
187
188 /// If Addr is compatible with the iN that will be used for an atomic
189 /// operation, bitcast it. Otherwise, create a temporary that is suitable
190 /// and copy the value across.
191 Address convertToAtomicIntPointer(Address Addr) const;
192
193 /// Turn an atomic-layout object into an r-value.
194 RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
195 SourceLocation loc, bool AsValue) const;
196
197 /// Converts a rvalue to integer value.
198 llvm::Value *convertRValueToInt(RValue RVal) const;
199
200 RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
201 AggValueSlot ResultSlot,
202 SourceLocation Loc, bool AsValue) const;
203
204 /// Copy an atomic r-value into atomic-layout memory.
205 void emitCopyIntoMemory(RValue rvalue) const;
206
207 /// Project an l-value down to the value field.
208 LValue projectValue() const {
209 assert(LVal.isSimple());
210 Address addr = getAtomicAddress();
211 if (hasPadding())
212 addr = CGF.Builder.CreateStructGEP(addr, 0);
213
214 return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
215 LVal.getBaseInfo(), LVal.getTBAAInfo());
216 }
217
218 /// Emits atomic load.
219 /// \returns Loaded value.
220 RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
221 bool AsValue, llvm::AtomicOrdering AO,
222 bool IsVolatile);
223
224 /// Emits atomic compare-and-exchange sequence.
225 /// \param Expected Expected value.
226 /// \param Desired Desired value.
227 /// \param Success Atomic ordering for success operation.
228 /// \param Failure Atomic ordering for failed operation.
229 /// \param IsWeak true if atomic operation is weak, false otherwise.
230 /// \returns Pair of values: previous value from storage (value type) and
231 /// boolean flag (i1 type) with true if success and false otherwise.
232 std::pair<RValue, llvm::Value *>
233 EmitAtomicCompareExchange(RValue Expected, RValue Desired,
234 llvm::AtomicOrdering Success =
235 llvm::AtomicOrdering::SequentiallyConsistent,
236 llvm::AtomicOrdering Failure =
237 llvm::AtomicOrdering::SequentiallyConsistent,
238 bool IsWeak = false);
239
240 /// Emits atomic update.
241 /// \param AO Atomic ordering.
242 /// \param UpdateOp Update operation for the current lvalue.
243 void EmitAtomicUpdate(llvm::AtomicOrdering AO,
244 const llvm::function_ref<RValue(RValue)> &UpdateOp,
245 bool IsVolatile);
246 /// Emits atomic update.
247 /// \param AO Atomic ordering.
248 void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
249 bool IsVolatile);
250
251 /// Materialize an atomic r-value in atomic-layout memory.
252 Address materializeRValue(RValue rvalue) const;
253
254 /// Creates temp alloca for intermediate operations on atomic value.
255 Address CreateTempAlloca() const;
256 private:
257 bool requiresMemSetZero(llvm::Type *type) const;
258
259
260 /// Emits atomic load as a libcall.
261 void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
262 llvm::AtomicOrdering AO, bool IsVolatile);
263 /// Emits atomic load as LLVM instruction.
264 llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
265 /// Emits atomic compare-and-exchange op as a libcall.
266 llvm::Value *EmitAtomicCompareExchangeLibcall(
267 llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
268 llvm::AtomicOrdering Success =
269 llvm::AtomicOrdering::SequentiallyConsistent,
270 llvm::AtomicOrdering Failure =
271 llvm::AtomicOrdering::SequentiallyConsistent);
272 /// Emits atomic compare-and-exchange op as LLVM instruction.
273 std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
274 llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
275 llvm::AtomicOrdering Success =
276 llvm::AtomicOrdering::SequentiallyConsistent,
277 llvm::AtomicOrdering Failure =
278 llvm::AtomicOrdering::SequentiallyConsistent,
279 bool IsWeak = false);
280 /// Emit atomic update as libcalls.
281 void
282 EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
283 const llvm::function_ref<RValue(RValue)> &UpdateOp,
284 bool IsVolatile);
285 /// Emit atomic update as LLVM instructions.
286 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
287 const llvm::function_ref<RValue(RValue)> &UpdateOp,
288 bool IsVolatile);
289 /// Emit atomic update as libcalls.
290 void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
291 bool IsVolatile);
292 /// Emit atomic update as LLVM instructions.
293 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
294 bool IsVolatile);
295 };
296}
297
298Address AtomicInfo::CreateTempAlloca() const {
299 Address TempAlloca = CGF.CreateMemTemp(
300 (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
301 : AtomicTy,
302 getAtomicAlignment(),
303 "atomic-temp");
304 // Cast to pointer to value type for bitfields.
305 if (LVal.isBitField())
307 TempAlloca, getAtomicAddress().getType(),
308 getAtomicAddress().getElementType());
309 return TempAlloca;
310}
311
313 StringRef fnName,
314 QualType resultType,
315 CallArgList &args) {
316 const CGFunctionInfo &fnInfo =
317 CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args);
318 llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
319 llvm::AttrBuilder fnAttrB(CGF.getLLVMContext());
320 fnAttrB.addAttribute(llvm::Attribute::NoUnwind);
321 fnAttrB.addAttribute(llvm::Attribute::WillReturn);
322 llvm::AttributeList fnAttrs = llvm::AttributeList::get(
323 CGF.getLLVMContext(), llvm::AttributeList::FunctionIndex, fnAttrB);
324
325 llvm::FunctionCallee fn =
326 CGF.CGM.CreateRuntimeFunction(fnTy, fnName, fnAttrs);
327 auto callee = CGCallee::forDirect(fn);
328 return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args);
329}
330
331/// Does a store of the given IR type modify the full expected width?
332static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
333 uint64_t expectedSize) {
334 return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
335}
336
337/// Does the atomic type require memsetting to zero before initialization?
338///
339/// The IR type is provided as a way of making certain queries faster.
340bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
341 // If the atomic type has size padding, we definitely need a memset.
342 if (hasPadding()) return true;
343
344 // Otherwise, do some simple heuristics to try to avoid it:
345 switch (getEvaluationKind()) {
346 // For scalars and complexes, check whether the store size of the
347 // type uses the full size.
348 case TEK_Scalar:
349 return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
350 case TEK_Complex:
351 return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
352 AtomicSizeInBits / 2);
353
354 // Padding in structs has an undefined bit pattern. User beware.
355 case TEK_Aggregate:
356 return false;
357 }
358 llvm_unreachable("bad evaluation kind");
359}
360
361bool AtomicInfo::emitMemSetZeroIfNecessary() const {
362 assert(LVal.isSimple());
363 Address addr = LVal.getAddress(CGF);
364 if (!requiresMemSetZero(addr.getElementType()))
365 return false;
366
368 addr.getPointer(), llvm::ConstantInt::get(CGF.Int8Ty, 0),
369 CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
370 LVal.getAlignment().getAsAlign());
371 return true;
372}
373
374static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
375 Address Dest, Address Ptr,
376 Address Val1, Address Val2,
377 uint64_t Size,
378 llvm::AtomicOrdering SuccessOrder,
379 llvm::AtomicOrdering FailureOrder,
380 llvm::SyncScope::ID Scope) {
381 // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
382 llvm::Value *Expected = CGF.Builder.CreateLoad(Val1);
383 llvm::Value *Desired = CGF.Builder.CreateLoad(Val2);
384
385 llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
386 Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder,
387 Scope);
388 Pair->setVolatile(E->isVolatile());
389 Pair->setWeak(IsWeak);
390
391 // Cmp holds the result of the compare-exchange operation: true on success,
392 // false on failure.
393 llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
394 llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
395
396 // This basic block is used to hold the store instruction if the operation
397 // failed.
398 llvm::BasicBlock *StoreExpectedBB =
399 CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
400
401 // This basic block is the exit point of the operation, we should end up
402 // here regardless of whether or not the operation succeeded.
403 llvm::BasicBlock *ContinueBB =
404 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
405
406 // Update Expected if Expected isn't equal to Old, otherwise branch to the
407 // exit point.
408 CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
409
410 CGF.Builder.SetInsertPoint(StoreExpectedBB);
411 // Update the memory at Expected with Old's value.
412 CGF.Builder.CreateStore(Old, Val1);
413 // Finally, branch to the exit point.
414 CGF.Builder.CreateBr(ContinueBB);
415
416 CGF.Builder.SetInsertPoint(ContinueBB);
417 // Update the memory at Dest with Cmp's value.
418 CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
419}
420
421/// Given an ordering required on success, emit all possible cmpxchg
422/// instructions to cope with the provided (but possibly only dynamically known)
423/// FailureOrder.
425 bool IsWeak, Address Dest, Address Ptr,
426 Address Val1, Address Val2,
427 llvm::Value *FailureOrderVal,
428 uint64_t Size,
429 llvm::AtomicOrdering SuccessOrder,
430 llvm::SyncScope::ID Scope) {
431 llvm::AtomicOrdering FailureOrder;
432 if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
433 auto FOS = FO->getSExtValue();
434 if (!llvm::isValidAtomicOrderingCABI(FOS))
435 FailureOrder = llvm::AtomicOrdering::Monotonic;
436 else
437 switch ((llvm::AtomicOrderingCABI)FOS) {
438 case llvm::AtomicOrderingCABI::relaxed:
439 // 31.7.2.18: "The failure argument shall not be memory_order_release
440 // nor memory_order_acq_rel". Fallback to monotonic.
441 case llvm::AtomicOrderingCABI::release:
442 case llvm::AtomicOrderingCABI::acq_rel:
443 FailureOrder = llvm::AtomicOrdering::Monotonic;
444 break;
445 case llvm::AtomicOrderingCABI::consume:
446 case llvm::AtomicOrderingCABI::acquire:
447 FailureOrder = llvm::AtomicOrdering::Acquire;
448 break;
449 case llvm::AtomicOrderingCABI::seq_cst:
450 FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent;
451 break;
452 }
453 // Prior to c++17, "the failure argument shall be no stronger than the
454 // success argument". This condition has been lifted and the only
455 // precondition is 31.7.2.18. Effectively treat this as a DR and skip
456 // language version checks.
457 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
458 FailureOrder, Scope);
459 return;
460 }
461
462 // Create all the relevant BB's
463 auto *MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
464 auto *AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
465 auto *SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
466 auto *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
467
468 // MonotonicBB is arbitrarily chosen as the default case; in practice, this
469 // doesn't matter unless someone is crazy enough to use something that
470 // doesn't fold to a constant for the ordering.
471 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
472 // Implemented as acquire, since it's the closest in LLVM.
473 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
474 AcquireBB);
475 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
476 AcquireBB);
477 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
478 SeqCstBB);
479
480 // Emit all the different atomics
481 CGF.Builder.SetInsertPoint(MonotonicBB);
482 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
483 Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope);
484 CGF.Builder.CreateBr(ContBB);
485
486 CGF.Builder.SetInsertPoint(AcquireBB);
487 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
488 llvm::AtomicOrdering::Acquire, Scope);
489 CGF.Builder.CreateBr(ContBB);
490
491 CGF.Builder.SetInsertPoint(SeqCstBB);
492 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
493 llvm::AtomicOrdering::SequentiallyConsistent, Scope);
494 CGF.Builder.CreateBr(ContBB);
495
496 CGF.Builder.SetInsertPoint(ContBB);
497}
498
499/// Duplicate the atomic min/max operation in conventional IR for the builtin
500/// variants that return the new rather than the original value.
501static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder,
503 bool IsSigned,
504 llvm::Value *OldVal,
505 llvm::Value *RHS) {
506 llvm::CmpInst::Predicate Pred;
507 switch (Op) {
508 default:
509 llvm_unreachable("Unexpected min/max operation");
510 case AtomicExpr::AO__atomic_max_fetch:
511 Pred = IsSigned ? llvm::CmpInst::ICMP_SGT : llvm::CmpInst::ICMP_UGT;
512 break;
513 case AtomicExpr::AO__atomic_min_fetch:
514 Pred = IsSigned ? llvm::CmpInst::ICMP_SLT : llvm::CmpInst::ICMP_ULT;
515 break;
516 }
517 llvm::Value *Cmp = Builder.CreateICmp(Pred, OldVal, RHS, "tst");
518 return Builder.CreateSelect(Cmp, OldVal, RHS, "newval");
519}
520
522 Address Ptr, Address Val1, Address Val2,
523 llvm::Value *IsWeak, llvm::Value *FailureOrder,
524 uint64_t Size, llvm::AtomicOrdering Order,
525 llvm::SyncScope::ID Scope) {
526 llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
527 bool PostOpMinMax = false;
528 unsigned PostOp = 0;
529
530 switch (E->getOp()) {
531 case AtomicExpr::AO__c11_atomic_init:
532 case AtomicExpr::AO__opencl_atomic_init:
533 llvm_unreachable("Already handled!");
534
535 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
536 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
537 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
538 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
539 FailureOrder, Size, Order, Scope);
540 return;
541 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
542 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
543 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
544 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
545 FailureOrder, Size, Order, Scope);
546 return;
547 case AtomicExpr::AO__atomic_compare_exchange:
548 case AtomicExpr::AO__atomic_compare_exchange_n: {
549 if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
550 emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
551 Val1, Val2, FailureOrder, Size, Order, Scope);
552 } else {
553 // Create all the relevant BB's
554 llvm::BasicBlock *StrongBB =
555 CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
556 llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
557 llvm::BasicBlock *ContBB =
558 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
559
560 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
561 SI->addCase(CGF.Builder.getInt1(false), StrongBB);
562
563 CGF.Builder.SetInsertPoint(StrongBB);
564 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
565 FailureOrder, Size, Order, Scope);
566 CGF.Builder.CreateBr(ContBB);
567
568 CGF.Builder.SetInsertPoint(WeakBB);
569 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
570 FailureOrder, Size, Order, Scope);
571 CGF.Builder.CreateBr(ContBB);
572
573 CGF.Builder.SetInsertPoint(ContBB);
574 }
575 return;
576 }
577 case AtomicExpr::AO__c11_atomic_load:
578 case AtomicExpr::AO__opencl_atomic_load:
579 case AtomicExpr::AO__hip_atomic_load:
580 case AtomicExpr::AO__atomic_load_n:
581 case AtomicExpr::AO__atomic_load: {
582 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
583 Load->setAtomic(Order, Scope);
584 Load->setVolatile(E->isVolatile());
585 CGF.Builder.CreateStore(Load, Dest);
586 return;
587 }
588
589 case AtomicExpr::AO__c11_atomic_store:
590 case AtomicExpr::AO__opencl_atomic_store:
591 case AtomicExpr::AO__hip_atomic_store:
592 case AtomicExpr::AO__atomic_store:
593 case AtomicExpr::AO__atomic_store_n: {
594 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
595 llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
596 Store->setAtomic(Order, Scope);
597 Store->setVolatile(E->isVolatile());
598 return;
599 }
600
601 case AtomicExpr::AO__c11_atomic_exchange:
602 case AtomicExpr::AO__hip_atomic_exchange:
603 case AtomicExpr::AO__opencl_atomic_exchange:
604 case AtomicExpr::AO__atomic_exchange_n:
605 case AtomicExpr::AO__atomic_exchange:
606 Op = llvm::AtomicRMWInst::Xchg;
607 break;
608
609 case AtomicExpr::AO__atomic_add_fetch:
610 PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FAdd
611 : llvm::Instruction::Add;
612 [[fallthrough]];
613 case AtomicExpr::AO__c11_atomic_fetch_add:
614 case AtomicExpr::AO__hip_atomic_fetch_add:
615 case AtomicExpr::AO__opencl_atomic_fetch_add:
616 case AtomicExpr::AO__atomic_fetch_add:
617 Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FAdd
618 : llvm::AtomicRMWInst::Add;
619 break;
620
621 case AtomicExpr::AO__atomic_sub_fetch:
622 PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FSub
623 : llvm::Instruction::Sub;
624 [[fallthrough]];
625 case AtomicExpr::AO__c11_atomic_fetch_sub:
626 case AtomicExpr::AO__hip_atomic_fetch_sub:
627 case AtomicExpr::AO__opencl_atomic_fetch_sub:
628 case AtomicExpr::AO__atomic_fetch_sub:
629 Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FSub
630 : llvm::AtomicRMWInst::Sub;
631 break;
632
633 case AtomicExpr::AO__atomic_min_fetch:
634 PostOpMinMax = true;
635 [[fallthrough]];
636 case AtomicExpr::AO__c11_atomic_fetch_min:
637 case AtomicExpr::AO__hip_atomic_fetch_min:
638 case AtomicExpr::AO__opencl_atomic_fetch_min:
639 case AtomicExpr::AO__atomic_fetch_min:
640 Op = E->getValueType()->isFloatingType()
641 ? llvm::AtomicRMWInst::FMin
643 ? llvm::AtomicRMWInst::Min
644 : llvm::AtomicRMWInst::UMin);
645 break;
646
647 case AtomicExpr::AO__atomic_max_fetch:
648 PostOpMinMax = true;
649 [[fallthrough]];
650 case AtomicExpr::AO__c11_atomic_fetch_max:
651 case AtomicExpr::AO__hip_atomic_fetch_max:
652 case AtomicExpr::AO__opencl_atomic_fetch_max:
653 case AtomicExpr::AO__atomic_fetch_max:
654 Op = E->getValueType()->isFloatingType()
655 ? llvm::AtomicRMWInst::FMax
657 ? llvm::AtomicRMWInst::Max
658 : llvm::AtomicRMWInst::UMax);
659 break;
660
661 case AtomicExpr::AO__atomic_and_fetch:
662 PostOp = llvm::Instruction::And;
663 [[fallthrough]];
664 case AtomicExpr::AO__c11_atomic_fetch_and:
665 case AtomicExpr::AO__hip_atomic_fetch_and:
666 case AtomicExpr::AO__opencl_atomic_fetch_and:
667 case AtomicExpr::AO__atomic_fetch_and:
668 Op = llvm::AtomicRMWInst::And;
669 break;
670
671 case AtomicExpr::AO__atomic_or_fetch:
672 PostOp = llvm::Instruction::Or;
673 [[fallthrough]];
674 case AtomicExpr::AO__c11_atomic_fetch_or:
675 case AtomicExpr::AO__hip_atomic_fetch_or:
676 case AtomicExpr::AO__opencl_atomic_fetch_or:
677 case AtomicExpr::AO__atomic_fetch_or:
678 Op = llvm::AtomicRMWInst::Or;
679 break;
680
681 case AtomicExpr::AO__atomic_xor_fetch:
682 PostOp = llvm::Instruction::Xor;
683 [[fallthrough]];
684 case AtomicExpr::AO__c11_atomic_fetch_xor:
685 case AtomicExpr::AO__hip_atomic_fetch_xor:
686 case AtomicExpr::AO__opencl_atomic_fetch_xor:
687 case AtomicExpr::AO__atomic_fetch_xor:
688 Op = llvm::AtomicRMWInst::Xor;
689 break;
690
691 case AtomicExpr::AO__atomic_nand_fetch:
692 PostOp = llvm::Instruction::And; // the NOT is special cased below
693 [[fallthrough]];
694 case AtomicExpr::AO__c11_atomic_fetch_nand:
695 case AtomicExpr::AO__atomic_fetch_nand:
696 Op = llvm::AtomicRMWInst::Nand;
697 break;
698 }
699
700 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
701 llvm::AtomicRMWInst *RMWI =
702 CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order, Scope);
703 RMWI->setVolatile(E->isVolatile());
704
705 // For __atomic_*_fetch operations, perform the operation again to
706 // determine the value which was written.
707 llvm::Value *Result = RMWI;
708 if (PostOpMinMax)
709 Result = EmitPostAtomicMinMax(CGF.Builder, E->getOp(),
711 RMWI, LoadVal1);
712 else if (PostOp)
713 Result = CGF.Builder.CreateBinOp((llvm::Instruction::BinaryOps)PostOp, RMWI,
714 LoadVal1);
715 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
716 Result = CGF.Builder.CreateNot(Result);
717 CGF.Builder.CreateStore(Result, Dest);
718}
719
720// This function emits any expression (scalar, complex, or aggregate)
721// into a temporary alloca.
722static Address
724 Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
725 CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
726 /*Init*/ true);
727 return DeclPtr;
728}
729
731 Address Ptr, Address Val1, Address Val2,
732 llvm::Value *IsWeak, llvm::Value *FailureOrder,
733 uint64_t Size, llvm::AtomicOrdering Order,
734 llvm::Value *Scope) {
735 auto ScopeModel = Expr->getScopeModel();
736
737 // LLVM atomic instructions always have synch scope. If clang atomic
738 // expression has no scope operand, use default LLVM synch scope.
739 if (!ScopeModel) {
740 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
741 Order, CGF.CGM.getLLVMContext().getOrInsertSyncScopeID(""));
742 return;
743 }
744
745 // Handle constant scope.
746 if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) {
747 auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID(
748 CGF.CGM.getLangOpts(), ScopeModel->map(SC->getZExtValue()),
749 Order, CGF.CGM.getLLVMContext());
750 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
751 Order, SCID);
752 return;
753 }
754
755 // Handle non-constant scope.
756 auto &Builder = CGF.Builder;
757 auto Scopes = ScopeModel->getRuntimeValues();
758 llvm::DenseMap<unsigned, llvm::BasicBlock *> BB;
759 for (auto S : Scopes)
760 BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn);
761
762 llvm::BasicBlock *ContBB =
763 CGF.createBasicBlock("atomic.scope.continue", CGF.CurFn);
764
765 auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false);
766 // If unsupported synch scope is encountered at run time, assume a fallback
767 // synch scope value.
768 auto FallBack = ScopeModel->getFallBackValue();
769 llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]);
770 for (auto S : Scopes) {
771 auto *B = BB[S];
772 if (S != FallBack)
773 SI->addCase(Builder.getInt32(S), B);
774
775 Builder.SetInsertPoint(B);
776 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
777 Order,
779 ScopeModel->map(S),
780 Order,
781 CGF.getLLVMContext()));
782 Builder.CreateBr(ContBB);
783 }
784
785 Builder.SetInsertPoint(ContBB);
786}
787
788static void
790 bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,
791 SourceLocation Loc, CharUnits SizeInChars) {
792 if (UseOptimizedLibcall) {
793 // Load value and pass it to the function directly.
794 CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy);
795 int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
796 ValTy =
797 CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
798 llvm::Type *ITy = llvm::IntegerType::get(CGF.getLLVMContext(), SizeInBits);
799 Address Ptr = Address(Val, ITy, Align);
800 Val = CGF.EmitLoadOfScalar(Ptr, false,
801 CGF.getContext().getPointerType(ValTy),
802 Loc);
803 // Coerce the value into an appropriately sized integer type.
804 Args.add(RValue::get(Val), ValTy);
805 } else {
806 // Non-optimized functions always take a reference.
807 Args.add(RValue::get(Val), CGF.getContext().VoidPtrTy);
808 }
809}
810
811RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
812 QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
813 QualType MemTy = AtomicTy;
814 if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
815 MemTy = AT->getValueType();
816 llvm::Value *IsWeak = nullptr, *OrderFail = nullptr;
817
818 Address Val1 = Address::invalid();
819 Address Val2 = Address::invalid();
820 Address Dest = Address::invalid();
822
823 if (E->getOp() == AtomicExpr::AO__c11_atomic_init ||
824 E->getOp() == AtomicExpr::AO__opencl_atomic_init) {
825 LValue lvalue = MakeAddrLValue(Ptr, AtomicTy);
826 EmitAtomicInit(E->getVal1(), lvalue);
827 return RValue::get(nullptr);
828 }
829
830 auto TInfo = getContext().getTypeInfoInChars(AtomicTy);
831 uint64_t Size = TInfo.Width.getQuantity();
832 unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();
833
834 bool Oversized = getContext().toBits(TInfo.Width) > MaxInlineWidthInBits;
835 bool Misaligned = (Ptr.getAlignment() % TInfo.Width) != 0;
836 bool UseLibcall = Misaligned | Oversized;
837 bool ShouldCastToIntPtrTy = true;
838
839 CharUnits MaxInlineWidth =
840 getContext().toCharUnitsFromBits(MaxInlineWidthInBits);
841
842 DiagnosticsEngine &Diags = CGM.getDiags();
843
844 if (Misaligned) {
845 Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned)
846 << (int)TInfo.Width.getQuantity()
847 << (int)Ptr.getAlignment().getQuantity();
848 }
849
850 if (Oversized) {
851 Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_oversized)
852 << (int)TInfo.Width.getQuantity() << (int)MaxInlineWidth.getQuantity();
853 }
854
855 llvm::Value *Order = EmitScalarExpr(E->getOrder());
856 llvm::Value *Scope =
857 E->getScopeModel() ? EmitScalarExpr(E->getScope()) : nullptr;
858
859 switch (E->getOp()) {
860 case AtomicExpr::AO__c11_atomic_init:
861 case AtomicExpr::AO__opencl_atomic_init:
862 llvm_unreachable("Already handled above with EmitAtomicInit!");
863
864 case AtomicExpr::AO__c11_atomic_load:
865 case AtomicExpr::AO__opencl_atomic_load:
866 case AtomicExpr::AO__hip_atomic_load:
867 case AtomicExpr::AO__atomic_load_n:
868 break;
869
870 case AtomicExpr::AO__atomic_load:
872 break;
873
874 case AtomicExpr::AO__atomic_store:
876 break;
877
878 case AtomicExpr::AO__atomic_exchange:
881 break;
882
883 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
884 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
885 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
886 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
887 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
888 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
889 case AtomicExpr::AO__atomic_compare_exchange_n:
890 case AtomicExpr::AO__atomic_compare_exchange:
892 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
894 else
895 Val2 = EmitValToTemp(*this, E->getVal2());
896 OrderFail = EmitScalarExpr(E->getOrderFail());
897 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n ||
898 E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
899 IsWeak = EmitScalarExpr(E->getWeak());
900 break;
901
902 case AtomicExpr::AO__c11_atomic_fetch_add:
903 case AtomicExpr::AO__c11_atomic_fetch_sub:
904 case AtomicExpr::AO__hip_atomic_fetch_add:
905 case AtomicExpr::AO__hip_atomic_fetch_sub:
906 case AtomicExpr::AO__opencl_atomic_fetch_add:
907 case AtomicExpr::AO__opencl_atomic_fetch_sub:
908 if (MemTy->isPointerType()) {
909 // For pointer arithmetic, we're required to do a bit of math:
910 // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
911 // ... but only for the C11 builtins. The GNU builtins expect the
912 // user to multiply by sizeof(T).
913 QualType Val1Ty = E->getVal1()->getType();
914 llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
915 CharUnits PointeeIncAmt =
917 Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
918 auto Temp = CreateMemTemp(Val1Ty, ".atomictmp");
919 Val1 = Temp;
920 EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty));
921 break;
922 }
923 [[fallthrough]];
924 case AtomicExpr::AO__atomic_fetch_add:
925 case AtomicExpr::AO__atomic_fetch_max:
926 case AtomicExpr::AO__atomic_fetch_min:
927 case AtomicExpr::AO__atomic_fetch_sub:
928 case AtomicExpr::AO__atomic_add_fetch:
929 case AtomicExpr::AO__atomic_max_fetch:
930 case AtomicExpr::AO__atomic_min_fetch:
931 case AtomicExpr::AO__atomic_sub_fetch:
932 case AtomicExpr::AO__c11_atomic_fetch_max:
933 case AtomicExpr::AO__c11_atomic_fetch_min:
934 case AtomicExpr::AO__opencl_atomic_fetch_max:
935 case AtomicExpr::AO__opencl_atomic_fetch_min:
936 case AtomicExpr::AO__hip_atomic_fetch_max:
937 case AtomicExpr::AO__hip_atomic_fetch_min:
938 ShouldCastToIntPtrTy = !MemTy->isFloatingType();
939 [[fallthrough]];
940
941 case AtomicExpr::AO__c11_atomic_store:
942 case AtomicExpr::AO__c11_atomic_exchange:
943 case AtomicExpr::AO__opencl_atomic_store:
944 case AtomicExpr::AO__hip_atomic_store:
945 case AtomicExpr::AO__opencl_atomic_exchange:
946 case AtomicExpr::AO__hip_atomic_exchange:
947 case AtomicExpr::AO__atomic_store_n:
948 case AtomicExpr::AO__atomic_exchange_n:
949 case AtomicExpr::AO__c11_atomic_fetch_and:
950 case AtomicExpr::AO__c11_atomic_fetch_or:
951 case AtomicExpr::AO__c11_atomic_fetch_xor:
952 case AtomicExpr::AO__c11_atomic_fetch_nand:
953 case AtomicExpr::AO__opencl_atomic_fetch_and:
954 case AtomicExpr::AO__opencl_atomic_fetch_or:
955 case AtomicExpr::AO__opencl_atomic_fetch_xor:
956 case AtomicExpr::AO__atomic_fetch_and:
957 case AtomicExpr::AO__hip_atomic_fetch_and:
958 case AtomicExpr::AO__atomic_fetch_or:
959 case AtomicExpr::AO__hip_atomic_fetch_or:
960 case AtomicExpr::AO__atomic_fetch_xor:
961 case AtomicExpr::AO__hip_atomic_fetch_xor:
962 case AtomicExpr::AO__atomic_fetch_nand:
963 case AtomicExpr::AO__atomic_and_fetch:
964 case AtomicExpr::AO__atomic_or_fetch:
965 case AtomicExpr::AO__atomic_xor_fetch:
966 case AtomicExpr::AO__atomic_nand_fetch:
967 Val1 = EmitValToTemp(*this, E->getVal1());
968 break;
969 }
970
971 QualType RValTy = E->getType().getUnqualifiedType();
972
973 // The inlined atomics only function on iN types, where N is a power of 2. We
974 // need to make sure (via temporaries if necessary) that all incoming values
975 // are compatible.
976 LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy);
977 AtomicInfo Atomics(*this, AtomicVal);
978
979 if (ShouldCastToIntPtrTy) {
980 Ptr = Atomics.castToAtomicIntPointer(Ptr);
981 if (Val1.isValid())
982 Val1 = Atomics.convertToAtomicIntPointer(Val1);
983 if (Val2.isValid())
984 Val2 = Atomics.convertToAtomicIntPointer(Val2);
985 }
986 if (Dest.isValid()) {
987 if (ShouldCastToIntPtrTy)
988 Dest = Atomics.castToAtomicIntPointer(Dest);
989 } else if (E->isCmpXChg())
990 Dest = CreateMemTemp(RValTy, "cmpxchg.bool");
991 else if (!RValTy->isVoidType()) {
992 Dest = Atomics.CreateTempAlloca();
993 if (ShouldCastToIntPtrTy)
994 Dest = Atomics.castToAtomicIntPointer(Dest);
995 }
996
997 // Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
998 if (UseLibcall) {
999 bool UseOptimizedLibcall = false;
1000 switch (E->getOp()) {
1001 case AtomicExpr::AO__c11_atomic_init:
1002 case AtomicExpr::AO__opencl_atomic_init:
1003 llvm_unreachable("Already handled above with EmitAtomicInit!");
1004
1005 case AtomicExpr::AO__c11_atomic_fetch_add:
1006 case AtomicExpr::AO__opencl_atomic_fetch_add:
1007 case AtomicExpr::AO__atomic_fetch_add:
1008 case AtomicExpr::AO__hip_atomic_fetch_add:
1009 case AtomicExpr::AO__c11_atomic_fetch_and:
1010 case AtomicExpr::AO__opencl_atomic_fetch_and:
1011 case AtomicExpr::AO__hip_atomic_fetch_and:
1012 case AtomicExpr::AO__atomic_fetch_and:
1013 case AtomicExpr::AO__c11_atomic_fetch_or:
1014 case AtomicExpr::AO__opencl_atomic_fetch_or:
1015 case AtomicExpr::AO__hip_atomic_fetch_or:
1016 case AtomicExpr::AO__atomic_fetch_or:
1017 case AtomicExpr::AO__c11_atomic_fetch_nand:
1018 case AtomicExpr::AO__atomic_fetch_nand:
1019 case AtomicExpr::AO__c11_atomic_fetch_sub:
1020 case AtomicExpr::AO__opencl_atomic_fetch_sub:
1021 case AtomicExpr::AO__atomic_fetch_sub:
1022 case AtomicExpr::AO__hip_atomic_fetch_sub:
1023 case AtomicExpr::AO__c11_atomic_fetch_xor:
1024 case AtomicExpr::AO__opencl_atomic_fetch_xor:
1025 case AtomicExpr::AO__opencl_atomic_fetch_min:
1026 case AtomicExpr::AO__opencl_atomic_fetch_max:
1027 case AtomicExpr::AO__atomic_fetch_xor:
1028 case AtomicExpr::AO__hip_atomic_fetch_xor:
1029 case AtomicExpr::AO__c11_atomic_fetch_max:
1030 case AtomicExpr::AO__c11_atomic_fetch_min:
1031 case AtomicExpr::AO__atomic_add_fetch:
1032 case AtomicExpr::AO__atomic_and_fetch:
1033 case AtomicExpr::AO__atomic_nand_fetch:
1034 case AtomicExpr::AO__atomic_or_fetch:
1035 case AtomicExpr::AO__atomic_sub_fetch:
1036 case AtomicExpr::AO__atomic_xor_fetch:
1037 case AtomicExpr::AO__atomic_fetch_max:
1038 case AtomicExpr::AO__hip_atomic_fetch_max:
1039 case AtomicExpr::AO__atomic_fetch_min:
1040 case AtomicExpr::AO__hip_atomic_fetch_min:
1041 case AtomicExpr::AO__atomic_max_fetch:
1042 case AtomicExpr::AO__atomic_min_fetch:
1043 // For these, only library calls for certain sizes exist.
1044 UseOptimizedLibcall = true;
1045 break;
1046
1047 case AtomicExpr::AO__atomic_load:
1048 case AtomicExpr::AO__atomic_store:
1049 case AtomicExpr::AO__atomic_exchange:
1050 case AtomicExpr::AO__atomic_compare_exchange:
1051 // Use the generic version if we don't know that the operand will be
1052 // suitably aligned for the optimized version.
1053 if (Misaligned)
1054 break;
1055 [[fallthrough]];
1056 case AtomicExpr::AO__c11_atomic_load:
1057 case AtomicExpr::AO__c11_atomic_store:
1058 case AtomicExpr::AO__c11_atomic_exchange:
1059 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1060 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1061 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
1062 case AtomicExpr::AO__opencl_atomic_load:
1063 case AtomicExpr::AO__hip_atomic_load:
1064 case AtomicExpr::AO__opencl_atomic_store:
1065 case AtomicExpr::AO__hip_atomic_store:
1066 case AtomicExpr::AO__opencl_atomic_exchange:
1067 case AtomicExpr::AO__hip_atomic_exchange:
1068 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1069 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
1070 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1071 case AtomicExpr::AO__atomic_load_n:
1072 case AtomicExpr::AO__atomic_store_n:
1073 case AtomicExpr::AO__atomic_exchange_n:
1074 case AtomicExpr::AO__atomic_compare_exchange_n:
1075 // Only use optimized library calls for sizes for which they exist.
1076 // FIXME: Size == 16 optimized library functions exist too.
1077 if (Size == 1 || Size == 2 || Size == 4 || Size == 8)
1078 UseOptimizedLibcall = true;
1079 break;
1080 }
1081
1082 CallArgList Args;
1083 if (!UseOptimizedLibcall) {
1084 // For non-optimized library calls, the size is the first parameter
1085 Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
1086 getContext().getSizeType());
1087 }
1088 // Atomic address is the first or second parameter
1089 // The OpenCL atomic library functions only accept pointer arguments to
1090 // generic address space.
1091 auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) {
1092 if (!E->isOpenCL())
1093 return V;
1094 auto AS = PT->castAs<PointerType>()->getPointeeType().getAddressSpace();
1095 if (AS == LangAS::opencl_generic)
1096 return V;
1098 auto *DestType = llvm::PointerType::get(getLLVMContext(), DestAS);
1099
1101 *this, V, AS, LangAS::opencl_generic, DestType, false);
1102 };
1103
1104 Args.add(RValue::get(CastToGenericAddrSpace(Ptr.getPointer(),
1105 E->getPtr()->getType())),
1107
1108 std::string LibCallName;
1109 QualType LoweredMemTy =
1110 MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy;
1111 QualType RetTy;
1112 bool HaveRetTy = false;
1113 llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
1114 bool PostOpMinMax = false;
1115 switch (E->getOp()) {
1116 case AtomicExpr::AO__c11_atomic_init:
1117 case AtomicExpr::AO__opencl_atomic_init:
1118 llvm_unreachable("Already handled!");
1119
1120 // There is only one libcall for compare an exchange, because there is no
1121 // optimisation benefit possible from a libcall version of a weak compare
1122 // and exchange.
1123 // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
1124 // void *desired, int success, int failure)
1125 // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired,
1126 // int success, int failure)
1127 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1128 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1129 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1130 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
1131 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1132 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
1133 case AtomicExpr::AO__atomic_compare_exchange:
1134 case AtomicExpr::AO__atomic_compare_exchange_n:
1135 LibCallName = "__atomic_compare_exchange";
1136 RetTy = getContext().BoolTy;
1137 HaveRetTy = true;
1138 Args.add(RValue::get(CastToGenericAddrSpace(Val1.getPointer(),
1139 E->getVal1()->getType())),
1141 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(),
1142 MemTy, E->getExprLoc(), TInfo.Width);
1143 Args.add(RValue::get(Order), getContext().IntTy);
1144 Order = OrderFail;
1145 break;
1146 // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
1147 // int order)
1148 // T __atomic_exchange_N(T *mem, T val, int order)
1149 case AtomicExpr::AO__c11_atomic_exchange:
1150 case AtomicExpr::AO__opencl_atomic_exchange:
1151 case AtomicExpr::AO__atomic_exchange_n:
1152 case AtomicExpr::AO__atomic_exchange:
1153 case AtomicExpr::AO__hip_atomic_exchange:
1154 LibCallName = "__atomic_exchange";
1155 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1156 MemTy, E->getExprLoc(), TInfo.Width);
1157 break;
1158 // void __atomic_store(size_t size, void *mem, void *val, int order)
1159 // void __atomic_store_N(T *mem, T val, int order)
1160 case AtomicExpr::AO__c11_atomic_store:
1161 case AtomicExpr::AO__opencl_atomic_store:
1162 case AtomicExpr::AO__hip_atomic_store:
1163 case AtomicExpr::AO__atomic_store:
1164 case AtomicExpr::AO__atomic_store_n:
1165 LibCallName = "__atomic_store";
1166 RetTy = getContext().VoidTy;
1167 HaveRetTy = true;
1168 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1169 MemTy, E->getExprLoc(), TInfo.Width);
1170 break;
1171 // void __atomic_load(size_t size, void *mem, void *return, int order)
1172 // T __atomic_load_N(T *mem, int order)
1173 case AtomicExpr::AO__c11_atomic_load:
1174 case AtomicExpr::AO__opencl_atomic_load:
1175 case AtomicExpr::AO__hip_atomic_load:
1176 case AtomicExpr::AO__atomic_load:
1177 case AtomicExpr::AO__atomic_load_n:
1178 LibCallName = "__atomic_load";
1179 break;
1180 // T __atomic_add_fetch_N(T *mem, T val, int order)
1181 // T __atomic_fetch_add_N(T *mem, T val, int order)
1182 case AtomicExpr::AO__atomic_add_fetch:
1183 PostOp = llvm::Instruction::Add;
1184 [[fallthrough]];
1185 case AtomicExpr::AO__c11_atomic_fetch_add:
1186 case AtomicExpr::AO__opencl_atomic_fetch_add:
1187 case AtomicExpr::AO__atomic_fetch_add:
1188 case AtomicExpr::AO__hip_atomic_fetch_add:
1189 LibCallName = "__atomic_fetch_add";
1190 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1191 LoweredMemTy, E->getExprLoc(), TInfo.Width);
1192 break;
1193 // T __atomic_and_fetch_N(T *mem, T val, int order)
1194 // T __atomic_fetch_and_N(T *mem, T val, int order)
1195 case AtomicExpr::AO__atomic_and_fetch:
1196 PostOp = llvm::Instruction::And;
1197 [[fallthrough]];
1198 case AtomicExpr::AO__c11_atomic_fetch_and:
1199 case AtomicExpr::AO__opencl_atomic_fetch_and:
1200 case AtomicExpr::AO__hip_atomic_fetch_and:
1201 case AtomicExpr::AO__atomic_fetch_and:
1202 LibCallName = "__atomic_fetch_and";
1203 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1204 MemTy, E->getExprLoc(), TInfo.Width);
1205 break;
1206 // T __atomic_or_fetch_N(T *mem, T val, int order)
1207 // T __atomic_fetch_or_N(T *mem, T val, int order)
1208 case AtomicExpr::AO__atomic_or_fetch:
1209 PostOp = llvm::Instruction::Or;
1210 [[fallthrough]];
1211 case AtomicExpr::AO__c11_atomic_fetch_or:
1212 case AtomicExpr::AO__opencl_atomic_fetch_or:
1213 case AtomicExpr::AO__hip_atomic_fetch_or:
1214 case AtomicExpr::AO__atomic_fetch_or:
1215 LibCallName = "__atomic_fetch_or";
1216 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1217 MemTy, E->getExprLoc(), TInfo.Width);
1218 break;
1219 // T __atomic_sub_fetch_N(T *mem, T val, int order)
1220 // T __atomic_fetch_sub_N(T *mem, T val, int order)
1221 case AtomicExpr::AO__atomic_sub_fetch:
1222 PostOp = llvm::Instruction::Sub;
1223 [[fallthrough]];
1224 case AtomicExpr::AO__c11_atomic_fetch_sub:
1225 case AtomicExpr::AO__opencl_atomic_fetch_sub:
1226 case AtomicExpr::AO__hip_atomic_fetch_sub:
1227 case AtomicExpr::AO__atomic_fetch_sub:
1228 LibCallName = "__atomic_fetch_sub";
1229 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1230 LoweredMemTy, E->getExprLoc(), TInfo.Width);
1231 break;
1232 // T __atomic_xor_fetch_N(T *mem, T val, int order)
1233 // T __atomic_fetch_xor_N(T *mem, T val, int order)
1234 case AtomicExpr::AO__atomic_xor_fetch:
1235 PostOp = llvm::Instruction::Xor;
1236 [[fallthrough]];
1237 case AtomicExpr::AO__c11_atomic_fetch_xor:
1238 case AtomicExpr::AO__opencl_atomic_fetch_xor:
1239 case AtomicExpr::AO__hip_atomic_fetch_xor:
1240 case AtomicExpr::AO__atomic_fetch_xor:
1241 LibCallName = "__atomic_fetch_xor";
1242 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1243 MemTy, E->getExprLoc(), TInfo.Width);
1244 break;
1245 case AtomicExpr::AO__atomic_min_fetch:
1246 PostOpMinMax = true;
1247 [[fallthrough]];
1248 case AtomicExpr::AO__c11_atomic_fetch_min:
1249 case AtomicExpr::AO__atomic_fetch_min:
1250 case AtomicExpr::AO__hip_atomic_fetch_min:
1251 case AtomicExpr::AO__opencl_atomic_fetch_min:
1252 LibCallName = E->getValueType()->isSignedIntegerType()
1253 ? "__atomic_fetch_min"
1254 : "__atomic_fetch_umin";
1255 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1256 LoweredMemTy, E->getExprLoc(), TInfo.Width);
1257 break;
1258 case AtomicExpr::AO__atomic_max_fetch:
1259 PostOpMinMax = true;
1260 [[fallthrough]];
1261 case AtomicExpr::AO__c11_atomic_fetch_max:
1262 case AtomicExpr::AO__atomic_fetch_max:
1263 case AtomicExpr::AO__hip_atomic_fetch_max:
1264 case AtomicExpr::AO__opencl_atomic_fetch_max:
1265 LibCallName = E->getValueType()->isSignedIntegerType()
1266 ? "__atomic_fetch_max"
1267 : "__atomic_fetch_umax";
1268 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1269 LoweredMemTy, E->getExprLoc(), TInfo.Width);
1270 break;
1271 // T __atomic_nand_fetch_N(T *mem, T val, int order)
1272 // T __atomic_fetch_nand_N(T *mem, T val, int order)
1273 case AtomicExpr::AO__atomic_nand_fetch:
1274 PostOp = llvm::Instruction::And; // the NOT is special cased below
1275 [[fallthrough]];
1276 case AtomicExpr::AO__c11_atomic_fetch_nand:
1277 case AtomicExpr::AO__atomic_fetch_nand:
1278 LibCallName = "__atomic_fetch_nand";
1279 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1280 MemTy, E->getExprLoc(), TInfo.Width);
1281 break;
1282 }
1283
1284 if (E->isOpenCL()) {
1285 LibCallName = std::string("__opencl") +
1286 StringRef(LibCallName).drop_front(1).str();
1287
1288 }
1289 // Optimized functions have the size in their name.
1290 if (UseOptimizedLibcall)
1291 LibCallName += "_" + llvm::utostr(Size);
1292 // By default, assume we return a value of the atomic type.
1293 if (!HaveRetTy) {
1294 if (UseOptimizedLibcall) {
1295 // Value is returned directly.
1296 // The function returns an appropriately sized integer type.
1298 getContext().toBits(TInfo.Width), /*Signed=*/false);
1299 } else {
1300 // Value is returned through parameter before the order.
1301 RetTy = getContext().VoidTy;
1303 }
1304 }
1305 // order is always the last parameter
1306 Args.add(RValue::get(Order),
1307 getContext().IntTy);
1308 if (E->isOpenCL())
1310
1311 // PostOp is only needed for the atomic_*_fetch operations, and
1312 // thus is only needed for and implemented in the
1313 // UseOptimizedLibcall codepath.
1314 assert(UseOptimizedLibcall || (!PostOp && !PostOpMinMax));
1315
1316 RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
1317 // The value is returned directly from the libcall.
1318 if (E->isCmpXChg())
1319 return Res;
1320
1321 // The value is returned directly for optimized libcalls but the expr
1322 // provided an out-param.
1323 if (UseOptimizedLibcall && Res.getScalarVal()) {
1324 llvm::Value *ResVal = Res.getScalarVal();
1325 if (PostOpMinMax) {
1326 llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal();
1327 ResVal = EmitPostAtomicMinMax(Builder, E->getOp(),
1329 ResVal, LoadVal1);
1330 } else if (PostOp) {
1331 llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal();
1332 ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1);
1333 }
1334 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
1335 ResVal = Builder.CreateNot(ResVal);
1336
1337 Builder.CreateStore(ResVal, Dest.withElementType(ResVal->getType()));
1338 }
1339
1340 if (RValTy->isVoidType())
1341 return RValue::get(nullptr);
1342
1344 RValTy, E->getExprLoc());
1345 }
1346
1347 bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
1348 E->getOp() == AtomicExpr::AO__opencl_atomic_store ||
1349 E->getOp() == AtomicExpr::AO__hip_atomic_store ||
1350 E->getOp() == AtomicExpr::AO__atomic_store ||
1351 E->getOp() == AtomicExpr::AO__atomic_store_n;
1352 bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
1353 E->getOp() == AtomicExpr::AO__opencl_atomic_load ||
1354 E->getOp() == AtomicExpr::AO__hip_atomic_load ||
1355 E->getOp() == AtomicExpr::AO__atomic_load ||
1356 E->getOp() == AtomicExpr::AO__atomic_load_n;
1357
1358 if (isa<llvm::ConstantInt>(Order)) {
1359 auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1360 // We should not ever get to a case where the ordering isn't a valid C ABI
1361 // value, but it's hard to enforce that in general.
1362 if (llvm::isValidAtomicOrderingCABI(ord))
1363 switch ((llvm::AtomicOrderingCABI)ord) {
1364 case llvm::AtomicOrderingCABI::relaxed:
1365 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1366 llvm::AtomicOrdering::Monotonic, Scope);
1367 break;
1368 case llvm::AtomicOrderingCABI::consume:
1369 case llvm::AtomicOrderingCABI::acquire:
1370 if (IsStore)
1371 break; // Avoid crashing on code with undefined behavior
1372 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1373 llvm::AtomicOrdering::Acquire, Scope);
1374 break;
1375 case llvm::AtomicOrderingCABI::release:
1376 if (IsLoad)
1377 break; // Avoid crashing on code with undefined behavior
1378 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1379 llvm::AtomicOrdering::Release, Scope);
1380 break;
1381 case llvm::AtomicOrderingCABI::acq_rel:
1382 if (IsLoad || IsStore)
1383 break; // Avoid crashing on code with undefined behavior
1384 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1385 llvm::AtomicOrdering::AcquireRelease, Scope);
1386 break;
1387 case llvm::AtomicOrderingCABI::seq_cst:
1388 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1389 llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1390 break;
1391 }
1392 if (RValTy->isVoidType())
1393 return RValue::get(nullptr);
1394
1396 RValTy, E->getExprLoc());
1397 }
1398
1399 // Long case, when Order isn't obviously constant.
1400
1401 // Create all the relevant BB's
1402 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
1403 *ReleaseBB = nullptr, *AcqRelBB = nullptr,
1404 *SeqCstBB = nullptr;
1405 MonotonicBB = createBasicBlock("monotonic", CurFn);
1406 if (!IsStore)
1407 AcquireBB = createBasicBlock("acquire", CurFn);
1408 if (!IsLoad)
1409 ReleaseBB = createBasicBlock("release", CurFn);
1410 if (!IsLoad && !IsStore)
1411 AcqRelBB = createBasicBlock("acqrel", CurFn);
1412 SeqCstBB = createBasicBlock("seqcst", CurFn);
1413 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1414
1415 // Create the switch for the split
1416 // MonotonicBB is arbitrarily chosen as the default case; in practice, this
1417 // doesn't matter unless someone is crazy enough to use something that
1418 // doesn't fold to a constant for the ordering.
1419 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1420 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
1421
1422 // Emit all the different atomics
1423 Builder.SetInsertPoint(MonotonicBB);
1424 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1425 llvm::AtomicOrdering::Monotonic, Scope);
1426 Builder.CreateBr(ContBB);
1427 if (!IsStore) {
1428 Builder.SetInsertPoint(AcquireBB);
1429 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1430 llvm::AtomicOrdering::Acquire, Scope);
1431 Builder.CreateBr(ContBB);
1432 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
1433 AcquireBB);
1434 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
1435 AcquireBB);
1436 }
1437 if (!IsLoad) {
1438 Builder.SetInsertPoint(ReleaseBB);
1439 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1440 llvm::AtomicOrdering::Release, Scope);
1441 Builder.CreateBr(ContBB);
1442 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release),
1443 ReleaseBB);
1444 }
1445 if (!IsLoad && !IsStore) {
1446 Builder.SetInsertPoint(AcqRelBB);
1447 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1448 llvm::AtomicOrdering::AcquireRelease, Scope);
1449 Builder.CreateBr(ContBB);
1450 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel),
1451 AcqRelBB);
1452 }
1453 Builder.SetInsertPoint(SeqCstBB);
1454 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1455 llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1456 Builder.CreateBr(ContBB);
1457 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
1458 SeqCstBB);
1459
1460 // Cleanup and return
1461 Builder.SetInsertPoint(ContBB);
1462 if (RValTy->isVoidType())
1463 return RValue::get(nullptr);
1464
1465 assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
1467 RValTy, E->getExprLoc());
1468}
1469
1470Address AtomicInfo::castToAtomicIntPointer(Address addr) const {
1471 llvm::IntegerType *ty =
1472 llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
1473 return addr.withElementType(ty);
1474}
1475
1476Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {
1477 llvm::Type *Ty = Addr.getElementType();
1478 uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
1479 if (SourceSizeInBits != AtomicSizeInBits) {
1480 Address Tmp = CreateTempAlloca();
1481 CGF.Builder.CreateMemCpy(Tmp, Addr,
1482 std::min(AtomicSizeInBits, SourceSizeInBits) / 8);
1483 Addr = Tmp;
1484 }
1485
1486 return castToAtomicIntPointer(Addr);
1487}
1488
1489RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
1490 AggValueSlot resultSlot,
1491 SourceLocation loc,
1492 bool asValue) const {
1493 if (LVal.isSimple()) {
1494 if (EvaluationKind == TEK_Aggregate)
1495 return resultSlot.asRValue();
1496
1497 // Drill into the padding structure if we have one.
1498 if (hasPadding())
1499 addr = CGF.Builder.CreateStructGEP(addr, 0);
1500
1501 // Otherwise, just convert the temporary to an r-value using the
1502 // normal conversion routine.
1503 return CGF.convertTempToRValue(addr, getValueType(), loc);
1504 }
1505 if (!asValue)
1506 // Get RValue from temp memory as atomic for non-simple lvalues
1507 return RValue::get(CGF.Builder.CreateLoad(addr));
1508 if (LVal.isBitField())
1509 return CGF.EmitLoadOfBitfieldLValue(
1510 LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(),
1511 LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1512 if (LVal.isVectorElt())
1513 return CGF.EmitLoadOfLValue(
1514 LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(),
1515 LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1516 assert(LVal.isExtVectorElt());
1518 addr, LVal.getExtVectorElts(), LVal.getType(),
1519 LVal.getBaseInfo(), TBAAAccessInfo()));
1520}
1521
1522RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
1523 AggValueSlot ResultSlot,
1524 SourceLocation Loc,
1525 bool AsValue) const {
1526 // Try not to in some easy cases.
1527 assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
1528 if (getEvaluationKind() == TEK_Scalar &&
1529 (((!LVal.isBitField() ||
1530 LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
1531 !hasPadding()) ||
1532 !AsValue)) {
1533 auto *ValTy = AsValue
1534 ? CGF.ConvertTypeForMem(ValueTy)
1535 : getAtomicAddress().getElementType();
1536 if (ValTy->isIntegerTy()) {
1537 assert(IntVal->getType() == ValTy && "Different integer types.");
1538 return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
1539 } else if (ValTy->isPointerTy())
1540 return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
1541 else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
1542 return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
1543 }
1544
1545 // Create a temporary. This needs to be big enough to hold the
1546 // atomic integer.
1547 Address Temp = Address::invalid();
1548 bool TempIsVolatile = false;
1549 if (AsValue && getEvaluationKind() == TEK_Aggregate) {
1550 assert(!ResultSlot.isIgnored());
1551 Temp = ResultSlot.getAddress();
1552 TempIsVolatile = ResultSlot.isVolatile();
1553 } else {
1554 Temp = CreateTempAlloca();
1555 }
1556
1557 // Slam the integer into the temporary.
1558 Address CastTemp = castToAtomicIntPointer(Temp);
1559 CGF.Builder.CreateStore(IntVal, CastTemp)
1560 ->setVolatile(TempIsVolatile);
1561
1562 return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue);
1563}
1564
1565void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1566 llvm::AtomicOrdering AO, bool) {
1567 // void __atomic_load(size_t size, void *mem, void *return, int order);
1568 CallArgList Args;
1569 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1570 Args.add(RValue::get(getAtomicPointer()), CGF.getContext().VoidPtrTy);
1571 Args.add(RValue::get(AddForLoaded), CGF.getContext().VoidPtrTy);
1572 Args.add(
1573 RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))),
1574 CGF.getContext().IntTy);
1575 emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1576}
1577
1578llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1579 bool IsVolatile) {
1580 // Okay, we're doing this natively.
1581 Address Addr = getAtomicAddressAsAtomicIntPointer();
1582 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
1583 Load->setAtomic(AO);
1584
1585 // Other decoration.
1586 if (IsVolatile)
1587 Load->setVolatile(true);
1589 return Load;
1590}
1591
1592/// An LValue is a candidate for having its loads and stores be made atomic if
1593/// we are operating under /volatile:ms *and* the LValue itself is volatile and
1594/// performing such an operation can be performed without a libcall.
1596 if (!CGM.getLangOpts().MSVolatile) return false;
1597 AtomicInfo AI(*this, LV);
1598 bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType());
1599 // An atomic is inline if we don't need to use a libcall.
1600 bool AtomicIsInline = !AI.shouldUseLibcall();
1601 // MSVC doesn't seem to do this for types wider than a pointer.
1602 if (getContext().getTypeSize(LV.getType()) >
1603 getContext().getTypeSize(getContext().getIntPtrType()))
1604 return false;
1605 return IsVolatile && AtomicIsInline;
1606}
1607
1609 AggValueSlot Slot) {
1610 llvm::AtomicOrdering AO;
1611 bool IsVolatile = LV.isVolatileQualified();
1612 if (LV.getType()->isAtomicType()) {
1613 AO = llvm::AtomicOrdering::SequentiallyConsistent;
1614 } else {
1615 AO = llvm::AtomicOrdering::Acquire;
1616 IsVolatile = true;
1617 }
1618 return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
1619}
1620
1621RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1622 bool AsValue, llvm::AtomicOrdering AO,
1623 bool IsVolatile) {
1624 // Check whether we should use a library call.
1625 if (shouldUseLibcall()) {
1626 Address TempAddr = Address::invalid();
1627 if (LVal.isSimple() && !ResultSlot.isIgnored()) {
1628 assert(getEvaluationKind() == TEK_Aggregate);
1629 TempAddr = ResultSlot.getAddress();
1630 } else
1631 TempAddr = CreateTempAlloca();
1632
1633 EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile);
1634
1635 // Okay, turn that back into the original value or whole atomic (for
1636 // non-simple lvalues) type.
1637 return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
1638 }
1639
1640 // Okay, we're doing this natively.
1641 auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1642
1643 // If we're ignoring an aggregate return, don't do anything.
1644 if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
1645 return RValue::getAggregate(Address::invalid(), false);
1646
1647 // Okay, turn that back into the original value or atomic (for non-simple
1648 // lvalues) type.
1649 return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
1650}
1651
1652/// Emit a load from an l-value of atomic type. Note that the r-value
1653/// we produce is an r-value of the atomic *value* type.
1655 llvm::AtomicOrdering AO, bool IsVolatile,
1656 AggValueSlot resultSlot) {
1657 AtomicInfo Atomics(*this, src);
1658 return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
1659 IsVolatile);
1660}
1661
1662/// Copy an r-value into memory as part of storing to an atomic type.
1663/// This needs to create a bit-pattern suitable for atomic operations.
1664void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1665 assert(LVal.isSimple());
1666 // If we have an r-value, the rvalue should be of the atomic type,
1667 // which means that the caller is responsible for having zeroed
1668 // any padding. Just do an aggregate copy of that type.
1669 if (rvalue.isAggregate()) {
1670 LValue Dest = CGF.MakeAddrLValue(getAtomicAddress(), getAtomicType());
1671 LValue Src = CGF.MakeAddrLValue(rvalue.getAggregateAddress(),
1672 getAtomicType());
1673 bool IsVolatile = rvalue.isVolatileQualified() ||
1674 LVal.isVolatileQualified();
1675 CGF.EmitAggregateCopy(Dest, Src, getAtomicType(),
1676 AggValueSlot::DoesNotOverlap, IsVolatile);
1677 return;
1678 }
1679
1680 // Okay, otherwise we're copying stuff.
1681
1682 // Zero out the buffer if necessary.
1683 emitMemSetZeroIfNecessary();
1684
1685 // Drill past the padding if present.
1686 LValue TempLVal = projectValue();
1687
1688 // Okay, store the rvalue in.
1689 if (rvalue.isScalar()) {
1690 CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
1691 } else {
1692 CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
1693 }
1694}
1695
1696
1697/// Materialize an r-value into memory for the purposes of storing it
1698/// to an atomic type.
1699Address AtomicInfo::materializeRValue(RValue rvalue) const {
1700 // Aggregate r-values are already in memory, and EmitAtomicStore
1701 // requires them to be values of the atomic type.
1702 if (rvalue.isAggregate())
1703 return rvalue.getAggregateAddress();
1704
1705 // Otherwise, make a temporary and materialize into it.
1706 LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType());
1707 AtomicInfo Atomics(CGF, TempLV);
1708 Atomics.emitCopyIntoMemory(rvalue);
1709 return TempLV.getAddress(CGF);
1710}
1711
1712llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
1713 // If we've got a scalar value of the right size, try to avoid going
1714 // through memory.
1715 if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) {
1716 llvm::Value *Value = RVal.getScalarVal();
1717 if (isa<llvm::IntegerType>(Value->getType()))
1718 return CGF.EmitToMemory(Value, ValueTy);
1719 else {
1720 llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1721 CGF.getLLVMContext(),
1722 LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1723 if (isa<llvm::PointerType>(Value->getType()))
1724 return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
1725 else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
1726 return CGF.Builder.CreateBitCast(Value, InputIntTy);
1727 }
1728 }
1729 // Otherwise, we need to go through memory.
1730 // Put the r-value in memory.
1731 Address Addr = materializeRValue(RVal);
1732
1733 // Cast the temporary to the atomic int type and pull a value out.
1734 Addr = castToAtomicIntPointer(Addr);
1735 return CGF.Builder.CreateLoad(Addr);
1736}
1737
1738std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1739 llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1740 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1741 // Do the atomic store.
1742 Address Addr = getAtomicAddressAsAtomicIntPointer();
1743 auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(),
1744 ExpectedVal, DesiredVal,
1745 Success, Failure);
1746 // Other decoration.
1747 Inst->setVolatile(LVal.isVolatileQualified());
1748 Inst->setWeak(IsWeak);
1749
1750 // Okay, turn that back into the original value type.
1751 auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
1752 auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
1753 return std::make_pair(PreviousVal, SuccessFailureVal);
1754}
1755
1756llvm::Value *
1757AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1758 llvm::Value *DesiredAddr,
1759 llvm::AtomicOrdering Success,
1760 llvm::AtomicOrdering Failure) {
1761 // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1762 // void *desired, int success, int failure);
1763 CallArgList Args;
1764 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1765 Args.add(RValue::get(getAtomicPointer()), CGF.getContext().VoidPtrTy);
1766 Args.add(RValue::get(ExpectedAddr), CGF.getContext().VoidPtrTy);
1767 Args.add(RValue::get(DesiredAddr), CGF.getContext().VoidPtrTy);
1768 Args.add(RValue::get(
1769 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))),
1770 CGF.getContext().IntTy);
1771 Args.add(RValue::get(
1772 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))),
1773 CGF.getContext().IntTy);
1774 auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1775 CGF.getContext().BoolTy, Args);
1776
1777 return SuccessFailureRVal.getScalarVal();
1778}
1779
1780std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1781 RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1782 llvm::AtomicOrdering Failure, bool IsWeak) {
1783 // Check whether we should use a library call.
1784 if (shouldUseLibcall()) {
1785 // Produce a source address.
1786 Address ExpectedAddr = materializeRValue(Expected);
1787 Address DesiredAddr = materializeRValue(Desired);
1788 auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1789 DesiredAddr.getPointer(),
1790 Success, Failure);
1791 return std::make_pair(
1792 convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1793 SourceLocation(), /*AsValue=*/false),
1794 Res);
1795 }
1796
1797 // If we've got a scalar value of the right size, try to avoid going
1798 // through memory.
1799 auto *ExpectedVal = convertRValueToInt(Expected);
1800 auto *DesiredVal = convertRValueToInt(Desired);
1801 auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1802 Failure, IsWeak);
1803 return std::make_pair(
1804 ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(),
1805 SourceLocation(), /*AsValue=*/false),
1806 Res.second);
1807}
1808
1809static void
1810EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1811 const llvm::function_ref<RValue(RValue)> &UpdateOp,
1812 Address DesiredAddr) {
1813 RValue UpRVal;
1814 LValue AtomicLVal = Atomics.getAtomicLValue();
1815 LValue DesiredLVal;
1816 if (AtomicLVal.isSimple()) {
1817 UpRVal = OldRVal;
1818 DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType());
1819 } else {
1820 // Build new lvalue for temp address.
1821 Address Ptr = Atomics.materializeRValue(OldRVal);
1822 LValue UpdateLVal;
1823 if (AtomicLVal.isBitField()) {
1824 UpdateLVal =
1825 LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
1826 AtomicLVal.getType(),
1827 AtomicLVal.getBaseInfo(),
1828 AtomicLVal.getTBAAInfo());
1829 DesiredLVal =
1830 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1831 AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1832 AtomicLVal.getTBAAInfo());
1833 } else if (AtomicLVal.isVectorElt()) {
1834 UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
1835 AtomicLVal.getType(),
1836 AtomicLVal.getBaseInfo(),
1837 AtomicLVal.getTBAAInfo());
1838 DesiredLVal = LValue::MakeVectorElt(
1839 DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(),
1840 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1841 } else {
1842 assert(AtomicLVal.isExtVectorElt());
1843 UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
1844 AtomicLVal.getType(),
1845 AtomicLVal.getBaseInfo(),
1846 AtomicLVal.getTBAAInfo());
1847 DesiredLVal = LValue::MakeExtVectorElt(
1848 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1849 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1850 }
1851 UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());
1852 }
1853 // Store new value in the corresponding memory area.
1854 RValue NewRVal = UpdateOp(UpRVal);
1855 if (NewRVal.isScalar()) {
1856 CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);
1857 } else {
1858 assert(NewRVal.isComplex());
1859 CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,
1860 /*isInit=*/false);
1861 }
1862}
1863
1864void AtomicInfo::EmitAtomicUpdateLibcall(
1865 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1866 bool IsVolatile) {
1867 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1868
1869 Address ExpectedAddr = CreateTempAlloca();
1870
1871 EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1872 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1873 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1874 CGF.EmitBlock(ContBB);
1875 Address DesiredAddr = CreateTempAlloca();
1876 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1877 requiresMemSetZero(getAtomicAddress().getElementType())) {
1878 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1879 CGF.Builder.CreateStore(OldVal, DesiredAddr);
1880 }
1881 auto OldRVal = convertAtomicTempToRValue(ExpectedAddr,
1883 SourceLocation(), /*AsValue=*/false);
1884 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
1885 auto *Res =
1886 EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1887 DesiredAddr.getPointer(),
1888 AO, Failure);
1889 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1890 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1891}
1892
1893void AtomicInfo::EmitAtomicUpdateOp(
1894 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1895 bool IsVolatile) {
1896 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1897
1898 // Do the atomic load.
1899 auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile);
1900 // For non-simple lvalues perform compare-and-swap procedure.
1901 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1902 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1903 auto *CurBB = CGF.Builder.GetInsertBlock();
1904 CGF.EmitBlock(ContBB);
1905 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1906 /*NumReservedValues=*/2);
1907 PHI->addIncoming(OldVal, CurBB);
1908 Address NewAtomicAddr = CreateTempAlloca();
1909 Address NewAtomicIntAddr = castToAtomicIntPointer(NewAtomicAddr);
1910 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1911 requiresMemSetZero(getAtomicAddress().getElementType())) {
1912 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1913 }
1914 auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(),
1915 SourceLocation(), /*AsValue=*/false);
1916 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
1917 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1918 // Try to write new value using cmpxchg operation.
1919 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1920 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1921 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1922 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1923}
1924
1925static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1926 RValue UpdateRVal, Address DesiredAddr) {
1927 LValue AtomicLVal = Atomics.getAtomicLValue();
1928 LValue DesiredLVal;
1929 // Build new lvalue for temp address.
1930 if (AtomicLVal.isBitField()) {
1931 DesiredLVal =
1932 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1933 AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1934 AtomicLVal.getTBAAInfo());
1935 } else if (AtomicLVal.isVectorElt()) {
1936 DesiredLVal =
1937 LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(),
1938 AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1939 AtomicLVal.getTBAAInfo());
1940 } else {
1941 assert(AtomicLVal.isExtVectorElt());
1942 DesiredLVal = LValue::MakeExtVectorElt(
1943 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1944 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1945 }
1946 // Store new value in the corresponding memory area.
1947 assert(UpdateRVal.isScalar());
1948 CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);
1949}
1950
1951void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1952 RValue UpdateRVal, bool IsVolatile) {
1953 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1954
1955 Address ExpectedAddr = CreateTempAlloca();
1956
1957 EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1958 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1959 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1960 CGF.EmitBlock(ContBB);
1961 Address DesiredAddr = CreateTempAlloca();
1962 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1963 requiresMemSetZero(getAtomicAddress().getElementType())) {
1964 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1965 CGF.Builder.CreateStore(OldVal, DesiredAddr);
1966 }
1967 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
1968 auto *Res =
1969 EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1970 DesiredAddr.getPointer(),
1971 AO, Failure);
1972 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1973 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1974}
1975
1976void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
1977 bool IsVolatile) {
1978 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1979
1980 // Do the atomic load.
1981 auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile);
1982 // For non-simple lvalues perform compare-and-swap procedure.
1983 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1984 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1985 auto *CurBB = CGF.Builder.GetInsertBlock();
1986 CGF.EmitBlock(ContBB);
1987 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1988 /*NumReservedValues=*/2);
1989 PHI->addIncoming(OldVal, CurBB);
1990 Address NewAtomicAddr = CreateTempAlloca();
1991 Address NewAtomicIntAddr = castToAtomicIntPointer(NewAtomicAddr);
1992 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1993 requiresMemSetZero(getAtomicAddress().getElementType())) {
1994 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1995 }
1996 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
1997 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1998 // Try to write new value using cmpxchg operation.
1999 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
2000 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
2001 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
2002 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
2003}
2004
2005void AtomicInfo::EmitAtomicUpdate(
2006 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
2007 bool IsVolatile) {
2008 if (shouldUseLibcall()) {
2009 EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
2010 } else {
2011 EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
2012 }
2013}
2014
2015void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
2016 bool IsVolatile) {
2017 if (shouldUseLibcall()) {
2018 EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
2019 } else {
2020 EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
2021 }
2022}
2023
2025 bool isInit) {
2026 bool IsVolatile = lvalue.isVolatileQualified();
2027 llvm::AtomicOrdering AO;
2028 if (lvalue.getType()->isAtomicType()) {
2029 AO = llvm::AtomicOrdering::SequentiallyConsistent;
2030 } else {
2031 AO = llvm::AtomicOrdering::Release;
2032 IsVolatile = true;
2033 }
2034 return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
2035}
2036
2037/// Emit a store to an l-value of atomic type.
2038///
2039/// Note that the r-value is expected to be an r-value *of the atomic
2040/// type*; this means that for aggregate r-values, it should include
2041/// storage for any padding that was necessary.
2043 llvm::AtomicOrdering AO, bool IsVolatile,
2044 bool isInit) {
2045 // If this is an aggregate r-value, it should agree in type except
2046 // maybe for address-space qualification.
2047 assert(!rvalue.isAggregate() ||
2049 dest.getAddress(*this).getElementType());
2050
2051 AtomicInfo atomics(*this, dest);
2052 LValue LVal = atomics.getAtomicLValue();
2053
2054 // If this is an initialization, just put the value there normally.
2055 if (LVal.isSimple()) {
2056 if (isInit) {
2057 atomics.emitCopyIntoMemory(rvalue);
2058 return;
2059 }
2060
2061 // Check whether we should use a library call.
2062 if (atomics.shouldUseLibcall()) {
2063 // Produce a source address.
2064 Address srcAddr = atomics.materializeRValue(rvalue);
2065
2066 // void __atomic_store(size_t size, void *mem, void *val, int order)
2067 CallArgList args;
2068 args.add(RValue::get(atomics.getAtomicSizeValue()),
2069 getContext().getSizeType());
2070 args.add(RValue::get(atomics.getAtomicPointer()), getContext().VoidPtrTy);
2071 args.add(RValue::get(srcAddr.getPointer()), getContext().VoidPtrTy);
2072 args.add(
2073 RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))),
2074 getContext().IntTy);
2075 emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
2076 return;
2077 }
2078
2079 // Okay, we're doing this natively.
2080 llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
2081
2082 // Do the atomic store.
2083 Address addr = atomics.castToAtomicIntPointer(atomics.getAtomicAddress());
2084 intValue = Builder.CreateIntCast(
2085 intValue, addr.getElementType(), /*isSigned=*/false);
2086 llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
2087
2088 if (AO == llvm::AtomicOrdering::Acquire)
2089 AO = llvm::AtomicOrdering::Monotonic;
2090 else if (AO == llvm::AtomicOrdering::AcquireRelease)
2091 AO = llvm::AtomicOrdering::Release;
2092 // Initializations don't need to be atomic.
2093 if (!isInit)
2094 store->setAtomic(AO);
2095
2096 // Other decoration.
2097 if (IsVolatile)
2098 store->setVolatile(true);
2100 return;
2101 }
2102
2103 // Emit simple atomic update operation.
2104 atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
2105}
2106
2107/// Emit a compare-and-exchange op for atomic type.
2108///
2109std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
2110 LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
2111 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
2112 AggValueSlot Slot) {
2113 // If this is an aggregate r-value, it should agree in type except
2114 // maybe for address-space qualification.
2115 assert(!Expected.isAggregate() ||
2116 Expected.getAggregateAddress().getElementType() ==
2117 Obj.getAddress(*this).getElementType());
2118 assert(!Desired.isAggregate() ||
2119 Desired.getAggregateAddress().getElementType() ==
2120 Obj.getAddress(*this).getElementType());
2121 AtomicInfo Atomics(*this, Obj);
2122
2123 return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
2124 IsWeak);
2125}
2126
2128 LValue LVal, llvm::AtomicOrdering AO,
2129 const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
2130 AtomicInfo Atomics(*this, LVal);
2131 Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
2132}
2133
2135 AtomicInfo atomics(*this, dest);
2136
2137 switch (atomics.getEvaluationKind()) {
2138 case TEK_Scalar: {
2139 llvm::Value *value = EmitScalarExpr(init);
2140 atomics.emitCopyIntoMemory(RValue::get(value));
2141 return;
2142 }
2143
2144 case TEK_Complex: {
2145 ComplexPairTy value = EmitComplexExpr(init);
2146 atomics.emitCopyIntoMemory(RValue::getComplex(value));
2147 return;
2148 }
2149
2150 case TEK_Aggregate: {
2151 // Fix up the destination if the initializer isn't an expression
2152 // of atomic type.
2153 bool Zeroed = false;
2154 if (!init->getType()->isAtomicType()) {
2155 Zeroed = atomics.emitMemSetZeroIfNecessary();
2156 dest = atomics.projectValue();
2157 }
2158
2159 // Evaluate the expression directly into the destination.
2161 dest, *this, AggValueSlot::IsNotDestructed,
2165
2166 EmitAggExpr(init, slot);
2167 return;
2168 }
2169 }
2170 llvm_unreachable("bad evaluation kind");
2171}
Defines the clang::ASTContext interface.
#define V(N, I)
Definition: ASTContext.h:3233
static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type, uint64_t expectedSize)
Does a store of the given IR type modify the full expected width?
Definition: CGAtomic.cpp:332
static llvm::Value * EmitPostAtomicMinMax(CGBuilderTy &Builder, AtomicExpr::AtomicOp Op, bool IsSigned, llvm::Value *OldVal, llvm::Value *RHS)
Duplicate the atomic min/max operation in conventional IR for the builtin variants that return the ne...
Definition: CGAtomic.cpp:501
static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal, const llvm::function_ref< RValue(RValue)> &UpdateOp, Address DesiredAddr)
Definition: CGAtomic.cpp:1810
static Address EmitValToTemp(CodeGenFunction &CGF, Expr *E)
Definition: CGAtomic.cpp:723
static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, Address Ptr, Address Val1, Address Val2, llvm::Value *IsWeak, llvm::Value *FailureOrder, uint64_t Size, llvm::AtomicOrdering Order, llvm::SyncScope::ID Scope)
Definition: CGAtomic.cpp:521
static void AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args, bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy, SourceLocation Loc, CharUnits SizeInChars)
Definition: CGAtomic.cpp:789
static RValue emitAtomicLibcall(CodeGenFunction &CGF, StringRef fnName, QualType resultType, CallArgList &args)
Definition: CGAtomic.cpp:312
static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, Address Dest, Address Ptr, Address Val1, Address Val2, llvm::Value *FailureOrderVal, uint64_t Size, llvm::AtomicOrdering SuccessOrder, llvm::SyncScope::ID Scope)
Given an ordering required on success, emit all possible cmpxchg instructions to cope with the provid...
Definition: CGAtomic.cpp:424
static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, Address Dest, Address Ptr, Address Val1, Address Val2, uint64_t Size, llvm::AtomicOrdering SuccessOrder, llvm::AtomicOrdering FailureOrder, llvm::SyncScope::ID Scope)
Definition: CGAtomic.cpp:374
CodeGenFunction::ComplexPairTy ComplexPairTy
__device__ int
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition: ASTContext.h:182
CharUnits getTypeAlignInChars(QualType T) const
Return the ABI-specified alignment of a (complete) type T, in characters.
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
CanQualType VoidPtrTy
Definition: ASTContext.h:1104
CanQualType BoolTy
Definition: ASTContext.h:1078
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth,...
CanQualType getSizeType() const
Return the unique type for "size_t" (C99 7.17), defined in <stddef.h>.
CanQualType IntTy
Definition: ASTContext.h:1086
TypeInfoChars getTypeInfoInChars(const Type *T) const
int64_t toBits(CharUnits CharSize) const
Convert a size in characters to a size in bits.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
CanQualType VoidTy
Definition: ASTContext.h:1077
QualType getExtVectorType(QualType VectorType, unsigned NumElts) const
Return the unique reference to an extended vector type of the specified element type and size.
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
unsigned getTargetAddressSpace(LangAS AS) const
QualType getIntPtrType() const
Return a type compatible with "intptr_t" (C99 7.18.1.4), as defined by the target.
AtomicExpr - Variadic atomic builtins: __atomic_exchange, __atomic_fetch_*, __atomic_load,...
Definition: Expr.h:6431
static std::unique_ptr< AtomicScopeModel > getScopeModel(AtomicOp Op)
Get atomic scope model for the atomic op code.
Definition: Expr.h:6554
Expr * getVal2() const
Definition: Expr.h:6483
Expr * getOrder() const
Definition: Expr.h:6466
QualType getValueType() const
Definition: Expr.cpp:5009
Expr * getScope() const
Definition: Expr.h:6469
bool isCmpXChg() const
Definition: Expr.h:6517
AtomicOp getOp() const
Definition: Expr.h:6495
bool isOpenCL() const
Definition: Expr.h:6528
Expr * getVal1() const
Definition: Expr.h:6473
Expr * getPtr() const
Definition: Expr.h:6463
Expr * getWeak() const
Definition: Expr.h:6489
SourceLocation getBeginLoc() const LLVM_READONLY
Definition: Expr.h:6536
Expr * getOrderFail() const
Definition: Expr.h:6479
bool isVolatile() const
Definition: Expr.h:6513
CharUnits - This is an opaque type for sizes expressed in character units.
Definition: CharUnits.h:38
bool isZero() const
isZero - Test whether the quantity equals zero.
Definition: CharUnits.h:122
llvm::Align getAsAlign() const
getAsAlign - Returns Quantity as a valid llvm::Align, Beware llvm::Align assumes power of two 8-bit b...
Definition: CharUnits.h:189
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
Definition: CharUnits.h:185
An aligned address.
Definition: Address.h:29
static Address invalid()
Definition: Address.h:46
CharUnits getAlignment() const
Return the alignment of this pointer.
Definition: Address.h:78
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Definition: Address.h:62
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
Definition: Address.h:100
llvm::Value * getPointer() const
Definition: Address.h:51
bool isValid() const
Definition: Address.h:47
An aggregate value slot.
Definition: CGValue.h:512
static AggValueSlot ignored()
ignored - Returns an aggregate value slot indicating that the aggregate value is being ignored.
Definition: CGValue.h:580
Address getAddress() const
Definition: CGValue.h:650
static AggValueSlot forLValue(const LValue &LV, CodeGenFunction &CGF, IsDestructed_t isDestructed, NeedsGCBarriers_t needsGC, IsAliased_t isAliased, Overlap_t mayOverlap, IsZeroed_t isZeroed=IsNotZeroed, IsSanitizerChecked_t isChecked=IsNotSanitizerChecked)
Definition: CGValue.h:610
RValue asRValue() const
Definition: CGValue.h:674
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
Definition: CGBuilder.h:97
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
Definition: CGBuilder.h:159
llvm::AtomicRMWInst * CreateAtomicRMW(llvm::AtomicRMWInst::BinOp Op, llvm::Value *Ptr, llvm::Value *Val, llvm::AtomicOrdering Ordering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
Definition: CGBuilder.h:144
llvm::CallInst * CreateMemSet(Address Dest, llvm::Value *Value, llvm::Value *Size, bool IsVolatile=false)
Definition: CGBuilder.h:329
Address CreateStructGEP(Address Addr, unsigned Index, const llvm::Twine &Name="")
Definition: CGBuilder.h:175
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
Definition: CGBuilder.h:71
llvm::CallInst * CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
Definition: CGBuilder.h:300
Address CreateAddrSpaceCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
Definition: CGBuilder.h:152
llvm::AtomicCmpXchgInst * CreateAtomicCmpXchg(llvm::Value *Ptr, llvm::Value *Cmp, llvm::Value *New, llvm::AtomicOrdering SuccessOrdering, llvm::AtomicOrdering FailureOrdering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
Definition: CGBuilder.h:132
static CGCallee forDirect(llvm::Constant *functionPtr, const CGCalleeInfo &abstractInfo=CGCalleeInfo())
Definition: CGCall.h:128
CGFunctionInfo - Class to encapsulate the information about a function definition.
CallArgList - Type for representing both the value and type of arguments in a call.
Definition: CGCall.h:257
void add(RValue rvalue, QualType type)
Definition: CGCall.h:281
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
llvm::Value * EmitFromMemory(llvm::Value *Value, QualType Ty)
EmitFromMemory - Change a scalar value from its memory representation to its value representation.
std::pair< RValue, llvm::Value * > EmitAtomicCompareExchange(LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, llvm::AtomicOrdering Success=llvm::AtomicOrdering::SequentiallyConsistent, llvm::AtomicOrdering Failure=llvm::AtomicOrdering::SequentiallyConsistent, bool IsWeak=false, AggValueSlot Slot=AggValueSlot::ignored())
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
RValue EmitAtomicLoad(LValue LV, SourceLocation SL, AggValueSlot Slot=AggValueSlot::ignored())
bool hasVolatileMember(QualType T)
hasVolatileMember - returns true if aggregate type has a volatile member.
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO, const llvm::function_ref< RValue(RValue)> &UpdateOp, bool IsVolatile)
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
ComplexPairTy EmitComplexExpr(const Expr *E, bool IgnoreReal=false, bool IgnoreImag=false)
EmitComplexExpr - Emit the computation of the specified expression of complex type,...
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
RValue convertTempToRValue(Address addr, QualType type, SourceLocation Loc)
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee, ReturnValueSlot ReturnValue, const CallArgList &Args, llvm::CallBase **callOrInvoke, bool IsMustTail, SourceLocation Loc)
EmitCall - Generate a call of the given function, expecting the given result type,...
llvm::Type * ConvertTypeForMem(QualType T)
void EmitAtomicInit(Expr *E, LValue lvalue)
const TargetInfo & getTarget() const
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
Address EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitPointerWithAlignment - Given an expression with a pointer type, emit the value and compute our be...
RValue EmitLoadOfExtVectorElementLValue(LValue V)
void EmitAggregateCopy(LValue Dest, LValue Src, QualType EltTy, AggValueSlot::Overlap_t MayOverlap, bool isVolatile=false)
EmitAggregateCopy - Emit an aggregate copy.
const TargetCodeGenInfo & getTargetHooks() const
void EmitAggExpr(const Expr *E, AggValueSlot AS)
EmitAggExpr - Emit the computation of the specified expression of aggregate type.
llvm::Value * EmitToMemory(llvm::Value *Value, QualType Ty)
EmitToMemory - Change a scalar value from its value representation to its in-memory representation.
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
Address CreateMemTemp(QualType T, const Twine &Name="tmp", Address *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit)
EmitStoreOfComplex - Store a complex number into the specified l-value.
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
llvm::LLVMContext & getLLVMContext()
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
bool LValueIsSuitableForInlineAtomic(LValue Src)
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
RValue EmitLoadOfBitfieldLValue(LValue LV, SourceLocation Loc)
This class organizes the cross-function state that is used while generating LLVM code.
llvm::FunctionCallee CreateRuntimeFunction(llvm::FunctionType *Ty, StringRef Name, llvm::AttributeList ExtraAttrs=llvm::AttributeList(), bool Local=false, bool AssumeConvergent=false)
Create or return a runtime function declaration with the specified type and name.
DiagnosticsEngine & getDiags() const
const LangOptions & getLangOpts() const
const llvm::DataLayout & getDataLayout() const
void DecorateInstructionWithTBAA(llvm::Instruction *Inst, TBAAAccessInfo TBAAInfo)
DecorateInstructionWithTBAA - Decorate the instruction with a TBAA tag.
llvm::LLVMContext & getLLVMContext()
llvm::ConstantInt * getSize(CharUnits numChars)
Emit the given number of characters as a value of type size_t.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
Definition: CGCall.cpp:1619
const CGFunctionInfo & arrangeBuiltinFunctionCall(QualType resultType, const CallArgList &args)
Definition: CGCall.cpp:654
LValue - This represents an lvalue references.
Definition: CGValue.h:171
bool isBitField() const
Definition: CGValue.h:268
llvm::Constant * getExtVectorElts() const
Definition: CGValue.h:398
static LValue MakeExtVectorElt(Address vecAddress, llvm::Constant *Elts, QualType type, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo)
Definition: CGValue.h:446
void setAlignment(CharUnits A)
Definition: CGValue.h:332
llvm::Value * getBitFieldPointer() const
Definition: CGValue.h:408
bool isVectorElt() const
Definition: CGValue.h:267
bool isSimple() const
Definition: CGValue.h:266
bool isVolatileQualified() const
Definition: CGValue.h:273
CharUnits getAlignment() const
Definition: CGValue.h:331
Address getAddress(CodeGenFunction &CGF) const
Definition: CGValue.h:350
bool isVolatile() const
Definition: CGValue.h:316
bool isGlobalReg() const
Definition: CGValue.h:270
llvm::Value * getVectorPointer() const
Definition: CGValue.h:367
bool isExtVectorElt() const
Definition: CGValue.h:269
llvm::Value * getVectorIdx() const
Definition: CGValue.h:371
LValueBaseInfo getBaseInfo() const
Definition: CGValue.h:334
llvm::Value * getPointer(CodeGenFunction &CGF) const
Definition: CGValue.h:346
QualType getType() const
Definition: CGValue.h:279
llvm::Value * getExtVectorPointer() const
Definition: CGValue.h:394
const CGBitFieldInfo & getBitFieldInfo() const
Definition: CGValue.h:409
TBAAAccessInfo getTBAAInfo() const
Definition: CGValue.h:323
Address getVectorAddress() const
Definition: CGValue.h:363
static LValue MakeBitfield(Address Addr, const CGBitFieldInfo &Info, QualType type, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo)
Create a new object to represent a bit-field access.
Definition: CGValue.h:466
static LValue MakeVectorElt(Address vecAddress, llvm::Value *Idx, QualType type, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo)
Definition: CGValue.h:432
Address getExtVectorAddress() const
Definition: CGValue.h:390
Address getBitFieldAddress() const
Definition: CGValue.h:404
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
Definition: CGValue.h:39
bool isScalar() const
Definition: CGValue.h:54
static RValue get(llvm::Value *V)
Definition: CGValue.h:89
static RValue getAggregate(Address addr, bool isVolatile=false)
Definition: CGValue.h:110
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
Definition: CGValue.h:96
bool isAggregate() const
Definition: CGValue.h:56
Address getAggregateAddress() const
getAggregateAddr() - Return the Value* of the address of the aggregate.
Definition: CGValue.h:73
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
Definition: CGValue.h:61
bool isComplex() const
Definition: CGValue.h:55
bool isVolatileQualified() const
Definition: CGValue.h:58
std::pair< llvm::Value *, llvm::Value * > getComplexVal() const
getComplexVal - Return the real/imag components of this complex value.
Definition: CGValue.h:68
ReturnValueSlot - Contains the address where the return value of a function can be stored,...
Definition: CGCall.h:355
virtual llvm::SyncScope::ID getLLVMSyncScopeID(const LangOptions &LangOpts, SyncScope Scope, llvm::AtomicOrdering Ordering, llvm::LLVMContext &Ctx) const
Get the syncscope used in LLVM IR.
Definition: TargetInfo.cpp:154
virtual llvm::Value * performAddrSpaceCast(CodeGen::CodeGenFunction &CGF, llvm::Value *V, LangAS SrcAddr, LangAS DestAddr, llvm::Type *DestTy, bool IsNonNull=false) const
Perform address space cast of an expression of pointer type.
Definition: TargetInfo.cpp:132
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:192
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
Definition: Diagnostic.h:1542
This represents one expression.
Definition: Expr.h:110
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
Definition: Expr.cpp:330
QualType getType() const
Definition: Expr.h:142
PointerType - C99 6.7.5.1 - Pointer Declarators.
Definition: Type.h:2840
A (possibly-)qualified type.
Definition: Type.h:736
bool isNull() const
Return true if this QualType doesn't point to a type yet.
Definition: Type.h:803
Qualifiers getQualifiers() const
Retrieve the set of qualifiers applied to this type.
Definition: Type.h:6787
QualType getUnqualifiedType() const
Retrieve the unqualified variant of the given type, removing as little sugar as possible.
Definition: Type.h:6840
Scope - A scope is a transient data structure that is used while parsing the program.
Definition: Scope.h:41
Encodes a location in the source.
unsigned getMaxAtomicInlineWidth() const
Return the maximum width lock-free atomic operation which can be inlined given the supported features...
Definition: TargetInfo.h:794
bool isVoidType() const
Definition: Type.h:7317
bool isSignedIntegerType() const
Return true if this is an integer type that is signed, according to C99 6.2.5p4 [char,...
Definition: Type.cpp:2087
bool isPointerType() const
Definition: Type.h:6999
const T * castAs() const
Member-template castAs<specific type>.
Definition: Type.h:7590
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
Definition: Type.cpp:655
bool isAtomicType() const
Definition: Type.h:7140
bool isFloatingType() const
Definition: Type.cpp:2190
const T * getAs() const
Member-template getAs<specific type>'.
Definition: Type.h:7523
QualType getType() const
Definition: Value.cpp:234
Represents a GCC generic vector type.
Definition: Type.h:3431
TypeEvaluationKind
The kind of evaluation to perform on values of a particular type.
const internal::VariadicAllOfMatcher< Type > type
Matches Types in the clang AST.
bool Load(InterpState &S, CodePtr OpPC)
Definition: Interp.h:1248
@ C
Languages that the frontend can parse and compile.
llvm::StringRef getAsString(SyncScope S)
Definition: SyncScope.h:55
unsigned long uint64_t
#define true
Definition: stdbool.h:21
Structure with information about how a bitfield should be accessed.
CharUnits StorageOffset
The offset of the bitfield storage from the start of the struct.
unsigned Offset
The offset within a contiguous run of bitfields that are represented as a single "field" within the L...
unsigned Size
The total size of the bit-field, in bits.
unsigned StorageSize
The storage size in bits which should be used when accessing this bitfield.
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
llvm::IntegerType * IntTy
int
uint64_t Width
Definition: ASTContext.h:153
unsigned Align
Definition: ASTContext.h:154