clang  6.0.0svn
CGAtomic.cpp
Go to the documentation of this file.
1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file contains the code for emitting atomic operations.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "CGCall.h"
15 #include "CGRecordLayout.h"
16 #include "CodeGenFunction.h"
17 #include "CodeGenModule.h"
18 #include "TargetInfo.h"
19 #include "clang/AST/ASTContext.h"
21 #include "llvm/ADT/DenseMap.h"
22 #include "llvm/IR/DataLayout.h"
23 #include "llvm/IR/Intrinsics.h"
24 #include "llvm/IR/Operator.h"
25 
26 using namespace clang;
27 using namespace CodeGen;
28 
29 namespace {
30  class AtomicInfo {
31  CodeGenFunction &CGF;
32  QualType AtomicTy;
33  QualType ValueTy;
34  uint64_t AtomicSizeInBits;
35  uint64_t ValueSizeInBits;
36  CharUnits AtomicAlign;
37  CharUnits ValueAlign;
38  CharUnits LValueAlign;
39  TypeEvaluationKind EvaluationKind;
40  bool UseLibcall;
41  LValue LVal;
42  CGBitFieldInfo BFI;
43  public:
44  AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
45  : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
46  EvaluationKind(TEK_Scalar), UseLibcall(true) {
47  assert(!lvalue.isGlobalReg());
48  ASTContext &C = CGF.getContext();
49  if (lvalue.isSimple()) {
50  AtomicTy = lvalue.getType();
51  if (auto *ATy = AtomicTy->getAs<AtomicType>())
52  ValueTy = ATy->getValueType();
53  else
54  ValueTy = AtomicTy;
55  EvaluationKind = CGF.getEvaluationKind(ValueTy);
56 
57  uint64_t ValueAlignInBits;
58  uint64_t AtomicAlignInBits;
59  TypeInfo ValueTI = C.getTypeInfo(ValueTy);
60  ValueSizeInBits = ValueTI.Width;
61  ValueAlignInBits = ValueTI.Align;
62 
63  TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
64  AtomicSizeInBits = AtomicTI.Width;
65  AtomicAlignInBits = AtomicTI.Align;
66 
67  assert(ValueSizeInBits <= AtomicSizeInBits);
68  assert(ValueAlignInBits <= AtomicAlignInBits);
69 
70  AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
71  ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
72  if (lvalue.getAlignment().isZero())
73  lvalue.setAlignment(AtomicAlign);
74 
75  LVal = lvalue;
76  } else if (lvalue.isBitField()) {
77  ValueTy = lvalue.getType();
78  ValueSizeInBits = C.getTypeSize(ValueTy);
79  auto &OrigBFI = lvalue.getBitFieldInfo();
80  auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
81  AtomicSizeInBits = C.toBits(
82  C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
83  .alignTo(lvalue.getAlignment()));
84  auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer());
85  auto OffsetInChars =
86  (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
87  lvalue.getAlignment();
88  VoidPtrAddr = CGF.Builder.CreateConstGEP1_64(
89  VoidPtrAddr, OffsetInChars.getQuantity());
91  VoidPtrAddr,
92  CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(),
93  "atomic_bitfield_base");
94  BFI = OrigBFI;
95  BFI.Offset = Offset;
96  BFI.StorageSize = AtomicSizeInBits;
97  BFI.StorageOffset += OffsetInChars;
98  LVal = LValue::MakeBitfield(Address(Addr, lvalue.getAlignment()),
99  BFI, lvalue.getType(), lvalue.getBaseInfo(),
100  lvalue.getTBAAInfo());
101  AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
102  if (AtomicTy.isNull()) {
103  llvm::APInt Size(
104  /*numBits=*/32,
105  C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
106  AtomicTy = C.getConstantArrayType(C.CharTy, Size, ArrayType::Normal,
107  /*IndexTypeQuals=*/0);
108  }
109  AtomicAlign = ValueAlign = lvalue.getAlignment();
110  } else if (lvalue.isVectorElt()) {
111  ValueTy = lvalue.getType()->getAs<VectorType>()->getElementType();
112  ValueSizeInBits = C.getTypeSize(ValueTy);
113  AtomicTy = lvalue.getType();
114  AtomicSizeInBits = C.getTypeSize(AtomicTy);
115  AtomicAlign = ValueAlign = lvalue.getAlignment();
116  LVal = lvalue;
117  } else {
118  assert(lvalue.isExtVectorElt());
119  ValueTy = lvalue.getType();
120  ValueSizeInBits = C.getTypeSize(ValueTy);
121  AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
122  lvalue.getType(), lvalue.getExtVectorAddress()
123  .getElementType()->getVectorNumElements());
124  AtomicSizeInBits = C.getTypeSize(AtomicTy);
125  AtomicAlign = ValueAlign = lvalue.getAlignment();
126  LVal = lvalue;
127  }
128  UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
129  AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
130  }
131 
132  QualType getAtomicType() const { return AtomicTy; }
133  QualType getValueType() const { return ValueTy; }
134  CharUnits getAtomicAlignment() const { return AtomicAlign; }
135  CharUnits getValueAlignment() const { return ValueAlign; }
136  uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
137  uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
138  TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
139  bool shouldUseLibcall() const { return UseLibcall; }
140  const LValue &getAtomicLValue() const { return LVal; }
141  llvm::Value *getAtomicPointer() const {
142  if (LVal.isSimple())
143  return LVal.getPointer();
144  else if (LVal.isBitField())
145  return LVal.getBitFieldPointer();
146  else if (LVal.isVectorElt())
147  return LVal.getVectorPointer();
148  assert(LVal.isExtVectorElt());
149  return LVal.getExtVectorPointer();
150  }
151  Address getAtomicAddress() const {
152  return Address(getAtomicPointer(), getAtomicAlignment());
153  }
154 
155  Address getAtomicAddressAsAtomicIntPointer() const {
156  return emitCastToAtomicIntPointer(getAtomicAddress());
157  }
158 
159  /// Is the atomic size larger than the underlying value type?
160  ///
161  /// Note that the absence of padding does not mean that atomic
162  /// objects are completely interchangeable with non-atomic
163  /// objects: we might have promoted the alignment of a type
164  /// without making it bigger.
165  bool hasPadding() const {
166  return (ValueSizeInBits != AtomicSizeInBits);
167  }
168 
169  bool emitMemSetZeroIfNecessary() const;
170 
171  llvm::Value *getAtomicSizeValue() const {
172  CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
173  return CGF.CGM.getSize(size);
174  }
175 
176  /// Cast the given pointer to an integer pointer suitable for atomic
177  /// operations if the source.
178  Address emitCastToAtomicIntPointer(Address Addr) const;
179 
180  /// If Addr is compatible with the iN that will be used for an atomic
181  /// operation, bitcast it. Otherwise, create a temporary that is suitable
182  /// and copy the value across.
183  Address convertToAtomicIntPointer(Address Addr) const;
184 
185  /// Turn an atomic-layout object into an r-value.
186  RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
187  SourceLocation loc, bool AsValue) const;
188 
189  /// \brief Converts a rvalue to integer value.
190  llvm::Value *convertRValueToInt(RValue RVal) const;
191 
192  RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
193  AggValueSlot ResultSlot,
194  SourceLocation Loc, bool AsValue) const;
195 
196  /// Copy an atomic r-value into atomic-layout memory.
197  void emitCopyIntoMemory(RValue rvalue) const;
198 
199  /// Project an l-value down to the value field.
200  LValue projectValue() const {
201  assert(LVal.isSimple());
202  Address addr = getAtomicAddress();
203  if (hasPadding())
204  addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits());
205 
206  return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
207  LVal.getBaseInfo(), LVal.getTBAAInfo());
208  }
209 
210  /// \brief Emits atomic load.
211  /// \returns Loaded value.
212  RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
213  bool AsValue, llvm::AtomicOrdering AO,
214  bool IsVolatile);
215 
216  /// \brief Emits atomic compare-and-exchange sequence.
217  /// \param Expected Expected value.
218  /// \param Desired Desired value.
219  /// \param Success Atomic ordering for success operation.
220  /// \param Failure Atomic ordering for failed operation.
221  /// \param IsWeak true if atomic operation is weak, false otherwise.
222  /// \returns Pair of values: previous value from storage (value type) and
223  /// boolean flag (i1 type) with true if success and false otherwise.
224  std::pair<RValue, llvm::Value *>
225  EmitAtomicCompareExchange(RValue Expected, RValue Desired,
226  llvm::AtomicOrdering Success =
227  llvm::AtomicOrdering::SequentiallyConsistent,
228  llvm::AtomicOrdering Failure =
229  llvm::AtomicOrdering::SequentiallyConsistent,
230  bool IsWeak = false);
231 
232  /// \brief Emits atomic update.
233  /// \param AO Atomic ordering.
234  /// \param UpdateOp Update operation for the current lvalue.
235  void EmitAtomicUpdate(llvm::AtomicOrdering AO,
236  const llvm::function_ref<RValue(RValue)> &UpdateOp,
237  bool IsVolatile);
238  /// \brief Emits atomic update.
239  /// \param AO Atomic ordering.
240  void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
241  bool IsVolatile);
242 
243  /// Materialize an atomic r-value in atomic-layout memory.
244  Address materializeRValue(RValue rvalue) const;
245 
246  /// \brief Creates temp alloca for intermediate operations on atomic value.
247  Address CreateTempAlloca() const;
248  private:
249  bool requiresMemSetZero(llvm::Type *type) const;
250 
251 
252  /// \brief Emits atomic load as a libcall.
253  void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
254  llvm::AtomicOrdering AO, bool IsVolatile);
255  /// \brief Emits atomic load as LLVM instruction.
256  llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
257  /// \brief Emits atomic compare-and-exchange op as a libcall.
258  llvm::Value *EmitAtomicCompareExchangeLibcall(
259  llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
260  llvm::AtomicOrdering Success =
261  llvm::AtomicOrdering::SequentiallyConsistent,
262  llvm::AtomicOrdering Failure =
263  llvm::AtomicOrdering::SequentiallyConsistent);
264  /// \brief Emits atomic compare-and-exchange op as LLVM instruction.
265  std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
266  llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
267  llvm::AtomicOrdering Success =
268  llvm::AtomicOrdering::SequentiallyConsistent,
269  llvm::AtomicOrdering Failure =
270  llvm::AtomicOrdering::SequentiallyConsistent,
271  bool IsWeak = false);
272  /// \brief Emit atomic update as libcalls.
273  void
274  EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
275  const llvm::function_ref<RValue(RValue)> &UpdateOp,
276  bool IsVolatile);
277  /// \brief Emit atomic update as LLVM instructions.
278  void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
279  const llvm::function_ref<RValue(RValue)> &UpdateOp,
280  bool IsVolatile);
281  /// \brief Emit atomic update as libcalls.
282  void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
283  bool IsVolatile);
284  /// \brief Emit atomic update as LLVM instructions.
285  void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
286  bool IsVolatile);
287  };
288 }
289 
290 Address AtomicInfo::CreateTempAlloca() const {
291  Address TempAlloca = CGF.CreateMemTemp(
292  (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
293  : AtomicTy,
294  getAtomicAlignment(),
295  "atomic-temp");
296  // Cast to pointer to value type for bitfields.
297  if (LVal.isBitField())
299  TempAlloca, getAtomicAddress().getType());
300  return TempAlloca;
301 }
302 
304  StringRef fnName,
305  QualType resultType,
306  CallArgList &args) {
307  const CGFunctionInfo &fnInfo =
308  CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args);
309  llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
310  llvm::Constant *fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName);
311  auto callee = CGCallee::forDirect(fn);
312  return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args);
313 }
314 
315 /// Does a store of the given IR type modify the full expected width?
317  uint64_t expectedSize) {
318  return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
319 }
320 
321 /// Does the atomic type require memsetting to zero before initialization?
322 ///
323 /// The IR type is provided as a way of making certain queries faster.
324 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
325  // If the atomic type has size padding, we definitely need a memset.
326  if (hasPadding()) return true;
327 
328  // Otherwise, do some simple heuristics to try to avoid it:
329  switch (getEvaluationKind()) {
330  // For scalars and complexes, check whether the store size of the
331  // type uses the full size.
332  case TEK_Scalar:
333  return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
334  case TEK_Complex:
335  return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
336  AtomicSizeInBits / 2);
337 
338  // Padding in structs has an undefined bit pattern. User beware.
339  case TEK_Aggregate:
340  return false;
341  }
342  llvm_unreachable("bad evaluation kind");
343 }
344 
345 bool AtomicInfo::emitMemSetZeroIfNecessary() const {
346  assert(LVal.isSimple());
347  llvm::Value *addr = LVal.getPointer();
348  if (!requiresMemSetZero(addr->getType()->getPointerElementType()))
349  return false;
350 
351  CGF.Builder.CreateMemSet(
352  addr, llvm::ConstantInt::get(CGF.Int8Ty, 0),
353  CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
354  LVal.getAlignment().getQuantity());
355  return true;
356 }
357 
358 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
359  Address Dest, Address Ptr,
360  Address Val1, Address Val2,
361  uint64_t Size,
362  llvm::AtomicOrdering SuccessOrder,
363  llvm::AtomicOrdering FailureOrder,
365  // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
366  llvm::Value *Expected = CGF.Builder.CreateLoad(Val1);
367  llvm::Value *Desired = CGF.Builder.CreateLoad(Val2);
368 
369  llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
370  Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder,
371  Scope);
372  Pair->setVolatile(E->isVolatile());
373  Pair->setWeak(IsWeak);
374 
375  // Cmp holds the result of the compare-exchange operation: true on success,
376  // false on failure.
377  llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
378  llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
379 
380  // This basic block is used to hold the store instruction if the operation
381  // failed.
382  llvm::BasicBlock *StoreExpectedBB =
383  CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
384 
385  // This basic block is the exit point of the operation, we should end up
386  // here regardless of whether or not the operation succeeded.
387  llvm::BasicBlock *ContinueBB =
388  CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
389 
390  // Update Expected if Expected isn't equal to Old, otherwise branch to the
391  // exit point.
392  CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
393 
394  CGF.Builder.SetInsertPoint(StoreExpectedBB);
395  // Update the memory at Expected with Old's value.
396  CGF.Builder.CreateStore(Old, Val1);
397  // Finally, branch to the exit point.
398  CGF.Builder.CreateBr(ContinueBB);
399 
400  CGF.Builder.SetInsertPoint(ContinueBB);
401  // Update the memory at Dest with Cmp's value.
402  CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
403 }
404 
405 /// Given an ordering required on success, emit all possible cmpxchg
406 /// instructions to cope with the provided (but possibly only dynamically known)
407 /// FailureOrder.
409  bool IsWeak, Address Dest, Address Ptr,
410  Address Val1, Address Val2,
411  llvm::Value *FailureOrderVal,
412  uint64_t Size,
413  llvm::AtomicOrdering SuccessOrder,
415  llvm::AtomicOrdering FailureOrder;
416  if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
417  auto FOS = FO->getSExtValue();
418  if (!llvm::isValidAtomicOrderingCABI(FOS))
419  FailureOrder = llvm::AtomicOrdering::Monotonic;
420  else
421  switch ((llvm::AtomicOrderingCABI)FOS) {
422  case llvm::AtomicOrderingCABI::relaxed:
423  case llvm::AtomicOrderingCABI::release:
424  case llvm::AtomicOrderingCABI::acq_rel:
425  FailureOrder = llvm::AtomicOrdering::Monotonic;
426  break;
427  case llvm::AtomicOrderingCABI::consume:
428  case llvm::AtomicOrderingCABI::acquire:
429  FailureOrder = llvm::AtomicOrdering::Acquire;
430  break;
431  case llvm::AtomicOrderingCABI::seq_cst:
432  FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent;
433  break;
434  }
435  if (isStrongerThan(FailureOrder, SuccessOrder)) {
436  // Don't assert on undefined behavior "failure argument shall be no
437  // stronger than the success argument".
438  FailureOrder =
439  llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder);
440  }
441  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
442  FailureOrder, Scope);
443  return;
444  }
445 
446  // Create all the relevant BB's
447  llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
448  *SeqCstBB = nullptr;
449  MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
450  if (SuccessOrder != llvm::AtomicOrdering::Monotonic &&
451  SuccessOrder != llvm::AtomicOrdering::Release)
452  AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
453  if (SuccessOrder == llvm::AtomicOrdering::SequentiallyConsistent)
454  SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
455 
456  llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
457 
458  llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
459 
460  // Emit all the different atomics
461 
462  // MonotonicBB is arbitrarily chosen as the default case; in practice, this
463  // doesn't matter unless someone is crazy enough to use something that
464  // doesn't fold to a constant for the ordering.
465  CGF.Builder.SetInsertPoint(MonotonicBB);
466  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
467  Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope);
468  CGF.Builder.CreateBr(ContBB);
469 
470  if (AcquireBB) {
471  CGF.Builder.SetInsertPoint(AcquireBB);
472  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
473  Size, SuccessOrder, llvm::AtomicOrdering::Acquire, Scope);
474  CGF.Builder.CreateBr(ContBB);
475  SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
476  AcquireBB);
477  SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
478  AcquireBB);
479  }
480  if (SeqCstBB) {
481  CGF.Builder.SetInsertPoint(SeqCstBB);
482  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
483  llvm::AtomicOrdering::SequentiallyConsistent, Scope);
484  CGF.Builder.CreateBr(ContBB);
485  SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
486  SeqCstBB);
487  }
488 
489  CGF.Builder.SetInsertPoint(ContBB);
490 }
491 
492 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
493  Address Ptr, Address Val1, Address Val2,
494  llvm::Value *IsWeak, llvm::Value *FailureOrder,
495  uint64_t Size, llvm::AtomicOrdering Order,
497  llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
498  llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
499 
500  switch (E->getOp()) {
501  case AtomicExpr::AO__c11_atomic_init:
502  case AtomicExpr::AO__opencl_atomic_init:
503  llvm_unreachable("Already handled!");
504 
505  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
506  case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
507  emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
508  FailureOrder, Size, Order, Scope);
509  return;
510  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
511  case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
512  emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
513  FailureOrder, Size, Order, Scope);
514  return;
515  case AtomicExpr::AO__atomic_compare_exchange:
516  case AtomicExpr::AO__atomic_compare_exchange_n: {
517  if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
518  emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
519  Val1, Val2, FailureOrder, Size, Order, Scope);
520  } else {
521  // Create all the relevant BB's
522  llvm::BasicBlock *StrongBB =
523  CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
524  llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
525  llvm::BasicBlock *ContBB =
526  CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
527 
528  llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
529  SI->addCase(CGF.Builder.getInt1(false), StrongBB);
530 
531  CGF.Builder.SetInsertPoint(StrongBB);
532  emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
533  FailureOrder, Size, Order, Scope);
534  CGF.Builder.CreateBr(ContBB);
535 
536  CGF.Builder.SetInsertPoint(WeakBB);
537  emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
538  FailureOrder, Size, Order, Scope);
539  CGF.Builder.CreateBr(ContBB);
540 
541  CGF.Builder.SetInsertPoint(ContBB);
542  }
543  return;
544  }
545  case AtomicExpr::AO__c11_atomic_load:
546  case AtomicExpr::AO__opencl_atomic_load:
547  case AtomicExpr::AO__atomic_load_n:
548  case AtomicExpr::AO__atomic_load: {
549  llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
550  Load->setAtomic(Order, Scope);
551  Load->setVolatile(E->isVolatile());
552  CGF.Builder.CreateStore(Load, Dest);
553  return;
554  }
555 
556  case AtomicExpr::AO__c11_atomic_store:
557  case AtomicExpr::AO__opencl_atomic_store:
558  case AtomicExpr::AO__atomic_store:
559  case AtomicExpr::AO__atomic_store_n: {
560  llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
561  llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
562  Store->setAtomic(Order, Scope);
563  Store->setVolatile(E->isVolatile());
564  return;
565  }
566 
567  case AtomicExpr::AO__c11_atomic_exchange:
568  case AtomicExpr::AO__opencl_atomic_exchange:
569  case AtomicExpr::AO__atomic_exchange_n:
570  case AtomicExpr::AO__atomic_exchange:
571  Op = llvm::AtomicRMWInst::Xchg;
572  break;
573 
574  case AtomicExpr::AO__atomic_add_fetch:
575  PostOp = llvm::Instruction::Add;
576  // Fall through.
577  case AtomicExpr::AO__c11_atomic_fetch_add:
578  case AtomicExpr::AO__opencl_atomic_fetch_add:
579  case AtomicExpr::AO__atomic_fetch_add:
580  Op = llvm::AtomicRMWInst::Add;
581  break;
582 
583  case AtomicExpr::AO__atomic_sub_fetch:
584  PostOp = llvm::Instruction::Sub;
585  // Fall through.
586  case AtomicExpr::AO__c11_atomic_fetch_sub:
587  case AtomicExpr::AO__opencl_atomic_fetch_sub:
588  case AtomicExpr::AO__atomic_fetch_sub:
589  Op = llvm::AtomicRMWInst::Sub;
590  break;
591 
592  case AtomicExpr::AO__opencl_atomic_fetch_min:
593  Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Min
594  : llvm::AtomicRMWInst::UMin;
595  break;
596 
597  case AtomicExpr::AO__opencl_atomic_fetch_max:
598  Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Max
599  : llvm::AtomicRMWInst::UMax;
600  break;
601 
602  case AtomicExpr::AO__atomic_and_fetch:
603  PostOp = llvm::Instruction::And;
604  // Fall through.
605  case AtomicExpr::AO__c11_atomic_fetch_and:
606  case AtomicExpr::AO__opencl_atomic_fetch_and:
607  case AtomicExpr::AO__atomic_fetch_and:
609  break;
610 
611  case AtomicExpr::AO__atomic_or_fetch:
612  PostOp = llvm::Instruction::Or;
613  // Fall through.
614  case AtomicExpr::AO__c11_atomic_fetch_or:
615  case AtomicExpr::AO__opencl_atomic_fetch_or:
616  case AtomicExpr::AO__atomic_fetch_or:
617  Op = llvm::AtomicRMWInst::Or;
618  break;
619 
620  case AtomicExpr::AO__atomic_xor_fetch:
621  PostOp = llvm::Instruction::Xor;
622  // Fall through.
623  case AtomicExpr::AO__c11_atomic_fetch_xor:
624  case AtomicExpr::AO__opencl_atomic_fetch_xor:
625  case AtomicExpr::AO__atomic_fetch_xor:
626  Op = llvm::AtomicRMWInst::Xor;
627  break;
628 
629  case AtomicExpr::AO__atomic_nand_fetch:
630  PostOp = llvm::Instruction::And; // the NOT is special cased below
631  // Fall through.
632  case AtomicExpr::AO__atomic_fetch_nand:
633  Op = llvm::AtomicRMWInst::Nand;
634  break;
635  }
636 
637  llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
638  llvm::AtomicRMWInst *RMWI =
639  CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order, Scope);
640  RMWI->setVolatile(E->isVolatile());
641 
642  // For __atomic_*_fetch operations, perform the operation again to
643  // determine the value which was written.
644  llvm::Value *Result = RMWI;
645  if (PostOp)
646  Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1);
647  if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
648  Result = CGF.Builder.CreateNot(Result);
649  CGF.Builder.CreateStore(Result, Dest);
650 }
651 
652 // This function emits any expression (scalar, complex, or aggregate)
653 // into a temporary alloca.
654 static Address
656  Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
657  CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
658  /*Init*/ true);
659  return DeclPtr;
660 }
661 
663  Address Ptr, Address Val1, Address Val2,
664  llvm::Value *IsWeak, llvm::Value *FailureOrder,
665  uint64_t Size, llvm::AtomicOrdering Order,
666  llvm::Value *Scope) {
667  auto ScopeModel = Expr->getScopeModel();
668 
669  // LLVM atomic instructions always have synch scope. If clang atomic
670  // expression has no scope operand, use default LLVM synch scope.
671  if (!ScopeModel) {
672  EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
673  Order, CGF.CGM.getLLVMContext().getOrInsertSyncScopeID(""));
674  return;
675  }
676 
677  // Handle constant scope.
678  if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) {
679  auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID(
680  ScopeModel->map(SC->getZExtValue()), CGF.CGM.getLLVMContext());
681  EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
682  Order, SCID);
683  return;
684  }
685 
686  // Handle non-constant scope.
687  auto &Builder = CGF.Builder;
688  auto Scopes = ScopeModel->getRuntimeValues();
689  llvm::DenseMap<unsigned, llvm::BasicBlock *> BB;
690  for (auto S : Scopes)
691  BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn);
692 
693  llvm::BasicBlock *ContBB =
694  CGF.createBasicBlock("atomic.scope.continue", CGF.CurFn);
695 
696  auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false);
697  // If unsupported synch scope is encountered at run time, assume a fallback
698  // synch scope value.
699  auto FallBack = ScopeModel->getFallBackValue();
700  llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]);
701  for (auto S : Scopes) {
702  auto *B = BB[S];
703  if (S != FallBack)
704  SI->addCase(Builder.getInt32(S), B);
705 
706  Builder.SetInsertPoint(B);
707  EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
708  Order,
709  CGF.getTargetHooks().getLLVMSyncScopeID(ScopeModel->map(S),
710  CGF.getLLVMContext()));
711  Builder.CreateBr(ContBB);
712  }
713 
714  Builder.SetInsertPoint(ContBB);
715 }
716 
717 static void
719  bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,
720  SourceLocation Loc, CharUnits SizeInChars) {
721  if (UseOptimizedLibcall) {
722  // Load value and pass it to the function directly.
723  CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy);
724  int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
725  ValTy =
726  CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
727  llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(),
728  SizeInBits)->getPointerTo();
729  Address Ptr = Address(CGF.Builder.CreateBitCast(Val, IPtrTy), Align);
730  Val = CGF.EmitLoadOfScalar(Ptr, false,
731  CGF.getContext().getPointerType(ValTy),
732  Loc);
733  // Coerce the value into an appropriately sized integer type.
734  Args.add(RValue::get(Val), ValTy);
735  } else {
736  // Non-optimized functions always take a reference.
737  Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)),
738  CGF.getContext().VoidPtrTy);
739  }
740 }
741 
743  QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
744  QualType MemTy = AtomicTy;
745  if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
746  MemTy = AT->getValueType();
747  llvm::Value *IsWeak = nullptr, *OrderFail = nullptr;
748 
749  Address Val1 = Address::invalid();
750  Address Val2 = Address::invalid();
751  Address Dest = Address::invalid();
752  Address Ptr = EmitPointerWithAlignment(E->getPtr());
753 
754  CharUnits sizeChars, alignChars;
755  std::tie(sizeChars, alignChars) = getContext().getTypeInfoInChars(AtomicTy);
756  uint64_t Size = sizeChars.getQuantity();
757  unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();
758  bool UseLibcall = ((Ptr.getAlignment() % sizeChars) != 0 ||
759  getContext().toBits(sizeChars) > MaxInlineWidthInBits);
760 
761  if (E->getOp() == AtomicExpr::AO__c11_atomic_init ||
762  E->getOp() == AtomicExpr::AO__opencl_atomic_init) {
763  LValue lvalue = MakeAddrLValue(Ptr, AtomicTy);
764  EmitAtomicInit(E->getVal1(), lvalue);
765  return RValue::get(nullptr);
766  }
767 
768  llvm::Value *Order = EmitScalarExpr(E->getOrder());
769  llvm::Value *Scope =
770  E->getScopeModel() ? EmitScalarExpr(E->getScope()) : nullptr;
771 
772  switch (E->getOp()) {
773  case AtomicExpr::AO__c11_atomic_init:
774  case AtomicExpr::AO__opencl_atomic_init:
775  llvm_unreachable("Already handled above with EmitAtomicInit!");
776 
777  case AtomicExpr::AO__c11_atomic_load:
778  case AtomicExpr::AO__opencl_atomic_load:
779  case AtomicExpr::AO__atomic_load_n:
780  break;
781 
782  case AtomicExpr::AO__atomic_load:
783  Dest = EmitPointerWithAlignment(E->getVal1());
784  break;
785 
786  case AtomicExpr::AO__atomic_store:
787  Val1 = EmitPointerWithAlignment(E->getVal1());
788  break;
789 
790  case AtomicExpr::AO__atomic_exchange:
791  Val1 = EmitPointerWithAlignment(E->getVal1());
792  Dest = EmitPointerWithAlignment(E->getVal2());
793  break;
794 
795  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
796  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
797  case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
798  case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
799  case AtomicExpr::AO__atomic_compare_exchange_n:
800  case AtomicExpr::AO__atomic_compare_exchange:
801  Val1 = EmitPointerWithAlignment(E->getVal1());
802  if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
803  Val2 = EmitPointerWithAlignment(E->getVal2());
804  else
805  Val2 = EmitValToTemp(*this, E->getVal2());
806  OrderFail = EmitScalarExpr(E->getOrderFail());
807  if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n ||
808  E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
809  IsWeak = EmitScalarExpr(E->getWeak());
810  break;
811 
812  case AtomicExpr::AO__c11_atomic_fetch_add:
813  case AtomicExpr::AO__c11_atomic_fetch_sub:
814  case AtomicExpr::AO__opencl_atomic_fetch_add:
815  case AtomicExpr::AO__opencl_atomic_fetch_sub:
816  if (MemTy->isPointerType()) {
817  // For pointer arithmetic, we're required to do a bit of math:
818  // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
819  // ... but only for the C11 builtins. The GNU builtins expect the
820  // user to multiply by sizeof(T).
821  QualType Val1Ty = E->getVal1()->getType();
822  llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
823  CharUnits PointeeIncAmt =
824  getContext().getTypeSizeInChars(MemTy->getPointeeType());
825  Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
826  auto Temp = CreateMemTemp(Val1Ty, ".atomictmp");
827  Val1 = Temp;
828  EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty));
829  break;
830  }
831  // Fall through.
832  case AtomicExpr::AO__atomic_fetch_add:
833  case AtomicExpr::AO__atomic_fetch_sub:
834  case AtomicExpr::AO__atomic_add_fetch:
835  case AtomicExpr::AO__atomic_sub_fetch:
836  case AtomicExpr::AO__c11_atomic_store:
837  case AtomicExpr::AO__c11_atomic_exchange:
838  case AtomicExpr::AO__opencl_atomic_store:
839  case AtomicExpr::AO__opencl_atomic_exchange:
840  case AtomicExpr::AO__atomic_store_n:
841  case AtomicExpr::AO__atomic_exchange_n:
842  case AtomicExpr::AO__c11_atomic_fetch_and:
843  case AtomicExpr::AO__c11_atomic_fetch_or:
844  case AtomicExpr::AO__c11_atomic_fetch_xor:
845  case AtomicExpr::AO__opencl_atomic_fetch_and:
846  case AtomicExpr::AO__opencl_atomic_fetch_or:
847  case AtomicExpr::AO__opencl_atomic_fetch_xor:
848  case AtomicExpr::AO__opencl_atomic_fetch_min:
849  case AtomicExpr::AO__opencl_atomic_fetch_max:
850  case AtomicExpr::AO__atomic_fetch_and:
851  case AtomicExpr::AO__atomic_fetch_or:
852  case AtomicExpr::AO__atomic_fetch_xor:
853  case AtomicExpr::AO__atomic_fetch_nand:
854  case AtomicExpr::AO__atomic_and_fetch:
855  case AtomicExpr::AO__atomic_or_fetch:
856  case AtomicExpr::AO__atomic_xor_fetch:
857  case AtomicExpr::AO__atomic_nand_fetch:
858  Val1 = EmitValToTemp(*this, E->getVal1());
859  break;
860  }
861 
862  QualType RValTy = E->getType().getUnqualifiedType();
863 
864  // The inlined atomics only function on iN types, where N is a power of 2. We
865  // need to make sure (via temporaries if necessary) that all incoming values
866  // are compatible.
867  LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy);
868  AtomicInfo Atomics(*this, AtomicVal);
869 
870  Ptr = Atomics.emitCastToAtomicIntPointer(Ptr);
871  if (Val1.isValid()) Val1 = Atomics.convertToAtomicIntPointer(Val1);
872  if (Val2.isValid()) Val2 = Atomics.convertToAtomicIntPointer(Val2);
873  if (Dest.isValid())
874  Dest = Atomics.emitCastToAtomicIntPointer(Dest);
875  else if (E->isCmpXChg())
876  Dest = CreateMemTemp(RValTy, "cmpxchg.bool");
877  else if (!RValTy->isVoidType())
878  Dest = Atomics.emitCastToAtomicIntPointer(Atomics.CreateTempAlloca());
879 
880  // Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
881  if (UseLibcall) {
882  bool UseOptimizedLibcall = false;
883  switch (E->getOp()) {
884  case AtomicExpr::AO__c11_atomic_init:
885  case AtomicExpr::AO__opencl_atomic_init:
886  llvm_unreachable("Already handled above with EmitAtomicInit!");
887 
888  case AtomicExpr::AO__c11_atomic_fetch_add:
889  case AtomicExpr::AO__opencl_atomic_fetch_add:
890  case AtomicExpr::AO__atomic_fetch_add:
891  case AtomicExpr::AO__c11_atomic_fetch_and:
892  case AtomicExpr::AO__opencl_atomic_fetch_and:
893  case AtomicExpr::AO__atomic_fetch_and:
894  case AtomicExpr::AO__c11_atomic_fetch_or:
895  case AtomicExpr::AO__opencl_atomic_fetch_or:
896  case AtomicExpr::AO__atomic_fetch_or:
897  case AtomicExpr::AO__atomic_fetch_nand:
898  case AtomicExpr::AO__c11_atomic_fetch_sub:
899  case AtomicExpr::AO__opencl_atomic_fetch_sub:
900  case AtomicExpr::AO__atomic_fetch_sub:
901  case AtomicExpr::AO__c11_atomic_fetch_xor:
902  case AtomicExpr::AO__opencl_atomic_fetch_xor:
903  case AtomicExpr::AO__opencl_atomic_fetch_min:
904  case AtomicExpr::AO__opencl_atomic_fetch_max:
905  case AtomicExpr::AO__atomic_fetch_xor:
906  case AtomicExpr::AO__atomic_add_fetch:
907  case AtomicExpr::AO__atomic_and_fetch:
908  case AtomicExpr::AO__atomic_nand_fetch:
909  case AtomicExpr::AO__atomic_or_fetch:
910  case AtomicExpr::AO__atomic_sub_fetch:
911  case AtomicExpr::AO__atomic_xor_fetch:
912  // For these, only library calls for certain sizes exist.
913  UseOptimizedLibcall = true;
914  break;
915 
916  case AtomicExpr::AO__c11_atomic_load:
917  case AtomicExpr::AO__c11_atomic_store:
918  case AtomicExpr::AO__c11_atomic_exchange:
919  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
920  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
921  case AtomicExpr::AO__opencl_atomic_load:
922  case AtomicExpr::AO__opencl_atomic_store:
923  case AtomicExpr::AO__opencl_atomic_exchange:
924  case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
925  case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
926  case AtomicExpr::AO__atomic_load_n:
927  case AtomicExpr::AO__atomic_load:
928  case AtomicExpr::AO__atomic_store_n:
929  case AtomicExpr::AO__atomic_store:
930  case AtomicExpr::AO__atomic_exchange_n:
931  case AtomicExpr::AO__atomic_exchange:
932  case AtomicExpr::AO__atomic_compare_exchange_n:
933  case AtomicExpr::AO__atomic_compare_exchange:
934  // Only use optimized library calls for sizes for which they exist.
935  if (Size == 1 || Size == 2 || Size == 4 || Size == 8)
936  UseOptimizedLibcall = true;
937  break;
938  }
939 
940  CallArgList Args;
941  if (!UseOptimizedLibcall) {
942  // For non-optimized library calls, the size is the first parameter
943  Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
944  getContext().getSizeType());
945  }
946  // Atomic address is the first or second parameter
947  // The OpenCL atomic library functions only accept pointer arguments to
948  // generic address space.
949  auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) {
950  if (!E->isOpenCL())
951  return V;
952  auto AS = PT->getAs<PointerType>()->getPointeeType().getAddressSpace();
953  if (AS == LangAS::opencl_generic)
954  return V;
955  auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic);
956  auto T = V->getType();
957  auto *DestType = T->getPointerElementType()->getPointerTo(DestAS);
958 
959  return getTargetHooks().performAddrSpaceCast(
960  *this, V, AS, LangAS::opencl_generic, DestType, false);
961  };
962 
963  Args.add(RValue::get(CastToGenericAddrSpace(
964  EmitCastToVoidPtr(Ptr.getPointer()), E->getPtr()->getType())),
965  getContext().VoidPtrTy);
966 
967  std::string LibCallName;
968  QualType LoweredMemTy =
969  MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy;
970  QualType RetTy;
971  bool HaveRetTy = false;
972  llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
973  switch (E->getOp()) {
974  case AtomicExpr::AO__c11_atomic_init:
975  case AtomicExpr::AO__opencl_atomic_init:
976  llvm_unreachable("Already handled!");
977 
978  // There is only one libcall for compare an exchange, because there is no
979  // optimisation benefit possible from a libcall version of a weak compare
980  // and exchange.
981  // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
982  // void *desired, int success, int failure)
983  // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired,
984  // int success, int failure)
985  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
986  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
987  case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
988  case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
989  case AtomicExpr::AO__atomic_compare_exchange:
990  case AtomicExpr::AO__atomic_compare_exchange_n:
991  LibCallName = "__atomic_compare_exchange";
992  RetTy = getContext().BoolTy;
993  HaveRetTy = true;
994  Args.add(
995  RValue::get(CastToGenericAddrSpace(
996  EmitCastToVoidPtr(Val1.getPointer()), E->getVal1()->getType())),
997  getContext().VoidPtrTy);
998  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(),
999  MemTy, E->getExprLoc(), sizeChars);
1000  Args.add(RValue::get(Order), getContext().IntTy);
1001  Order = OrderFail;
1002  break;
1003  // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
1004  // int order)
1005  // T __atomic_exchange_N(T *mem, T val, int order)
1006  case AtomicExpr::AO__c11_atomic_exchange:
1007  case AtomicExpr::AO__opencl_atomic_exchange:
1008  case AtomicExpr::AO__atomic_exchange_n:
1009  case AtomicExpr::AO__atomic_exchange:
1010  LibCallName = "__atomic_exchange";
1011  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1012  MemTy, E->getExprLoc(), sizeChars);
1013  break;
1014  // void __atomic_store(size_t size, void *mem, void *val, int order)
1015  // void __atomic_store_N(T *mem, T val, int order)
1016  case AtomicExpr::AO__c11_atomic_store:
1017  case AtomicExpr::AO__opencl_atomic_store:
1018  case AtomicExpr::AO__atomic_store:
1019  case AtomicExpr::AO__atomic_store_n:
1020  LibCallName = "__atomic_store";
1021  RetTy = getContext().VoidTy;
1022  HaveRetTy = true;
1023  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1024  MemTy, E->getExprLoc(), sizeChars);
1025  break;
1026  // void __atomic_load(size_t size, void *mem, void *return, int order)
1027  // T __atomic_load_N(T *mem, int order)
1028  case AtomicExpr::AO__c11_atomic_load:
1029  case AtomicExpr::AO__opencl_atomic_load:
1030  case AtomicExpr::AO__atomic_load:
1031  case AtomicExpr::AO__atomic_load_n:
1032  LibCallName = "__atomic_load";
1033  break;
1034  // T __atomic_add_fetch_N(T *mem, T val, int order)
1035  // T __atomic_fetch_add_N(T *mem, T val, int order)
1036  case AtomicExpr::AO__atomic_add_fetch:
1037  PostOp = llvm::Instruction::Add;
1038  // Fall through.
1039  case AtomicExpr::AO__c11_atomic_fetch_add:
1040  case AtomicExpr::AO__opencl_atomic_fetch_add:
1041  case AtomicExpr::AO__atomic_fetch_add:
1042  LibCallName = "__atomic_fetch_add";
1043  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1044  LoweredMemTy, E->getExprLoc(), sizeChars);
1045  break;
1046  // T __atomic_and_fetch_N(T *mem, T val, int order)
1047  // T __atomic_fetch_and_N(T *mem, T val, int order)
1048  case AtomicExpr::AO__atomic_and_fetch:
1049  PostOp = llvm::Instruction::And;
1050  // Fall through.
1051  case AtomicExpr::AO__c11_atomic_fetch_and:
1052  case AtomicExpr::AO__opencl_atomic_fetch_and:
1053  case AtomicExpr::AO__atomic_fetch_and:
1054  LibCallName = "__atomic_fetch_and";
1055  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1056  MemTy, E->getExprLoc(), sizeChars);
1057  break;
1058  // T __atomic_or_fetch_N(T *mem, T val, int order)
1059  // T __atomic_fetch_or_N(T *mem, T val, int order)
1060  case AtomicExpr::AO__atomic_or_fetch:
1061  PostOp = llvm::Instruction::Or;
1062  // Fall through.
1063  case AtomicExpr::AO__c11_atomic_fetch_or:
1064  case AtomicExpr::AO__opencl_atomic_fetch_or:
1065  case AtomicExpr::AO__atomic_fetch_or:
1066  LibCallName = "__atomic_fetch_or";
1067  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1068  MemTy, E->getExprLoc(), sizeChars);
1069  break;
1070  // T __atomic_sub_fetch_N(T *mem, T val, int order)
1071  // T __atomic_fetch_sub_N(T *mem, T val, int order)
1072  case AtomicExpr::AO__atomic_sub_fetch:
1073  PostOp = llvm::Instruction::Sub;
1074  // Fall through.
1075  case AtomicExpr::AO__c11_atomic_fetch_sub:
1076  case AtomicExpr::AO__opencl_atomic_fetch_sub:
1077  case AtomicExpr::AO__atomic_fetch_sub:
1078  LibCallName = "__atomic_fetch_sub";
1079  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1080  LoweredMemTy, E->getExprLoc(), sizeChars);
1081  break;
1082  // T __atomic_xor_fetch_N(T *mem, T val, int order)
1083  // T __atomic_fetch_xor_N(T *mem, T val, int order)
1084  case AtomicExpr::AO__atomic_xor_fetch:
1085  PostOp = llvm::Instruction::Xor;
1086  // Fall through.
1087  case AtomicExpr::AO__c11_atomic_fetch_xor:
1088  case AtomicExpr::AO__opencl_atomic_fetch_xor:
1089  case AtomicExpr::AO__atomic_fetch_xor:
1090  LibCallName = "__atomic_fetch_xor";
1091  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1092  MemTy, E->getExprLoc(), sizeChars);
1093  break;
1094  case AtomicExpr::AO__opencl_atomic_fetch_min:
1095  LibCallName = E->getValueType()->isSignedIntegerType()
1096  ? "__atomic_fetch_min"
1097  : "__atomic_fetch_umin";
1098  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1099  LoweredMemTy, E->getExprLoc(), sizeChars);
1100  break;
1101  case AtomicExpr::AO__opencl_atomic_fetch_max:
1102  LibCallName = E->getValueType()->isSignedIntegerType()
1103  ? "__atomic_fetch_max"
1104  : "__atomic_fetch_umax";
1105  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1106  LoweredMemTy, E->getExprLoc(), sizeChars);
1107  break;
1108  // T __atomic_nand_fetch_N(T *mem, T val, int order)
1109  // T __atomic_fetch_nand_N(T *mem, T val, int order)
1110  case AtomicExpr::AO__atomic_nand_fetch:
1111  PostOp = llvm::Instruction::And; // the NOT is special cased below
1112  // Fall through.
1113  case AtomicExpr::AO__atomic_fetch_nand:
1114  LibCallName = "__atomic_fetch_nand";
1115  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1116  MemTy, E->getExprLoc(), sizeChars);
1117  break;
1118  }
1119 
1120  if (E->isOpenCL()) {
1121  LibCallName = std::string("__opencl") +
1122  StringRef(LibCallName).drop_front(1).str();
1123 
1124  }
1125  // Optimized functions have the size in their name.
1126  if (UseOptimizedLibcall)
1127  LibCallName += "_" + llvm::utostr(Size);
1128  // By default, assume we return a value of the atomic type.
1129  if (!HaveRetTy) {
1130  if (UseOptimizedLibcall) {
1131  // Value is returned directly.
1132  // The function returns an appropriately sized integer type.
1133  RetTy = getContext().getIntTypeForBitwidth(
1134  getContext().toBits(sizeChars), /*Signed=*/false);
1135  } else {
1136  // Value is returned through parameter before the order.
1137  RetTy = getContext().VoidTy;
1138  Args.add(RValue::get(EmitCastToVoidPtr(Dest.getPointer())),
1139  getContext().VoidPtrTy);
1140  }
1141  }
1142  // order is always the last parameter
1143  Args.add(RValue::get(Order),
1144  getContext().IntTy);
1145  if (E->isOpenCL())
1146  Args.add(RValue::get(Scope), getContext().IntTy);
1147 
1148  // PostOp is only needed for the atomic_*_fetch operations, and
1149  // thus is only needed for and implemented in the
1150  // UseOptimizedLibcall codepath.
1151  assert(UseOptimizedLibcall || !PostOp);
1152 
1153  RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
1154  // The value is returned directly from the libcall.
1155  if (E->isCmpXChg())
1156  return Res;
1157 
1158  // The value is returned directly for optimized libcalls but the expr
1159  // provided an out-param.
1160  if (UseOptimizedLibcall && Res.getScalarVal()) {
1161  llvm::Value *ResVal = Res.getScalarVal();
1162  if (PostOp) {
1163  llvm::Value *LoadVal1 = Args[1].RV.getScalarVal();
1164  ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1);
1165  }
1166  if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
1167  ResVal = Builder.CreateNot(ResVal);
1168 
1169  Builder.CreateStore(
1170  ResVal,
1171  Builder.CreateBitCast(Dest, ResVal->getType()->getPointerTo()));
1172  }
1173 
1174  if (RValTy->isVoidType())
1175  return RValue::get(nullptr);
1176 
1177  return convertTempToRValue(
1178  Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()),
1179  RValTy, E->getExprLoc());
1180  }
1181 
1182  bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
1183  E->getOp() == AtomicExpr::AO__opencl_atomic_store ||
1184  E->getOp() == AtomicExpr::AO__atomic_store ||
1185  E->getOp() == AtomicExpr::AO__atomic_store_n;
1186  bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
1187  E->getOp() == AtomicExpr::AO__opencl_atomic_load ||
1188  E->getOp() == AtomicExpr::AO__atomic_load ||
1189  E->getOp() == AtomicExpr::AO__atomic_load_n;
1190 
1191  if (isa<llvm::ConstantInt>(Order)) {
1192  auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1193  // We should not ever get to a case where the ordering isn't a valid C ABI
1194  // value, but it's hard to enforce that in general.
1195  if (llvm::isValidAtomicOrderingCABI(ord))
1196  switch ((llvm::AtomicOrderingCABI)ord) {
1197  case llvm::AtomicOrderingCABI::relaxed:
1198  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1199  llvm::AtomicOrdering::Monotonic, Scope);
1200  break;
1201  case llvm::AtomicOrderingCABI::consume:
1202  case llvm::AtomicOrderingCABI::acquire:
1203  if (IsStore)
1204  break; // Avoid crashing on code with undefined behavior
1205  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1206  llvm::AtomicOrdering::Acquire, Scope);
1207  break;
1208  case llvm::AtomicOrderingCABI::release:
1209  if (IsLoad)
1210  break; // Avoid crashing on code with undefined behavior
1211  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1212  llvm::AtomicOrdering::Release, Scope);
1213  break;
1214  case llvm::AtomicOrderingCABI::acq_rel:
1215  if (IsLoad || IsStore)
1216  break; // Avoid crashing on code with undefined behavior
1217  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1218  llvm::AtomicOrdering::AcquireRelease, Scope);
1219  break;
1220  case llvm::AtomicOrderingCABI::seq_cst:
1221  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1222  llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1223  break;
1224  }
1225  if (RValTy->isVoidType())
1226  return RValue::get(nullptr);
1227 
1228  return convertTempToRValue(
1229  Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo(
1230  Dest.getAddressSpace())),
1231  RValTy, E->getExprLoc());
1232  }
1233 
1234  // Long case, when Order isn't obviously constant.
1235 
1236  // Create all the relevant BB's
1237  llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
1238  *ReleaseBB = nullptr, *AcqRelBB = nullptr,
1239  *SeqCstBB = nullptr;
1240  MonotonicBB = createBasicBlock("monotonic", CurFn);
1241  if (!IsStore)
1242  AcquireBB = createBasicBlock("acquire", CurFn);
1243  if (!IsLoad)
1244  ReleaseBB = createBasicBlock("release", CurFn);
1245  if (!IsLoad && !IsStore)
1246  AcqRelBB = createBasicBlock("acqrel", CurFn);
1247  SeqCstBB = createBasicBlock("seqcst", CurFn);
1248  llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1249 
1250  // Create the switch for the split
1251  // MonotonicBB is arbitrarily chosen as the default case; in practice, this
1252  // doesn't matter unless someone is crazy enough to use something that
1253  // doesn't fold to a constant for the ordering.
1254  Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1255  llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
1256 
1257  // Emit all the different atomics
1258  Builder.SetInsertPoint(MonotonicBB);
1259  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1260  llvm::AtomicOrdering::Monotonic, Scope);
1261  Builder.CreateBr(ContBB);
1262  if (!IsStore) {
1263  Builder.SetInsertPoint(AcquireBB);
1264  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1265  llvm::AtomicOrdering::Acquire, Scope);
1266  Builder.CreateBr(ContBB);
1267  SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
1268  AcquireBB);
1269  SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
1270  AcquireBB);
1271  }
1272  if (!IsLoad) {
1273  Builder.SetInsertPoint(ReleaseBB);
1274  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1275  llvm::AtomicOrdering::Release, Scope);
1276  Builder.CreateBr(ContBB);
1277  SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release),
1278  ReleaseBB);
1279  }
1280  if (!IsLoad && !IsStore) {
1281  Builder.SetInsertPoint(AcqRelBB);
1282  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1283  llvm::AtomicOrdering::AcquireRelease, Scope);
1284  Builder.CreateBr(ContBB);
1285  SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel),
1286  AcqRelBB);
1287  }
1288  Builder.SetInsertPoint(SeqCstBB);
1289  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1290  llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1291  Builder.CreateBr(ContBB);
1292  SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
1293  SeqCstBB);
1294 
1295  // Cleanup and return
1296  Builder.SetInsertPoint(ContBB);
1297  if (RValTy->isVoidType())
1298  return RValue::get(nullptr);
1299 
1300  assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
1301  return convertTempToRValue(
1302  Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo(
1303  Dest.getAddressSpace())),
1304  RValTy, E->getExprLoc());
1305 }
1306 
1307 Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const {
1308  unsigned addrspace =
1309  cast<llvm::PointerType>(addr.getPointer()->getType())->getAddressSpace();
1310  llvm::IntegerType *ty =
1311  llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
1312  return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace));
1313 }
1314 
1315 Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {
1316  llvm::Type *Ty = Addr.getElementType();
1317  uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
1318  if (SourceSizeInBits != AtomicSizeInBits) {
1319  Address Tmp = CreateTempAlloca();
1320  CGF.Builder.CreateMemCpy(Tmp, Addr,
1321  std::min(AtomicSizeInBits, SourceSizeInBits) / 8);
1322  Addr = Tmp;
1323  }
1324 
1325  return emitCastToAtomicIntPointer(Addr);
1326 }
1327 
1328 RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
1329  AggValueSlot resultSlot,
1330  SourceLocation loc,
1331  bool asValue) const {
1332  if (LVal.isSimple()) {
1333  if (EvaluationKind == TEK_Aggregate)
1334  return resultSlot.asRValue();
1335 
1336  // Drill into the padding structure if we have one.
1337  if (hasPadding())
1338  addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits());
1339 
1340  // Otherwise, just convert the temporary to an r-value using the
1341  // normal conversion routine.
1342  return CGF.convertTempToRValue(addr, getValueType(), loc);
1343  }
1344  if (!asValue)
1345  // Get RValue from temp memory as atomic for non-simple lvalues
1346  return RValue::get(CGF.Builder.CreateLoad(addr));
1347  if (LVal.isBitField())
1348  return CGF.EmitLoadOfBitfieldLValue(
1349  LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(),
1350  LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1351  if (LVal.isVectorElt())
1352  return CGF.EmitLoadOfLValue(
1353  LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(),
1354  LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1355  assert(LVal.isExtVectorElt());
1357  addr, LVal.getExtVectorElts(), LVal.getType(),
1358  LVal.getBaseInfo(), TBAAAccessInfo()));
1359 }
1360 
1361 RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
1362  AggValueSlot ResultSlot,
1363  SourceLocation Loc,
1364  bool AsValue) const {
1365  // Try not to in some easy cases.
1366  assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
1367  if (getEvaluationKind() == TEK_Scalar &&
1368  (((!LVal.isBitField() ||
1369  LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
1370  !hasPadding()) ||
1371  !AsValue)) {
1372  auto *ValTy = AsValue
1373  ? CGF.ConvertTypeForMem(ValueTy)
1374  : getAtomicAddress().getType()->getPointerElementType();
1375  if (ValTy->isIntegerTy()) {
1376  assert(IntVal->getType() == ValTy && "Different integer types.");
1377  return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
1378  } else if (ValTy->isPointerTy())
1379  return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
1380  else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
1381  return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
1382  }
1383 
1384  // Create a temporary. This needs to be big enough to hold the
1385  // atomic integer.
1386  Address Temp = Address::invalid();
1387  bool TempIsVolatile = false;
1388  if (AsValue && getEvaluationKind() == TEK_Aggregate) {
1389  assert(!ResultSlot.isIgnored());
1390  Temp = ResultSlot.getAddress();
1391  TempIsVolatile = ResultSlot.isVolatile();
1392  } else {
1393  Temp = CreateTempAlloca();
1394  }
1395 
1396  // Slam the integer into the temporary.
1397  Address CastTemp = emitCastToAtomicIntPointer(Temp);
1398  CGF.Builder.CreateStore(IntVal, CastTemp)
1399  ->setVolatile(TempIsVolatile);
1400 
1401  return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue);
1402 }
1403 
1404 void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1405  llvm::AtomicOrdering AO, bool) {
1406  // void __atomic_load(size_t size, void *mem, void *return, int order);
1407  CallArgList Args;
1408  Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1409  Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1410  CGF.getContext().VoidPtrTy);
1411  Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)),
1412  CGF.getContext().VoidPtrTy);
1413  Args.add(
1414  RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))),
1415  CGF.getContext().IntTy);
1416  emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1417 }
1418 
1419 llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1420  bool IsVolatile) {
1421  // Okay, we're doing this natively.
1422  Address Addr = getAtomicAddressAsAtomicIntPointer();
1423  llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
1424  Load->setAtomic(AO);
1425 
1426  // Other decoration.
1427  if (IsVolatile)
1428  Load->setVolatile(true);
1429  CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo());
1430  return Load;
1431 }
1432 
1433 /// An LValue is a candidate for having its loads and stores be made atomic if
1434 /// we are operating under /volatile:ms *and* the LValue itself is volatile and
1435 /// performing such an operation can be performed without a libcall.
1437  if (!CGM.getCodeGenOpts().MSVolatile) return false;
1438  AtomicInfo AI(*this, LV);
1439  bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType());
1440  // An atomic is inline if we don't need to use a libcall.
1441  bool AtomicIsInline = !AI.shouldUseLibcall();
1442  // MSVC doesn't seem to do this for types wider than a pointer.
1443  if (getContext().getTypeSize(LV.getType()) >
1444  getContext().getTypeSize(getContext().getIntPtrType()))
1445  return false;
1446  return IsVolatile && AtomicIsInline;
1447 }
1448 
1450  AggValueSlot Slot) {
1451  llvm::AtomicOrdering AO;
1452  bool IsVolatile = LV.isVolatileQualified();
1453  if (LV.getType()->isAtomicType()) {
1454  AO = llvm::AtomicOrdering::SequentiallyConsistent;
1455  } else {
1456  AO = llvm::AtomicOrdering::Acquire;
1457  IsVolatile = true;
1458  }
1459  return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
1460 }
1461 
1462 RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1463  bool AsValue, llvm::AtomicOrdering AO,
1464  bool IsVolatile) {
1465  // Check whether we should use a library call.
1466  if (shouldUseLibcall()) {
1467  Address TempAddr = Address::invalid();
1468  if (LVal.isSimple() && !ResultSlot.isIgnored()) {
1469  assert(getEvaluationKind() == TEK_Aggregate);
1470  TempAddr = ResultSlot.getAddress();
1471  } else
1472  TempAddr = CreateTempAlloca();
1473 
1474  EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile);
1475 
1476  // Okay, turn that back into the original value or whole atomic (for
1477  // non-simple lvalues) type.
1478  return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
1479  }
1480 
1481  // Okay, we're doing this natively.
1482  auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1483 
1484  // If we're ignoring an aggregate return, don't do anything.
1485  if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
1486  return RValue::getAggregate(Address::invalid(), false);
1487 
1488  // Okay, turn that back into the original value or atomic (for non-simple
1489  // lvalues) type.
1490  return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
1491 }
1492 
1493 /// Emit a load from an l-value of atomic type. Note that the r-value
1494 /// we produce is an r-value of the atomic *value* type.
1496  llvm::AtomicOrdering AO, bool IsVolatile,
1497  AggValueSlot resultSlot) {
1498  AtomicInfo Atomics(*this, src);
1499  return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
1500  IsVolatile);
1501 }
1502 
1503 /// Copy an r-value into memory as part of storing to an atomic type.
1504 /// This needs to create a bit-pattern suitable for atomic operations.
1505 void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1506  assert(LVal.isSimple());
1507  // If we have an r-value, the rvalue should be of the atomic type,
1508  // which means that the caller is responsible for having zeroed
1509  // any padding. Just do an aggregate copy of that type.
1510  if (rvalue.isAggregate()) {
1511  CGF.EmitAggregateCopy(getAtomicAddress(),
1512  rvalue.getAggregateAddress(),
1513  getAtomicType(),
1514  (rvalue.isVolatileQualified()
1515  || LVal.isVolatileQualified()));
1516  return;
1517  }
1518 
1519  // Okay, otherwise we're copying stuff.
1520 
1521  // Zero out the buffer if necessary.
1522  emitMemSetZeroIfNecessary();
1523 
1524  // Drill past the padding if present.
1525  LValue TempLVal = projectValue();
1526 
1527  // Okay, store the rvalue in.
1528  if (rvalue.isScalar()) {
1529  CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
1530  } else {
1531  CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
1532  }
1533 }
1534 
1535 
1536 /// Materialize an r-value into memory for the purposes of storing it
1537 /// to an atomic type.
1538 Address AtomicInfo::materializeRValue(RValue rvalue) const {
1539  // Aggregate r-values are already in memory, and EmitAtomicStore
1540  // requires them to be values of the atomic type.
1541  if (rvalue.isAggregate())
1542  return rvalue.getAggregateAddress();
1543 
1544  // Otherwise, make a temporary and materialize into it.
1545  LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType());
1546  AtomicInfo Atomics(CGF, TempLV);
1547  Atomics.emitCopyIntoMemory(rvalue);
1548  return TempLV.getAddress();
1549 }
1550 
1551 llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
1552  // If we've got a scalar value of the right size, try to avoid going
1553  // through memory.
1554  if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) {
1555  llvm::Value *Value = RVal.getScalarVal();
1556  if (isa<llvm::IntegerType>(Value->getType()))
1557  return CGF.EmitToMemory(Value, ValueTy);
1558  else {
1559  llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1560  CGF.getLLVMContext(),
1561  LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1562  if (isa<llvm::PointerType>(Value->getType()))
1563  return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
1564  else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
1565  return CGF.Builder.CreateBitCast(Value, InputIntTy);
1566  }
1567  }
1568  // Otherwise, we need to go through memory.
1569  // Put the r-value in memory.
1570  Address Addr = materializeRValue(RVal);
1571 
1572  // Cast the temporary to the atomic int type and pull a value out.
1573  Addr = emitCastToAtomicIntPointer(Addr);
1574  return CGF.Builder.CreateLoad(Addr);
1575 }
1576 
1577 std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1578  llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1579  llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1580  // Do the atomic store.
1581  Address Addr = getAtomicAddressAsAtomicIntPointer();
1582  auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(),
1583  ExpectedVal, DesiredVal,
1584  Success, Failure);
1585  // Other decoration.
1586  Inst->setVolatile(LVal.isVolatileQualified());
1587  Inst->setWeak(IsWeak);
1588 
1589  // Okay, turn that back into the original value type.
1590  auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
1591  auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
1592  return std::make_pair(PreviousVal, SuccessFailureVal);
1593 }
1594 
1595 llvm::Value *
1596 AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1597  llvm::Value *DesiredAddr,
1598  llvm::AtomicOrdering Success,
1599  llvm::AtomicOrdering Failure) {
1600  // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1601  // void *desired, int success, int failure);
1602  CallArgList Args;
1603  Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1604  Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1605  CGF.getContext().VoidPtrTy);
1606  Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)),
1607  CGF.getContext().VoidPtrTy);
1608  Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)),
1609  CGF.getContext().VoidPtrTy);
1610  Args.add(RValue::get(
1611  llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))),
1612  CGF.getContext().IntTy);
1613  Args.add(RValue::get(
1614  llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))),
1615  CGF.getContext().IntTy);
1616  auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1617  CGF.getContext().BoolTy, Args);
1618 
1619  return SuccessFailureRVal.getScalarVal();
1620 }
1621 
1622 std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1623  RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1624  llvm::AtomicOrdering Failure, bool IsWeak) {
1625  if (isStrongerThan(Failure, Success))
1626  // Don't assert on undefined behavior "failure argument shall be no stronger
1627  // than the success argument".
1628  Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success);
1629 
1630  // Check whether we should use a library call.
1631  if (shouldUseLibcall()) {
1632  // Produce a source address.
1633  Address ExpectedAddr = materializeRValue(Expected);
1634  Address DesiredAddr = materializeRValue(Desired);
1635  auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1636  DesiredAddr.getPointer(),
1637  Success, Failure);
1638  return std::make_pair(
1639  convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1640  SourceLocation(), /*AsValue=*/false),
1641  Res);
1642  }
1643 
1644  // If we've got a scalar value of the right size, try to avoid going
1645  // through memory.
1646  auto *ExpectedVal = convertRValueToInt(Expected);
1647  auto *DesiredVal = convertRValueToInt(Desired);
1648  auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1649  Failure, IsWeak);
1650  return std::make_pair(
1651  ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(),
1652  SourceLocation(), /*AsValue=*/false),
1653  Res.second);
1654 }
1655 
1656 static void
1657 EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1658  const llvm::function_ref<RValue(RValue)> &UpdateOp,
1659  Address DesiredAddr) {
1660  RValue UpRVal;
1661  LValue AtomicLVal = Atomics.getAtomicLValue();
1662  LValue DesiredLVal;
1663  if (AtomicLVal.isSimple()) {
1664  UpRVal = OldRVal;
1665  DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType());
1666  } else {
1667  // Build new lvalue for temp address
1668  Address Ptr = Atomics.materializeRValue(OldRVal);
1669  LValue UpdateLVal;
1670  if (AtomicLVal.isBitField()) {
1671  UpdateLVal =
1672  LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
1673  AtomicLVal.getType(),
1674  AtomicLVal.getBaseInfo(),
1675  AtomicLVal.getTBAAInfo());
1676  DesiredLVal =
1677  LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1678  AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1679  AtomicLVal.getTBAAInfo());
1680  } else if (AtomicLVal.isVectorElt()) {
1681  UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
1682  AtomicLVal.getType(),
1683  AtomicLVal.getBaseInfo(),
1684  AtomicLVal.getTBAAInfo());
1685  DesiredLVal = LValue::MakeVectorElt(
1686  DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(),
1687  AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1688  } else {
1689  assert(AtomicLVal.isExtVectorElt());
1690  UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
1691  AtomicLVal.getType(),
1692  AtomicLVal.getBaseInfo(),
1693  AtomicLVal.getTBAAInfo());
1694  DesiredLVal = LValue::MakeExtVectorElt(
1695  DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1696  AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1697  }
1698  UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());
1699  }
1700  // Store new value in the corresponding memory area
1701  RValue NewRVal = UpdateOp(UpRVal);
1702  if (NewRVal.isScalar()) {
1703  CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);
1704  } else {
1705  assert(NewRVal.isComplex());
1706  CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,
1707  /*isInit=*/false);
1708  }
1709 }
1710 
1711 void AtomicInfo::EmitAtomicUpdateLibcall(
1712  llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1713  bool IsVolatile) {
1714  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1715 
1716  Address ExpectedAddr = CreateTempAlloca();
1717 
1718  EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1719  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1720  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1721  CGF.EmitBlock(ContBB);
1722  Address DesiredAddr = CreateTempAlloca();
1723  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1724  requiresMemSetZero(getAtomicAddress().getElementType())) {
1725  auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1726  CGF.Builder.CreateStore(OldVal, DesiredAddr);
1727  }
1728  auto OldRVal = convertAtomicTempToRValue(ExpectedAddr,
1730  SourceLocation(), /*AsValue=*/false);
1731  EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
1732  auto *Res =
1733  EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1734  DesiredAddr.getPointer(),
1735  AO, Failure);
1736  CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1737  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1738 }
1739 
1740 void AtomicInfo::EmitAtomicUpdateOp(
1741  llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1742  bool IsVolatile) {
1743  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1744 
1745  // Do the atomic load.
1746  auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
1747  // For non-simple lvalues perform compare-and-swap procedure.
1748  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1749  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1750  auto *CurBB = CGF.Builder.GetInsertBlock();
1751  CGF.EmitBlock(ContBB);
1752  llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1753  /*NumReservedValues=*/2);
1754  PHI->addIncoming(OldVal, CurBB);
1755  Address NewAtomicAddr = CreateTempAlloca();
1756  Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1757  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1758  requiresMemSetZero(getAtomicAddress().getElementType())) {
1759  CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1760  }
1761  auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(),
1762  SourceLocation(), /*AsValue=*/false);
1763  EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
1764  auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1765  // Try to write new value using cmpxchg operation
1766  auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1767  PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1768  CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1769  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1770 }
1771 
1772 static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1773  RValue UpdateRVal, Address DesiredAddr) {
1774  LValue AtomicLVal = Atomics.getAtomicLValue();
1775  LValue DesiredLVal;
1776  // Build new lvalue for temp address
1777  if (AtomicLVal.isBitField()) {
1778  DesiredLVal =
1779  LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1780  AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1781  AtomicLVal.getTBAAInfo());
1782  } else if (AtomicLVal.isVectorElt()) {
1783  DesiredLVal =
1784  LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(),
1785  AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1786  AtomicLVal.getTBAAInfo());
1787  } else {
1788  assert(AtomicLVal.isExtVectorElt());
1789  DesiredLVal = LValue::MakeExtVectorElt(
1790  DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1791  AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1792  }
1793  // Store new value in the corresponding memory area
1794  assert(UpdateRVal.isScalar());
1795  CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);
1796 }
1797 
1798 void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1799  RValue UpdateRVal, bool IsVolatile) {
1800  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1801 
1802  Address ExpectedAddr = CreateTempAlloca();
1803 
1804  EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1805  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1806  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1807  CGF.EmitBlock(ContBB);
1808  Address DesiredAddr = CreateTempAlloca();
1809  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1810  requiresMemSetZero(getAtomicAddress().getElementType())) {
1811  auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1812  CGF.Builder.CreateStore(OldVal, DesiredAddr);
1813  }
1814  EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
1815  auto *Res =
1816  EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1817  DesiredAddr.getPointer(),
1818  AO, Failure);
1819  CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1820  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1821 }
1822 
1823 void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
1824  bool IsVolatile) {
1825  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1826 
1827  // Do the atomic load.
1828  auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
1829  // For non-simple lvalues perform compare-and-swap procedure.
1830  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1831  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1832  auto *CurBB = CGF.Builder.GetInsertBlock();
1833  CGF.EmitBlock(ContBB);
1834  llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1835  /*NumReservedValues=*/2);
1836  PHI->addIncoming(OldVal, CurBB);
1837  Address NewAtomicAddr = CreateTempAlloca();
1838  Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1839  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1840  requiresMemSetZero(getAtomicAddress().getElementType())) {
1841  CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1842  }
1843  EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
1844  auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1845  // Try to write new value using cmpxchg operation
1846  auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1847  PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1848  CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1849  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1850 }
1851 
1852 void AtomicInfo::EmitAtomicUpdate(
1853  llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1854  bool IsVolatile) {
1855  if (shouldUseLibcall()) {
1856  EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
1857  } else {
1858  EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
1859  }
1860 }
1861 
1862 void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
1863  bool IsVolatile) {
1864  if (shouldUseLibcall()) {
1865  EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
1866  } else {
1867  EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
1868  }
1869 }
1870 
1872  bool isInit) {
1873  bool IsVolatile = lvalue.isVolatileQualified();
1874  llvm::AtomicOrdering AO;
1875  if (lvalue.getType()->isAtomicType()) {
1876  AO = llvm::AtomicOrdering::SequentiallyConsistent;
1877  } else {
1878  AO = llvm::AtomicOrdering::Release;
1879  IsVolatile = true;
1880  }
1881  return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
1882 }
1883 
1884 /// Emit a store to an l-value of atomic type.
1885 ///
1886 /// Note that the r-value is expected to be an r-value *of the atomic
1887 /// type*; this means that for aggregate r-values, it should include
1888 /// storage for any padding that was necessary.
1890  llvm::AtomicOrdering AO, bool IsVolatile,
1891  bool isInit) {
1892  // If this is an aggregate r-value, it should agree in type except
1893  // maybe for address-space qualification.
1894  assert(!rvalue.isAggregate() ||
1896  == dest.getAddress().getElementType());
1897 
1898  AtomicInfo atomics(*this, dest);
1899  LValue LVal = atomics.getAtomicLValue();
1900 
1901  // If this is an initialization, just put the value there normally.
1902  if (LVal.isSimple()) {
1903  if (isInit) {
1904  atomics.emitCopyIntoMemory(rvalue);
1905  return;
1906  }
1907 
1908  // Check whether we should use a library call.
1909  if (atomics.shouldUseLibcall()) {
1910  // Produce a source address.
1911  Address srcAddr = atomics.materializeRValue(rvalue);
1912 
1913  // void __atomic_store(size_t size, void *mem, void *val, int order)
1914  CallArgList args;
1915  args.add(RValue::get(atomics.getAtomicSizeValue()),
1916  getContext().getSizeType());
1917  args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())),
1918  getContext().VoidPtrTy);
1919  args.add(RValue::get(EmitCastToVoidPtr(srcAddr.getPointer())),
1920  getContext().VoidPtrTy);
1921  args.add(
1922  RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))),
1923  getContext().IntTy);
1924  emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
1925  return;
1926  }
1927 
1928  // Okay, we're doing this natively.
1929  llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
1930 
1931  // Do the atomic store.
1932  Address addr =
1933  atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress());
1934  intValue = Builder.CreateIntCast(
1935  intValue, addr.getElementType(), /*isSigned=*/false);
1936  llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
1937 
1938  // Initializations don't need to be atomic.
1939  if (!isInit)
1940  store->setAtomic(AO);
1941 
1942  // Other decoration.
1943  if (IsVolatile)
1944  store->setVolatile(true);
1945  CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo());
1946  return;
1947  }
1948 
1949  // Emit simple atomic update operation.
1950  atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
1951 }
1952 
1953 /// Emit a compare-and-exchange op for atomic type.
1954 ///
1955 std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
1956  LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
1957  llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
1958  AggValueSlot Slot) {
1959  // If this is an aggregate r-value, it should agree in type except
1960  // maybe for address-space qualification.
1961  assert(!Expected.isAggregate() ||
1962  Expected.getAggregateAddress().getElementType() ==
1963  Obj.getAddress().getElementType());
1964  assert(!Desired.isAggregate() ||
1965  Desired.getAggregateAddress().getElementType() ==
1966  Obj.getAddress().getElementType());
1967  AtomicInfo Atomics(*this, Obj);
1968 
1969  return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
1970  IsWeak);
1971 }
1972 
1974  LValue LVal, llvm::AtomicOrdering AO,
1975  const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
1976  AtomicInfo Atomics(*this, LVal);
1977  Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
1978 }
1979 
1981  AtomicInfo atomics(*this, dest);
1982 
1983  switch (atomics.getEvaluationKind()) {
1984  case TEK_Scalar: {
1985  llvm::Value *value = EmitScalarExpr(init);
1986  atomics.emitCopyIntoMemory(RValue::get(value));
1987  return;
1988  }
1989 
1990  case TEK_Complex: {
1991  ComplexPairTy value = EmitComplexExpr(init);
1992  atomics.emitCopyIntoMemory(RValue::getComplex(value));
1993  return;
1994  }
1995 
1996  case TEK_Aggregate: {
1997  // Fix up the destination if the initializer isn't an expression
1998  // of atomic type.
1999  bool Zeroed = false;
2000  if (!init->getType()->isAtomicType()) {
2001  Zeroed = atomics.emitMemSetZeroIfNecessary();
2002  dest = atomics.projectValue();
2003  }
2004 
2005  // Evaluate the expression directly into the destination.
2010  Zeroed ? AggValueSlot::IsZeroed :
2012 
2013  EmitAggExpr(init, slot);
2014  return;
2015  }
2016  }
2017  llvm_unreachable("bad evaluation kind");
2018 }
bool isAggregate() const
Definition: CGValue.h:54
const llvm::DataLayout & getDataLayout() const
llvm::Value * getVectorPointer() const
Definition: CGValue.h:340
ReturnValueSlot - Contains the address where the return value of a function can be stored...
Definition: CGCall.h:281
Expr * getVal2() const
Definition: Expr.h:5156
Defines the clang::ASTContext interface.
std::pair< RValue, llvm::Value * > EmitAtomicCompareExchange(LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, llvm::AtomicOrdering Success=llvm::AtomicOrdering::SequentiallyConsistent, llvm::AtomicOrdering Failure=llvm::AtomicOrdering::SequentiallyConsistent, bool IsWeak=false, AggValueSlot Slot=AggValueSlot::ignored())
Emit a compare-and-exchange op for atomic type.
Definition: CGAtomic.cpp:1955
llvm::IntegerType * IntTy
int
Address getAddress() const
Definition: CGValue.h:561
PointerType - C99 6.7.5.1 - Pointer Declarators.
Definition: Type.h:2222
CanQualType VoidPtrTy
Definition: ASTContext.h:981
A (possibly-)qualified type.
Definition: Type.h:614
llvm::Type * ConvertTypeForMem(QualType T)
llvm::LLVMContext & getLLVMContext()
static AggValueSlot forLValue(const LValue &LV, IsDestructed_t isDestructed, NeedsGCBarriers_t needsGC, IsAliased_t isAliased, IsZeroed_t isZeroed=IsNotZeroed)
Definition: CGValue.h:523
void setAlignment(CharUnits A)
Definition: CGValue.h:319
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee...
Definition: Type.cpp:435
bool isVolatile() const
Definition: CGValue.h:303
const void * Store
Store - This opaque type encapsulates an immutable mapping from locations to values.
Definition: StoreRef.h:26
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
Definition: CGExpr.cpp:1791
bool isZero() const
isZero - Test whether the quantity equals zero.
Definition: CharUnits.h:116
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
Definition: CGAtomic.cpp:1871
static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, Address Ptr, Address Val1, Address Val2, llvm::Value *IsWeak, llvm::Value *FailureOrder, uint64_t Size, llvm::AtomicOrdering Order, llvm::SyncScope::ID Scope)
Definition: CGAtomic.cpp:492
TypeEvaluationKind
The kind of evaluation to perform on values of a particular type.
const T * getAs() const
Member-template getAs<specific type>&#39;.
Definition: Type.h:6099
static LValue MakeVectorElt(Address vecAddress, llvm::Value *Idx, QualType type, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo)
Definition: CGValue.h:382
Expr * getVal1() const
Definition: Expr.h:5146
llvm::Value * getPointer() const
Definition: Address.h:38
unsigned getAddressSpace() const
Return the address space that this address resides in.
Definition: Address.h:57
Address getAddress() const
Definition: CGValue.h:329
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition: ASTContext.h:128
bool isVolatileQualified() const
Definition: CGValue.h:260
CharUnits getAlignment() const
Definition: CGValue.h:318
void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO, const llvm::function_ref< RValue(RValue)> &UpdateOp, bool IsVolatile)
Definition: CGAtomic.cpp:1973
RValue EmitLoadOfExtVectorElementLValue(LValue V)
Definition: CGExpr.cpp:1715
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
unsigned Size
The total size of the bit-field, in bits.
CharUnits - This is an opaque type for sizes expressed in character units.
Definition: CharUnits.h:38
uint32_t Offset
Definition: CacheTokens.cpp:43
Expr * getPtr() const
Definition: Expr.h:5136
CharUnits StorageOffset
The offset of the bitfield storage from the start of the struct.
CharUnits getAlignment() const
Return the alignment of this pointer.
Definition: Address.h:67
static LValue MakeExtVectorElt(Address vecAddress, llvm::Constant *Elts, QualType type, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo)
Definition: CGValue.h:394
bool isComplex() const
Definition: CGValue.h:53
llvm::CallInst * CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
Definition: CGBuilder.h:259
Scope - A scope is a transient data structure that is used while parsing the program.
Definition: Scope.h:39
static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal, const llvm::function_ref< RValue(RValue)> &UpdateOp, Address DesiredAddr)
Definition: CGAtomic.cpp:1657
unsigned Align
Definition: ASTContext.h:118
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
void EmitAtomicInit(Expr *E, LValue lvalue)
Definition: CGAtomic.cpp:1980
unsigned Offset
The offset within a contiguous run of bitfields that are represented as a single "field" within the L...
bool isSimple() const
Definition: CGValue.h:254
RValue EmitAtomicExpr(AtomicExpr *E)
Definition: CGAtomic.cpp:742
RValue convertTempToRValue(Address addr, QualType type, SourceLocation Loc)
Given the address of a temporary variable, produce an r-value of its type.
Definition: CGExpr.cpp:4593
bool isCmpXChg() const
Definition: Expr.h:5180
Address getAggregateAddress() const
getAggregateAddr() - Return the Value* of the address of the aggregate.
Definition: CGValue.h:71
static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type, uint64_t expectedSize)
Does a store of the given IR type modify the full expected width?
Definition: CGAtomic.cpp:316
llvm::Constant * CreateRuntimeFunction(llvm::FunctionType *Ty, StringRef Name, llvm::AttributeList ExtraAttrs=llvm::AttributeList(), bool Local=false)
Create a new runtime function with the specified type and name.
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
bool isVectorElt() const
Definition: CGValue.h:255
bool isValid() const
Definition: Address.h:36
Expr * getScope() const
Definition: Expr.h:5142
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
Definition: CGExpr.cpp:199
LValueBaseInfo getBaseInfo() const
Definition: CGValue.h:321
RValue - This trivial value class is used to represent the result of an expression that is evaluated...
Definition: CGValue.h:39
Address getExtVectorAddress() const
Definition: CGValue.h:344
static std::unique_ptr< AtomicScopeModel > getScopeModel(AtomicOp Op)
Get atomic scope model for the atomic op code.
Definition: Expr.h:5215
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
Definition: CharUnits.h:179
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
Expr - This represents one expression.
Definition: Expr.h:106
llvm::StringRef getAsString(SyncScope S)
Definition: SyncScope.h:51
static Address invalid()
Definition: Address.h:35
static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, Address Dest, Address Ptr, Address Val1, Address Val2, uint64_t Size, llvm::AtomicOrdering SuccessOrder, llvm::AtomicOrdering FailureOrder, llvm::SyncScope::ID Scope)
Definition: CGAtomic.cpp:358
const FunctionProtoType * T
std::pair< llvm::Value *, llvm::Value * > getComplexVal() const
getComplexVal - Return the real/imag components of this complex value.
Definition: CGValue.h:66
static CGCallee forDirect(llvm::Constant *functionPtr, const CGCalleeInfo &abstractInfo=CGCalleeInfo())
Definition: CGCall.h:125
llvm::PointerType * getType() const
Return the type of the pointer value.
Definition: Address.h:44
Expr * getOrder() const
Definition: Expr.h:5139
CharUnits getTypeAlignInChars(QualType T) const
Return the ABI-specified alignment of a (complete) type T, in characters.
llvm::Value * EmitToMemory(llvm::Value *Value, QualType Ty)
EmitToMemory - Change a scalar value from its value representation to its in-memory representation...
Definition: CGExpr.cpp:1552
void add(RValue rvalue, QualType type, bool needscopy=false)
Definition: CGCall.h:207
static LValue MakeBitfield(Address Addr, const CGBitFieldInfo &Info, QualType type, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo)
Create a new object to represent a bit-field access.
Definition: CGValue.h:412
char __ovld __cnfn min(char x, char y)
Returns y if y < x, otherwise it returns x.
llvm::LLVMContext & getLLVMContext()
const CGFunctionInfo & arrangeBuiltinFunctionCall(QualType resultType, const CallArgList &args)
Definition: CGCall.cpp:627
bool isSignedIntegerType() const
Return true if this is an integer type that is signed, according to C99 6.2.5p4 [char, signed char, short, int, long..], or an enum decl which has a signed representation.
Definition: Type.cpp:1765
QualType getType() const
Definition: Expr.h:128
bool isVolatile() const
Definition: Expr.h:5176
TBAAAccessInfo getTBAAInfo() const
Definition: CGValue.h:310
llvm::CallInst * CreateMemSet(Address Dest, llvm::Value *Value, llvm::Value *Size, bool IsVolatile=false)
Definition: CGBuilder.h:281
Represents a GCC generic vector type.
Definition: Type.h:2838
AtomicOp getOp() const
Definition: Expr.h:5168
llvm::Value * EmitCastToVoidPtr(llvm::Value *value)
Emit a cast to void* in the appropriate address space.
Definition: CGExpr.cpp:50
RValue EmitLoadOfBitfieldLValue(LValue LV, SourceLocation Loc)
Definition: CGExpr.cpp:1683
RValue EmitAtomicLoad(LValue LV, SourceLocation SL, AggValueSlot Slot=AggValueSlot::ignored())
Definition: CGAtomic.cpp:1449
The l-value was considered opaque, so the alignment was determined from a type.
Address CreateBitCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
Definition: CGBuilder.h:142
Encodes a location in the source.
bool LValueIsSuitableForInlineAtomic(LValue Src)
An LValue is a candidate for having its loads and stores be made atomic if we are operating under /vo...
Definition: CGAtomic.cpp:1436
llvm::Value * EmitFromMemory(llvm::Value *Value, QualType Ty)
EmitFromMemory - Change a scalar value from its memory representation to its value representation...
Definition: CGExpr.cpp:1566
const CGBitFieldInfo & getBitFieldInfo() const
Definition: CGValue.h:361
An aggregate value slot.
Definition: CGValue.h:440
QualType getExtVectorType(QualType VectorType, unsigned NumElts) const
Return the unique reference to an extended vector type of the specified element type and size...
AtomicExpr - Variadic atomic builtins: __atomic_exchange, __atomic_fetch_*, __atomic_load, __atomic_store, and __atomic_compare_exchange_*, for the similarly-named C++11 instructions, and __c11 variants for <stdatomic.h>, and corresponding __opencl_atomic_* for OpenCL 2.0.
Definition: Expr.h:5104
CanQualType VoidTy
Definition: ASTContext.h:965
static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, Address Dest, Address Ptr, Address Val1, Address Val2, llvm::Value *FailureOrderVal, uint64_t Size, llvm::AtomicOrdering SuccessOrder, llvm::SyncScope::ID Scope)
Given an ordering required on success, emit all possible cmpxchg instructions to cope with the provid...
Definition: CGAtomic.cpp:408
An aligned address.
Definition: Address.h:25
QualType getType() const
Definition: CGValue.h:266
const TargetCodeGenInfo & getTargetHooks() const
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
Definition: Expr.cpp:216
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
Definition: CGValue.h:59
CGFunctionInfo - Class to encapsulate the information about a function definition.
This class organizes the cross-function state that is used while generating LLVM code.
Dataflow Directional Tag Classes.
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
Definition: CGValue.h:93
static AggValueSlot ignored()
ignored - Returns an aggregate value slot indicating that the aggregate value is being ignored...
Definition: CGValue.h:486
Address CreateStructGEP(Address Addr, unsigned Index, CharUnits Offset, const llvm::Twine &Name="")
Definition: CGBuilder.h:172
bool isOpenCL() const
Definition: Expr.h:5189
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
Definition: CGBuilder.h:70
unsigned StorageSize
The storage size in bits which should be used when accessing this bitfield.
void EmitAggregateCopy(Address DestPtr, Address SrcPtr, QualType EltTy, bool isVolatile=false, bool isAssignment=false)
EmitAggregateCopy - Emit an aggregate copy.
Definition: CGExprAgg.cpp:1544
bool isBitField() const
Definition: CGValue.h:256
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
Definition: CGBuilder.h:108
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
llvm::Value * getExtVectorPointer() const
Definition: CGValue.h:347
Expr * getOrderFail() const
Definition: Expr.h:5152
bool isVolatileQualified() const
Definition: CGValue.h:56
static void AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args, bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy, SourceLocation Loc, CharUnits SizeInChars)
Definition: CGAtomic.cpp:718
void DecorateInstructionWithTBAA(llvm::Instruction *Inst, TBAAAccessInfo TBAAInfo)
DecorateInstructionWithTBAA - Decorate the instruction with a TBAA tag.
void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit)
EmitStoreOfComplex - Store a complex number into the specified l-value.
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Definition: Address.h:52
bool isAtomicType() const
Definition: Type.h:5851
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
Definition: CGStmt.cpp:436
llvm::ConstantInt * getSize(CharUnits numChars)
Emit the given number of characters as a value of type size_t.
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
int64_t toBits(CharUnits CharSize) const
Convert a size in characters to a size in bits.
virtual llvm::SyncScope::ID getLLVMSyncScopeID(SyncScope S, llvm::LLVMContext &C) const
Get the syncscope used in LLVM IR.
Definition: TargetInfo.cpp:454
QualType getUnqualifiedType() const
Retrieve the unqualified variant of the given type, removing as little sugar as possible.
Definition: Type.h:5623
llvm::Value * getBitFieldPointer() const
Definition: CGValue.h:360
true
A convenience builder class for complex constant initializers, especially for anonymous global struct...
bool isVoidType() const
Definition: Type.h:5963
Qualifiers getQualifiers() const
Retrieve the set of qualifiers applied to this type.
Definition: Type.h:5570
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
Definition: CGExpr.cpp:1637
bool isGlobalReg() const
Definition: CGValue.h:258
uint64_t Width
Definition: ASTContext.h:117
CanQualType IntTy
Definition: ASTContext.h:973
static RValue get(llvm::Value *V)
Definition: CGValue.h:86
bool isPointerType() const
Definition: Type.h:5766
bool isExtVectorElt() const
Definition: CGValue.h:257
static RValue emitAtomicLibcall(CodeGenFunction &CGF, StringRef fnName, QualType resultType, CallArgList &args)
Definition: CGAtomic.cpp:303
static RValue getAggregate(Address addr, bool isVolatile=false)
Definition: CGValue.h:107
LValue - This represents an lvalue references.
Definition: CGValue.h:172
RValue asRValue() const
Definition: CGValue.h:577
CanQualType BoolTy
Definition: ASTContext.h:966
QualType getValueType() const
Definition: Expr.cpp:4060
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
Definition: CGBuilder.h:164
static LValue MakeAddr(Address address, QualType type, ASTContext &Context, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo)
Definition: CGValue.h:369
llvm::Value * getVectorIdx() const
Definition: CGValue.h:341
CallArgList - Type for representing both the value and type of arguments in a call.
Definition: CGCall.h:182
Address CreateMemTemp(QualType T, const Twine &Name="tmp", bool CastToDefaultAddrSpace=true)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignment...
Definition: CGExpr.cpp:124
static Address EmitValToTemp(CodeGenFunction &CGF, Expr *E)
Definition: CGAtomic.cpp:655
llvm::Value * getPointer() const
Definition: CGValue.h:325
bool isScalar() const
Definition: CGValue.h:52
Expr * getWeak() const
Definition: Expr.h:5162
RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee, ReturnValueSlot ReturnValue, const CallArgList &Args, llvm::Instruction **callOrInvoke=nullptr)
EmitCall - Generate a call of the given function, expecting the given result type, and using the given argument list which specifies both the LLVM arguments and the types they were derived from.
Definition: CGCall.cpp:3708
llvm::Constant * getExtVectorElts() const
Definition: CGValue.h:351
Structure with information about how a bitfield should be accessed.
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth, signed/unsigned.
CanQualType getSizeType() const
Return the unique type for "size_t" (C99 7.17), defined in <stddef.h>.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
Definition: CGCall.cpp:1524