clang  9.0.0svn
CGAtomic.cpp
Go to the documentation of this file.
1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the code for emitting atomic operations.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "CGCall.h"
14 #include "CGRecordLayout.h"
15 #include "CodeGenFunction.h"
16 #include "CodeGenModule.h"
17 #include "TargetInfo.h"
18 #include "clang/AST/ASTContext.h"
21 #include "llvm/ADT/DenseMap.h"
22 #include "llvm/IR/DataLayout.h"
23 #include "llvm/IR/Intrinsics.h"
24 #include "llvm/IR/Operator.h"
25 
26 using namespace clang;
27 using namespace CodeGen;
28 
29 namespace {
30  class AtomicInfo {
31  CodeGenFunction &CGF;
32  QualType AtomicTy;
33  QualType ValueTy;
34  uint64_t AtomicSizeInBits;
35  uint64_t ValueSizeInBits;
36  CharUnits AtomicAlign;
37  CharUnits ValueAlign;
38  CharUnits LValueAlign;
39  TypeEvaluationKind EvaluationKind;
40  bool UseLibcall;
41  LValue LVal;
42  CGBitFieldInfo BFI;
43  public:
44  AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
45  : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
46  EvaluationKind(TEK_Scalar), UseLibcall(true) {
47  assert(!lvalue.isGlobalReg());
48  ASTContext &C = CGF.getContext();
49  if (lvalue.isSimple()) {
50  AtomicTy = lvalue.getType();
51  if (auto *ATy = AtomicTy->getAs<AtomicType>())
52  ValueTy = ATy->getValueType();
53  else
54  ValueTy = AtomicTy;
55  EvaluationKind = CGF.getEvaluationKind(ValueTy);
56 
57  uint64_t ValueAlignInBits;
58  uint64_t AtomicAlignInBits;
59  TypeInfo ValueTI = C.getTypeInfo(ValueTy);
60  ValueSizeInBits = ValueTI.Width;
61  ValueAlignInBits = ValueTI.Align;
62 
63  TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
64  AtomicSizeInBits = AtomicTI.Width;
65  AtomicAlignInBits = AtomicTI.Align;
66 
67  assert(ValueSizeInBits <= AtomicSizeInBits);
68  assert(ValueAlignInBits <= AtomicAlignInBits);
69 
70  AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
71  ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
72  if (lvalue.getAlignment().isZero())
73  lvalue.setAlignment(AtomicAlign);
74 
75  LVal = lvalue;
76  } else if (lvalue.isBitField()) {
77  ValueTy = lvalue.getType();
78  ValueSizeInBits = C.getTypeSize(ValueTy);
79  auto &OrigBFI = lvalue.getBitFieldInfo();
80  auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
81  AtomicSizeInBits = C.toBits(
82  C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
83  .alignTo(lvalue.getAlignment()));
84  auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer());
85  auto OffsetInChars =
86  (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
87  lvalue.getAlignment();
88  VoidPtrAddr = CGF.Builder.CreateConstGEP1_64(
89  VoidPtrAddr, OffsetInChars.getQuantity());
91  VoidPtrAddr,
92  CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(),
93  "atomic_bitfield_base");
94  BFI = OrigBFI;
95  BFI.Offset = Offset;
96  BFI.StorageSize = AtomicSizeInBits;
97  BFI.StorageOffset += OffsetInChars;
98  LVal = LValue::MakeBitfield(Address(Addr, lvalue.getAlignment()),
99  BFI, lvalue.getType(), lvalue.getBaseInfo(),
100  lvalue.getTBAAInfo());
101  AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
102  if (AtomicTy.isNull()) {
103  llvm::APInt Size(
104  /*numBits=*/32,
105  C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
106  AtomicTy = C.getConstantArrayType(C.CharTy, Size, ArrayType::Normal,
107  /*IndexTypeQuals=*/0);
108  }
109  AtomicAlign = ValueAlign = lvalue.getAlignment();
110  } else if (lvalue.isVectorElt()) {
111  ValueTy = lvalue.getType()->getAs<VectorType>()->getElementType();
112  ValueSizeInBits = C.getTypeSize(ValueTy);
113  AtomicTy = lvalue.getType();
114  AtomicSizeInBits = C.getTypeSize(AtomicTy);
115  AtomicAlign = ValueAlign = lvalue.getAlignment();
116  LVal = lvalue;
117  } else {
118  assert(lvalue.isExtVectorElt());
119  ValueTy = lvalue.getType();
120  ValueSizeInBits = C.getTypeSize(ValueTy);
121  AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
122  lvalue.getType(), lvalue.getExtVectorAddress()
123  .getElementType()->getVectorNumElements());
124  AtomicSizeInBits = C.getTypeSize(AtomicTy);
125  AtomicAlign = ValueAlign = lvalue.getAlignment();
126  LVal = lvalue;
127  }
128  UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
129  AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
130  }
131 
132  QualType getAtomicType() const { return AtomicTy; }
133  QualType getValueType() const { return ValueTy; }
134  CharUnits getAtomicAlignment() const { return AtomicAlign; }
135  CharUnits getValueAlignment() const { return ValueAlign; }
136  uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
137  uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
138  TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
139  bool shouldUseLibcall() const { return UseLibcall; }
140  const LValue &getAtomicLValue() const { return LVal; }
141  llvm::Value *getAtomicPointer() const {
142  if (LVal.isSimple())
143  return LVal.getPointer();
144  else if (LVal.isBitField())
145  return LVal.getBitFieldPointer();
146  else if (LVal.isVectorElt())
147  return LVal.getVectorPointer();
148  assert(LVal.isExtVectorElt());
149  return LVal.getExtVectorPointer();
150  }
151  Address getAtomicAddress() const {
152  return Address(getAtomicPointer(), getAtomicAlignment());
153  }
154 
155  Address getAtomicAddressAsAtomicIntPointer() const {
156  return emitCastToAtomicIntPointer(getAtomicAddress());
157  }
158 
159  /// Is the atomic size larger than the underlying value type?
160  ///
161  /// Note that the absence of padding does not mean that atomic
162  /// objects are completely interchangeable with non-atomic
163  /// objects: we might have promoted the alignment of a type
164  /// without making it bigger.
165  bool hasPadding() const {
166  return (ValueSizeInBits != AtomicSizeInBits);
167  }
168 
169  bool emitMemSetZeroIfNecessary() const;
170 
171  llvm::Value *getAtomicSizeValue() const {
172  CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
173  return CGF.CGM.getSize(size);
174  }
175 
176  /// Cast the given pointer to an integer pointer suitable for atomic
177  /// operations if the source.
178  Address emitCastToAtomicIntPointer(Address Addr) const;
179 
180  /// If Addr is compatible with the iN that will be used for an atomic
181  /// operation, bitcast it. Otherwise, create a temporary that is suitable
182  /// and copy the value across.
183  Address convertToAtomicIntPointer(Address Addr) const;
184 
185  /// Turn an atomic-layout object into an r-value.
186  RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
187  SourceLocation loc, bool AsValue) const;
188 
189  /// Converts a rvalue to integer value.
190  llvm::Value *convertRValueToInt(RValue RVal) const;
191 
192  RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
193  AggValueSlot ResultSlot,
194  SourceLocation Loc, bool AsValue) const;
195 
196  /// Copy an atomic r-value into atomic-layout memory.
197  void emitCopyIntoMemory(RValue rvalue) const;
198 
199  /// Project an l-value down to the value field.
200  LValue projectValue() const {
201  assert(LVal.isSimple());
202  Address addr = getAtomicAddress();
203  if (hasPadding())
204  addr = CGF.Builder.CreateStructGEP(addr, 0);
205 
206  return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
207  LVal.getBaseInfo(), LVal.getTBAAInfo());
208  }
209 
210  /// Emits atomic load.
211  /// \returns Loaded value.
212  RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
213  bool AsValue, llvm::AtomicOrdering AO,
214  bool IsVolatile);
215 
216  /// Emits atomic compare-and-exchange sequence.
217  /// \param Expected Expected value.
218  /// \param Desired Desired value.
219  /// \param Success Atomic ordering for success operation.
220  /// \param Failure Atomic ordering for failed operation.
221  /// \param IsWeak true if atomic operation is weak, false otherwise.
222  /// \returns Pair of values: previous value from storage (value type) and
223  /// boolean flag (i1 type) with true if success and false otherwise.
224  std::pair<RValue, llvm::Value *>
225  EmitAtomicCompareExchange(RValue Expected, RValue Desired,
226  llvm::AtomicOrdering Success =
227  llvm::AtomicOrdering::SequentiallyConsistent,
228  llvm::AtomicOrdering Failure =
229  llvm::AtomicOrdering::SequentiallyConsistent,
230  bool IsWeak = false);
231 
232  /// Emits atomic update.
233  /// \param AO Atomic ordering.
234  /// \param UpdateOp Update operation for the current lvalue.
235  void EmitAtomicUpdate(llvm::AtomicOrdering AO,
236  const llvm::function_ref<RValue(RValue)> &UpdateOp,
237  bool IsVolatile);
238  /// Emits atomic update.
239  /// \param AO Atomic ordering.
240  void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
241  bool IsVolatile);
242 
243  /// Materialize an atomic r-value in atomic-layout memory.
244  Address materializeRValue(RValue rvalue) const;
245 
246  /// Creates temp alloca for intermediate operations on atomic value.
247  Address CreateTempAlloca() const;
248  private:
249  bool requiresMemSetZero(llvm::Type *type) const;
250 
251 
252  /// Emits atomic load as a libcall.
253  void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
254  llvm::AtomicOrdering AO, bool IsVolatile);
255  /// Emits atomic load as LLVM instruction.
256  llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
257  /// Emits atomic compare-and-exchange op as a libcall.
258  llvm::Value *EmitAtomicCompareExchangeLibcall(
259  llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
260  llvm::AtomicOrdering Success =
261  llvm::AtomicOrdering::SequentiallyConsistent,
262  llvm::AtomicOrdering Failure =
263  llvm::AtomicOrdering::SequentiallyConsistent);
264  /// Emits atomic compare-and-exchange op as LLVM instruction.
265  std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
266  llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
267  llvm::AtomicOrdering Success =
268  llvm::AtomicOrdering::SequentiallyConsistent,
269  llvm::AtomicOrdering Failure =
270  llvm::AtomicOrdering::SequentiallyConsistent,
271  bool IsWeak = false);
272  /// Emit atomic update as libcalls.
273  void
274  EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
275  const llvm::function_ref<RValue(RValue)> &UpdateOp,
276  bool IsVolatile);
277  /// Emit atomic update as LLVM instructions.
278  void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
279  const llvm::function_ref<RValue(RValue)> &UpdateOp,
280  bool IsVolatile);
281  /// Emit atomic update as libcalls.
282  void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
283  bool IsVolatile);
284  /// Emit atomic update as LLVM instructions.
285  void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
286  bool IsVolatile);
287  };
288 }
289 
290 Address AtomicInfo::CreateTempAlloca() const {
291  Address TempAlloca = CGF.CreateMemTemp(
292  (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
293  : AtomicTy,
294  getAtomicAlignment(),
295  "atomic-temp");
296  // Cast to pointer to value type for bitfields.
297  if (LVal.isBitField())
299  TempAlloca, getAtomicAddress().getType());
300  return TempAlloca;
301 }
302 
304  StringRef fnName,
305  QualType resultType,
306  CallArgList &args) {
307  const CGFunctionInfo &fnInfo =
308  CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args);
309  llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
310  llvm::FunctionCallee fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName);
311  auto callee = CGCallee::forDirect(fn);
312  return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args);
313 }
314 
315 /// Does a store of the given IR type modify the full expected width?
317  uint64_t expectedSize) {
318  return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
319 }
320 
321 /// Does the atomic type require memsetting to zero before initialization?
322 ///
323 /// The IR type is provided as a way of making certain queries faster.
324 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
325  // If the atomic type has size padding, we definitely need a memset.
326  if (hasPadding()) return true;
327 
328  // Otherwise, do some simple heuristics to try to avoid it:
329  switch (getEvaluationKind()) {
330  // For scalars and complexes, check whether the store size of the
331  // type uses the full size.
332  case TEK_Scalar:
333  return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
334  case TEK_Complex:
335  return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
336  AtomicSizeInBits / 2);
337 
338  // Padding in structs has an undefined bit pattern. User beware.
339  case TEK_Aggregate:
340  return false;
341  }
342  llvm_unreachable("bad evaluation kind");
343 }
344 
345 bool AtomicInfo::emitMemSetZeroIfNecessary() const {
346  assert(LVal.isSimple());
347  llvm::Value *addr = LVal.getPointer();
348  if (!requiresMemSetZero(addr->getType()->getPointerElementType()))
349  return false;
350 
351  CGF.Builder.CreateMemSet(
352  addr, llvm::ConstantInt::get(CGF.Int8Ty, 0),
353  CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
354  LVal.getAlignment().getQuantity());
355  return true;
356 }
357 
358 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
359  Address Dest, Address Ptr,
360  Address Val1, Address Val2,
361  uint64_t Size,
362  llvm::AtomicOrdering SuccessOrder,
363  llvm::AtomicOrdering FailureOrder,
365  // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
366  llvm::Value *Expected = CGF.Builder.CreateLoad(Val1);
367  llvm::Value *Desired = CGF.Builder.CreateLoad(Val2);
368 
369  llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
370  Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder,
371  Scope);
372  Pair->setVolatile(E->isVolatile());
373  Pair->setWeak(IsWeak);
374 
375  // Cmp holds the result of the compare-exchange operation: true on success,
376  // false on failure.
377  llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
378  llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
379 
380  // This basic block is used to hold the store instruction if the operation
381  // failed.
382  llvm::BasicBlock *StoreExpectedBB =
383  CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
384 
385  // This basic block is the exit point of the operation, we should end up
386  // here regardless of whether or not the operation succeeded.
387  llvm::BasicBlock *ContinueBB =
388  CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
389 
390  // Update Expected if Expected isn't equal to Old, otherwise branch to the
391  // exit point.
392  CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
393 
394  CGF.Builder.SetInsertPoint(StoreExpectedBB);
395  // Update the memory at Expected with Old's value.
396  CGF.Builder.CreateStore(Old, Val1);
397  // Finally, branch to the exit point.
398  CGF.Builder.CreateBr(ContinueBB);
399 
400  CGF.Builder.SetInsertPoint(ContinueBB);
401  // Update the memory at Dest with Cmp's value.
402  CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
403 }
404 
405 /// Given an ordering required on success, emit all possible cmpxchg
406 /// instructions to cope with the provided (but possibly only dynamically known)
407 /// FailureOrder.
409  bool IsWeak, Address Dest, Address Ptr,
410  Address Val1, Address Val2,
411  llvm::Value *FailureOrderVal,
412  uint64_t Size,
413  llvm::AtomicOrdering SuccessOrder,
415  llvm::AtomicOrdering FailureOrder;
416  if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
417  auto FOS = FO->getSExtValue();
418  if (!llvm::isValidAtomicOrderingCABI(FOS))
419  FailureOrder = llvm::AtomicOrdering::Monotonic;
420  else
421  switch ((llvm::AtomicOrderingCABI)FOS) {
422  case llvm::AtomicOrderingCABI::relaxed:
423  case llvm::AtomicOrderingCABI::release:
424  case llvm::AtomicOrderingCABI::acq_rel:
425  FailureOrder = llvm::AtomicOrdering::Monotonic;
426  break;
427  case llvm::AtomicOrderingCABI::consume:
428  case llvm::AtomicOrderingCABI::acquire:
429  FailureOrder = llvm::AtomicOrdering::Acquire;
430  break;
431  case llvm::AtomicOrderingCABI::seq_cst:
432  FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent;
433  break;
434  }
435  if (isStrongerThan(FailureOrder, SuccessOrder)) {
436  // Don't assert on undefined behavior "failure argument shall be no
437  // stronger than the success argument".
438  FailureOrder =
439  llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder);
440  }
441  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
442  FailureOrder, Scope);
443  return;
444  }
445 
446  // Create all the relevant BB's
447  llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
448  *SeqCstBB = nullptr;
449  MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
450  if (SuccessOrder != llvm::AtomicOrdering::Monotonic &&
451  SuccessOrder != llvm::AtomicOrdering::Release)
452  AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
453  if (SuccessOrder == llvm::AtomicOrdering::SequentiallyConsistent)
454  SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
455 
456  llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
457 
458  llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
459 
460  // Emit all the different atomics
461 
462  // MonotonicBB is arbitrarily chosen as the default case; in practice, this
463  // doesn't matter unless someone is crazy enough to use something that
464  // doesn't fold to a constant for the ordering.
465  CGF.Builder.SetInsertPoint(MonotonicBB);
466  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
467  Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope);
468  CGF.Builder.CreateBr(ContBB);
469 
470  if (AcquireBB) {
471  CGF.Builder.SetInsertPoint(AcquireBB);
472  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
473  Size, SuccessOrder, llvm::AtomicOrdering::Acquire, Scope);
474  CGF.Builder.CreateBr(ContBB);
475  SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
476  AcquireBB);
477  SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
478  AcquireBB);
479  }
480  if (SeqCstBB) {
481  CGF.Builder.SetInsertPoint(SeqCstBB);
482  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
483  llvm::AtomicOrdering::SequentiallyConsistent, Scope);
484  CGF.Builder.CreateBr(ContBB);
485  SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
486  SeqCstBB);
487  }
488 
489  CGF.Builder.SetInsertPoint(ContBB);
490 }
491 
492 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
493  Address Ptr, Address Val1, Address Val2,
494  llvm::Value *IsWeak, llvm::Value *FailureOrder,
495  uint64_t Size, llvm::AtomicOrdering Order,
497  llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
498  llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
499 
500  switch (E->getOp()) {
501  case AtomicExpr::AO__c11_atomic_init:
502  case AtomicExpr::AO__opencl_atomic_init:
503  llvm_unreachable("Already handled!");
504 
505  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
506  case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
507  emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
508  FailureOrder, Size, Order, Scope);
509  return;
510  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
511  case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
512  emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
513  FailureOrder, Size, Order, Scope);
514  return;
515  case AtomicExpr::AO__atomic_compare_exchange:
516  case AtomicExpr::AO__atomic_compare_exchange_n: {
517  if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
518  emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
519  Val1, Val2, FailureOrder, Size, Order, Scope);
520  } else {
521  // Create all the relevant BB's
522  llvm::BasicBlock *StrongBB =
523  CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
524  llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
525  llvm::BasicBlock *ContBB =
526  CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
527 
528  llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
529  SI->addCase(CGF.Builder.getInt1(false), StrongBB);
530 
531  CGF.Builder.SetInsertPoint(StrongBB);
532  emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
533  FailureOrder, Size, Order, Scope);
534  CGF.Builder.CreateBr(ContBB);
535 
536  CGF.Builder.SetInsertPoint(WeakBB);
537  emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
538  FailureOrder, Size, Order, Scope);
539  CGF.Builder.CreateBr(ContBB);
540 
541  CGF.Builder.SetInsertPoint(ContBB);
542  }
543  return;
544  }
545  case AtomicExpr::AO__c11_atomic_load:
546  case AtomicExpr::AO__opencl_atomic_load:
547  case AtomicExpr::AO__atomic_load_n:
548  case AtomicExpr::AO__atomic_load: {
549  llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
550  Load->setAtomic(Order, Scope);
551  Load->setVolatile(E->isVolatile());
552  CGF.Builder.CreateStore(Load, Dest);
553  return;
554  }
555 
556  case AtomicExpr::AO__c11_atomic_store:
557  case AtomicExpr::AO__opencl_atomic_store:
558  case AtomicExpr::AO__atomic_store:
559  case AtomicExpr::AO__atomic_store_n: {
560  llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
561  llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
562  Store->setAtomic(Order, Scope);
563  Store->setVolatile(E->isVolatile());
564  return;
565  }
566 
567  case AtomicExpr::AO__c11_atomic_exchange:
568  case AtomicExpr::AO__opencl_atomic_exchange:
569  case AtomicExpr::AO__atomic_exchange_n:
570  case AtomicExpr::AO__atomic_exchange:
571  Op = llvm::AtomicRMWInst::Xchg;
572  break;
573 
574  case AtomicExpr::AO__atomic_add_fetch:
575  PostOp = llvm::Instruction::Add;
576  LLVM_FALLTHROUGH;
577  case AtomicExpr::AO__c11_atomic_fetch_add:
578  case AtomicExpr::AO__opencl_atomic_fetch_add:
579  case AtomicExpr::AO__atomic_fetch_add:
580  Op = llvm::AtomicRMWInst::Add;
581  break;
582 
583  case AtomicExpr::AO__atomic_sub_fetch:
584  PostOp = llvm::Instruction::Sub;
585  LLVM_FALLTHROUGH;
586  case AtomicExpr::AO__c11_atomic_fetch_sub:
587  case AtomicExpr::AO__opencl_atomic_fetch_sub:
588  case AtomicExpr::AO__atomic_fetch_sub:
589  Op = llvm::AtomicRMWInst::Sub;
590  break;
591 
592  case AtomicExpr::AO__opencl_atomic_fetch_min:
593  case AtomicExpr::AO__atomic_fetch_min:
594  Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Min
595  : llvm::AtomicRMWInst::UMin;
596  break;
597 
598  case AtomicExpr::AO__opencl_atomic_fetch_max:
599  case AtomicExpr::AO__atomic_fetch_max:
600  Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Max
601  : llvm::AtomicRMWInst::UMax;
602  break;
603 
604  case AtomicExpr::AO__atomic_and_fetch:
605  PostOp = llvm::Instruction::And;
606  LLVM_FALLTHROUGH;
607  case AtomicExpr::AO__c11_atomic_fetch_and:
608  case AtomicExpr::AO__opencl_atomic_fetch_and:
609  case AtomicExpr::AO__atomic_fetch_and:
611  break;
612 
613  case AtomicExpr::AO__atomic_or_fetch:
614  PostOp = llvm::Instruction::Or;
615  LLVM_FALLTHROUGH;
616  case AtomicExpr::AO__c11_atomic_fetch_or:
617  case AtomicExpr::AO__opencl_atomic_fetch_or:
618  case AtomicExpr::AO__atomic_fetch_or:
619  Op = llvm::AtomicRMWInst::Or;
620  break;
621 
622  case AtomicExpr::AO__atomic_xor_fetch:
623  PostOp = llvm::Instruction::Xor;
624  LLVM_FALLTHROUGH;
625  case AtomicExpr::AO__c11_atomic_fetch_xor:
626  case AtomicExpr::AO__opencl_atomic_fetch_xor:
627  case AtomicExpr::AO__atomic_fetch_xor:
628  Op = llvm::AtomicRMWInst::Xor;
629  break;
630 
631  case AtomicExpr::AO__atomic_nand_fetch:
632  PostOp = llvm::Instruction::And; // the NOT is special cased below
633  LLVM_FALLTHROUGH;
634  case AtomicExpr::AO__atomic_fetch_nand:
635  Op = llvm::AtomicRMWInst::Nand;
636  break;
637  }
638 
639  llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
640  llvm::AtomicRMWInst *RMWI =
641  CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order, Scope);
642  RMWI->setVolatile(E->isVolatile());
643 
644  // For __atomic_*_fetch operations, perform the operation again to
645  // determine the value which was written.
646  llvm::Value *Result = RMWI;
647  if (PostOp)
648  Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1);
649  if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
650  Result = CGF.Builder.CreateNot(Result);
651  CGF.Builder.CreateStore(Result, Dest);
652 }
653 
654 // This function emits any expression (scalar, complex, or aggregate)
655 // into a temporary alloca.
656 static Address
658  Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
659  CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
660  /*Init*/ true);
661  return DeclPtr;
662 }
663 
665  Address Ptr, Address Val1, Address Val2,
666  llvm::Value *IsWeak, llvm::Value *FailureOrder,
667  uint64_t Size, llvm::AtomicOrdering Order,
668  llvm::Value *Scope) {
669  auto ScopeModel = Expr->getScopeModel();
670 
671  // LLVM atomic instructions always have synch scope. If clang atomic
672  // expression has no scope operand, use default LLVM synch scope.
673  if (!ScopeModel) {
674  EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
675  Order, CGF.CGM.getLLVMContext().getOrInsertSyncScopeID(""));
676  return;
677  }
678 
679  // Handle constant scope.
680  if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) {
681  auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID(
682  ScopeModel->map(SC->getZExtValue()), CGF.CGM.getLLVMContext());
683  EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
684  Order, SCID);
685  return;
686  }
687 
688  // Handle non-constant scope.
689  auto &Builder = CGF.Builder;
690  auto Scopes = ScopeModel->getRuntimeValues();
691  llvm::DenseMap<unsigned, llvm::BasicBlock *> BB;
692  for (auto S : Scopes)
693  BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn);
694 
695  llvm::BasicBlock *ContBB =
696  CGF.createBasicBlock("atomic.scope.continue", CGF.CurFn);
697 
698  auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false);
699  // If unsupported synch scope is encountered at run time, assume a fallback
700  // synch scope value.
701  auto FallBack = ScopeModel->getFallBackValue();
702  llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]);
703  for (auto S : Scopes) {
704  auto *B = BB[S];
705  if (S != FallBack)
706  SI->addCase(Builder.getInt32(S), B);
707 
708  Builder.SetInsertPoint(B);
709  EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
710  Order,
711  CGF.getTargetHooks().getLLVMSyncScopeID(ScopeModel->map(S),
712  CGF.getLLVMContext()));
713  Builder.CreateBr(ContBB);
714  }
715 
716  Builder.SetInsertPoint(ContBB);
717 }
718 
719 static void
721  bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,
722  SourceLocation Loc, CharUnits SizeInChars) {
723  if (UseOptimizedLibcall) {
724  // Load value and pass it to the function directly.
725  CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy);
726  int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
727  ValTy =
728  CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
729  llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(),
730  SizeInBits)->getPointerTo();
731  Address Ptr = Address(CGF.Builder.CreateBitCast(Val, IPtrTy), Align);
732  Val = CGF.EmitLoadOfScalar(Ptr, false,
733  CGF.getContext().getPointerType(ValTy),
734  Loc);
735  // Coerce the value into an appropriately sized integer type.
736  Args.add(RValue::get(Val), ValTy);
737  } else {
738  // Non-optimized functions always take a reference.
739  Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)),
740  CGF.getContext().VoidPtrTy);
741  }
742 }
743 
745  QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
746  QualType MemTy = AtomicTy;
747  if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
748  MemTy = AT->getValueType();
749  llvm::Value *IsWeak = nullptr, *OrderFail = nullptr;
750 
751  Address Val1 = Address::invalid();
752  Address Val2 = Address::invalid();
753  Address Dest = Address::invalid();
754  Address Ptr = EmitPointerWithAlignment(E->getPtr());
755 
756  if (E->getOp() == AtomicExpr::AO__c11_atomic_init ||
757  E->getOp() == AtomicExpr::AO__opencl_atomic_init) {
758  LValue lvalue = MakeAddrLValue(Ptr, AtomicTy);
759  EmitAtomicInit(E->getVal1(), lvalue);
760  return RValue::get(nullptr);
761  }
762 
763  CharUnits sizeChars, alignChars;
764  std::tie(sizeChars, alignChars) = getContext().getTypeInfoInChars(AtomicTy);
765  uint64_t Size = sizeChars.getQuantity();
766  unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();
767 
768  bool Oversized = getContext().toBits(sizeChars) > MaxInlineWidthInBits;
769  bool Misaligned = (Ptr.getAlignment() % sizeChars) != 0;
770  bool UseLibcall = Misaligned | Oversized;
771 
772  if (UseLibcall) {
773  CGM.getDiags().Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned)
774  << !Oversized;
775  }
776 
777  llvm::Value *Order = EmitScalarExpr(E->getOrder());
778  llvm::Value *Scope =
779  E->getScopeModel() ? EmitScalarExpr(E->getScope()) : nullptr;
780 
781  switch (E->getOp()) {
782  case AtomicExpr::AO__c11_atomic_init:
783  case AtomicExpr::AO__opencl_atomic_init:
784  llvm_unreachable("Already handled above with EmitAtomicInit!");
785 
786  case AtomicExpr::AO__c11_atomic_load:
787  case AtomicExpr::AO__opencl_atomic_load:
788  case AtomicExpr::AO__atomic_load_n:
789  break;
790 
791  case AtomicExpr::AO__atomic_load:
792  Dest = EmitPointerWithAlignment(E->getVal1());
793  break;
794 
795  case AtomicExpr::AO__atomic_store:
796  Val1 = EmitPointerWithAlignment(E->getVal1());
797  break;
798 
799  case AtomicExpr::AO__atomic_exchange:
800  Val1 = EmitPointerWithAlignment(E->getVal1());
801  Dest = EmitPointerWithAlignment(E->getVal2());
802  break;
803 
804  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
805  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
806  case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
807  case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
808  case AtomicExpr::AO__atomic_compare_exchange_n:
809  case AtomicExpr::AO__atomic_compare_exchange:
810  Val1 = EmitPointerWithAlignment(E->getVal1());
811  if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
812  Val2 = EmitPointerWithAlignment(E->getVal2());
813  else
814  Val2 = EmitValToTemp(*this, E->getVal2());
815  OrderFail = EmitScalarExpr(E->getOrderFail());
816  if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n ||
817  E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
818  IsWeak = EmitScalarExpr(E->getWeak());
819  break;
820 
821  case AtomicExpr::AO__c11_atomic_fetch_add:
822  case AtomicExpr::AO__c11_atomic_fetch_sub:
823  case AtomicExpr::AO__opencl_atomic_fetch_add:
824  case AtomicExpr::AO__opencl_atomic_fetch_sub:
825  if (MemTy->isPointerType()) {
826  // For pointer arithmetic, we're required to do a bit of math:
827  // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
828  // ... but only for the C11 builtins. The GNU builtins expect the
829  // user to multiply by sizeof(T).
830  QualType Val1Ty = E->getVal1()->getType();
831  llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
832  CharUnits PointeeIncAmt =
833  getContext().getTypeSizeInChars(MemTy->getPointeeType());
834  Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
835  auto Temp = CreateMemTemp(Val1Ty, ".atomictmp");
836  Val1 = Temp;
837  EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty));
838  break;
839  }
840  LLVM_FALLTHROUGH;
841  case AtomicExpr::AO__atomic_fetch_add:
842  case AtomicExpr::AO__atomic_fetch_sub:
843  case AtomicExpr::AO__atomic_add_fetch:
844  case AtomicExpr::AO__atomic_sub_fetch:
845  case AtomicExpr::AO__c11_atomic_store:
846  case AtomicExpr::AO__c11_atomic_exchange:
847  case AtomicExpr::AO__opencl_atomic_store:
848  case AtomicExpr::AO__opencl_atomic_exchange:
849  case AtomicExpr::AO__atomic_store_n:
850  case AtomicExpr::AO__atomic_exchange_n:
851  case AtomicExpr::AO__c11_atomic_fetch_and:
852  case AtomicExpr::AO__c11_atomic_fetch_or:
853  case AtomicExpr::AO__c11_atomic_fetch_xor:
854  case AtomicExpr::AO__opencl_atomic_fetch_and:
855  case AtomicExpr::AO__opencl_atomic_fetch_or:
856  case AtomicExpr::AO__opencl_atomic_fetch_xor:
857  case AtomicExpr::AO__opencl_atomic_fetch_min:
858  case AtomicExpr::AO__opencl_atomic_fetch_max:
859  case AtomicExpr::AO__atomic_fetch_and:
860  case AtomicExpr::AO__atomic_fetch_or:
861  case AtomicExpr::AO__atomic_fetch_xor:
862  case AtomicExpr::AO__atomic_fetch_nand:
863  case AtomicExpr::AO__atomic_and_fetch:
864  case AtomicExpr::AO__atomic_or_fetch:
865  case AtomicExpr::AO__atomic_xor_fetch:
866  case AtomicExpr::AO__atomic_nand_fetch:
867  case AtomicExpr::AO__atomic_fetch_min:
868  case AtomicExpr::AO__atomic_fetch_max:
869  Val1 = EmitValToTemp(*this, E->getVal1());
870  break;
871  }
872 
873  QualType RValTy = E->getType().getUnqualifiedType();
874 
875  // The inlined atomics only function on iN types, where N is a power of 2. We
876  // need to make sure (via temporaries if necessary) that all incoming values
877  // are compatible.
878  LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy);
879  AtomicInfo Atomics(*this, AtomicVal);
880 
881  Ptr = Atomics.emitCastToAtomicIntPointer(Ptr);
882  if (Val1.isValid()) Val1 = Atomics.convertToAtomicIntPointer(Val1);
883  if (Val2.isValid()) Val2 = Atomics.convertToAtomicIntPointer(Val2);
884  if (Dest.isValid())
885  Dest = Atomics.emitCastToAtomicIntPointer(Dest);
886  else if (E->isCmpXChg())
887  Dest = CreateMemTemp(RValTy, "cmpxchg.bool");
888  else if (!RValTy->isVoidType())
889  Dest = Atomics.emitCastToAtomicIntPointer(Atomics.CreateTempAlloca());
890 
891  // Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
892  if (UseLibcall) {
893  bool UseOptimizedLibcall = false;
894  switch (E->getOp()) {
895  case AtomicExpr::AO__c11_atomic_init:
896  case AtomicExpr::AO__opencl_atomic_init:
897  llvm_unreachable("Already handled above with EmitAtomicInit!");
898 
899  case AtomicExpr::AO__c11_atomic_fetch_add:
900  case AtomicExpr::AO__opencl_atomic_fetch_add:
901  case AtomicExpr::AO__atomic_fetch_add:
902  case AtomicExpr::AO__c11_atomic_fetch_and:
903  case AtomicExpr::AO__opencl_atomic_fetch_and:
904  case AtomicExpr::AO__atomic_fetch_and:
905  case AtomicExpr::AO__c11_atomic_fetch_or:
906  case AtomicExpr::AO__opencl_atomic_fetch_or:
907  case AtomicExpr::AO__atomic_fetch_or:
908  case AtomicExpr::AO__atomic_fetch_nand:
909  case AtomicExpr::AO__c11_atomic_fetch_sub:
910  case AtomicExpr::AO__opencl_atomic_fetch_sub:
911  case AtomicExpr::AO__atomic_fetch_sub:
912  case AtomicExpr::AO__c11_atomic_fetch_xor:
913  case AtomicExpr::AO__opencl_atomic_fetch_xor:
914  case AtomicExpr::AO__opencl_atomic_fetch_min:
915  case AtomicExpr::AO__opencl_atomic_fetch_max:
916  case AtomicExpr::AO__atomic_fetch_xor:
917  case AtomicExpr::AO__atomic_add_fetch:
918  case AtomicExpr::AO__atomic_and_fetch:
919  case AtomicExpr::AO__atomic_nand_fetch:
920  case AtomicExpr::AO__atomic_or_fetch:
921  case AtomicExpr::AO__atomic_sub_fetch:
922  case AtomicExpr::AO__atomic_xor_fetch:
923  case AtomicExpr::AO__atomic_fetch_min:
924  case AtomicExpr::AO__atomic_fetch_max:
925  // For these, only library calls for certain sizes exist.
926  UseOptimizedLibcall = true;
927  break;
928 
929  case AtomicExpr::AO__atomic_load:
930  case AtomicExpr::AO__atomic_store:
931  case AtomicExpr::AO__atomic_exchange:
932  case AtomicExpr::AO__atomic_compare_exchange:
933  // Use the generic version if we don't know that the operand will be
934  // suitably aligned for the optimized version.
935  if (Misaligned)
936  break;
937  LLVM_FALLTHROUGH;
938  case AtomicExpr::AO__c11_atomic_load:
939  case AtomicExpr::AO__c11_atomic_store:
940  case AtomicExpr::AO__c11_atomic_exchange:
941  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
942  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
943  case AtomicExpr::AO__opencl_atomic_load:
944  case AtomicExpr::AO__opencl_atomic_store:
945  case AtomicExpr::AO__opencl_atomic_exchange:
946  case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
947  case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
948  case AtomicExpr::AO__atomic_load_n:
949  case AtomicExpr::AO__atomic_store_n:
950  case AtomicExpr::AO__atomic_exchange_n:
951  case AtomicExpr::AO__atomic_compare_exchange_n:
952  // Only use optimized library calls for sizes for which they exist.
953  // FIXME: Size == 16 optimized library functions exist too.
954  if (Size == 1 || Size == 2 || Size == 4 || Size == 8)
955  UseOptimizedLibcall = true;
956  break;
957  }
958 
959  CallArgList Args;
960  if (!UseOptimizedLibcall) {
961  // For non-optimized library calls, the size is the first parameter
962  Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
963  getContext().getSizeType());
964  }
965  // Atomic address is the first or second parameter
966  // The OpenCL atomic library functions only accept pointer arguments to
967  // generic address space.
968  auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) {
969  if (!E->isOpenCL())
970  return V;
971  auto AS = PT->getAs<PointerType>()->getPointeeType().getAddressSpace();
972  if (AS == LangAS::opencl_generic)
973  return V;
974  auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic);
975  auto T = V->getType();
976  auto *DestType = T->getPointerElementType()->getPointerTo(DestAS);
977 
978  return getTargetHooks().performAddrSpaceCast(
979  *this, V, AS, LangAS::opencl_generic, DestType, false);
980  };
981 
982  Args.add(RValue::get(CastToGenericAddrSpace(
983  EmitCastToVoidPtr(Ptr.getPointer()), E->getPtr()->getType())),
984  getContext().VoidPtrTy);
985 
986  std::string LibCallName;
987  QualType LoweredMemTy =
988  MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy;
989  QualType RetTy;
990  bool HaveRetTy = false;
991  llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
992  switch (E->getOp()) {
993  case AtomicExpr::AO__c11_atomic_init:
994  case AtomicExpr::AO__opencl_atomic_init:
995  llvm_unreachable("Already handled!");
996 
997  // There is only one libcall for compare an exchange, because there is no
998  // optimisation benefit possible from a libcall version of a weak compare
999  // and exchange.
1000  // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
1001  // void *desired, int success, int failure)
1002  // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired,
1003  // int success, int failure)
1004  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1005  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1006  case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1007  case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1008  case AtomicExpr::AO__atomic_compare_exchange:
1009  case AtomicExpr::AO__atomic_compare_exchange_n:
1010  LibCallName = "__atomic_compare_exchange";
1011  RetTy = getContext().BoolTy;
1012  HaveRetTy = true;
1013  Args.add(
1014  RValue::get(CastToGenericAddrSpace(
1015  EmitCastToVoidPtr(Val1.getPointer()), E->getVal1()->getType())),
1016  getContext().VoidPtrTy);
1017  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(),
1018  MemTy, E->getExprLoc(), sizeChars);
1019  Args.add(RValue::get(Order), getContext().IntTy);
1020  Order = OrderFail;
1021  break;
1022  // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
1023  // int order)
1024  // T __atomic_exchange_N(T *mem, T val, int order)
1025  case AtomicExpr::AO__c11_atomic_exchange:
1026  case AtomicExpr::AO__opencl_atomic_exchange:
1027  case AtomicExpr::AO__atomic_exchange_n:
1028  case AtomicExpr::AO__atomic_exchange:
1029  LibCallName = "__atomic_exchange";
1030  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1031  MemTy, E->getExprLoc(), sizeChars);
1032  break;
1033  // void __atomic_store(size_t size, void *mem, void *val, int order)
1034  // void __atomic_store_N(T *mem, T val, int order)
1035  case AtomicExpr::AO__c11_atomic_store:
1036  case AtomicExpr::AO__opencl_atomic_store:
1037  case AtomicExpr::AO__atomic_store:
1038  case AtomicExpr::AO__atomic_store_n:
1039  LibCallName = "__atomic_store";
1040  RetTy = getContext().VoidTy;
1041  HaveRetTy = true;
1042  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1043  MemTy, E->getExprLoc(), sizeChars);
1044  break;
1045  // void __atomic_load(size_t size, void *mem, void *return, int order)
1046  // T __atomic_load_N(T *mem, int order)
1047  case AtomicExpr::AO__c11_atomic_load:
1048  case AtomicExpr::AO__opencl_atomic_load:
1049  case AtomicExpr::AO__atomic_load:
1050  case AtomicExpr::AO__atomic_load_n:
1051  LibCallName = "__atomic_load";
1052  break;
1053  // T __atomic_add_fetch_N(T *mem, T val, int order)
1054  // T __atomic_fetch_add_N(T *mem, T val, int order)
1055  case AtomicExpr::AO__atomic_add_fetch:
1056  PostOp = llvm::Instruction::Add;
1057  LLVM_FALLTHROUGH;
1058  case AtomicExpr::AO__c11_atomic_fetch_add:
1059  case AtomicExpr::AO__opencl_atomic_fetch_add:
1060  case AtomicExpr::AO__atomic_fetch_add:
1061  LibCallName = "__atomic_fetch_add";
1062  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1063  LoweredMemTy, E->getExprLoc(), sizeChars);
1064  break;
1065  // T __atomic_and_fetch_N(T *mem, T val, int order)
1066  // T __atomic_fetch_and_N(T *mem, T val, int order)
1067  case AtomicExpr::AO__atomic_and_fetch:
1068  PostOp = llvm::Instruction::And;
1069  LLVM_FALLTHROUGH;
1070  case AtomicExpr::AO__c11_atomic_fetch_and:
1071  case AtomicExpr::AO__opencl_atomic_fetch_and:
1072  case AtomicExpr::AO__atomic_fetch_and:
1073  LibCallName = "__atomic_fetch_and";
1074  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1075  MemTy, E->getExprLoc(), sizeChars);
1076  break;
1077  // T __atomic_or_fetch_N(T *mem, T val, int order)
1078  // T __atomic_fetch_or_N(T *mem, T val, int order)
1079  case AtomicExpr::AO__atomic_or_fetch:
1080  PostOp = llvm::Instruction::Or;
1081  LLVM_FALLTHROUGH;
1082  case AtomicExpr::AO__c11_atomic_fetch_or:
1083  case AtomicExpr::AO__opencl_atomic_fetch_or:
1084  case AtomicExpr::AO__atomic_fetch_or:
1085  LibCallName = "__atomic_fetch_or";
1086  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1087  MemTy, E->getExprLoc(), sizeChars);
1088  break;
1089  // T __atomic_sub_fetch_N(T *mem, T val, int order)
1090  // T __atomic_fetch_sub_N(T *mem, T val, int order)
1091  case AtomicExpr::AO__atomic_sub_fetch:
1092  PostOp = llvm::Instruction::Sub;
1093  LLVM_FALLTHROUGH;
1094  case AtomicExpr::AO__c11_atomic_fetch_sub:
1095  case AtomicExpr::AO__opencl_atomic_fetch_sub:
1096  case AtomicExpr::AO__atomic_fetch_sub:
1097  LibCallName = "__atomic_fetch_sub";
1098  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1099  LoweredMemTy, E->getExprLoc(), sizeChars);
1100  break;
1101  // T __atomic_xor_fetch_N(T *mem, T val, int order)
1102  // T __atomic_fetch_xor_N(T *mem, T val, int order)
1103  case AtomicExpr::AO__atomic_xor_fetch:
1104  PostOp = llvm::Instruction::Xor;
1105  LLVM_FALLTHROUGH;
1106  case AtomicExpr::AO__c11_atomic_fetch_xor:
1107  case AtomicExpr::AO__opencl_atomic_fetch_xor:
1108  case AtomicExpr::AO__atomic_fetch_xor:
1109  LibCallName = "__atomic_fetch_xor";
1110  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1111  MemTy, E->getExprLoc(), sizeChars);
1112  break;
1113  case AtomicExpr::AO__atomic_fetch_min:
1114  case AtomicExpr::AO__opencl_atomic_fetch_min:
1115  LibCallName = E->getValueType()->isSignedIntegerType()
1116  ? "__atomic_fetch_min"
1117  : "__atomic_fetch_umin";
1118  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1119  LoweredMemTy, E->getExprLoc(), sizeChars);
1120  break;
1121  case AtomicExpr::AO__atomic_fetch_max:
1122  case AtomicExpr::AO__opencl_atomic_fetch_max:
1123  LibCallName = E->getValueType()->isSignedIntegerType()
1124  ? "__atomic_fetch_max"
1125  : "__atomic_fetch_umax";
1126  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1127  LoweredMemTy, E->getExprLoc(), sizeChars);
1128  break;
1129  // T __atomic_nand_fetch_N(T *mem, T val, int order)
1130  // T __atomic_fetch_nand_N(T *mem, T val, int order)
1131  case AtomicExpr::AO__atomic_nand_fetch:
1132  PostOp = llvm::Instruction::And; // the NOT is special cased below
1133  LLVM_FALLTHROUGH;
1134  case AtomicExpr::AO__atomic_fetch_nand:
1135  LibCallName = "__atomic_fetch_nand";
1136  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1137  MemTy, E->getExprLoc(), sizeChars);
1138  break;
1139  }
1140 
1141  if (E->isOpenCL()) {
1142  LibCallName = std::string("__opencl") +
1143  StringRef(LibCallName).drop_front(1).str();
1144 
1145  }
1146  // Optimized functions have the size in their name.
1147  if (UseOptimizedLibcall)
1148  LibCallName += "_" + llvm::utostr(Size);
1149  // By default, assume we return a value of the atomic type.
1150  if (!HaveRetTy) {
1151  if (UseOptimizedLibcall) {
1152  // Value is returned directly.
1153  // The function returns an appropriately sized integer type.
1154  RetTy = getContext().getIntTypeForBitwidth(
1155  getContext().toBits(sizeChars), /*Signed=*/false);
1156  } else {
1157  // Value is returned through parameter before the order.
1158  RetTy = getContext().VoidTy;
1159  Args.add(RValue::get(EmitCastToVoidPtr(Dest.getPointer())),
1160  getContext().VoidPtrTy);
1161  }
1162  }
1163  // order is always the last parameter
1164  Args.add(RValue::get(Order),
1165  getContext().IntTy);
1166  if (E->isOpenCL())
1167  Args.add(RValue::get(Scope), getContext().IntTy);
1168 
1169  // PostOp is only needed for the atomic_*_fetch operations, and
1170  // thus is only needed for and implemented in the
1171  // UseOptimizedLibcall codepath.
1172  assert(UseOptimizedLibcall || !PostOp);
1173 
1174  RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
1175  // The value is returned directly from the libcall.
1176  if (E->isCmpXChg())
1177  return Res;
1178 
1179  // The value is returned directly for optimized libcalls but the expr
1180  // provided an out-param.
1181  if (UseOptimizedLibcall && Res.getScalarVal()) {
1182  llvm::Value *ResVal = Res.getScalarVal();
1183  if (PostOp) {
1184  llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal();
1185  ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1);
1186  }
1187  if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
1188  ResVal = Builder.CreateNot(ResVal);
1189 
1190  Builder.CreateStore(
1191  ResVal,
1192  Builder.CreateBitCast(Dest, ResVal->getType()->getPointerTo()));
1193  }
1194 
1195  if (RValTy->isVoidType())
1196  return RValue::get(nullptr);
1197 
1198  return convertTempToRValue(
1199  Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()),
1200  RValTy, E->getExprLoc());
1201  }
1202 
1203  bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
1204  E->getOp() == AtomicExpr::AO__opencl_atomic_store ||
1205  E->getOp() == AtomicExpr::AO__atomic_store ||
1206  E->getOp() == AtomicExpr::AO__atomic_store_n;
1207  bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
1208  E->getOp() == AtomicExpr::AO__opencl_atomic_load ||
1209  E->getOp() == AtomicExpr::AO__atomic_load ||
1210  E->getOp() == AtomicExpr::AO__atomic_load_n;
1211 
1212  if (isa<llvm::ConstantInt>(Order)) {
1213  auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1214  // We should not ever get to a case where the ordering isn't a valid C ABI
1215  // value, but it's hard to enforce that in general.
1216  if (llvm::isValidAtomicOrderingCABI(ord))
1217  switch ((llvm::AtomicOrderingCABI)ord) {
1218  case llvm::AtomicOrderingCABI::relaxed:
1219  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1220  llvm::AtomicOrdering::Monotonic, Scope);
1221  break;
1222  case llvm::AtomicOrderingCABI::consume:
1223  case llvm::AtomicOrderingCABI::acquire:
1224  if (IsStore)
1225  break; // Avoid crashing on code with undefined behavior
1226  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1227  llvm::AtomicOrdering::Acquire, Scope);
1228  break;
1229  case llvm::AtomicOrderingCABI::release:
1230  if (IsLoad)
1231  break; // Avoid crashing on code with undefined behavior
1232  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1233  llvm::AtomicOrdering::Release, Scope);
1234  break;
1235  case llvm::AtomicOrderingCABI::acq_rel:
1236  if (IsLoad || IsStore)
1237  break; // Avoid crashing on code with undefined behavior
1238  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1239  llvm::AtomicOrdering::AcquireRelease, Scope);
1240  break;
1241  case llvm::AtomicOrderingCABI::seq_cst:
1242  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1243  llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1244  break;
1245  }
1246  if (RValTy->isVoidType())
1247  return RValue::get(nullptr);
1248 
1249  return convertTempToRValue(
1250  Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo(
1251  Dest.getAddressSpace())),
1252  RValTy, E->getExprLoc());
1253  }
1254 
1255  // Long case, when Order isn't obviously constant.
1256 
1257  // Create all the relevant BB's
1258  llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
1259  *ReleaseBB = nullptr, *AcqRelBB = nullptr,
1260  *SeqCstBB = nullptr;
1261  MonotonicBB = createBasicBlock("monotonic", CurFn);
1262  if (!IsStore)
1263  AcquireBB = createBasicBlock("acquire", CurFn);
1264  if (!IsLoad)
1265  ReleaseBB = createBasicBlock("release", CurFn);
1266  if (!IsLoad && !IsStore)
1267  AcqRelBB = createBasicBlock("acqrel", CurFn);
1268  SeqCstBB = createBasicBlock("seqcst", CurFn);
1269  llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1270 
1271  // Create the switch for the split
1272  // MonotonicBB is arbitrarily chosen as the default case; in practice, this
1273  // doesn't matter unless someone is crazy enough to use something that
1274  // doesn't fold to a constant for the ordering.
1275  Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1276  llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
1277 
1278  // Emit all the different atomics
1279  Builder.SetInsertPoint(MonotonicBB);
1280  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1281  llvm::AtomicOrdering::Monotonic, Scope);
1282  Builder.CreateBr(ContBB);
1283  if (!IsStore) {
1284  Builder.SetInsertPoint(AcquireBB);
1285  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1286  llvm::AtomicOrdering::Acquire, Scope);
1287  Builder.CreateBr(ContBB);
1288  SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
1289  AcquireBB);
1290  SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
1291  AcquireBB);
1292  }
1293  if (!IsLoad) {
1294  Builder.SetInsertPoint(ReleaseBB);
1295  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1296  llvm::AtomicOrdering::Release, Scope);
1297  Builder.CreateBr(ContBB);
1298  SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release),
1299  ReleaseBB);
1300  }
1301  if (!IsLoad && !IsStore) {
1302  Builder.SetInsertPoint(AcqRelBB);
1303  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1304  llvm::AtomicOrdering::AcquireRelease, Scope);
1305  Builder.CreateBr(ContBB);
1306  SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel),
1307  AcqRelBB);
1308  }
1309  Builder.SetInsertPoint(SeqCstBB);
1310  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1311  llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1312  Builder.CreateBr(ContBB);
1313  SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
1314  SeqCstBB);
1315 
1316  // Cleanup and return
1317  Builder.SetInsertPoint(ContBB);
1318  if (RValTy->isVoidType())
1319  return RValue::get(nullptr);
1320 
1321  assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
1322  return convertTempToRValue(
1323  Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo(
1324  Dest.getAddressSpace())),
1325  RValTy, E->getExprLoc());
1326 }
1327 
1328 Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const {
1329  unsigned addrspace =
1330  cast<llvm::PointerType>(addr.getPointer()->getType())->getAddressSpace();
1331  llvm::IntegerType *ty =
1332  llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
1333  return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace));
1334 }
1335 
1336 Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {
1337  llvm::Type *Ty = Addr.getElementType();
1338  uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
1339  if (SourceSizeInBits != AtomicSizeInBits) {
1340  Address Tmp = CreateTempAlloca();
1341  CGF.Builder.CreateMemCpy(Tmp, Addr,
1342  std::min(AtomicSizeInBits, SourceSizeInBits) / 8);
1343  Addr = Tmp;
1344  }
1345 
1346  return emitCastToAtomicIntPointer(Addr);
1347 }
1348 
1349 RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
1350  AggValueSlot resultSlot,
1351  SourceLocation loc,
1352  bool asValue) const {
1353  if (LVal.isSimple()) {
1354  if (EvaluationKind == TEK_Aggregate)
1355  return resultSlot.asRValue();
1356 
1357  // Drill into the padding structure if we have one.
1358  if (hasPadding())
1359  addr = CGF.Builder.CreateStructGEP(addr, 0);
1360 
1361  // Otherwise, just convert the temporary to an r-value using the
1362  // normal conversion routine.
1363  return CGF.convertTempToRValue(addr, getValueType(), loc);
1364  }
1365  if (!asValue)
1366  // Get RValue from temp memory as atomic for non-simple lvalues
1367  return RValue::get(CGF.Builder.CreateLoad(addr));
1368  if (LVal.isBitField())
1369  return CGF.EmitLoadOfBitfieldLValue(
1370  LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(),
1371  LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1372  if (LVal.isVectorElt())
1373  return CGF.EmitLoadOfLValue(
1374  LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(),
1375  LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1376  assert(LVal.isExtVectorElt());
1378  addr, LVal.getExtVectorElts(), LVal.getType(),
1379  LVal.getBaseInfo(), TBAAAccessInfo()));
1380 }
1381 
1382 RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
1383  AggValueSlot ResultSlot,
1384  SourceLocation Loc,
1385  bool AsValue) const {
1386  // Try not to in some easy cases.
1387  assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
1388  if (getEvaluationKind() == TEK_Scalar &&
1389  (((!LVal.isBitField() ||
1390  LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
1391  !hasPadding()) ||
1392  !AsValue)) {
1393  auto *ValTy = AsValue
1394  ? CGF.ConvertTypeForMem(ValueTy)
1395  : getAtomicAddress().getType()->getPointerElementType();
1396  if (ValTy->isIntegerTy()) {
1397  assert(IntVal->getType() == ValTy && "Different integer types.");
1398  return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
1399  } else if (ValTy->isPointerTy())
1400  return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
1401  else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
1402  return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
1403  }
1404 
1405  // Create a temporary. This needs to be big enough to hold the
1406  // atomic integer.
1407  Address Temp = Address::invalid();
1408  bool TempIsVolatile = false;
1409  if (AsValue && getEvaluationKind() == TEK_Aggregate) {
1410  assert(!ResultSlot.isIgnored());
1411  Temp = ResultSlot.getAddress();
1412  TempIsVolatile = ResultSlot.isVolatile();
1413  } else {
1414  Temp = CreateTempAlloca();
1415  }
1416 
1417  // Slam the integer into the temporary.
1418  Address CastTemp = emitCastToAtomicIntPointer(Temp);
1419  CGF.Builder.CreateStore(IntVal, CastTemp)
1420  ->setVolatile(TempIsVolatile);
1421 
1422  return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue);
1423 }
1424 
1425 void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1426  llvm::AtomicOrdering AO, bool) {
1427  // void __atomic_load(size_t size, void *mem, void *return, int order);
1428  CallArgList Args;
1429  Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1430  Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1431  CGF.getContext().VoidPtrTy);
1432  Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)),
1433  CGF.getContext().VoidPtrTy);
1434  Args.add(
1435  RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))),
1436  CGF.getContext().IntTy);
1437  emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1438 }
1439 
1440 llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1441  bool IsVolatile) {
1442  // Okay, we're doing this natively.
1443  Address Addr = getAtomicAddressAsAtomicIntPointer();
1444  llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
1445  Load->setAtomic(AO);
1446 
1447  // Other decoration.
1448  if (IsVolatile)
1449  Load->setVolatile(true);
1450  CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo());
1451  return Load;
1452 }
1453 
1454 /// An LValue is a candidate for having its loads and stores be made atomic if
1455 /// we are operating under /volatile:ms *and* the LValue itself is volatile and
1456 /// performing such an operation can be performed without a libcall.
1458  if (!CGM.getCodeGenOpts().MSVolatile) return false;
1459  AtomicInfo AI(*this, LV);
1460  bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType());
1461  // An atomic is inline if we don't need to use a libcall.
1462  bool AtomicIsInline = !AI.shouldUseLibcall();
1463  // MSVC doesn't seem to do this for types wider than a pointer.
1464  if (getContext().getTypeSize(LV.getType()) >
1465  getContext().getTypeSize(getContext().getIntPtrType()))
1466  return false;
1467  return IsVolatile && AtomicIsInline;
1468 }
1469 
1471  AggValueSlot Slot) {
1472  llvm::AtomicOrdering AO;
1473  bool IsVolatile = LV.isVolatileQualified();
1474  if (LV.getType()->isAtomicType()) {
1475  AO = llvm::AtomicOrdering::SequentiallyConsistent;
1476  } else {
1477  AO = llvm::AtomicOrdering::Acquire;
1478  IsVolatile = true;
1479  }
1480  return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
1481 }
1482 
1483 RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1484  bool AsValue, llvm::AtomicOrdering AO,
1485  bool IsVolatile) {
1486  // Check whether we should use a library call.
1487  if (shouldUseLibcall()) {
1488  Address TempAddr = Address::invalid();
1489  if (LVal.isSimple() && !ResultSlot.isIgnored()) {
1490  assert(getEvaluationKind() == TEK_Aggregate);
1491  TempAddr = ResultSlot.getAddress();
1492  } else
1493  TempAddr = CreateTempAlloca();
1494 
1495  EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile);
1496 
1497  // Okay, turn that back into the original value or whole atomic (for
1498  // non-simple lvalues) type.
1499  return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
1500  }
1501 
1502  // Okay, we're doing this natively.
1503  auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1504 
1505  // If we're ignoring an aggregate return, don't do anything.
1506  if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
1507  return RValue::getAggregate(Address::invalid(), false);
1508 
1509  // Okay, turn that back into the original value or atomic (for non-simple
1510  // lvalues) type.
1511  return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
1512 }
1513 
1514 /// Emit a load from an l-value of atomic type. Note that the r-value
1515 /// we produce is an r-value of the atomic *value* type.
1517  llvm::AtomicOrdering AO, bool IsVolatile,
1518  AggValueSlot resultSlot) {
1519  AtomicInfo Atomics(*this, src);
1520  return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
1521  IsVolatile);
1522 }
1523 
1524 /// Copy an r-value into memory as part of storing to an atomic type.
1525 /// This needs to create a bit-pattern suitable for atomic operations.
1526 void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1527  assert(LVal.isSimple());
1528  // If we have an r-value, the rvalue should be of the atomic type,
1529  // which means that the caller is responsible for having zeroed
1530  // any padding. Just do an aggregate copy of that type.
1531  if (rvalue.isAggregate()) {
1532  LValue Dest = CGF.MakeAddrLValue(getAtomicAddress(), getAtomicType());
1533  LValue Src = CGF.MakeAddrLValue(rvalue.getAggregateAddress(),
1534  getAtomicType());
1535  bool IsVolatile = rvalue.isVolatileQualified() ||
1536  LVal.isVolatileQualified();
1537  CGF.EmitAggregateCopy(Dest, Src, getAtomicType(),
1538  AggValueSlot::DoesNotOverlap, IsVolatile);
1539  return;
1540  }
1541 
1542  // Okay, otherwise we're copying stuff.
1543 
1544  // Zero out the buffer if necessary.
1545  emitMemSetZeroIfNecessary();
1546 
1547  // Drill past the padding if present.
1548  LValue TempLVal = projectValue();
1549 
1550  // Okay, store the rvalue in.
1551  if (rvalue.isScalar()) {
1552  CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
1553  } else {
1554  CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
1555  }
1556 }
1557 
1558 
1559 /// Materialize an r-value into memory for the purposes of storing it
1560 /// to an atomic type.
1561 Address AtomicInfo::materializeRValue(RValue rvalue) const {
1562  // Aggregate r-values are already in memory, and EmitAtomicStore
1563  // requires them to be values of the atomic type.
1564  if (rvalue.isAggregate())
1565  return rvalue.getAggregateAddress();
1566 
1567  // Otherwise, make a temporary and materialize into it.
1568  LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType());
1569  AtomicInfo Atomics(CGF, TempLV);
1570  Atomics.emitCopyIntoMemory(rvalue);
1571  return TempLV.getAddress();
1572 }
1573 
1574 llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
1575  // If we've got a scalar value of the right size, try to avoid going
1576  // through memory.
1577  if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) {
1578  llvm::Value *Value = RVal.getScalarVal();
1579  if (isa<llvm::IntegerType>(Value->getType()))
1580  return CGF.EmitToMemory(Value, ValueTy);
1581  else {
1582  llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1583  CGF.getLLVMContext(),
1584  LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1585  if (isa<llvm::PointerType>(Value->getType()))
1586  return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
1587  else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
1588  return CGF.Builder.CreateBitCast(Value, InputIntTy);
1589  }
1590  }
1591  // Otherwise, we need to go through memory.
1592  // Put the r-value in memory.
1593  Address Addr = materializeRValue(RVal);
1594 
1595  // Cast the temporary to the atomic int type and pull a value out.
1596  Addr = emitCastToAtomicIntPointer(Addr);
1597  return CGF.Builder.CreateLoad(Addr);
1598 }
1599 
1600 std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1601  llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1602  llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1603  // Do the atomic store.
1604  Address Addr = getAtomicAddressAsAtomicIntPointer();
1605  auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(),
1606  ExpectedVal, DesiredVal,
1607  Success, Failure);
1608  // Other decoration.
1609  Inst->setVolatile(LVal.isVolatileQualified());
1610  Inst->setWeak(IsWeak);
1611 
1612  // Okay, turn that back into the original value type.
1613  auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
1614  auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
1615  return std::make_pair(PreviousVal, SuccessFailureVal);
1616 }
1617 
1618 llvm::Value *
1619 AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1620  llvm::Value *DesiredAddr,
1621  llvm::AtomicOrdering Success,
1622  llvm::AtomicOrdering Failure) {
1623  // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1624  // void *desired, int success, int failure);
1625  CallArgList Args;
1626  Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1627  Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1628  CGF.getContext().VoidPtrTy);
1629  Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)),
1630  CGF.getContext().VoidPtrTy);
1631  Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)),
1632  CGF.getContext().VoidPtrTy);
1633  Args.add(RValue::get(
1634  llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))),
1635  CGF.getContext().IntTy);
1636  Args.add(RValue::get(
1637  llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))),
1638  CGF.getContext().IntTy);
1639  auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1640  CGF.getContext().BoolTy, Args);
1641 
1642  return SuccessFailureRVal.getScalarVal();
1643 }
1644 
1645 std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1646  RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1647  llvm::AtomicOrdering Failure, bool IsWeak) {
1648  if (isStrongerThan(Failure, Success))
1649  // Don't assert on undefined behavior "failure argument shall be no stronger
1650  // than the success argument".
1651  Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success);
1652 
1653  // Check whether we should use a library call.
1654  if (shouldUseLibcall()) {
1655  // Produce a source address.
1656  Address ExpectedAddr = materializeRValue(Expected);
1657  Address DesiredAddr = materializeRValue(Desired);
1658  auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1659  DesiredAddr.getPointer(),
1660  Success, Failure);
1661  return std::make_pair(
1662  convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1663  SourceLocation(), /*AsValue=*/false),
1664  Res);
1665  }
1666 
1667  // If we've got a scalar value of the right size, try to avoid going
1668  // through memory.
1669  auto *ExpectedVal = convertRValueToInt(Expected);
1670  auto *DesiredVal = convertRValueToInt(Desired);
1671  auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1672  Failure, IsWeak);
1673  return std::make_pair(
1674  ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(),
1675  SourceLocation(), /*AsValue=*/false),
1676  Res.second);
1677 }
1678 
1679 static void
1680 EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1681  const llvm::function_ref<RValue(RValue)> &UpdateOp,
1682  Address DesiredAddr) {
1683  RValue UpRVal;
1684  LValue AtomicLVal = Atomics.getAtomicLValue();
1685  LValue DesiredLVal;
1686  if (AtomicLVal.isSimple()) {
1687  UpRVal = OldRVal;
1688  DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType());
1689  } else {
1690  // Build new lvalue for temp address
1691  Address Ptr = Atomics.materializeRValue(OldRVal);
1692  LValue UpdateLVal;
1693  if (AtomicLVal.isBitField()) {
1694  UpdateLVal =
1695  LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
1696  AtomicLVal.getType(),
1697  AtomicLVal.getBaseInfo(),
1698  AtomicLVal.getTBAAInfo());
1699  DesiredLVal =
1700  LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1701  AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1702  AtomicLVal.getTBAAInfo());
1703  } else if (AtomicLVal.isVectorElt()) {
1704  UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
1705  AtomicLVal.getType(),
1706  AtomicLVal.getBaseInfo(),
1707  AtomicLVal.getTBAAInfo());
1708  DesiredLVal = LValue::MakeVectorElt(
1709  DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(),
1710  AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1711  } else {
1712  assert(AtomicLVal.isExtVectorElt());
1713  UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
1714  AtomicLVal.getType(),
1715  AtomicLVal.getBaseInfo(),
1716  AtomicLVal.getTBAAInfo());
1717  DesiredLVal = LValue::MakeExtVectorElt(
1718  DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1719  AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1720  }
1721  UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());
1722  }
1723  // Store new value in the corresponding memory area
1724  RValue NewRVal = UpdateOp(UpRVal);
1725  if (NewRVal.isScalar()) {
1726  CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);
1727  } else {
1728  assert(NewRVal.isComplex());
1729  CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,
1730  /*isInit=*/false);
1731  }
1732 }
1733 
1734 void AtomicInfo::EmitAtomicUpdateLibcall(
1735  llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1736  bool IsVolatile) {
1737  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1738 
1739  Address ExpectedAddr = CreateTempAlloca();
1740 
1741  EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1742  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1743  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1744  CGF.EmitBlock(ContBB);
1745  Address DesiredAddr = CreateTempAlloca();
1746  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1747  requiresMemSetZero(getAtomicAddress().getElementType())) {
1748  auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1749  CGF.Builder.CreateStore(OldVal, DesiredAddr);
1750  }
1751  auto OldRVal = convertAtomicTempToRValue(ExpectedAddr,
1753  SourceLocation(), /*AsValue=*/false);
1754  EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
1755  auto *Res =
1756  EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1757  DesiredAddr.getPointer(),
1758  AO, Failure);
1759  CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1760  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1761 }
1762 
1763 void AtomicInfo::EmitAtomicUpdateOp(
1764  llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1765  bool IsVolatile) {
1766  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1767 
1768  // Do the atomic load.
1769  auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
1770  // For non-simple lvalues perform compare-and-swap procedure.
1771  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1772  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1773  auto *CurBB = CGF.Builder.GetInsertBlock();
1774  CGF.EmitBlock(ContBB);
1775  llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1776  /*NumReservedValues=*/2);
1777  PHI->addIncoming(OldVal, CurBB);
1778  Address NewAtomicAddr = CreateTempAlloca();
1779  Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1780  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1781  requiresMemSetZero(getAtomicAddress().getElementType())) {
1782  CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1783  }
1784  auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(),
1785  SourceLocation(), /*AsValue=*/false);
1786  EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
1787  auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1788  // Try to write new value using cmpxchg operation
1789  auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1790  PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1791  CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1792  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1793 }
1794 
1795 static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1796  RValue UpdateRVal, Address DesiredAddr) {
1797  LValue AtomicLVal = Atomics.getAtomicLValue();
1798  LValue DesiredLVal;
1799  // Build new lvalue for temp address
1800  if (AtomicLVal.isBitField()) {
1801  DesiredLVal =
1802  LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1803  AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1804  AtomicLVal.getTBAAInfo());
1805  } else if (AtomicLVal.isVectorElt()) {
1806  DesiredLVal =
1807  LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(),
1808  AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1809  AtomicLVal.getTBAAInfo());
1810  } else {
1811  assert(AtomicLVal.isExtVectorElt());
1812  DesiredLVal = LValue::MakeExtVectorElt(
1813  DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1814  AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1815  }
1816  // Store new value in the corresponding memory area
1817  assert(UpdateRVal.isScalar());
1818  CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);
1819 }
1820 
1821 void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1822  RValue UpdateRVal, bool IsVolatile) {
1823  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1824 
1825  Address ExpectedAddr = CreateTempAlloca();
1826 
1827  EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1828  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1829  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1830  CGF.EmitBlock(ContBB);
1831  Address DesiredAddr = CreateTempAlloca();
1832  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1833  requiresMemSetZero(getAtomicAddress().getElementType())) {
1834  auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1835  CGF.Builder.CreateStore(OldVal, DesiredAddr);
1836  }
1837  EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
1838  auto *Res =
1839  EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1840  DesiredAddr.getPointer(),
1841  AO, Failure);
1842  CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1843  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1844 }
1845 
1846 void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
1847  bool IsVolatile) {
1848  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1849 
1850  // Do the atomic load.
1851  auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
1852  // For non-simple lvalues perform compare-and-swap procedure.
1853  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1854  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1855  auto *CurBB = CGF.Builder.GetInsertBlock();
1856  CGF.EmitBlock(ContBB);
1857  llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1858  /*NumReservedValues=*/2);
1859  PHI->addIncoming(OldVal, CurBB);
1860  Address NewAtomicAddr = CreateTempAlloca();
1861  Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1862  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1863  requiresMemSetZero(getAtomicAddress().getElementType())) {
1864  CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1865  }
1866  EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
1867  auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1868  // Try to write new value using cmpxchg operation
1869  auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1870  PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1871  CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1872  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1873 }
1874 
1875 void AtomicInfo::EmitAtomicUpdate(
1876  llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1877  bool IsVolatile) {
1878  if (shouldUseLibcall()) {
1879  EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
1880  } else {
1881  EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
1882  }
1883 }
1884 
1885 void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
1886  bool IsVolatile) {
1887  if (shouldUseLibcall()) {
1888  EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
1889  } else {
1890  EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
1891  }
1892 }
1893 
1895  bool isInit) {
1896  bool IsVolatile = lvalue.isVolatileQualified();
1897  llvm::AtomicOrdering AO;
1898  if (lvalue.getType()->isAtomicType()) {
1899  AO = llvm::AtomicOrdering::SequentiallyConsistent;
1900  } else {
1901  AO = llvm::AtomicOrdering::Release;
1902  IsVolatile = true;
1903  }
1904  return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
1905 }
1906 
1907 /// Emit a store to an l-value of atomic type.
1908 ///
1909 /// Note that the r-value is expected to be an r-value *of the atomic
1910 /// type*; this means that for aggregate r-values, it should include
1911 /// storage for any padding that was necessary.
1913  llvm::AtomicOrdering AO, bool IsVolatile,
1914  bool isInit) {
1915  // If this is an aggregate r-value, it should agree in type except
1916  // maybe for address-space qualification.
1917  assert(!rvalue.isAggregate() ||
1919  == dest.getAddress().getElementType());
1920 
1921  AtomicInfo atomics(*this, dest);
1922  LValue LVal = atomics.getAtomicLValue();
1923 
1924  // If this is an initialization, just put the value there normally.
1925  if (LVal.isSimple()) {
1926  if (isInit) {
1927  atomics.emitCopyIntoMemory(rvalue);
1928  return;
1929  }
1930 
1931  // Check whether we should use a library call.
1932  if (atomics.shouldUseLibcall()) {
1933  // Produce a source address.
1934  Address srcAddr = atomics.materializeRValue(rvalue);
1935 
1936  // void __atomic_store(size_t size, void *mem, void *val, int order)
1937  CallArgList args;
1938  args.add(RValue::get(atomics.getAtomicSizeValue()),
1939  getContext().getSizeType());
1940  args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())),
1941  getContext().VoidPtrTy);
1942  args.add(RValue::get(EmitCastToVoidPtr(srcAddr.getPointer())),
1943  getContext().VoidPtrTy);
1944  args.add(
1945  RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))),
1946  getContext().IntTy);
1947  emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
1948  return;
1949  }
1950 
1951  // Okay, we're doing this natively.
1952  llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
1953 
1954  // Do the atomic store.
1955  Address addr =
1956  atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress());
1957  intValue = Builder.CreateIntCast(
1958  intValue, addr.getElementType(), /*isSigned=*/false);
1959  llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
1960 
1961  // Initializations don't need to be atomic.
1962  if (!isInit)
1963  store->setAtomic(AO);
1964 
1965  // Other decoration.
1966  if (IsVolatile)
1967  store->setVolatile(true);
1968  CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo());
1969  return;
1970  }
1971 
1972  // Emit simple atomic update operation.
1973  atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
1974 }
1975 
1976 /// Emit a compare-and-exchange op for atomic type.
1977 ///
1978 std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
1979  LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
1980  llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
1981  AggValueSlot Slot) {
1982  // If this is an aggregate r-value, it should agree in type except
1983  // maybe for address-space qualification.
1984  assert(!Expected.isAggregate() ||
1985  Expected.getAggregateAddress().getElementType() ==
1986  Obj.getAddress().getElementType());
1987  assert(!Desired.isAggregate() ||
1988  Desired.getAggregateAddress().getElementType() ==
1989  Obj.getAddress().getElementType());
1990  AtomicInfo Atomics(*this, Obj);
1991 
1992  return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
1993  IsWeak);
1994 }
1995 
1997  LValue LVal, llvm::AtomicOrdering AO,
1998  const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
1999  AtomicInfo Atomics(*this, LVal);
2000  Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
2001 }
2002 
2004  AtomicInfo atomics(*this, dest);
2005 
2006  switch (atomics.getEvaluationKind()) {
2007  case TEK_Scalar: {
2008  llvm::Value *value = EmitScalarExpr(init);
2009  atomics.emitCopyIntoMemory(RValue::get(value));
2010  return;
2011  }
2012 
2013  case TEK_Complex: {
2014  ComplexPairTy value = EmitComplexExpr(init);
2015  atomics.emitCopyIntoMemory(RValue::getComplex(value));
2016  return;
2017  }
2018 
2019  case TEK_Aggregate: {
2020  // Fix up the destination if the initializer isn't an expression
2021  // of atomic type.
2022  bool Zeroed = false;
2023  if (!init->getType()->isAtomicType()) {
2024  Zeroed = atomics.emitMemSetZeroIfNecessary();
2025  dest = atomics.projectValue();
2026  }
2027 
2028  // Evaluate the expression directly into the destination.
2034  Zeroed ? AggValueSlot::IsZeroed :
2036 
2037  EmitAggExpr(init, slot);
2038  return;
2039  }
2040  }
2041  llvm_unreachable("bad evaluation kind");
2042 }
bool isAggregate() const
Definition: CGValue.h:53
const llvm::DataLayout & getDataLayout() const
llvm::Value * getVectorPointer() const
Definition: CGValue.h:337
ReturnValueSlot - Contains the address where the return value of a function can be stored...
Definition: CGCall.h:363
Expr * getVal2() const
Definition: Expr.h:5692
Address CreateStructGEP(Address Addr, unsigned Index, const llvm::Twine &Name="")
Definition: CGBuilder.h:178
Defines the clang::ASTContext interface.
std::pair< RValue, llvm::Value * > EmitAtomicCompareExchange(LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, llvm::AtomicOrdering Success=llvm::AtomicOrdering::SequentiallyConsistent, llvm::AtomicOrdering Failure=llvm::AtomicOrdering::SequentiallyConsistent, bool IsWeak=false, AggValueSlot Slot=AggValueSlot::ignored())
Emit a compare-and-exchange op for atomic type.
Definition: CGAtomic.cpp:1978
llvm::IntegerType * IntTy
int
Address getAddress() const
Definition: CGValue.h:582
PointerType - C99 6.7.5.1 - Pointer Declarators.
Definition: Type.h:2547
CanQualType VoidPtrTy
Definition: ASTContext.h:1043
A (possibly-)qualified type.
Definition: Type.h:639
llvm::Type * ConvertTypeForMem(QualType T)
Address CreateMemTemp(QualType T, const Twine &Name="tmp", Address *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
Definition: CGExpr.cpp:138
llvm::LLVMContext & getLLVMContext()
void setAlignment(CharUnits A)
Definition: CGValue.h:316
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee...
Definition: Type.cpp:505
bool isVolatile() const
Definition: CGValue.h:300
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
Definition: CGExpr.cpp:1918
bool isZero() const
isZero - Test whether the quantity equals zero.
Definition: CharUnits.h:115
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
Definition: CGAtomic.cpp:1894
static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, Address Ptr, Address Val1, Address Val2, llvm::Value *IsWeak, llvm::Value *FailureOrder, uint64_t Size, llvm::AtomicOrdering Order, llvm::SyncScope::ID Scope)
Definition: CGAtomic.cpp:492
TypeEvaluationKind
The kind of evaluation to perform on values of a particular type.
const T * getAs() const
Member-template getAs<specific type>&#39;.
Definition: Type.h:6766
const void * Store
Store - This opaque type encapsulates an immutable mapping from locations to values.
Definition: StoreRef.h:27
static LValue MakeVectorElt(Address vecAddress, llvm::Value *Idx, QualType type, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo)
Definition: CGValue.h:379
Expr * getVal1() const
Definition: Expr.h:5682
llvm::Value * getPointer() const
Definition: Address.h:37
unsigned getAddressSpace() const
Return the address space that this address resides in.
Definition: Address.h:56
void add(RValue rvalue, QualType type)
Definition: CGCall.h:287
Address getAddress() const
Definition: CGValue.h:326
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition: ASTContext.h:154
bool isVolatileQualified() const
Definition: CGValue.h:257
CharUnits getAlignment() const
Definition: CGValue.h:315
void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO, const llvm::function_ref< RValue(RValue)> &UpdateOp, bool IsVolatile)
Definition: CGAtomic.cpp:1996
RValue EmitLoadOfExtVectorElementLValue(LValue V)
Definition: CGExpr.cpp:1843
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
unsigned Size
The total size of the bit-field, in bits.
CharUnits - This is an opaque type for sizes expressed in character units.
Definition: CharUnits.h:37
Expr * getPtr() const
Definition: Expr.h:5672
CharUnits StorageOffset
The offset of the bitfield storage from the start of the struct.
CharUnits getAlignment() const
Return the alignment of this pointer.
Definition: Address.h:66
static LValue MakeExtVectorElt(Address vecAddress, llvm::Constant *Elts, QualType type, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo)
Definition: CGValue.h:391
bool isComplex() const
Definition: CGValue.h:52
llvm::CallInst * CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
Definition: CGBuilder.h:274
Scope - A scope is a transient data structure that is used while parsing the program.
Definition: Scope.h:40
static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal, const llvm::function_ref< RValue(RValue)> &UpdateOp, Address DesiredAddr)
Definition: CGAtomic.cpp:1680
unsigned Align
Definition: ASTContext.h:144
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
void EmitAtomicInit(Expr *E, LValue lvalue)
Definition: CGAtomic.cpp:2003
unsigned Offset
The offset within a contiguous run of bitfields that are represented as a single "field" within the L...
bool isSimple() const
Definition: CGValue.h:251
RValue EmitAtomicExpr(AtomicExpr *E)
Definition: CGAtomic.cpp:744
RValue convertTempToRValue(Address addr, QualType type, SourceLocation Loc)
Given the address of a temporary variable, produce an r-value of its type.
Definition: CGExpr.cpp:4812
bool isCmpXChg() const
Definition: Expr.h:5716
Address getAggregateAddress() const
getAggregateAddr() - Return the Value* of the address of the aggregate.
Definition: CGValue.h:70
static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type, uint64_t expectedSize)
Does a store of the given IR type modify the full expected width?
Definition: CGAtomic.cpp:316
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
bool isVectorElt() const
Definition: CGValue.h:252
bool isValid() const
Definition: Address.h:35
Expr * getScope() const
Definition: Expr.h:5678
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
Definition: CGExpr.cpp:222
LValueBaseInfo getBaseInfo() const
Definition: CGValue.h:318
RValue - This trivial value class is used to represent the result of an expression that is evaluated...
Definition: CGValue.h:38
Address getExtVectorAddress() const
Definition: CGValue.h:341
static std::unique_ptr< AtomicScopeModel > getScopeModel(AtomicOp Op)
Get atomic scope model for the atomic op code.
Definition: Expr.h:5751
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
Definition: CharUnits.h:178
unsigned Offset
Definition: Format.cpp:1676
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
This represents one expression.
Definition: Expr.h:108
llvm::StringRef getAsString(SyncScope S)
Definition: SyncScope.h:50
static Address invalid()
Definition: Address.h:34
static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, Address Dest, Address Ptr, Address Val1, Address Val2, uint64_t Size, llvm::AtomicOrdering SuccessOrder, llvm::AtomicOrdering FailureOrder, llvm::SyncScope::ID Scope)
Definition: CGAtomic.cpp:358
std::pair< llvm::Value *, llvm::Value * > getComplexVal() const
getComplexVal - Return the real/imag components of this complex value.
Definition: CGValue.h:65
static CGCallee forDirect(llvm::Constant *functionPtr, const CGCalleeInfo &abstractInfo=CGCalleeInfo())
Definition: CGCall.h:133
llvm::PointerType * getType() const
Return the type of the pointer value.
Definition: Address.h:43
Expr * getOrder() const
Definition: Expr.h:5675
CharUnits getTypeAlignInChars(QualType T) const
Return the ABI-specified alignment of a (complete) type T, in characters.
llvm::Value * EmitToMemory(llvm::Value *Value, QualType Ty)
EmitToMemory - Change a scalar value from its value representation to its in-memory representation...
Definition: CGExpr.cpp:1682
static LValue MakeBitfield(Address Addr, const CGBitFieldInfo &Info, QualType type, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo)
Create a new object to represent a bit-field access.
Definition: CGValue.h:409
llvm::LLVMContext & getLLVMContext()
const CGFunctionInfo & arrangeBuiltinFunctionCall(QualType resultType, const CallArgList &args)
Definition: CGCall.cpp:639
bool isSignedIntegerType() const
Return true if this is an integer type that is signed, according to C99 6.2.5p4 [char, signed char, short, int, long..], or an enum decl which has a signed representation.
Definition: Type.cpp:1866
QualType getType() const
Definition: Expr.h:130
bool isVolatile() const
Definition: Expr.h:5712
TBAAAccessInfo getTBAAInfo() const
Definition: CGValue.h:307
llvm::CallInst * CreateMemSet(Address Dest, llvm::Value *Value, llvm::Value *Size, bool IsVolatile=false)
Definition: CGBuilder.h:296
Represents a GCC generic vector type.
Definition: Type.h:3178
AtomicOp getOp() const
Definition: Expr.h:5704
llvm::Value * EmitCastToVoidPtr(llvm::Value *value)
Emit a cast to void* in the appropriate address space.
Definition: CGExpr.cpp:49
RValue EmitLoadOfBitfieldLValue(LValue LV, SourceLocation Loc)
Definition: CGExpr.cpp:1811
RValue EmitAtomicLoad(LValue LV, SourceLocation SL, AggValueSlot Slot=AggValueSlot::ignored())
Definition: CGAtomic.cpp:1470
The l-value was considered opaque, so the alignment was determined from a type.
Address CreateBitCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
Definition: CGBuilder.h:141
Encodes a location in the source.
bool LValueIsSuitableForInlineAtomic(LValue Src)
An LValue is a candidate for having its loads and stores be made atomic if we are operating under /vo...
Definition: CGAtomic.cpp:1457
llvm::Value * EmitFromMemory(llvm::Value *Value, QualType Ty)
EmitFromMemory - Change a scalar value from its memory representation to its value representation...
Definition: CGExpr.cpp:1696
RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee, ReturnValueSlot ReturnValue, const CallArgList &Args, llvm::CallBase **callOrInvoke, SourceLocation Loc)
EmitCall - Generate a call of the given function, expecting the given result type, and using the given argument list which specifies both the LLVM arguments and the types they were derived from.
Definition: CGCall.cpp:3787
const CGBitFieldInfo & getBitFieldInfo() const
Definition: CGValue.h:358
An aggregate value slot.
Definition: CGValue.h:436
QualType getExtVectorType(QualType VectorType, unsigned NumElts) const
Return the unique reference to an extended vector type of the specified element type and size...
AtomicExpr - Variadic atomic builtins: __atomic_exchange, __atomic_fetch_*, __atomic_load, __atomic_store, and __atomic_compare_exchange_*, for the similarly-named C++11 instructions, and __c11 variants for <stdatomic.h>, and corresponding __opencl_atomic_* for OpenCL 2.0.
Definition: Expr.h:5640
CanQualType VoidTy
Definition: ASTContext.h:1015
static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, Address Dest, Address Ptr, Address Val1, Address Val2, llvm::Value *FailureOrderVal, uint64_t Size, llvm::AtomicOrdering SuccessOrder, llvm::SyncScope::ID Scope)
Given an ordering required on success, emit all possible cmpxchg instructions to cope with the provid...
Definition: CGAtomic.cpp:408
An aligned address.
Definition: Address.h:24
QualType getType() const
Definition: CGValue.h:263
const TargetCodeGenInfo & getTargetHooks() const
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
Definition: Expr.cpp:214
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
Definition: CGValue.h:58
CGFunctionInfo - Class to encapsulate the information about a function definition.
This class organizes the cross-function state that is used while generating LLVM code.
Dataflow Directional Tag Classes.
SourceLocation getBeginLoc() const LLVM_READONLY
Definition: Expr.h:5733
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
Definition: CGValue.h:92
static AggValueSlot ignored()
ignored - Returns an aggregate value slot indicating that the aggregate value is being ignored...
Definition: CGValue.h:498
bool isOpenCL() const
Definition: Expr.h:5725
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
Definition: CGBuilder.h:69
unsigned StorageSize
The storage size in bits which should be used when accessing this bitfield.
bool isBitField() const
Definition: CGValue.h:253
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
Definition: CGBuilder.h:107
static AggValueSlot forLValue(const LValue &LV, IsDestructed_t isDestructed, NeedsGCBarriers_t needsGC, IsAliased_t isAliased, Overlap_t mayOverlap, IsZeroed_t isZeroed=IsNotZeroed, IsSanitizerChecked_t isChecked=IsNotSanitizerChecked)
Definition: CGValue.h:539
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
llvm::Value * getExtVectorPointer() const
Definition: CGValue.h:344
Expr * getOrderFail() const
Definition: Expr.h:5688
bool isVolatileQualified() const
Definition: CGValue.h:55
static void AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args, bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy, SourceLocation Loc, CharUnits SizeInChars)
Definition: CGAtomic.cpp:720
void DecorateInstructionWithTBAA(llvm::Instruction *Inst, TBAAAccessInfo TBAAInfo)
DecorateInstructionWithTBAA - Decorate the instruction with a TBAA tag.
void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit)
EmitStoreOfComplex - Store a complex number into the specified l-value.
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Definition: Address.h:51
bool isAtomicType() const
Definition: Type.h:6416
llvm::FunctionCallee CreateRuntimeFunction(llvm::FunctionType *Ty, StringRef Name, llvm::AttributeList ExtraAttrs=llvm::AttributeList(), bool Local=false)
Create or return a runtime function declaration with the specified type and name. ...
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
Definition: CGStmt.cpp:450
llvm::ConstantInt * getSize(CharUnits numChars)
Emit the given number of characters as a value of type size_t.
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
int64_t toBits(CharUnits CharSize) const
Convert a size in characters to a size in bits.
virtual llvm::SyncScope::ID getLLVMSyncScopeID(SyncScope S, llvm::LLVMContext &C) const
Get the syncscope used in LLVM IR.
Definition: TargetInfo.cpp:465
QualType getUnqualifiedType() const
Retrieve the unqualified variant of the given type, removing as little sugar as possible.
Definition: Type.h:6162
llvm::Value * getBitFieldPointer() const
Definition: CGValue.h:357
true
A convenience builder class for complex constant initializers, especially for anonymous global struct...
bool isVoidType() const
Definition: Type.h:6558
Qualifiers getQualifiers() const
Retrieve the set of qualifiers applied to this type.
Definition: Type.h:6109
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
Definition: CGExpr.cpp:1765
bool isGlobalReg() const
Definition: CGValue.h:255
uint64_t Width
Definition: ASTContext.h:143
CanQualType IntTy
Definition: ASTContext.h:1024
void EmitAggregateCopy(LValue Dest, LValue Src, QualType EltTy, AggValueSlot::Overlap_t MayOverlap, bool isVolatile=false)
EmitAggregateCopy - Emit an aggregate copy.
Definition: CGExprAgg.cpp:1867
static RValue get(llvm::Value *V)
Definition: CGValue.h:85
bool isPointerType() const
Definition: Type.h:6306
__DEVICE__ int min(int __a, int __b)
bool isExtVectorElt() const
Definition: CGValue.h:254
static RValue emitAtomicLibcall(CodeGenFunction &CGF, StringRef fnName, QualType resultType, CallArgList &args)
Definition: CGAtomic.cpp:303
static RValue getAggregate(Address addr, bool isVolatile=false)
Definition: CGValue.h:106
LValue - This represents an lvalue references.
Definition: CGValue.h:166
RValue asRValue() const
Definition: CGValue.h:606
CanQualType BoolTy
Definition: ASTContext.h:1016
QualType getValueType() const
Definition: Expr.cpp:4285
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
Definition: CGBuilder.h:163
static LValue MakeAddr(Address address, QualType type, ASTContext &Context, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo)
Definition: CGValue.h:366
llvm::Value * getVectorIdx() const
Definition: CGValue.h:338
CallArgList - Type for representing both the value and type of arguments in a call.
Definition: CGCall.h:262
static Address EmitValToTemp(CodeGenFunction &CGF, Expr *E)
Definition: CGAtomic.cpp:657
llvm::Value * getPointer() const
Definition: CGValue.h:322
bool isScalar() const
Definition: CGValue.h:51
Expr * getWeak() const
Definition: Expr.h:5698
llvm::Constant * getExtVectorElts() const
Definition: CGValue.h:348
Structure with information about how a bitfield should be accessed.
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth, signed/unsigned.
CanQualType getSizeType() const
Return the unique type for "size_t" (C99 7.17), defined in <stddef.h>.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
Definition: CGCall.cpp:1541