clang  15.0.0git
CGAtomic.cpp
Go to the documentation of this file.
1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the code for emitting atomic operations.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "CGCall.h"
14 #include "CGRecordLayout.h"
15 #include "CodeGenFunction.h"
16 #include "CodeGenModule.h"
17 #include "TargetInfo.h"
18 #include "clang/AST/ASTContext.h"
21 #include "llvm/ADT/DenseMap.h"
22 #include "llvm/IR/DataLayout.h"
23 #include "llvm/IR/Intrinsics.h"
24 #include "llvm/IR/Operator.h"
25 
26 using namespace clang;
27 using namespace CodeGen;
28 
29 namespace {
30  class AtomicInfo {
31  CodeGenFunction &CGF;
32  QualType AtomicTy;
33  QualType ValueTy;
34  uint64_t AtomicSizeInBits;
35  uint64_t ValueSizeInBits;
36  CharUnits AtomicAlign;
37  CharUnits ValueAlign;
38  TypeEvaluationKind EvaluationKind;
39  bool UseLibcall;
40  LValue LVal;
41  CGBitFieldInfo BFI;
42  public:
43  AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
44  : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
45  EvaluationKind(TEK_Scalar), UseLibcall(true) {
46  assert(!lvalue.isGlobalReg());
47  ASTContext &C = CGF.getContext();
48  if (lvalue.isSimple()) {
49  AtomicTy = lvalue.getType();
50  if (auto *ATy = AtomicTy->getAs<AtomicType>())
51  ValueTy = ATy->getValueType();
52  else
53  ValueTy = AtomicTy;
54  EvaluationKind = CGF.getEvaluationKind(ValueTy);
55 
56  uint64_t ValueAlignInBits;
57  uint64_t AtomicAlignInBits;
58  TypeInfo ValueTI = C.getTypeInfo(ValueTy);
59  ValueSizeInBits = ValueTI.Width;
60  ValueAlignInBits = ValueTI.Align;
61 
62  TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
63  AtomicSizeInBits = AtomicTI.Width;
64  AtomicAlignInBits = AtomicTI.Align;
65 
66  assert(ValueSizeInBits <= AtomicSizeInBits);
67  assert(ValueAlignInBits <= AtomicAlignInBits);
68 
69  AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
70  ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
71  if (lvalue.getAlignment().isZero())
72  lvalue.setAlignment(AtomicAlign);
73 
74  LVal = lvalue;
75  } else if (lvalue.isBitField()) {
76  ValueTy = lvalue.getType();
77  ValueSizeInBits = C.getTypeSize(ValueTy);
78  auto &OrigBFI = lvalue.getBitFieldInfo();
79  auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
80  AtomicSizeInBits = C.toBits(
81  C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
82  .alignTo(lvalue.getAlignment()));
83  auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer());
84  auto OffsetInChars =
85  (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
86  lvalue.getAlignment();
87  VoidPtrAddr = CGF.Builder.CreateConstGEP1_64(
88  CGF.Int8Ty, VoidPtrAddr, OffsetInChars.getQuantity());
89  llvm::Type *IntTy = CGF.Builder.getIntNTy(AtomicSizeInBits);
91  VoidPtrAddr, IntTy->getPointerTo(), "atomic_bitfield_base");
92  BFI = OrigBFI;
93  BFI.Offset = Offset;
94  BFI.StorageSize = AtomicSizeInBits;
95  BFI.StorageOffset += OffsetInChars;
96  LVal = LValue::MakeBitfield(Address(Addr, IntTy, lvalue.getAlignment()),
97  BFI, lvalue.getType(), lvalue.getBaseInfo(),
98  lvalue.getTBAAInfo());
99  AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
100  if (AtomicTy.isNull()) {
102  /*numBits=*/32,
103  C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
104  AtomicTy =
105  C.getConstantArrayType(C.CharTy, Size, nullptr, ArrayType::Normal,
106  /*IndexTypeQuals=*/0);
107  }
108  AtomicAlign = ValueAlign = lvalue.getAlignment();
109  } else if (lvalue.isVectorElt()) {
110  ValueTy = lvalue.getType()->castAs<VectorType>()->getElementType();
111  ValueSizeInBits = C.getTypeSize(ValueTy);
112  AtomicTy = lvalue.getType();
113  AtomicSizeInBits = C.getTypeSize(AtomicTy);
114  AtomicAlign = ValueAlign = lvalue.getAlignment();
115  LVal = lvalue;
116  } else {
117  assert(lvalue.isExtVectorElt());
118  ValueTy = lvalue.getType();
119  ValueSizeInBits = C.getTypeSize(ValueTy);
120  AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
121  lvalue.getType(), cast<llvm::FixedVectorType>(
123  ->getNumElements());
124  AtomicSizeInBits = C.getTypeSize(AtomicTy);
125  AtomicAlign = ValueAlign = lvalue.getAlignment();
126  LVal = lvalue;
127  }
128  UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
129  AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
130  }
131 
132  QualType getAtomicType() const { return AtomicTy; }
133  QualType getValueType() const { return ValueTy; }
134  CharUnits getAtomicAlignment() const { return AtomicAlign; }
135  uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
136  uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
137  TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
138  bool shouldUseLibcall() const { return UseLibcall; }
139  const LValue &getAtomicLValue() const { return LVal; }
140  llvm::Value *getAtomicPointer() const {
141  if (LVal.isSimple())
142  return LVal.getPointer(CGF);
143  else if (LVal.isBitField())
144  return LVal.getBitFieldPointer();
145  else if (LVal.isVectorElt())
146  return LVal.getVectorPointer();
147  assert(LVal.isExtVectorElt());
148  return LVal.getExtVectorPointer();
149  }
150  Address getAtomicAddress() const {
151  llvm::Type *ElTy;
152  if (LVal.isSimple())
153  ElTy = LVal.getAddress(CGF).getElementType();
154  else if (LVal.isBitField())
155  ElTy = LVal.getBitFieldAddress().getElementType();
156  else if (LVal.isVectorElt())
157  ElTy = LVal.getVectorAddress().getElementType();
158  else
159  ElTy = LVal.getExtVectorAddress().getElementType();
160  return Address(getAtomicPointer(), ElTy, getAtomicAlignment());
161  }
162 
163  Address getAtomicAddressAsAtomicIntPointer() const {
164  return emitCastToAtomicIntPointer(getAtomicAddress());
165  }
166 
167  /// Is the atomic size larger than the underlying value type?
168  ///
169  /// Note that the absence of padding does not mean that atomic
170  /// objects are completely interchangeable with non-atomic
171  /// objects: we might have promoted the alignment of a type
172  /// without making it bigger.
173  bool hasPadding() const {
174  return (ValueSizeInBits != AtomicSizeInBits);
175  }
176 
177  bool emitMemSetZeroIfNecessary() const;
178 
179  llvm::Value *getAtomicSizeValue() const {
180  CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
181  return CGF.CGM.getSize(size);
182  }
183 
184  /// Cast the given pointer to an integer pointer suitable for atomic
185  /// operations if the source.
186  Address emitCastToAtomicIntPointer(Address Addr) const;
187 
188  /// If Addr is compatible with the iN that will be used for an atomic
189  /// operation, bitcast it. Otherwise, create a temporary that is suitable
190  /// and copy the value across.
191  Address convertToAtomicIntPointer(Address Addr) const;
192 
193  /// Turn an atomic-layout object into an r-value.
194  RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
195  SourceLocation loc, bool AsValue) const;
196 
197  /// Converts a rvalue to integer value.
198  llvm::Value *convertRValueToInt(RValue RVal) const;
199 
200  RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
201  AggValueSlot ResultSlot,
202  SourceLocation Loc, bool AsValue) const;
203 
204  /// Copy an atomic r-value into atomic-layout memory.
205  void emitCopyIntoMemory(RValue rvalue) const;
206 
207  /// Project an l-value down to the value field.
208  LValue projectValue() const {
209  assert(LVal.isSimple());
210  Address addr = getAtomicAddress();
211  if (hasPadding())
212  addr = CGF.Builder.CreateStructGEP(addr, 0);
213 
214  return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
215  LVal.getBaseInfo(), LVal.getTBAAInfo());
216  }
217 
218  /// Emits atomic load.
219  /// \returns Loaded value.
220  RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
221  bool AsValue, llvm::AtomicOrdering AO,
222  bool IsVolatile);
223 
224  /// Emits atomic compare-and-exchange sequence.
225  /// \param Expected Expected value.
226  /// \param Desired Desired value.
227  /// \param Success Atomic ordering for success operation.
228  /// \param Failure Atomic ordering for failed operation.
229  /// \param IsWeak true if atomic operation is weak, false otherwise.
230  /// \returns Pair of values: previous value from storage (value type) and
231  /// boolean flag (i1 type) with true if success and false otherwise.
232  std::pair<RValue, llvm::Value *>
233  EmitAtomicCompareExchange(RValue Expected, RValue Desired,
234  llvm::AtomicOrdering Success =
235  llvm::AtomicOrdering::SequentiallyConsistent,
236  llvm::AtomicOrdering Failure =
237  llvm::AtomicOrdering::SequentiallyConsistent,
238  bool IsWeak = false);
239 
240  /// Emits atomic update.
241  /// \param AO Atomic ordering.
242  /// \param UpdateOp Update operation for the current lvalue.
243  void EmitAtomicUpdate(llvm::AtomicOrdering AO,
244  const llvm::function_ref<RValue(RValue)> &UpdateOp,
245  bool IsVolatile);
246  /// Emits atomic update.
247  /// \param AO Atomic ordering.
248  void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
249  bool IsVolatile);
250 
251  /// Materialize an atomic r-value in atomic-layout memory.
252  Address materializeRValue(RValue rvalue) const;
253 
254  /// Creates temp alloca for intermediate operations on atomic value.
255  Address CreateTempAlloca() const;
256  private:
257  bool requiresMemSetZero(llvm::Type *type) const;
258 
259 
260  /// Emits atomic load as a libcall.
261  void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
262  llvm::AtomicOrdering AO, bool IsVolatile);
263  /// Emits atomic load as LLVM instruction.
264  llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
265  /// Emits atomic compare-and-exchange op as a libcall.
266  llvm::Value *EmitAtomicCompareExchangeLibcall(
267  llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
268  llvm::AtomicOrdering Success =
269  llvm::AtomicOrdering::SequentiallyConsistent,
270  llvm::AtomicOrdering Failure =
271  llvm::AtomicOrdering::SequentiallyConsistent);
272  /// Emits atomic compare-and-exchange op as LLVM instruction.
273  std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
274  llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
275  llvm::AtomicOrdering Success =
276  llvm::AtomicOrdering::SequentiallyConsistent,
277  llvm::AtomicOrdering Failure =
278  llvm::AtomicOrdering::SequentiallyConsistent,
279  bool IsWeak = false);
280  /// Emit atomic update as libcalls.
281  void
282  EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
283  const llvm::function_ref<RValue(RValue)> &UpdateOp,
284  bool IsVolatile);
285  /// Emit atomic update as LLVM instructions.
286  void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
287  const llvm::function_ref<RValue(RValue)> &UpdateOp,
288  bool IsVolatile);
289  /// Emit atomic update as libcalls.
290  void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
291  bool IsVolatile);
292  /// Emit atomic update as LLVM instructions.
293  void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
294  bool IsVolatile);
295  };
296 }
297 
298 Address AtomicInfo::CreateTempAlloca() const {
299  Address TempAlloca = CGF.CreateMemTemp(
300  (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
301  : AtomicTy,
302  getAtomicAlignment(),
303  "atomic-temp");
304  // Cast to pointer to value type for bitfields.
305  if (LVal.isBitField())
307  TempAlloca, getAtomicAddress().getType(),
308  getAtomicAddress().getElementType());
309  return TempAlloca;
310 }
311 
313  StringRef fnName,
314  QualType resultType,
315  CallArgList &args) {
316  const CGFunctionInfo &fnInfo =
317  CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args);
318  llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
319  llvm::AttrBuilder fnAttrB(CGF.getLLVMContext());
320  fnAttrB.addAttribute(llvm::Attribute::NoUnwind);
321  fnAttrB.addAttribute(llvm::Attribute::WillReturn);
322  llvm::AttributeList fnAttrs = llvm::AttributeList::get(
323  CGF.getLLVMContext(), llvm::AttributeList::FunctionIndex, fnAttrB);
324 
325  llvm::FunctionCallee fn =
326  CGF.CGM.CreateRuntimeFunction(fnTy, fnName, fnAttrs);
327  auto callee = CGCallee::forDirect(fn);
328  return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args);
329 }
330 
331 /// Does a store of the given IR type modify the full expected width?
332 static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
333  uint64_t expectedSize) {
334  return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
335 }
336 
337 /// Does the atomic type require memsetting to zero before initialization?
338 ///
339 /// The IR type is provided as a way of making certain queries faster.
340 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
341  // If the atomic type has size padding, we definitely need a memset.
342  if (hasPadding()) return true;
343 
344  // Otherwise, do some simple heuristics to try to avoid it:
345  switch (getEvaluationKind()) {
346  // For scalars and complexes, check whether the store size of the
347  // type uses the full size.
348  case TEK_Scalar:
349  return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
350  case TEK_Complex:
351  return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
352  AtomicSizeInBits / 2);
353 
354  // Padding in structs has an undefined bit pattern. User beware.
355  case TEK_Aggregate:
356  return false;
357  }
358  llvm_unreachable("bad evaluation kind");
359 }
360 
361 bool AtomicInfo::emitMemSetZeroIfNecessary() const {
362  assert(LVal.isSimple());
363  Address addr = LVal.getAddress(CGF);
364  if (!requiresMemSetZero(addr.getElementType()))
365  return false;
366 
367  CGF.Builder.CreateMemSet(
368  addr.getPointer(), llvm::ConstantInt::get(CGF.Int8Ty, 0),
369  CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
370  LVal.getAlignment().getAsAlign());
371  return true;
372 }
373 
374 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
375  Address Dest, Address Ptr,
376  Address Val1, Address Val2,
377  uint64_t Size,
378  llvm::AtomicOrdering SuccessOrder,
379  llvm::AtomicOrdering FailureOrder,
381  // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
382  llvm::Value *Expected = CGF.Builder.CreateLoad(Val1);
383  llvm::Value *Desired = CGF.Builder.CreateLoad(Val2);
384 
385  llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
386  Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder,
387  Scope);
388  Pair->setVolatile(E->isVolatile());
389  Pair->setWeak(IsWeak);
390 
391  // Cmp holds the result of the compare-exchange operation: true on success,
392  // false on failure.
393  llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
394  llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
395 
396  // This basic block is used to hold the store instruction if the operation
397  // failed.
398  llvm::BasicBlock *StoreExpectedBB =
399  CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
400 
401  // This basic block is the exit point of the operation, we should end up
402  // here regardless of whether or not the operation succeeded.
403  llvm::BasicBlock *ContinueBB =
404  CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
405 
406  // Update Expected if Expected isn't equal to Old, otherwise branch to the
407  // exit point.
408  CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
409 
410  CGF.Builder.SetInsertPoint(StoreExpectedBB);
411  // Update the memory at Expected with Old's value.
412  CGF.Builder.CreateStore(Old, Val1);
413  // Finally, branch to the exit point.
414  CGF.Builder.CreateBr(ContinueBB);
415 
416  CGF.Builder.SetInsertPoint(ContinueBB);
417  // Update the memory at Dest with Cmp's value.
418  CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
419 }
420 
421 /// Given an ordering required on success, emit all possible cmpxchg
422 /// instructions to cope with the provided (but possibly only dynamically known)
423 /// FailureOrder.
425  bool IsWeak, Address Dest, Address Ptr,
426  Address Val1, Address Val2,
427  llvm::Value *FailureOrderVal,
428  uint64_t Size,
429  llvm::AtomicOrdering SuccessOrder,
431  llvm::AtomicOrdering FailureOrder;
432  if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
433  auto FOS = FO->getSExtValue();
434  if (!llvm::isValidAtomicOrderingCABI(FOS))
435  FailureOrder = llvm::AtomicOrdering::Monotonic;
436  else
437  switch ((llvm::AtomicOrderingCABI)FOS) {
438  case llvm::AtomicOrderingCABI::relaxed:
439  // 31.7.2.18: "The failure argument shall not be memory_order_release
440  // nor memory_order_acq_rel". Fallback to monotonic.
441  case llvm::AtomicOrderingCABI::release:
442  case llvm::AtomicOrderingCABI::acq_rel:
443  FailureOrder = llvm::AtomicOrdering::Monotonic;
444  break;
445  case llvm::AtomicOrderingCABI::consume:
446  case llvm::AtomicOrderingCABI::acquire:
447  FailureOrder = llvm::AtomicOrdering::Acquire;
448  break;
449  case llvm::AtomicOrderingCABI::seq_cst:
450  FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent;
451  break;
452  }
453  // Prior to c++17, "the failure argument shall be no stronger than the
454  // success argument". This condition has been lifted and the only
455  // precondition is 31.7.2.18. Effectively treat this as a DR and skip
456  // language version checks.
457  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
458  FailureOrder, Scope);
459  return;
460  }
461 
462  // Create all the relevant BB's
463  auto *MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
464  auto *AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
465  auto *SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
466  auto *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
467 
468  // MonotonicBB is arbitrarily chosen as the default case; in practice, this
469  // doesn't matter unless someone is crazy enough to use something that
470  // doesn't fold to a constant for the ordering.
471  llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
472  // Implemented as acquire, since it's the closest in LLVM.
473  SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
474  AcquireBB);
475  SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
476  AcquireBB);
477  SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
478  SeqCstBB);
479 
480  // Emit all the different atomics
481  CGF.Builder.SetInsertPoint(MonotonicBB);
482  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
483  Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope);
484  CGF.Builder.CreateBr(ContBB);
485 
486  CGF.Builder.SetInsertPoint(AcquireBB);
487  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
488  llvm::AtomicOrdering::Acquire, Scope);
489  CGF.Builder.CreateBr(ContBB);
490 
491  CGF.Builder.SetInsertPoint(SeqCstBB);
492  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
493  llvm::AtomicOrdering::SequentiallyConsistent, Scope);
494  CGF.Builder.CreateBr(ContBB);
495 
496  CGF.Builder.SetInsertPoint(ContBB);
497 }
498 
499 /// Duplicate the atomic min/max operation in conventional IR for the builtin
500 /// variants that return the new rather than the original value.
501 static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder,
503  bool IsSigned,
504  llvm::Value *OldVal,
505  llvm::Value *RHS) {
506  llvm::CmpInst::Predicate Pred;
507  switch (Op) {
508  default:
509  llvm_unreachable("Unexpected min/max operation");
510  case AtomicExpr::AO__atomic_max_fetch:
511  Pred = IsSigned ? llvm::CmpInst::ICMP_SGT : llvm::CmpInst::ICMP_UGT;
512  break;
513  case AtomicExpr::AO__atomic_min_fetch:
514  Pred = IsSigned ? llvm::CmpInst::ICMP_SLT : llvm::CmpInst::ICMP_ULT;
515  break;
516  }
517  llvm::Value *Cmp = Builder.CreateICmp(Pred, OldVal, RHS, "tst");
518  return Builder.CreateSelect(Cmp, OldVal, RHS, "newval");
519 }
520 
521 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
522  Address Ptr, Address Val1, Address Val2,
523  llvm::Value *IsWeak, llvm::Value *FailureOrder,
524  uint64_t Size, llvm::AtomicOrdering Order,
526  llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
527  bool PostOpMinMax = false;
528  unsigned PostOp = 0;
529 
530  switch (E->getOp()) {
531  case AtomicExpr::AO__c11_atomic_init:
532  case AtomicExpr::AO__opencl_atomic_init:
533  llvm_unreachable("Already handled!");
534 
535  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
536  case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
537  case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
538  emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
539  FailureOrder, Size, Order, Scope);
540  return;
541  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
542  case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
543  case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
544  emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
545  FailureOrder, Size, Order, Scope);
546  return;
547  case AtomicExpr::AO__atomic_compare_exchange:
548  case AtomicExpr::AO__atomic_compare_exchange_n: {
549  if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
550  emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
551  Val1, Val2, FailureOrder, Size, Order, Scope);
552  } else {
553  // Create all the relevant BB's
554  llvm::BasicBlock *StrongBB =
555  CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
556  llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
557  llvm::BasicBlock *ContBB =
558  CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
559 
560  llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
561  SI->addCase(CGF.Builder.getInt1(false), StrongBB);
562 
563  CGF.Builder.SetInsertPoint(StrongBB);
564  emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
565  FailureOrder, Size, Order, Scope);
566  CGF.Builder.CreateBr(ContBB);
567 
568  CGF.Builder.SetInsertPoint(WeakBB);
569  emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
570  FailureOrder, Size, Order, Scope);
571  CGF.Builder.CreateBr(ContBB);
572 
573  CGF.Builder.SetInsertPoint(ContBB);
574  }
575  return;
576  }
577  case AtomicExpr::AO__c11_atomic_load:
578  case AtomicExpr::AO__opencl_atomic_load:
579  case AtomicExpr::AO__hip_atomic_load:
580  case AtomicExpr::AO__atomic_load_n:
581  case AtomicExpr::AO__atomic_load: {
582  llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
583  Load->setAtomic(Order, Scope);
584  Load->setVolatile(E->isVolatile());
585  CGF.Builder.CreateStore(Load, Dest);
586  return;
587  }
588 
589  case AtomicExpr::AO__c11_atomic_store:
590  case AtomicExpr::AO__opencl_atomic_store:
591  case AtomicExpr::AO__hip_atomic_store:
592  case AtomicExpr::AO__atomic_store:
593  case AtomicExpr::AO__atomic_store_n: {
594  llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
595  llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
596  Store->setAtomic(Order, Scope);
597  Store->setVolatile(E->isVolatile());
598  return;
599  }
600 
601  case AtomicExpr::AO__c11_atomic_exchange:
602  case AtomicExpr::AO__hip_atomic_exchange:
603  case AtomicExpr::AO__opencl_atomic_exchange:
604  case AtomicExpr::AO__atomic_exchange_n:
605  case AtomicExpr::AO__atomic_exchange:
606  Op = llvm::AtomicRMWInst::Xchg;
607  break;
608 
609  case AtomicExpr::AO__atomic_add_fetch:
610  PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FAdd
612  LLVM_FALLTHROUGH;
613  case AtomicExpr::AO__c11_atomic_fetch_add:
614  case AtomicExpr::AO__hip_atomic_fetch_add:
615  case AtomicExpr::AO__opencl_atomic_fetch_add:
616  case AtomicExpr::AO__atomic_fetch_add:
617  Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FAdd
619  break;
620 
621  case AtomicExpr::AO__atomic_sub_fetch:
622  PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FSub
624  LLVM_FALLTHROUGH;
625  case AtomicExpr::AO__c11_atomic_fetch_sub:
626  case AtomicExpr::AO__opencl_atomic_fetch_sub:
627  case AtomicExpr::AO__atomic_fetch_sub:
628  Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FSub
630  break;
631 
632  case AtomicExpr::AO__atomic_min_fetch:
633  PostOpMinMax = true;
634  LLVM_FALLTHROUGH;
635  case AtomicExpr::AO__c11_atomic_fetch_min:
636  case AtomicExpr::AO__hip_atomic_fetch_min:
637  case AtomicExpr::AO__opencl_atomic_fetch_min:
638  case AtomicExpr::AO__atomic_fetch_min:
639  Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Min
640  : llvm::AtomicRMWInst::UMin;
641  break;
642 
643  case AtomicExpr::AO__atomic_max_fetch:
644  PostOpMinMax = true;
645  LLVM_FALLTHROUGH;
646  case AtomicExpr::AO__c11_atomic_fetch_max:
647  case AtomicExpr::AO__hip_atomic_fetch_max:
648  case AtomicExpr::AO__opencl_atomic_fetch_max:
649  case AtomicExpr::AO__atomic_fetch_max:
650  Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Max
651  : llvm::AtomicRMWInst::UMax;
652  break;
653 
654  case AtomicExpr::AO__atomic_and_fetch:
655  PostOp = llvm::Instruction::And;
656  LLVM_FALLTHROUGH;
657  case AtomicExpr::AO__c11_atomic_fetch_and:
658  case AtomicExpr::AO__hip_atomic_fetch_and:
659  case AtomicExpr::AO__opencl_atomic_fetch_and:
660  case AtomicExpr::AO__atomic_fetch_and:
662  break;
663 
664  case AtomicExpr::AO__atomic_or_fetch:
665  PostOp = llvm::Instruction::Or;
666  LLVM_FALLTHROUGH;
667  case AtomicExpr::AO__c11_atomic_fetch_or:
668  case AtomicExpr::AO__hip_atomic_fetch_or:
669  case AtomicExpr::AO__opencl_atomic_fetch_or:
670  case AtomicExpr::AO__atomic_fetch_or:
671  Op = llvm::AtomicRMWInst::Or;
672  break;
673 
674  case AtomicExpr::AO__atomic_xor_fetch:
675  PostOp = llvm::Instruction::Xor;
676  LLVM_FALLTHROUGH;
677  case AtomicExpr::AO__c11_atomic_fetch_xor:
678  case AtomicExpr::AO__hip_atomic_fetch_xor:
679  case AtomicExpr::AO__opencl_atomic_fetch_xor:
680  case AtomicExpr::AO__atomic_fetch_xor:
681  Op = llvm::AtomicRMWInst::Xor;
682  break;
683 
684  case AtomicExpr::AO__atomic_nand_fetch:
685  PostOp = llvm::Instruction::And; // the NOT is special cased below
686  LLVM_FALLTHROUGH;
687  case AtomicExpr::AO__c11_atomic_fetch_nand:
688  case AtomicExpr::AO__atomic_fetch_nand:
689  Op = llvm::AtomicRMWInst::Nand;
690  break;
691  }
692 
693  llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
694  llvm::AtomicRMWInst *RMWI =
695  CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order, Scope);
696  RMWI->setVolatile(E->isVolatile());
697 
698  // For __atomic_*_fetch operations, perform the operation again to
699  // determine the value which was written.
700  llvm::Value *Result = RMWI;
701  if (PostOpMinMax)
702  Result = EmitPostAtomicMinMax(CGF.Builder, E->getOp(),
704  RMWI, LoadVal1);
705  else if (PostOp)
706  Result = CGF.Builder.CreateBinOp((llvm::Instruction::BinaryOps)PostOp, RMWI,
707  LoadVal1);
708  if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
709  Result = CGF.Builder.CreateNot(Result);
710  CGF.Builder.CreateStore(Result, Dest);
711 }
712 
713 // This function emits any expression (scalar, complex, or aggregate)
714 // into a temporary alloca.
715 static Address
717  Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
718  CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
719  /*Init*/ true);
720  return DeclPtr;
721 }
722 
724  Address Ptr, Address Val1, Address Val2,
725  llvm::Value *IsWeak, llvm::Value *FailureOrder,
726  uint64_t Size, llvm::AtomicOrdering Order,
727  llvm::Value *Scope) {
728  auto ScopeModel = Expr->getScopeModel();
729 
730  // LLVM atomic instructions always have synch scope. If clang atomic
731  // expression has no scope operand, use default LLVM synch scope.
732  if (!ScopeModel) {
733  EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
734  Order, CGF.CGM.getLLVMContext().getOrInsertSyncScopeID(""));
735  return;
736  }
737 
738  // Handle constant scope.
739  if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) {
740  auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID(
741  CGF.CGM.getLangOpts(), ScopeModel->map(SC->getZExtValue()),
742  Order, CGF.CGM.getLLVMContext());
743  EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
744  Order, SCID);
745  return;
746  }
747 
748  // Handle non-constant scope.
749  auto &Builder = CGF.Builder;
750  auto Scopes = ScopeModel->getRuntimeValues();
751  llvm::DenseMap<unsigned, llvm::BasicBlock *> BB;
752  for (auto S : Scopes)
753  BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn);
754 
755  llvm::BasicBlock *ContBB =
756  CGF.createBasicBlock("atomic.scope.continue", CGF.CurFn);
757 
758  auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false);
759  // If unsupported synch scope is encountered at run time, assume a fallback
760  // synch scope value.
761  auto FallBack = ScopeModel->getFallBackValue();
762  llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]);
763  for (auto S : Scopes) {
764  auto *B = BB[S];
765  if (S != FallBack)
766  SI->addCase(Builder.getInt32(S), B);
767 
768  Builder.SetInsertPoint(B);
769  EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
770  Order,
772  ScopeModel->map(S),
773  Order,
774  CGF.getLLVMContext()));
775  Builder.CreateBr(ContBB);
776  }
777 
778  Builder.SetInsertPoint(ContBB);
779 }
780 
781 static void
783  bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,
784  SourceLocation Loc, CharUnits SizeInChars) {
785  if (UseOptimizedLibcall) {
786  // Load value and pass it to the function directly.
787  CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy);
788  int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
789  ValTy =
790  CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
791  llvm::Type *ITy = llvm::IntegerType::get(CGF.getLLVMContext(), SizeInBits);
792  Address Ptr = Address(CGF.Builder.CreateBitCast(Val, ITy->getPointerTo()),
793  ITy, Align);
794  Val = CGF.EmitLoadOfScalar(Ptr, false,
795  CGF.getContext().getPointerType(ValTy),
796  Loc);
797  // Coerce the value into an appropriately sized integer type.
798  Args.add(RValue::get(Val), ValTy);
799  } else {
800  // Non-optimized functions always take a reference.
801  Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)),
802  CGF.getContext().VoidPtrTy);
803  }
804 }
805 
807  QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
808  QualType MemTy = AtomicTy;
809  if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
810  MemTy = AT->getValueType();
811  llvm::Value *IsWeak = nullptr, *OrderFail = nullptr;
812 
813  Address Val1 = Address::invalid();
814  Address Val2 = Address::invalid();
815  Address Dest = Address::invalid();
817 
818  if (E->getOp() == AtomicExpr::AO__c11_atomic_init ||
819  E->getOp() == AtomicExpr::AO__opencl_atomic_init) {
820  LValue lvalue = MakeAddrLValue(Ptr, AtomicTy);
821  EmitAtomicInit(E->getVal1(), lvalue);
822  return RValue::get(nullptr);
823  }
824 
825  auto TInfo = getContext().getTypeInfoInChars(AtomicTy);
826  uint64_t Size = TInfo.Width.getQuantity();
827  unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();
828 
829  bool Oversized = getContext().toBits(TInfo.Width) > MaxInlineWidthInBits;
830  bool Misaligned = (Ptr.getAlignment() % TInfo.Width) != 0;
831  bool UseLibcall = Misaligned | Oversized;
832  bool ShouldCastToIntPtrTy = true;
833 
834  CharUnits MaxInlineWidth =
835  getContext().toCharUnitsFromBits(MaxInlineWidthInBits);
836 
837  DiagnosticsEngine &Diags = CGM.getDiags();
838 
839  if (Misaligned) {
840  Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned)
841  << (int)TInfo.Width.getQuantity()
842  << (int)Ptr.getAlignment().getQuantity();
843  }
844 
845  if (Oversized) {
846  Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_oversized)
847  << (int)TInfo.Width.getQuantity() << (int)MaxInlineWidth.getQuantity();
848  }
849 
850  llvm::Value *Order = EmitScalarExpr(E->getOrder());
851  llvm::Value *Scope =
852  E->getScopeModel() ? EmitScalarExpr(E->getScope()) : nullptr;
853 
854  switch (E->getOp()) {
855  case AtomicExpr::AO__c11_atomic_init:
856  case AtomicExpr::AO__opencl_atomic_init:
857  llvm_unreachable("Already handled above with EmitAtomicInit!");
858 
859  case AtomicExpr::AO__c11_atomic_load:
860  case AtomicExpr::AO__opencl_atomic_load:
861  case AtomicExpr::AO__hip_atomic_load:
862  case AtomicExpr::AO__atomic_load_n:
863  break;
864 
865  case AtomicExpr::AO__atomic_load:
866  Dest = EmitPointerWithAlignment(E->getVal1());
867  break;
868 
869  case AtomicExpr::AO__atomic_store:
870  Val1 = EmitPointerWithAlignment(E->getVal1());
871  break;
872 
873  case AtomicExpr::AO__atomic_exchange:
874  Val1 = EmitPointerWithAlignment(E->getVal1());
875  Dest = EmitPointerWithAlignment(E->getVal2());
876  break;
877 
878  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
879  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
880  case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
881  case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
882  case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
883  case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
884  case AtomicExpr::AO__atomic_compare_exchange_n:
885  case AtomicExpr::AO__atomic_compare_exchange:
886  Val1 = EmitPointerWithAlignment(E->getVal1());
887  if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
888  Val2 = EmitPointerWithAlignment(E->getVal2());
889  else
890  Val2 = EmitValToTemp(*this, E->getVal2());
891  OrderFail = EmitScalarExpr(E->getOrderFail());
892  if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n ||
893  E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
894  IsWeak = EmitScalarExpr(E->getWeak());
895  break;
896 
897  case AtomicExpr::AO__c11_atomic_fetch_add:
898  case AtomicExpr::AO__c11_atomic_fetch_sub:
899  case AtomicExpr::AO__hip_atomic_fetch_add:
900  case AtomicExpr::AO__opencl_atomic_fetch_add:
901  case AtomicExpr::AO__opencl_atomic_fetch_sub:
902  if (MemTy->isPointerType()) {
903  // For pointer arithmetic, we're required to do a bit of math:
904  // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
905  // ... but only for the C11 builtins. The GNU builtins expect the
906  // user to multiply by sizeof(T).
907  QualType Val1Ty = E->getVal1()->getType();
908  llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
909  CharUnits PointeeIncAmt =
911  Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
912  auto Temp = CreateMemTemp(Val1Ty, ".atomictmp");
913  Val1 = Temp;
914  EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty));
915  break;
916  }
917  LLVM_FALLTHROUGH;
918  case AtomicExpr::AO__atomic_fetch_add:
919  case AtomicExpr::AO__atomic_fetch_sub:
920  case AtomicExpr::AO__atomic_add_fetch:
921  case AtomicExpr::AO__atomic_sub_fetch:
922  ShouldCastToIntPtrTy = !MemTy->isFloatingType();
923  LLVM_FALLTHROUGH;
924 
925  case AtomicExpr::AO__c11_atomic_store:
926  case AtomicExpr::AO__c11_atomic_exchange:
927  case AtomicExpr::AO__opencl_atomic_store:
928  case AtomicExpr::AO__hip_atomic_store:
929  case AtomicExpr::AO__opencl_atomic_exchange:
930  case AtomicExpr::AO__hip_atomic_exchange:
931  case AtomicExpr::AO__atomic_store_n:
932  case AtomicExpr::AO__atomic_exchange_n:
933  case AtomicExpr::AO__c11_atomic_fetch_and:
934  case AtomicExpr::AO__c11_atomic_fetch_or:
935  case AtomicExpr::AO__c11_atomic_fetch_xor:
936  case AtomicExpr::AO__c11_atomic_fetch_nand:
937  case AtomicExpr::AO__c11_atomic_fetch_max:
938  case AtomicExpr::AO__c11_atomic_fetch_min:
939  case AtomicExpr::AO__opencl_atomic_fetch_and:
940  case AtomicExpr::AO__opencl_atomic_fetch_or:
941  case AtomicExpr::AO__opencl_atomic_fetch_xor:
942  case AtomicExpr::AO__opencl_atomic_fetch_min:
943  case AtomicExpr::AO__opencl_atomic_fetch_max:
944  case AtomicExpr::AO__atomic_fetch_and:
945  case AtomicExpr::AO__hip_atomic_fetch_and:
946  case AtomicExpr::AO__atomic_fetch_or:
947  case AtomicExpr::AO__hip_atomic_fetch_or:
948  case AtomicExpr::AO__atomic_fetch_xor:
949  case AtomicExpr::AO__hip_atomic_fetch_xor:
950  case AtomicExpr::AO__atomic_fetch_nand:
951  case AtomicExpr::AO__atomic_and_fetch:
952  case AtomicExpr::AO__atomic_or_fetch:
953  case AtomicExpr::AO__atomic_xor_fetch:
954  case AtomicExpr::AO__atomic_nand_fetch:
955  case AtomicExpr::AO__atomic_max_fetch:
956  case AtomicExpr::AO__atomic_min_fetch:
957  case AtomicExpr::AO__atomic_fetch_max:
958  case AtomicExpr::AO__hip_atomic_fetch_max:
959  case AtomicExpr::AO__atomic_fetch_min:
960  case AtomicExpr::AO__hip_atomic_fetch_min:
961  Val1 = EmitValToTemp(*this, E->getVal1());
962  break;
963  }
964 
965  QualType RValTy = E->getType().getUnqualifiedType();
966 
967  // The inlined atomics only function on iN types, where N is a power of 2. We
968  // need to make sure (via temporaries if necessary) that all incoming values
969  // are compatible.
970  LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy);
971  AtomicInfo Atomics(*this, AtomicVal);
972 
973  if (ShouldCastToIntPtrTy) {
974  Ptr = Atomics.emitCastToAtomicIntPointer(Ptr);
975  if (Val1.isValid())
976  Val1 = Atomics.convertToAtomicIntPointer(Val1);
977  if (Val2.isValid())
978  Val2 = Atomics.convertToAtomicIntPointer(Val2);
979  }
980  if (Dest.isValid()) {
981  if (ShouldCastToIntPtrTy)
982  Dest = Atomics.emitCastToAtomicIntPointer(Dest);
983  } else if (E->isCmpXChg())
984  Dest = CreateMemTemp(RValTy, "cmpxchg.bool");
985  else if (!RValTy->isVoidType()) {
986  Dest = Atomics.CreateTempAlloca();
987  if (ShouldCastToIntPtrTy)
988  Dest = Atomics.emitCastToAtomicIntPointer(Dest);
989  }
990 
991  // Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
992  if (UseLibcall) {
993  bool UseOptimizedLibcall = false;
994  switch (E->getOp()) {
995  case AtomicExpr::AO__c11_atomic_init:
996  case AtomicExpr::AO__opencl_atomic_init:
997  llvm_unreachable("Already handled above with EmitAtomicInit!");
998 
999  case AtomicExpr::AO__c11_atomic_fetch_add:
1000  case AtomicExpr::AO__opencl_atomic_fetch_add:
1001  case AtomicExpr::AO__atomic_fetch_add:
1002  case AtomicExpr::AO__hip_atomic_fetch_add:
1003  case AtomicExpr::AO__c11_atomic_fetch_and:
1004  case AtomicExpr::AO__opencl_atomic_fetch_and:
1005  case AtomicExpr::AO__hip_atomic_fetch_and:
1006  case AtomicExpr::AO__atomic_fetch_and:
1007  case AtomicExpr::AO__c11_atomic_fetch_or:
1008  case AtomicExpr::AO__opencl_atomic_fetch_or:
1009  case AtomicExpr::AO__hip_atomic_fetch_or:
1010  case AtomicExpr::AO__atomic_fetch_or:
1011  case AtomicExpr::AO__c11_atomic_fetch_nand:
1012  case AtomicExpr::AO__atomic_fetch_nand:
1013  case AtomicExpr::AO__c11_atomic_fetch_sub:
1014  case AtomicExpr::AO__opencl_atomic_fetch_sub:
1015  case AtomicExpr::AO__atomic_fetch_sub:
1016  case AtomicExpr::AO__c11_atomic_fetch_xor:
1017  case AtomicExpr::AO__opencl_atomic_fetch_xor:
1018  case AtomicExpr::AO__opencl_atomic_fetch_min:
1019  case AtomicExpr::AO__opencl_atomic_fetch_max:
1020  case AtomicExpr::AO__atomic_fetch_xor:
1021  case AtomicExpr::AO__hip_atomic_fetch_xor:
1022  case AtomicExpr::AO__c11_atomic_fetch_max:
1023  case AtomicExpr::AO__c11_atomic_fetch_min:
1024  case AtomicExpr::AO__atomic_add_fetch:
1025  case AtomicExpr::AO__atomic_and_fetch:
1026  case AtomicExpr::AO__atomic_nand_fetch:
1027  case AtomicExpr::AO__atomic_or_fetch:
1028  case AtomicExpr::AO__atomic_sub_fetch:
1029  case AtomicExpr::AO__atomic_xor_fetch:
1030  case AtomicExpr::AO__atomic_fetch_max:
1031  case AtomicExpr::AO__hip_atomic_fetch_max:
1032  case AtomicExpr::AO__atomic_fetch_min:
1033  case AtomicExpr::AO__hip_atomic_fetch_min:
1034  case AtomicExpr::AO__atomic_max_fetch:
1035  case AtomicExpr::AO__atomic_min_fetch:
1036  // For these, only library calls for certain sizes exist.
1037  UseOptimizedLibcall = true;
1038  break;
1039 
1040  case AtomicExpr::AO__atomic_load:
1041  case AtomicExpr::AO__atomic_store:
1042  case AtomicExpr::AO__atomic_exchange:
1043  case AtomicExpr::AO__atomic_compare_exchange:
1044  // Use the generic version if we don't know that the operand will be
1045  // suitably aligned for the optimized version.
1046  if (Misaligned)
1047  break;
1048  LLVM_FALLTHROUGH;
1049  case AtomicExpr::AO__c11_atomic_load:
1050  case AtomicExpr::AO__c11_atomic_store:
1051  case AtomicExpr::AO__c11_atomic_exchange:
1052  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1053  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1054  case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
1055  case AtomicExpr::AO__opencl_atomic_load:
1056  case AtomicExpr::AO__hip_atomic_load:
1057  case AtomicExpr::AO__opencl_atomic_store:
1058  case AtomicExpr::AO__hip_atomic_store:
1059  case AtomicExpr::AO__opencl_atomic_exchange:
1060  case AtomicExpr::AO__hip_atomic_exchange:
1061  case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1062  case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
1063  case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1064  case AtomicExpr::AO__atomic_load_n:
1065  case AtomicExpr::AO__atomic_store_n:
1066  case AtomicExpr::AO__atomic_exchange_n:
1067  case AtomicExpr::AO__atomic_compare_exchange_n:
1068  // Only use optimized library calls for sizes for which they exist.
1069  // FIXME: Size == 16 optimized library functions exist too.
1070  if (Size == 1 || Size == 2 || Size == 4 || Size == 8)
1071  UseOptimizedLibcall = true;
1072  break;
1073  }
1074 
1075  CallArgList Args;
1076  if (!UseOptimizedLibcall) {
1077  // For non-optimized library calls, the size is the first parameter
1078  Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
1079  getContext().getSizeType());
1080  }
1081  // Atomic address is the first or second parameter
1082  // The OpenCL atomic library functions only accept pointer arguments to
1083  // generic address space.
1084  auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) {
1085  if (!E->isOpenCL())
1086  return V;
1087  auto AS = PT->castAs<PointerType>()->getPointeeType().getAddressSpace();
1088  if (AS == LangAS::opencl_generic)
1089  return V;
1091  auto T = llvm::cast<llvm::PointerType>(V->getType());
1092  auto *DestType = llvm::PointerType::getWithSamePointeeType(T, DestAS);
1093 
1095  *this, V, AS, LangAS::opencl_generic, DestType, false);
1096  };
1097 
1098  Args.add(RValue::get(CastToGenericAddrSpace(
1099  EmitCastToVoidPtr(Ptr.getPointer()), E->getPtr()->getType())),
1100  getContext().VoidPtrTy);
1101 
1102  std::string LibCallName;
1103  QualType LoweredMemTy =
1104  MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy;
1105  QualType RetTy;
1106  bool HaveRetTy = false;
1107  llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
1108  bool PostOpMinMax = false;
1109  switch (E->getOp()) {
1110  case AtomicExpr::AO__c11_atomic_init:
1111  case AtomicExpr::AO__opencl_atomic_init:
1112  llvm_unreachable("Already handled!");
1113 
1114  // There is only one libcall for compare an exchange, because there is no
1115  // optimisation benefit possible from a libcall version of a weak compare
1116  // and exchange.
1117  // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
1118  // void *desired, int success, int failure)
1119  // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired,
1120  // int success, int failure)
1121  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1122  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1123  case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1124  case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
1125  case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1126  case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
1127  case AtomicExpr::AO__atomic_compare_exchange:
1128  case AtomicExpr::AO__atomic_compare_exchange_n:
1129  LibCallName = "__atomic_compare_exchange";
1130  RetTy = getContext().BoolTy;
1131  HaveRetTy = true;
1132  Args.add(
1133  RValue::get(CastToGenericAddrSpace(
1134  EmitCastToVoidPtr(Val1.getPointer()), E->getVal1()->getType())),
1135  getContext().VoidPtrTy);
1136  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(),
1137  MemTy, E->getExprLoc(), TInfo.Width);
1138  Args.add(RValue::get(Order), getContext().IntTy);
1139  Order = OrderFail;
1140  break;
1141  // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
1142  // int order)
1143  // T __atomic_exchange_N(T *mem, T val, int order)
1144  case AtomicExpr::AO__c11_atomic_exchange:
1145  case AtomicExpr::AO__opencl_atomic_exchange:
1146  case AtomicExpr::AO__atomic_exchange_n:
1147  case AtomicExpr::AO__atomic_exchange:
1148  case AtomicExpr::AO__hip_atomic_exchange:
1149  LibCallName = "__atomic_exchange";
1150  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1151  MemTy, E->getExprLoc(), TInfo.Width);
1152  break;
1153  // void __atomic_store(size_t size, void *mem, void *val, int order)
1154  // void __atomic_store_N(T *mem, T val, int order)
1155  case AtomicExpr::AO__c11_atomic_store:
1156  case AtomicExpr::AO__opencl_atomic_store:
1157  case AtomicExpr::AO__hip_atomic_store:
1158  case AtomicExpr::AO__atomic_store:
1159  case AtomicExpr::AO__atomic_store_n:
1160  LibCallName = "__atomic_store";
1161  RetTy = getContext().VoidTy;
1162  HaveRetTy = true;
1163  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1164  MemTy, E->getExprLoc(), TInfo.Width);
1165  break;
1166  // void __atomic_load(size_t size, void *mem, void *return, int order)
1167  // T __atomic_load_N(T *mem, int order)
1168  case AtomicExpr::AO__c11_atomic_load:
1169  case AtomicExpr::AO__opencl_atomic_load:
1170  case AtomicExpr::AO__hip_atomic_load:
1171  case AtomicExpr::AO__atomic_load:
1172  case AtomicExpr::AO__atomic_load_n:
1173  LibCallName = "__atomic_load";
1174  break;
1175  // T __atomic_add_fetch_N(T *mem, T val, int order)
1176  // T __atomic_fetch_add_N(T *mem, T val, int order)
1177  case AtomicExpr::AO__atomic_add_fetch:
1178  PostOp = llvm::Instruction::Add;
1179  LLVM_FALLTHROUGH;
1180  case AtomicExpr::AO__c11_atomic_fetch_add:
1181  case AtomicExpr::AO__opencl_atomic_fetch_add:
1182  case AtomicExpr::AO__atomic_fetch_add:
1183  case AtomicExpr::AO__hip_atomic_fetch_add:
1184  LibCallName = "__atomic_fetch_add";
1185  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1186  LoweredMemTy, E->getExprLoc(), TInfo.Width);
1187  break;
1188  // T __atomic_and_fetch_N(T *mem, T val, int order)
1189  // T __atomic_fetch_and_N(T *mem, T val, int order)
1190  case AtomicExpr::AO__atomic_and_fetch:
1191  PostOp = llvm::Instruction::And;
1192  LLVM_FALLTHROUGH;
1193  case AtomicExpr::AO__c11_atomic_fetch_and:
1194  case AtomicExpr::AO__opencl_atomic_fetch_and:
1195  case AtomicExpr::AO__hip_atomic_fetch_and:
1196  case AtomicExpr::AO__atomic_fetch_and:
1197  LibCallName = "__atomic_fetch_and";
1198  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1199  MemTy, E->getExprLoc(), TInfo.Width);
1200  break;
1201  // T __atomic_or_fetch_N(T *mem, T val, int order)
1202  // T __atomic_fetch_or_N(T *mem, T val, int order)
1203  case AtomicExpr::AO__atomic_or_fetch:
1204  PostOp = llvm::Instruction::Or;
1205  LLVM_FALLTHROUGH;
1206  case AtomicExpr::AO__c11_atomic_fetch_or:
1207  case AtomicExpr::AO__opencl_atomic_fetch_or:
1208  case AtomicExpr::AO__hip_atomic_fetch_or:
1209  case AtomicExpr::AO__atomic_fetch_or:
1210  LibCallName = "__atomic_fetch_or";
1211  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1212  MemTy, E->getExprLoc(), TInfo.Width);
1213  break;
1214  // T __atomic_sub_fetch_N(T *mem, T val, int order)
1215  // T __atomic_fetch_sub_N(T *mem, T val, int order)
1216  case AtomicExpr::AO__atomic_sub_fetch:
1217  PostOp = llvm::Instruction::Sub;
1218  LLVM_FALLTHROUGH;
1219  case AtomicExpr::AO__c11_atomic_fetch_sub:
1220  case AtomicExpr::AO__opencl_atomic_fetch_sub:
1221  case AtomicExpr::AO__atomic_fetch_sub:
1222  LibCallName = "__atomic_fetch_sub";
1223  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1224  LoweredMemTy, E->getExprLoc(), TInfo.Width);
1225  break;
1226  // T __atomic_xor_fetch_N(T *mem, T val, int order)
1227  // T __atomic_fetch_xor_N(T *mem, T val, int order)
1228  case AtomicExpr::AO__atomic_xor_fetch:
1229  PostOp = llvm::Instruction::Xor;
1230  LLVM_FALLTHROUGH;
1231  case AtomicExpr::AO__c11_atomic_fetch_xor:
1232  case AtomicExpr::AO__opencl_atomic_fetch_xor:
1233  case AtomicExpr::AO__hip_atomic_fetch_xor:
1234  case AtomicExpr::AO__atomic_fetch_xor:
1235  LibCallName = "__atomic_fetch_xor";
1236  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1237  MemTy, E->getExprLoc(), TInfo.Width);
1238  break;
1239  case AtomicExpr::AO__atomic_min_fetch:
1240  PostOpMinMax = true;
1241  LLVM_FALLTHROUGH;
1242  case AtomicExpr::AO__c11_atomic_fetch_min:
1243  case AtomicExpr::AO__atomic_fetch_min:
1244  case AtomicExpr::AO__hip_atomic_fetch_min:
1245  case AtomicExpr::AO__opencl_atomic_fetch_min:
1246  LibCallName = E->getValueType()->isSignedIntegerType()
1247  ? "__atomic_fetch_min"
1248  : "__atomic_fetch_umin";
1249  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1250  LoweredMemTy, E->getExprLoc(), TInfo.Width);
1251  break;
1252  case AtomicExpr::AO__atomic_max_fetch:
1253  PostOpMinMax = true;
1254  LLVM_FALLTHROUGH;
1255  case AtomicExpr::AO__c11_atomic_fetch_max:
1256  case AtomicExpr::AO__atomic_fetch_max:
1257  case AtomicExpr::AO__hip_atomic_fetch_max:
1258  case AtomicExpr::AO__opencl_atomic_fetch_max:
1259  LibCallName = E->getValueType()->isSignedIntegerType()
1260  ? "__atomic_fetch_max"
1261  : "__atomic_fetch_umax";
1262  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1263  LoweredMemTy, E->getExprLoc(), TInfo.Width);
1264  break;
1265  // T __atomic_nand_fetch_N(T *mem, T val, int order)
1266  // T __atomic_fetch_nand_N(T *mem, T val, int order)
1267  case AtomicExpr::AO__atomic_nand_fetch:
1268  PostOp = llvm::Instruction::And; // the NOT is special cased below
1269  LLVM_FALLTHROUGH;
1270  case AtomicExpr::AO__c11_atomic_fetch_nand:
1271  case AtomicExpr::AO__atomic_fetch_nand:
1272  LibCallName = "__atomic_fetch_nand";
1273  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1274  MemTy, E->getExprLoc(), TInfo.Width);
1275  break;
1276  }
1277 
1278  if (E->isOpenCL()) {
1279  LibCallName = std::string("__opencl") +
1280  StringRef(LibCallName).drop_front(1).str();
1281 
1282  }
1283  // Optimized functions have the size in their name.
1284  if (UseOptimizedLibcall)
1285  LibCallName += "_" + llvm::utostr(Size);
1286  // By default, assume we return a value of the atomic type.
1287  if (!HaveRetTy) {
1288  if (UseOptimizedLibcall) {
1289  // Value is returned directly.
1290  // The function returns an appropriately sized integer type.
1292  getContext().toBits(TInfo.Width), /*Signed=*/false);
1293  } else {
1294  // Value is returned through parameter before the order.
1295  RetTy = getContext().VoidTy;
1297  getContext().VoidPtrTy);
1298  }
1299  }
1300  // order is always the last parameter
1301  Args.add(RValue::get(Order),
1302  getContext().IntTy);
1303  if (E->isOpenCL())
1304  Args.add(RValue::get(Scope), getContext().IntTy);
1305 
1306  // PostOp is only needed for the atomic_*_fetch operations, and
1307  // thus is only needed for and implemented in the
1308  // UseOptimizedLibcall codepath.
1309  assert(UseOptimizedLibcall || (!PostOp && !PostOpMinMax));
1310 
1311  RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
1312  // The value is returned directly from the libcall.
1313  if (E->isCmpXChg())
1314  return Res;
1315 
1316  // The value is returned directly for optimized libcalls but the expr
1317  // provided an out-param.
1318  if (UseOptimizedLibcall && Res.getScalarVal()) {
1319  llvm::Value *ResVal = Res.getScalarVal();
1320  if (PostOpMinMax) {
1321  llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal();
1322  ResVal = EmitPostAtomicMinMax(Builder, E->getOp(),
1324  ResVal, LoadVal1);
1325  } else if (PostOp) {
1326  llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal();
1327  ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1);
1328  }
1329  if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
1330  ResVal = Builder.CreateNot(ResVal);
1331 
1333  ResVal, Builder.CreateElementBitCast(Dest, ResVal->getType()));
1334  }
1335 
1336  if (RValTy->isVoidType())
1337  return RValue::get(nullptr);
1338 
1339  return convertTempToRValue(
1341  RValTy, E->getExprLoc());
1342  }
1343 
1344  bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
1345  E->getOp() == AtomicExpr::AO__opencl_atomic_store ||
1346  E->getOp() == AtomicExpr::AO__hip_atomic_store ||
1347  E->getOp() == AtomicExpr::AO__atomic_store ||
1348  E->getOp() == AtomicExpr::AO__atomic_store_n;
1349  bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
1350  E->getOp() == AtomicExpr::AO__opencl_atomic_load ||
1351  E->getOp() == AtomicExpr::AO__hip_atomic_load ||
1352  E->getOp() == AtomicExpr::AO__atomic_load ||
1353  E->getOp() == AtomicExpr::AO__atomic_load_n;
1354 
1355  if (isa<llvm::ConstantInt>(Order)) {
1356  auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1357  // We should not ever get to a case where the ordering isn't a valid C ABI
1358  // value, but it's hard to enforce that in general.
1359  if (llvm::isValidAtomicOrderingCABI(ord))
1360  switch ((llvm::AtomicOrderingCABI)ord) {
1361  case llvm::AtomicOrderingCABI::relaxed:
1362  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1363  llvm::AtomicOrdering::Monotonic, Scope);
1364  break;
1365  case llvm::AtomicOrderingCABI::consume:
1366  case llvm::AtomicOrderingCABI::acquire:
1367  if (IsStore)
1368  break; // Avoid crashing on code with undefined behavior
1369  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1370  llvm::AtomicOrdering::Acquire, Scope);
1371  break;
1372  case llvm::AtomicOrderingCABI::release:
1373  if (IsLoad)
1374  break; // Avoid crashing on code with undefined behavior
1375  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1376  llvm::AtomicOrdering::Release, Scope);
1377  break;
1378  case llvm::AtomicOrderingCABI::acq_rel:
1379  if (IsLoad || IsStore)
1380  break; // Avoid crashing on code with undefined behavior
1381  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1382  llvm::AtomicOrdering::AcquireRelease, Scope);
1383  break;
1384  case llvm::AtomicOrderingCABI::seq_cst:
1385  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1386  llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1387  break;
1388  }
1389  if (RValTy->isVoidType())
1390  return RValue::get(nullptr);
1391 
1392  return convertTempToRValue(
1394  RValTy, E->getExprLoc());
1395  }
1396 
1397  // Long case, when Order isn't obviously constant.
1398 
1399  // Create all the relevant BB's
1400  llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
1401  *ReleaseBB = nullptr, *AcqRelBB = nullptr,
1402  *SeqCstBB = nullptr;
1403  MonotonicBB = createBasicBlock("monotonic", CurFn);
1404  if (!IsStore)
1405  AcquireBB = createBasicBlock("acquire", CurFn);
1406  if (!IsLoad)
1407  ReleaseBB = createBasicBlock("release", CurFn);
1408  if (!IsLoad && !IsStore)
1409  AcqRelBB = createBasicBlock("acqrel", CurFn);
1410  SeqCstBB = createBasicBlock("seqcst", CurFn);
1411  llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1412 
1413  // Create the switch for the split
1414  // MonotonicBB is arbitrarily chosen as the default case; in practice, this
1415  // doesn't matter unless someone is crazy enough to use something that
1416  // doesn't fold to a constant for the ordering.
1417  Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1418  llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
1419 
1420  // Emit all the different atomics
1421  Builder.SetInsertPoint(MonotonicBB);
1422  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1423  llvm::AtomicOrdering::Monotonic, Scope);
1424  Builder.CreateBr(ContBB);
1425  if (!IsStore) {
1426  Builder.SetInsertPoint(AcquireBB);
1427  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1428  llvm::AtomicOrdering::Acquire, Scope);
1429  Builder.CreateBr(ContBB);
1430  SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
1431  AcquireBB);
1432  SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
1433  AcquireBB);
1434  }
1435  if (!IsLoad) {
1436  Builder.SetInsertPoint(ReleaseBB);
1437  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1438  llvm::AtomicOrdering::Release, Scope);
1439  Builder.CreateBr(ContBB);
1440  SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release),
1441  ReleaseBB);
1442  }
1443  if (!IsLoad && !IsStore) {
1444  Builder.SetInsertPoint(AcqRelBB);
1445  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1446  llvm::AtomicOrdering::AcquireRelease, Scope);
1447  Builder.CreateBr(ContBB);
1448  SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel),
1449  AcqRelBB);
1450  }
1451  Builder.SetInsertPoint(SeqCstBB);
1452  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1453  llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1454  Builder.CreateBr(ContBB);
1455  SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
1456  SeqCstBB);
1457 
1458  // Cleanup and return
1459  Builder.SetInsertPoint(ContBB);
1460  if (RValTy->isVoidType())
1461  return RValue::get(nullptr);
1462 
1463  assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
1464  return convertTempToRValue(
1466  RValTy, E->getExprLoc());
1467 }
1468 
1469 Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const {
1470  llvm::IntegerType *ty =
1471  llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
1472  return CGF.Builder.CreateElementBitCast(addr, ty);
1473 }
1474 
1475 Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {
1476  llvm::Type *Ty = Addr.getElementType();
1477  uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
1478  if (SourceSizeInBits != AtomicSizeInBits) {
1479  Address Tmp = CreateTempAlloca();
1480  CGF.Builder.CreateMemCpy(Tmp, Addr,
1481  std::min(AtomicSizeInBits, SourceSizeInBits) / 8);
1482  Addr = Tmp;
1483  }
1484 
1485  return emitCastToAtomicIntPointer(Addr);
1486 }
1487 
1488 RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
1489  AggValueSlot resultSlot,
1490  SourceLocation loc,
1491  bool asValue) const {
1492  if (LVal.isSimple()) {
1493  if (EvaluationKind == TEK_Aggregate)
1494  return resultSlot.asRValue();
1495 
1496  // Drill into the padding structure if we have one.
1497  if (hasPadding())
1498  addr = CGF.Builder.CreateStructGEP(addr, 0);
1499 
1500  // Otherwise, just convert the temporary to an r-value using the
1501  // normal conversion routine.
1502  return CGF.convertTempToRValue(addr, getValueType(), loc);
1503  }
1504  if (!asValue)
1505  // Get RValue from temp memory as atomic for non-simple lvalues
1506  return RValue::get(CGF.Builder.CreateLoad(addr));
1507  if (LVal.isBitField())
1508  return CGF.EmitLoadOfBitfieldLValue(
1509  LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(),
1510  LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1511  if (LVal.isVectorElt())
1512  return CGF.EmitLoadOfLValue(
1513  LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(),
1514  LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1515  assert(LVal.isExtVectorElt());
1517  addr, LVal.getExtVectorElts(), LVal.getType(),
1518  LVal.getBaseInfo(), TBAAAccessInfo()));
1519 }
1520 
1521 RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
1522  AggValueSlot ResultSlot,
1523  SourceLocation Loc,
1524  bool AsValue) const {
1525  // Try not to in some easy cases.
1526  assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
1527  if (getEvaluationKind() == TEK_Scalar &&
1528  (((!LVal.isBitField() ||
1529  LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
1530  !hasPadding()) ||
1531  !AsValue)) {
1532  auto *ValTy = AsValue
1533  ? CGF.ConvertTypeForMem(ValueTy)
1534  : getAtomicAddress().getElementType();
1535  if (ValTy->isIntegerTy()) {
1536  assert(IntVal->getType() == ValTy && "Different integer types.");
1537  return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
1538  } else if (ValTy->isPointerTy())
1539  return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
1540  else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
1541  return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
1542  }
1543 
1544  // Create a temporary. This needs to be big enough to hold the
1545  // atomic integer.
1546  Address Temp = Address::invalid();
1547  bool TempIsVolatile = false;
1548  if (AsValue && getEvaluationKind() == TEK_Aggregate) {
1549  assert(!ResultSlot.isIgnored());
1550  Temp = ResultSlot.getAddress();
1551  TempIsVolatile = ResultSlot.isVolatile();
1552  } else {
1553  Temp = CreateTempAlloca();
1554  }
1555 
1556  // Slam the integer into the temporary.
1557  Address CastTemp = emitCastToAtomicIntPointer(Temp);
1558  CGF.Builder.CreateStore(IntVal, CastTemp)
1559  ->setVolatile(TempIsVolatile);
1560 
1561  return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue);
1562 }
1563 
1564 void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1565  llvm::AtomicOrdering AO, bool) {
1566  // void __atomic_load(size_t size, void *mem, void *return, int order);
1567  CallArgList Args;
1568  Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1569  Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1570  CGF.getContext().VoidPtrTy);
1571  Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)),
1572  CGF.getContext().VoidPtrTy);
1573  Args.add(
1574  RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))),
1575  CGF.getContext().IntTy);
1576  emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1577 }
1578 
1579 llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1580  bool IsVolatile) {
1581  // Okay, we're doing this natively.
1582  Address Addr = getAtomicAddressAsAtomicIntPointer();
1583  llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
1584  Load->setAtomic(AO);
1585 
1586  // Other decoration.
1587  if (IsVolatile)
1588  Load->setVolatile(true);
1590  return Load;
1591 }
1592 
1593 /// An LValue is a candidate for having its loads and stores be made atomic if
1594 /// we are operating under /volatile:ms *and* the LValue itself is volatile and
1595 /// performing such an operation can be performed without a libcall.
1597  if (!CGM.getCodeGenOpts().MSVolatile) return false;
1598  AtomicInfo AI(*this, LV);
1599  bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType());
1600  // An atomic is inline if we don't need to use a libcall.
1601  bool AtomicIsInline = !AI.shouldUseLibcall();
1602  // MSVC doesn't seem to do this for types wider than a pointer.
1603  if (getContext().getTypeSize(LV.getType()) >
1604  getContext().getTypeSize(getContext().getIntPtrType()))
1605  return false;
1606  return IsVolatile && AtomicIsInline;
1607 }
1608 
1610  AggValueSlot Slot) {
1611  llvm::AtomicOrdering AO;
1612  bool IsVolatile = LV.isVolatileQualified();
1613  if (LV.getType()->isAtomicType()) {
1614  AO = llvm::AtomicOrdering::SequentiallyConsistent;
1615  } else {
1616  AO = llvm::AtomicOrdering::Acquire;
1617  IsVolatile = true;
1618  }
1619  return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
1620 }
1621 
1622 RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1623  bool AsValue, llvm::AtomicOrdering AO,
1624  bool IsVolatile) {
1625  // Check whether we should use a library call.
1626  if (shouldUseLibcall()) {
1627  Address TempAddr = Address::invalid();
1628  if (LVal.isSimple() && !ResultSlot.isIgnored()) {
1629  assert(getEvaluationKind() == TEK_Aggregate);
1630  TempAddr = ResultSlot.getAddress();
1631  } else
1632  TempAddr = CreateTempAlloca();
1633 
1634  EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile);
1635 
1636  // Okay, turn that back into the original value or whole atomic (for
1637  // non-simple lvalues) type.
1638  return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
1639  }
1640 
1641  // Okay, we're doing this natively.
1642  auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1643 
1644  // If we're ignoring an aggregate return, don't do anything.
1645  if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
1646  return RValue::getAggregate(Address::invalid(), false);
1647 
1648  // Okay, turn that back into the original value or atomic (for non-simple
1649  // lvalues) type.
1650  return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
1651 }
1652 
1653 /// Emit a load from an l-value of atomic type. Note that the r-value
1654 /// we produce is an r-value of the atomic *value* type.
1656  llvm::AtomicOrdering AO, bool IsVolatile,
1657  AggValueSlot resultSlot) {
1658  AtomicInfo Atomics(*this, src);
1659  return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
1660  IsVolatile);
1661 }
1662 
1663 /// Copy an r-value into memory as part of storing to an atomic type.
1664 /// This needs to create a bit-pattern suitable for atomic operations.
1665 void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1666  assert(LVal.isSimple());
1667  // If we have an r-value, the rvalue should be of the atomic type,
1668  // which means that the caller is responsible for having zeroed
1669  // any padding. Just do an aggregate copy of that type.
1670  if (rvalue.isAggregate()) {
1671  LValue Dest = CGF.MakeAddrLValue(getAtomicAddress(), getAtomicType());
1672  LValue Src = CGF.MakeAddrLValue(rvalue.getAggregateAddress(),
1673  getAtomicType());
1674  bool IsVolatile = rvalue.isVolatileQualified() ||
1675  LVal.isVolatileQualified();
1676  CGF.EmitAggregateCopy(Dest, Src, getAtomicType(),
1677  AggValueSlot::DoesNotOverlap, IsVolatile);
1678  return;
1679  }
1680 
1681  // Okay, otherwise we're copying stuff.
1682 
1683  // Zero out the buffer if necessary.
1684  emitMemSetZeroIfNecessary();
1685 
1686  // Drill past the padding if present.
1687  LValue TempLVal = projectValue();
1688 
1689  // Okay, store the rvalue in.
1690  if (rvalue.isScalar()) {
1691  CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
1692  } else {
1693  CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
1694  }
1695 }
1696 
1697 
1698 /// Materialize an r-value into memory for the purposes of storing it
1699 /// to an atomic type.
1700 Address AtomicInfo::materializeRValue(RValue rvalue) const {
1701  // Aggregate r-values are already in memory, and EmitAtomicStore
1702  // requires them to be values of the atomic type.
1703  if (rvalue.isAggregate())
1704  return rvalue.getAggregateAddress();
1705 
1706  // Otherwise, make a temporary and materialize into it.
1707  LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType());
1708  AtomicInfo Atomics(CGF, TempLV);
1709  Atomics.emitCopyIntoMemory(rvalue);
1710  return TempLV.getAddress(CGF);
1711 }
1712 
1713 llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
1714  // If we've got a scalar value of the right size, try to avoid going
1715  // through memory.
1716  if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) {
1717  llvm::Value *Value = RVal.getScalarVal();
1718  if (isa<llvm::IntegerType>(Value->getType()))
1719  return CGF.EmitToMemory(Value, ValueTy);
1720  else {
1721  llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1722  CGF.getLLVMContext(),
1723  LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1724  if (isa<llvm::PointerType>(Value->getType()))
1725  return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
1726  else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
1727  return CGF.Builder.CreateBitCast(Value, InputIntTy);
1728  }
1729  }
1730  // Otherwise, we need to go through memory.
1731  // Put the r-value in memory.
1732  Address Addr = materializeRValue(RVal);
1733 
1734  // Cast the temporary to the atomic int type and pull a value out.
1735  Addr = emitCastToAtomicIntPointer(Addr);
1736  return CGF.Builder.CreateLoad(Addr);
1737 }
1738 
1739 std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1740  llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1741  llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1742  // Do the atomic store.
1743  Address Addr = getAtomicAddressAsAtomicIntPointer();
1744  auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(),
1745  ExpectedVal, DesiredVal,
1746  Success, Failure);
1747  // Other decoration.
1748  Inst->setVolatile(LVal.isVolatileQualified());
1749  Inst->setWeak(IsWeak);
1750 
1751  // Okay, turn that back into the original value type.
1752  auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
1753  auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
1754  return std::make_pair(PreviousVal, SuccessFailureVal);
1755 }
1756 
1757 llvm::Value *
1758 AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1759  llvm::Value *DesiredAddr,
1760  llvm::AtomicOrdering Success,
1761  llvm::AtomicOrdering Failure) {
1762  // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1763  // void *desired, int success, int failure);
1764  CallArgList Args;
1765  Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1766  Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1767  CGF.getContext().VoidPtrTy);
1768  Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)),
1769  CGF.getContext().VoidPtrTy);
1770  Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)),
1771  CGF.getContext().VoidPtrTy);
1772  Args.add(RValue::get(
1773  llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))),
1774  CGF.getContext().IntTy);
1775  Args.add(RValue::get(
1776  llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))),
1777  CGF.getContext().IntTy);
1778  auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1779  CGF.getContext().BoolTy, Args);
1780 
1781  return SuccessFailureRVal.getScalarVal();
1782 }
1783 
1784 std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1785  RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1786  llvm::AtomicOrdering Failure, bool IsWeak) {
1787  // Check whether we should use a library call.
1788  if (shouldUseLibcall()) {
1789  // Produce a source address.
1790  Address ExpectedAddr = materializeRValue(Expected);
1791  Address DesiredAddr = materializeRValue(Desired);
1792  auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1793  DesiredAddr.getPointer(),
1794  Success, Failure);
1795  return std::make_pair(
1796  convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1797  SourceLocation(), /*AsValue=*/false),
1798  Res);
1799  }
1800 
1801  // If we've got a scalar value of the right size, try to avoid going
1802  // through memory.
1803  auto *ExpectedVal = convertRValueToInt(Expected);
1804  auto *DesiredVal = convertRValueToInt(Desired);
1805  auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1806  Failure, IsWeak);
1807  return std::make_pair(
1808  ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(),
1809  SourceLocation(), /*AsValue=*/false),
1810  Res.second);
1811 }
1812 
1813 static void
1814 EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1815  const llvm::function_ref<RValue(RValue)> &UpdateOp,
1816  Address DesiredAddr) {
1817  RValue UpRVal;
1818  LValue AtomicLVal = Atomics.getAtomicLValue();
1819  LValue DesiredLVal;
1820  if (AtomicLVal.isSimple()) {
1821  UpRVal = OldRVal;
1822  DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType());
1823  } else {
1824  // Build new lvalue for temp address.
1825  Address Ptr = Atomics.materializeRValue(OldRVal);
1826  LValue UpdateLVal;
1827  if (AtomicLVal.isBitField()) {
1828  UpdateLVal =
1829  LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
1830  AtomicLVal.getType(),
1831  AtomicLVal.getBaseInfo(),
1832  AtomicLVal.getTBAAInfo());
1833  DesiredLVal =
1834  LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1835  AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1836  AtomicLVal.getTBAAInfo());
1837  } else if (AtomicLVal.isVectorElt()) {
1838  UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
1839  AtomicLVal.getType(),
1840  AtomicLVal.getBaseInfo(),
1841  AtomicLVal.getTBAAInfo());
1842  DesiredLVal = LValue::MakeVectorElt(
1843  DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(),
1844  AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1845  } else {
1846  assert(AtomicLVal.isExtVectorElt());
1847  UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
1848  AtomicLVal.getType(),
1849  AtomicLVal.getBaseInfo(),
1850  AtomicLVal.getTBAAInfo());
1851  DesiredLVal = LValue::MakeExtVectorElt(
1852  DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1853  AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1854  }
1855  UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());
1856  }
1857  // Store new value in the corresponding memory area.
1858  RValue NewRVal = UpdateOp(UpRVal);
1859  if (NewRVal.isScalar()) {
1860  CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);
1861  } else {
1862  assert(NewRVal.isComplex());
1863  CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,
1864  /*isInit=*/false);
1865  }
1866 }
1867 
1868 void AtomicInfo::EmitAtomicUpdateLibcall(
1869  llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1870  bool IsVolatile) {
1871  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1872 
1873  Address ExpectedAddr = CreateTempAlloca();
1874 
1875  EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1876  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1877  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1878  CGF.EmitBlock(ContBB);
1879  Address DesiredAddr = CreateTempAlloca();
1880  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1881  requiresMemSetZero(getAtomicAddress().getElementType())) {
1882  auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1883  CGF.Builder.CreateStore(OldVal, DesiredAddr);
1884  }
1885  auto OldRVal = convertAtomicTempToRValue(ExpectedAddr,
1887  SourceLocation(), /*AsValue=*/false);
1888  EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
1889  auto *Res =
1890  EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1891  DesiredAddr.getPointer(),
1892  AO, Failure);
1893  CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1894  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1895 }
1896 
1897 void AtomicInfo::EmitAtomicUpdateOp(
1898  llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1899  bool IsVolatile) {
1900  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1901 
1902  // Do the atomic load.
1903  auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile);
1904  // For non-simple lvalues perform compare-and-swap procedure.
1905  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1906  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1907  auto *CurBB = CGF.Builder.GetInsertBlock();
1908  CGF.EmitBlock(ContBB);
1909  llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1910  /*NumReservedValues=*/2);
1911  PHI->addIncoming(OldVal, CurBB);
1912  Address NewAtomicAddr = CreateTempAlloca();
1913  Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1914  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1915  requiresMemSetZero(getAtomicAddress().getElementType())) {
1916  CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1917  }
1918  auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(),
1919  SourceLocation(), /*AsValue=*/false);
1920  EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
1921  auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1922  // Try to write new value using cmpxchg operation.
1923  auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1924  PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1925  CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1926  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1927 }
1928 
1929 static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1930  RValue UpdateRVal, Address DesiredAddr) {
1931  LValue AtomicLVal = Atomics.getAtomicLValue();
1932  LValue DesiredLVal;
1933  // Build new lvalue for temp address.
1934  if (AtomicLVal.isBitField()) {
1935  DesiredLVal =
1936  LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1937  AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1938  AtomicLVal.getTBAAInfo());
1939  } else if (AtomicLVal.isVectorElt()) {
1940  DesiredLVal =
1941  LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(),
1942  AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1943  AtomicLVal.getTBAAInfo());
1944  } else {
1945  assert(AtomicLVal.isExtVectorElt());
1946  DesiredLVal = LValue::MakeExtVectorElt(
1947  DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1948  AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1949  }
1950  // Store new value in the corresponding memory area.
1951  assert(UpdateRVal.isScalar());
1952  CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);
1953 }
1954 
1955 void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1956  RValue UpdateRVal, bool IsVolatile) {
1957  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1958 
1959  Address ExpectedAddr = CreateTempAlloca();
1960 
1961  EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1962  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1963  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1964  CGF.EmitBlock(ContBB);
1965  Address DesiredAddr = CreateTempAlloca();
1966  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1967  requiresMemSetZero(getAtomicAddress().getElementType())) {
1968  auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1969  CGF.Builder.CreateStore(OldVal, DesiredAddr);
1970  }
1971  EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
1972  auto *Res =
1973  EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1974  DesiredAddr.getPointer(),
1975  AO, Failure);
1976  CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1977  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1978 }
1979 
1980 void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
1981  bool IsVolatile) {
1982  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1983 
1984  // Do the atomic load.
1985  auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile);
1986  // For non-simple lvalues perform compare-and-swap procedure.
1987  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1988  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1989  auto *CurBB = CGF.Builder.GetInsertBlock();
1990  CGF.EmitBlock(ContBB);
1991  llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1992  /*NumReservedValues=*/2);
1993  PHI->addIncoming(OldVal, CurBB);
1994  Address NewAtomicAddr = CreateTempAlloca();
1995  Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1996  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1997  requiresMemSetZero(getAtomicAddress().getElementType())) {
1998  CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1999  }
2000  EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
2001  auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
2002  // Try to write new value using cmpxchg operation.
2003  auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
2004  PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
2005  CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
2006  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
2007 }
2008 
2009 void AtomicInfo::EmitAtomicUpdate(
2010  llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
2011  bool IsVolatile) {
2012  if (shouldUseLibcall()) {
2013  EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
2014  } else {
2015  EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
2016  }
2017 }
2018 
2019 void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
2020  bool IsVolatile) {
2021  if (shouldUseLibcall()) {
2022  EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
2023  } else {
2024  EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
2025  }
2026 }
2027 
2029  bool isInit) {
2030  bool IsVolatile = lvalue.isVolatileQualified();
2031  llvm::AtomicOrdering AO;
2032  if (lvalue.getType()->isAtomicType()) {
2033  AO = llvm::AtomicOrdering::SequentiallyConsistent;
2034  } else {
2035  AO = llvm::AtomicOrdering::Release;
2036  IsVolatile = true;
2037  }
2038  return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
2039 }
2040 
2041 /// Emit a store to an l-value of atomic type.
2042 ///
2043 /// Note that the r-value is expected to be an r-value *of the atomic
2044 /// type*; this means that for aggregate r-values, it should include
2045 /// storage for any padding that was necessary.
2047  llvm::AtomicOrdering AO, bool IsVolatile,
2048  bool isInit) {
2049  // If this is an aggregate r-value, it should agree in type except
2050  // maybe for address-space qualification.
2051  assert(!rvalue.isAggregate() ||
2052  rvalue.getAggregateAddress().getElementType() ==
2053  dest.getAddress(*this).getElementType());
2054 
2055  AtomicInfo atomics(*this, dest);
2056  LValue LVal = atomics.getAtomicLValue();
2057 
2058  // If this is an initialization, just put the value there normally.
2059  if (LVal.isSimple()) {
2060  if (isInit) {
2061  atomics.emitCopyIntoMemory(rvalue);
2062  return;
2063  }
2064 
2065  // Check whether we should use a library call.
2066  if (atomics.shouldUseLibcall()) {
2067  // Produce a source address.
2068  Address srcAddr = atomics.materializeRValue(rvalue);
2069 
2070  // void __atomic_store(size_t size, void *mem, void *val, int order)
2071  CallArgList args;
2072  args.add(RValue::get(atomics.getAtomicSizeValue()),
2073  getContext().getSizeType());
2074  args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())),
2075  getContext().VoidPtrTy);
2076  args.add(RValue::get(EmitCastToVoidPtr(srcAddr.getPointer())),
2077  getContext().VoidPtrTy);
2078  args.add(
2079  RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))),
2080  getContext().IntTy);
2081  emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
2082  return;
2083  }
2084 
2085  // Okay, we're doing this natively.
2086  llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
2087 
2088  // Do the atomic store.
2089  Address addr =
2090  atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress());
2091  intValue = Builder.CreateIntCast(
2092  intValue, addr.getElementType(), /*isSigned=*/false);
2093  llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
2094 
2095  if (AO == llvm::AtomicOrdering::Acquire)
2096  AO = llvm::AtomicOrdering::Monotonic;
2097  else if (AO == llvm::AtomicOrdering::AcquireRelease)
2098  AO = llvm::AtomicOrdering::Release;
2099  // Initializations don't need to be atomic.
2100  if (!isInit)
2101  store->setAtomic(AO);
2102 
2103  // Other decoration.
2104  if (IsVolatile)
2105  store->setVolatile(true);
2107  return;
2108  }
2109 
2110  // Emit simple atomic update operation.
2111  atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
2112 }
2113 
2114 /// Emit a compare-and-exchange op for atomic type.
2115 ///
2116 std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
2117  LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
2118  llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
2119  AggValueSlot Slot) {
2120  // If this is an aggregate r-value, it should agree in type except
2121  // maybe for address-space qualification.
2122  assert(!Expected.isAggregate() ||
2123  Expected.getAggregateAddress().getElementType() ==
2124  Obj.getAddress(*this).getElementType());
2125  assert(!Desired.isAggregate() ||
2126  Desired.getAggregateAddress().getElementType() ==
2127  Obj.getAddress(*this).getElementType());
2128  AtomicInfo Atomics(*this, Obj);
2129 
2130  return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
2131  IsWeak);
2132 }
2133 
2135  LValue LVal, llvm::AtomicOrdering AO,
2136  const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
2137  AtomicInfo Atomics(*this, LVal);
2138  Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
2139 }
2140 
2142  AtomicInfo atomics(*this, dest);
2143 
2144  switch (atomics.getEvaluationKind()) {
2145  case TEK_Scalar: {
2146  llvm::Value *value = EmitScalarExpr(init);
2147  atomics.emitCopyIntoMemory(RValue::get(value));
2148  return;
2149  }
2150 
2151  case TEK_Complex: {
2152  ComplexPairTy value = EmitComplexExpr(init);
2153  atomics.emitCopyIntoMemory(RValue::getComplex(value));
2154  return;
2155  }
2156 
2157  case TEK_Aggregate: {
2158  // Fix up the destination if the initializer isn't an expression
2159  // of atomic type.
2160  bool Zeroed = false;
2161  if (!init->getType()->isAtomicType()) {
2162  Zeroed = atomics.emitMemSetZeroIfNecessary();
2163  dest = atomics.projectValue();
2164  }
2165 
2166  // Evaluate the expression directly into the destination.
2168  dest, *this, AggValueSlot::IsNotDestructed,
2172 
2173  EmitAggExpr(init, slot);
2174  return;
2175  }
2176  }
2177  llvm_unreachable("bad evaluation kind");
2178 }
clang::CodeGen::CodeGenFunction::ConvertTypeForMem
llvm::Type * ConvertTypeForMem(QualType T)
Definition: CodeGenFunction.cpp:208
clang::ASTContext::getTypeSizeInChars
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
Definition: ASTContext.cpp:2462
clang::AtomicExpr::getPtr
Expr * getPtr() const
Definition: Expr.h:6266
clang::AtomicExpr::getVal2
Expr * getVal2() const
Definition: Expr.h:6286
clang::CodeGen::CGBuilderTy::CreateMemCpy
llvm::CallInst * CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
Definition: CGBuilder.h:309
clang::AtomicExpr
AtomicExpr - Variadic atomic builtins: __atomic_exchange, __atomic_fetch_*, __atomic_load,...
Definition: Expr.h:6234
clang::CodeGen::CodeGenTypeCache::SizeTy
llvm::IntegerType * SizeTy
Definition: CodeGenTypeCache.h:50
clang::CodeGen::CodeGenFunction::getTypeSize
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
Definition: CGStmtOpenMP.cpp:305
clang::CodeGen::CodeGenFunction::EmitAtomicCompareExchange
std::pair< RValue, llvm::Value * > EmitAtomicCompareExchange(LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, llvm::AtomicOrdering Success=llvm::AtomicOrdering::SequentiallyConsistent, llvm::AtomicOrdering Failure=llvm::AtomicOrdering::SequentiallyConsistent, bool IsWeak=false, AggValueSlot Slot=AggValueSlot::ignored())
Emit a compare-and-exchange op for atomic type.
Definition: CGAtomic.cpp:2116
clang::interp::Add
bool Add(InterpState &S, CodePtr OpPC)
Definition: Interp.h:134
clang::CharUnits::getAsAlign
llvm::Align getAsAlign() const
getAsAlign - Returns Quantity as a valid llvm::Align, Beware llvm::Align assumes power of two 8-bit b...
Definition: CharUnits.h:183
clang::interp::APInt
llvm::APInt APInt
Definition: Integral.h:27
clang::CodeGen::CodeGenTypeCache::IntTy
llvm::IntegerType * IntTy
int
Definition: CodeGenTypeCache.h:42
EmitPostAtomicMinMax
static llvm::Value * EmitPostAtomicMinMax(CGBuilderTy &Builder, AtomicExpr::AtomicOp Op, bool IsSigned, llvm::Value *OldVal, llvm::Value *RHS)
Duplicate the atomic min/max operation in conventional IR for the builtin variants that return the ne...
Definition: CGAtomic.cpp:501
clang::CodeGen::TEK_Aggregate
@ TEK_Aggregate
Definition: CodeGenFunction.h:109
clang::CodeGen::RValue
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
Definition: CGValue.h:39
clang::CodeGen::CodeGenModule::CreateRuntimeFunction
llvm::FunctionCallee CreateRuntimeFunction(llvm::FunctionType *Ty, StringRef Name, llvm::AttributeList ExtraAttrs=llvm::AttributeList(), bool Local=false, bool AssumeConvergent=false)
Create or return a runtime function declaration with the specified type and name.
Definition: CodeGenModule.cpp:4025
type
clang::CodeGen::RValue::getAggregate
static RValue getAggregate(Address addr, bool isVolatile=false)
Definition: CGValue.h:110
CodeGenFunction.h
string
string(SUBSTRING ${CMAKE_CURRENT_BINARY_DIR} 0 ${PATH_LIB_START} PATH_HEAD) string(SUBSTRING $
Definition: CMakeLists.txt:22
clang::CodeGen::CodeGenFunction::EmitCall
RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee, ReturnValueSlot ReturnValue, const CallArgList &Args, llvm::CallBase **callOrInvoke, bool IsMustTail, SourceLocation Loc)
EmitCall - Generate a call of the given function, expecting the given result type,...
Definition: CGCall.cpp:4687
clang::AtomicExpr::getValueType
QualType getValueType() const
Definition: Expr.cpp:4786
clang::CodeGen::AggValueSlot::DoesNotNeedGCBarriers
@ DoesNotNeedGCBarriers
Definition: CGValue.h:554
clang::CodeGen::Address::getAlignment
CharUnits getAlignment() const
Return the alignment of this pointer.
Definition: Address.h:120
clang::CodeGen::LValue::MakeBitfield
static LValue MakeBitfield(Address Addr, const CGBitFieldInfo &Info, QualType type, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo)
Create a new object to represent a bit-field access.
Definition: CGValue.h:448
clang::ASTContext::VoidTy
CanQualType VoidTy
Definition: ASTContext.h:1096
clang::AtomicExpr::getOrder
Expr * getOrder() const
Definition: Expr.h:6269
clang::SourceLocation
Encodes a location in the source.
Definition: SourceLocation.h:86
clang::QualType::getQualifiers
Qualifiers getQualifiers() const
Retrieve the set of qualifiers applied to this type.
Definition: Type.h:6527
clang::AtomicExpr::getOp
AtomicOp getOp() const
Definition: Expr.h:6298
clang::CodeGen::LValue::getAddress
Address getAddress(CodeGenFunction &CGF) const
Definition: CGValue.h:341
clang::CodeGen::CodeGenFunction::EmitAtomicStore
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
Definition: CGAtomic.cpp:2028
clang::CodeGen::CodeGenModule::getSize
llvm::ConstantInt * getSize(CharUnits numChars)
Emit the given number of characters as a value of type size_t.
Definition: CodeGenModule.cpp:1076
clang::CodeGen::TargetCodeGenInfo::getLLVMSyncScopeID
virtual llvm::SyncScope::ID getLLVMSyncScopeID(const LangOptions &LangOpts, SyncScope Scope, llvm::AtomicOrdering Ordering, llvm::LLVMContext &Ctx) const
Get the syncscope used in LLVM IR.
Definition: TargetInfo.cpp:523
CGRecordLayout.h
clang::CodeGen::LValue::getVectorIdx
llvm::Value * getVectorIdx() const
Definition: CGValue.h:359
clang::QualType
A (possibly-)qualified type.
Definition: Type.h:675
clang::ASTContext::getTargetAddressSpace
unsigned getTargetAddressSpace(QualType T) const
Definition: ASTContext.cpp:11998
clang::CodeGen::CodeGenFunction::EmitLoadOfExtVectorElementLValue
RValue EmitLoadOfExtVectorElementLValue(LValue V)
Definition: CGExpr.cpp:2047
AttributeLangSupport::C
@ C
Definition: SemaDeclAttr.cpp:55
clang::CodeGen::Address::isValid
bool isValid() const
Definition: Address.h:91
clang::Type::isFloatingType
bool isFloatingType() const
Definition: Type.cpp:2121
clang::DiagnosticsEngine
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:192
int
__device__ int
Definition: __clang_hip_libdevice_declares.h:63
clang::TypeInfo::Width
uint64_t Width
Definition: ASTContext.h:179
clang::CodeGen::CodeGenFunction::createBasicBlock
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
Definition: CodeGenFunction.h:2430
clang::CodeGen::LValue::getVectorAddress
Address getVectorAddress() const
Definition: CGValue.h:352
clang::CodeGen::CGBuilderTy::CreateStore
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
Definition: CGBuilder.h:97
clang::AtomicExpr::getScopeModel
static std::unique_ptr< AtomicScopeModel > getScopeModel(AtomicOp Op)
Get atomic scope model for the atomic op code.
Definition: Expr.h:6347
TargetInfo.h
clang::CodeGen::CodeGenModule::getLangOpts
const LangOptions & getLangOpts() const
Definition: CodeGenModule.h:698
clang::Type::isVoidType
bool isVoidType() const
Definition: Type.h:7037
llvm::Expected
Definition: LLVM.h:41
clang::CodeGen::CodeGenFunction::EmitStoreOfScalar
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
Definition: CodeGenFunction.h:3848
clang::CodeGen::CGBitFieldInfo::Size
unsigned Size
The total size of the bit-field, in bits.
Definition: CGRecordLayout.h:71
clang::AtomicExpr::isOpenCL
bool isOpenCL() const
Definition: Expr.h:6321
clang::ento::Store
const void * Store
Store - This opaque type encapsulates an immutable mapping from locations to values.
Definition: StoreRef.h:27
clang::CodeGen::CGBuilderTy
Definition: CGBuilder.h:44
clang::CodeGen::CGBuilderTy::CreateStructGEP
Address CreateStructGEP(Address Addr, unsigned Index, const llvm::Twine &Name="")
Definition: CGBuilder.h:189
clang::ASTContext::toBits
int64_t toBits(CharUnits CharSize) const
Convert a size in characters to a size in bits.
Definition: ASTContext.cpp:2456
clang::AtomicExpr::getBeginLoc
SourceLocation getBeginLoc() const LLVM_READONLY
Definition: Expr.h:6329
clang::AtomicExpr::getOrderFail
Expr * getOrderFail() const
Definition: Expr.h:6282
clang::ArrayType::Normal
@ Normal
Definition: Type.h:2903
clang::CodeGen::CallArgList::add
void add(RValue rvalue, QualType type)
Definition: CGCall.h:286
clang::CodeGen::CodeGenFunction::EmitStoreThroughLValue
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
Definition: CGExpr.cpp:2118
clang::CodeGen::CodeGenFunction::Builder
CGBuilderTy Builder
Definition: CodeGenFunction.h:269
clang::CodeGen::CodeGenTypeCache::VoidPtrTy
llvm::PointerType * VoidPtrTy
Definition: CodeGenTypeCache.h:56
Offset
unsigned Offset
Definition: Format.cpp:2570
clang::TargetInfo::getMaxAtomicInlineWidth
unsigned getMaxAtomicInlineWidth() const
Return the maximum width lock-free atomic operation which can be inlined given the supported features...
Definition: TargetInfo.h:745
clang::CodeGen::TypeEvaluationKind
TypeEvaluationKind
The kind of evaluation to perform on values of a particular type.
Definition: CodeGenFunction.h:106
clang::ASTContext::getTypeAlignInChars
CharUnits getTypeAlignInChars(QualType T) const
Return the ABI-specified alignment of a (complete) type T, in characters.
Definition: ASTContext.cpp:2471
clang::CodeGen::CodeGenFunction::EmitLoadOfScalar
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
Definition: CodeGenFunction.h:3826
clang::CodeGen::LValue::MakeAddr
static LValue MakeAddr(Address address, QualType type, ASTContext &Context, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo)
Definition: CGValue.h:402
clang::CodeGen::CodeGenFunction::ComplexPairTy
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
Definition: CodeGenFunction.h:267
clang::CodeGen::ReturnValueSlot
ReturnValueSlot - Contains the address where the return value of a function can be stored,...
Definition: CGCall.h:360
clang::CodeGen::AggValueSlot::IsZeroed
@ IsZeroed
Definition: CGValue.h:552
CGCall.h
clang::TypeInfo
Definition: ASTContext.h:178
clang::getAsString
llvm::StringRef getAsString(SyncScope S)
Definition: SyncScope.h:55
V
#define V(N, I)
Definition: ASTContext.h:3176
clang::CodeGen::CGBitFieldInfo::Offset
unsigned Offset
The offset within a contiguous run of bitfields that are represented as a single "field" within the L...
Definition: CGRecordLayout.h:68
clang::ASTContext::getExtVectorType
QualType getExtVectorType(QualType VectorType, unsigned NumElts) const
Return the unique reference to an extended vector type of the specified element type and size.
Definition: ASTContext.cpp:4056
min
__DEVICE__ int min(int __a, int __b)
Definition: __clang_cuda_math.h:197
emitAtomicLibcall
static RValue emitAtomicLibcall(CodeGenFunction &CGF, StringRef fnName, QualType resultType, CallArgList &args)
Definition: CGAtomic.cpp:312
clang::CodeGen::TBAAAccessInfo
Definition: CodeGenTBAA.h:41
clang::CodeGen::CGBuilderTy::CreatePointerBitCastOrAddrSpaceCast
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
Definition: CGBuilder.h:173
clang::CodeGen::AggValueSlot::asRValue
RValue asRValue() const
Definition: CGValue.h:651
clang::CodeGen::LValue::MakeVectorElt
static LValue MakeVectorElt(Address vecAddress, llvm::Value *Idx, QualType type, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo)
Definition: CGValue.h:416
clang::CodeGen::AggValueSlot::ignored
static AggValueSlot ignored()
ignored - Returns an aggregate value slot indicating that the aggregate value is being ignored.
Definition: CGValue.h:559
clang::CodeGen::LValue::isVolatileQualified
bool isVolatileQualified() const
Definition: CGValue.h:272
clang::Scope
Scope - A scope is a transient data structure that is used while parsing the program.
Definition: Scope.h:40
clang::CodeGen::CodeGenFunction::EmitCastToVoidPtr
llvm::Value * EmitCastToVoidPtr(llvm::Value *value)
Emit a cast to void* in the appropriate address space.
Definition: CGExpr.cpp:54
clang::CodeGen::LValue::isBitField
bool isBitField() const
Definition: CGValue.h:267
clang::VectorType
Represents a GCC generic vector type.
Definition: Type.h:3242
clang::CodeGen::CodeGenModule::getCodeGenOpts
const CodeGenOptions & getCodeGenOpts() const
Definition: CodeGenModule.h:703
clang::CodeGen::Address::getType
llvm::PointerType * getType() const
Return the type of the pointer value.
Definition: Address.h:99
clang::CodeGen::LValue::getBitFieldInfo
const CGBitFieldInfo & getBitFieldInfo() const
Definition: CGValue.h:394
clang::ASTContext
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition: ASTContext.h:208
clang::ASTContext::getSizeType
CanQualType getSizeType() const
Return the unique type for "size_t" (C99 7.17), defined in <stddef.h>.
Definition: ASTContext.cpp:5823
CGFunctionInfo.h
clang::CodeGen::CodeGenFunction::EmitAtomicUpdate
void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO, const llvm::function_ref< RValue(RValue)> &UpdateOp, bool IsVolatile)
Definition: CGAtomic.cpp:2134
clang::CodeGen::RValue::isAggregate
bool isAggregate() const
Definition: CGValue.h:56
clang::CodeGen::TEK_Complex
@ TEK_Complex
Definition: CodeGenFunction.h:108
clang::CodeGen::LValue::MakeExtVectorElt
static LValue MakeExtVectorElt(Address vecAddress, llvm::Constant *Elts, QualType type, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo)
Definition: CGValue.h:429
clang::Type::getAs
const T * getAs() const
Member-template getAs<specific type>'.
Definition: Type.h:7243
clang::CodeGen::RValue::isVolatileQualified
bool isVolatileQualified() const
Definition: CGValue.h:58
emitAtomicCmpXchgFailureSet
static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, Address Dest, Address Ptr, Address Val1, Address Val2, llvm::Value *FailureOrderVal, uint64_t Size, llvm::AtomicOrdering SuccessOrder, llvm::SyncScope::ID Scope)
Given an ordering required on success, emit all possible cmpxchg instructions to cope with the provid...
Definition: CGAtomic.cpp:424
clang::CodeGen::CodeGenFunction::getTarget
const TargetInfo & getTarget() const
Definition: CodeGenFunction.h:2024
clang::CodeGen::CodeGenFunction::EmitAtomicExpr
RValue EmitAtomicExpr(AtomicExpr *E)
Definition: CGAtomic.cpp:806
clang::CodeGen::CodeGenTypeCache::VoidTy
llvm::Type * VoidTy
void
Definition: CodeGenTypeCache.h:34
clang::CodeGen::LValue::isVolatile
bool isVolatile() const
Definition: CGValue.h:315
clang::CodeGen::LValue::getType
QualType getType() const
Definition: CGValue.h:278
clang::interp::Load
bool Load(InterpState &S, CodePtr OpPC)
Definition: Interp.h:618
clang::CodeGen::LValue::getBitFieldPointer
llvm::Value * getBitFieldPointer() const
Definition: CGValue.h:393
clang::CodeGen::CodeGenFunction::convertTempToRValue
RValue convertTempToRValue(Address addr, QualType type, SourceLocation Loc)
Given the address of a temporary variable, produce an r-value of its type.
Definition: CGExpr.cpp:5529
ASTContext.h
clang::ASTContext::getIntTypeForBitwidth
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth,...
Definition: ASTContext.cpp:11728
clang::Type::getPointeeType
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
Definition: Type.cpp:625
clang::CodeGen::CodeGenModule::getTypes
CodeGenTypes & getTypes()
Definition: CodeGenModule.h:721
clang::CodeGen::CodeGenFunction::EmitFromMemory
llvm::Value * EmitFromMemory(llvm::Value *Value, QualType Ty)
EmitFromMemory - Change a scalar value from its memory representation to its value representation.
Definition: CGExpr.cpp:1815
clang::CodeGen::CodeGenFunction::EmitAtomicLoad
RValue EmitAtomicLoad(LValue LV, SourceLocation SL, AggValueSlot Slot=AggValueSlot::ignored())
Definition: CGAtomic.cpp:1609
emitAtomicCmpXchg
static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, Address Dest, Address Ptr, Address Val1, Address Val2, uint64_t Size, llvm::AtomicOrdering SuccessOrder, llvm::AtomicOrdering FailureOrder, llvm::SyncScope::ID Scope)
Definition: CGAtomic.cpp:374
clang::CodeGen::CodeGenFunction::getLLVMContext
llvm::LLVMContext & getLLVMContext()
Definition: CodeGenFunction.h:2025
clang::CodeGen::CGBitFieldInfo::StorageOffset
CharUnits StorageOffset
The offset of the bitfield storage from the start of the struct.
Definition: CGRecordLayout.h:81
clang::Type::isAtomicType
bool isAtomicType() const
Definition: Type.h:6889
clang::CodeGen::CodeGenFunction::getContext
ASTContext & getContext() const
Definition: CodeGenFunction.h:1982
clang::CodeGen::Address
An aligned address.
Definition: Address.h:74
CodeGenModule.h
clang::CodeGen::RValue::isComplex
bool isComplex() const
Definition: CGValue.h:55
clang::CodeGen::CodeGenFunction::getEvaluationKind
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
Definition: CodeGenFunction.cpp:216
clang::AtomicExpr::AtomicOp
AtomicOp
Definition: Expr.h:6236
clang::CodeGen::LValue::getBaseInfo
LValueBaseInfo getBaseInfo() const
Definition: CGValue.h:333
clang::CodeGen::CodeGenFunction::EmitAtomicInit
void EmitAtomicInit(Expr *E, LValue lvalue)
Definition: CGAtomic.cpp:2141
clang::CodeGen::CodeGenFunction::EmitAnyExprToMem
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
Definition: CGExpr.cpp:241
clang::CodeGen::CGCallee::forDirect
static CGCallee forDirect(llvm::Constant *functionPtr, const CGCalleeInfo &abstractInfo=CGCalleeInfo())
Definition: CGCall.h:133
clang::CodeGen::LValue::getTBAAInfo
TBAAAccessInfo getTBAAInfo() const
Definition: CGValue.h:322
clang::QualType::getUnqualifiedType
QualType getUnqualifiedType() const
Retrieve the unqualified variant of the given type, removing as little sugar as possible.
Definition: Type.h:6580
clang::CodeGen::CodeGenFunction::EmitLoadOfBitfieldLValue
RValue EmitLoadOfBitfieldLValue(LValue LV, SourceLocation Loc)
Definition: CGExpr.cpp:2010
clang::CodeGen::LValue::setAlignment
void setAlignment(CharUnits A)
Definition: CGValue.h:331
clang::CodeGen::CodeGenFunction::LValueIsSuitableForInlineAtomic
bool LValueIsSuitableForInlineAtomic(LValue Src)
An LValue is a candidate for having its loads and stores be made atomic if we are operating under /vo...
Definition: CGAtomic.cpp:1596
clang::CodeGen::CGBitFieldInfo
Structure with information about how a bitfield should be accessed.
Definition: CGRecordLayout.h:65
clang::CodeGen::CodeGenFunction::EmitToMemory
llvm::Value * EmitToMemory(llvm::Value *Value, QualType Ty)
EmitToMemory - Change a scalar value from its value representation to its in-memory representation.
Definition: CGExpr.cpp:1801
clang::CodeGen::CodeGenTypes::GetFunctionType
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
Definition: CGCall.cpp:1613
clang::CodeGen::CodeGenModule::getDataLayout
const llvm::DataLayout & getDataLayout() const
Definition: CodeGenModule.h:706
clang::Type::castAs
const T * castAs() const
Member-template castAs<specific type>.
Definition: Type.h:7310
EmitAtomicOp
static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, Address Ptr, Address Val1, Address Val2, llvm::Value *IsWeak, llvm::Value *FailureOrder, uint64_t Size, llvm::AtomicOrdering Order, llvm::SyncScope::ID Scope)
Definition: CGAtomic.cpp:521
clang::prec::And
@ And
Definition: OperatorPrecedence.h:35
clang::CodeGen::LValue
LValue - This represents an lvalue references.
Definition: CGValue.h:171
clang::CodeGen::LValue::isVectorElt
bool isVectorElt() const
Definition: CGValue.h:266
clang::CodeGen::LValue::isSimple
bool isSimple() const
Definition: CGValue.h:265
clang::CodeGen::CodeGenFunction::getTargetHooks
const TargetCodeGenInfo & getTargetHooks() const
Definition: CodeGenFunction.h:2026
EmitValToTemp
static Address EmitValToTemp(CodeGenFunction &CGF, Expr *E)
Definition: CGAtomic.cpp:716
clang::ASTContext::VoidPtrTy
CanQualType VoidPtrTy
Definition: ASTContext.h:1123
clang::Type::isPointerType
bool isPointerType() const
Definition: Type.h:6748
clang::AtomicExpr::isCmpXChg
bool isCmpXChg() const
Definition: Expr.h:6310
clang::syntax::NodeRole::Size
@ Size
clang::AtomicExpr::isVolatile
bool isVolatile() const
Definition: Expr.h:6306
clang::CodeGen::Address::getPointer
llvm::Value * getPointer() const
Definition: Address.h:93
clang::CodeGen::CodeGenFunction
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
Definition: CodeGenFunction.h:231
clang::ASTContext::IntTy
CanQualType IntTy
Definition: ASTContext.h:1105
clang::CharUnits::isZero
bool isZero() const
isZero - Test whether the quantity equals zero.
Definition: CharUnits.h:116
clang::CodeGen::AggValueSlot::IsNotAliased
@ IsNotAliased
Definition: CGValue.h:550
isFullSizeType
static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type, uint64_t expectedSize)
Does a store of the given IR type modify the full expected width?
Definition: CGAtomic.cpp:332
clang::Expr::getExprLoc
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
Definition: Expr.cpp:247
clang::CodeGen::CodeGenModule
This class organizes the cross-function state that is used while generating LLVM code.
Definition: CodeGenModule.h:275
clang::ASTContext::getIntPtrType
QualType getIntPtrType() const
Return a type compatible with "intptr_t" (C99 7.18.1.4), as defined by the target.
Definition: ASTContext.cpp:5857
clang::QualType::isNull
bool isNull() const
Return true if this QualType doesn't point to a type yet.
Definition: Type.h:740
clang::CodeGen::LValue::getExtVectorPointer
llvm::Value * getExtVectorPointer() const
Definition: CGValue.h:380
clang::CodeGen::CodeGenFunction::EmitComplexExpr
ComplexPairTy EmitComplexExpr(const Expr *E, bool IgnoreReal=false, bool IgnoreImag=false)
EmitComplexExpr - Emit the computation of the specified expression of complex type,...
Definition: CGExprComplex.cpp:1120
Value
Value
Definition: UninitializedValues.cpp:102
clang::CodeGen::CodeGenFunction::EmitAggExpr
void EmitAggExpr(const Expr *E, AggValueSlot AS)
EmitAggExpr - Emit the computation of the specified expression of aggregate type.
Definition: CGExprAgg.cpp:1993
clang::CodeGen::Address::invalid
static Address invalid()
Definition: Address.h:90
clang::CodeGen::AggValueSlot::forLValue
static AggValueSlot forLValue(const LValue &LV, CodeGenFunction &CGF, IsDestructed_t isDestructed, NeedsGCBarriers_t needsGC, IsAliased_t isAliased, Overlap_t mayOverlap, IsZeroed_t isZeroed=IsNotZeroed, IsSanitizerChecked_t isChecked=IsNotSanitizerChecked)
Definition: CGValue.h:587
clang::CodeGen::LValue::getExtVectorElts
llvm::Constant * getExtVectorElts() const
Definition: CGValue.h:384
clang::AtomicExpr::getScope
Expr * getScope() const
Definition: Expr.h:6272
clang::CodeGen::AggValueSlot::isVolatile
bool isVolatile() const
Definition: CGValue.h:604
AddDirectArgument
static void AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args, bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy, SourceLocation Loc, CharUnits SizeInChars)
Definition: CGAtomic.cpp:782
clang::CodeGen::AggValueSlot
An aggregate value slot.
Definition: CGValue.h:491
clang::Type::isSignedIntegerType
bool isSignedIntegerType() const
Return true if this is an integer type that is signed, according to C99 6.2.5p4 [char,...
Definition: Type.cpp:2018
clang::CodeGen::CGBitFieldInfo::StorageSize
unsigned StorageSize
The storage size in bits which should be used when accessing this bitfield.
Definition: CGRecordLayout.h:78
clang::CodeGen::RValue::getComplexVal
std::pair< llvm::Value *, llvm::Value * > getComplexVal() const
getComplexVal - Return the real/imag components of this complex value.
Definition: CGValue.h:68
clang::CodeGen::Address::getElementType
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Definition: Address.h:104
clang::CodeGen::CodeGenFunction::MakeAddrLValue
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
Definition: CodeGenFunction.h:2492
clang::CodeGen::CodeGenModule::DecorateInstructionWithTBAA
void DecorateInstructionWithTBAA(llvm::Instruction *Inst, TBAAAccessInfo TBAAInfo)
DecorateInstructionWithTBAA - Decorate the instruction with a TBAA tag.
Definition: CodeGenModule.cpp:1040
clang::CodeGen::LValue::getPointer
llvm::Value * getPointer(CodeGenFunction &CGF) const
Definition: CGValue.h:337
clang::CodeGen::CGBuilderTy::CreateElementBitCast
Address CreateElementBitCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
Cast the element type of the given address to a different type, preserving information like the align...
Definition: CGBuilder.h:165
clang::CodeGen::CodeGenFunction::EmitScalarExpr
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
Definition: CGExprScalar.cpp:4869
clang::CodeGen::CGFunctionInfo
CGFunctionInfo - Class to encapsulate the information about a function definition.
Definition: CGFunctionInfo.h:546
clang::CodeGen::CodeGenFunction::EmitLoadOfLValue
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
Definition: CGExpr.cpp:1948
clang::CodeGen::AggValueSlot::isIgnored
bool isIgnored() const
Definition: CGValue.h:631
clang::CodeGen::RValue::getComplex
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
Definition: CGValue.h:96
clang::PointerType
PointerType - C99 6.7.5.1 - Pointer Declarators.
Definition: Type.h:2653
clang::CodeGen::CodeGenFunction::EmitStoreOfComplex
void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit)
EmitStoreOfComplex - Store a complex number into the specified l-value.
Definition: CGExprComplex.cpp:1139
clang::CodeGen::CodeGenFunction::CGM
CodeGenModule & CGM
Definition: CodeGenFunction.h:261
clang::CodeGen::TEK_Scalar
@ TEK_Scalar
Definition: CodeGenFunction.h:107
clang::CodeGen::CodeGenFunction::CurFn
llvm::Function * CurFn
Definition: CodeGenFunction.h:325
clang::Builtin::ID
ID
Definition: Builtins.h:51
EmitAtomicUpdateValue
static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal, const llvm::function_ref< RValue(RValue)> &UpdateOp, Address DesiredAddr)
Definition: CGAtomic.cpp:1814
clang
Definition: CalledOnceCheck.h:17
clang::TypeInfo::Align
unsigned Align
Definition: ASTContext.h:180
clang::CodeGen::CGBuilderTy::CreateAtomicCmpXchg
llvm::AtomicCmpXchgInst * CreateAtomicCmpXchg(llvm::Value *Ptr, llvm::Value *Cmp, llvm::Value *New, llvm::AtomicOrdering SuccessOrdering, llvm::AtomicOrdering FailureOrdering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
Definition: CGBuilder.h:135
clang::CodeGen::RValue::get
static RValue get(llvm::Value *V)
Definition: CGValue.h:89
clang::ASTContext::toCharUnitsFromBits
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
Definition: ASTContext.cpp:2451
clang::CodeGen::CodeGenFunction::EmitAggregateCopy
void EmitAggregateCopy(LValue Dest, LValue Src, QualType EltTy, AggValueSlot::Overlap_t MayOverlap, bool isVolatile=false)
EmitAggregateCopy - Emit an aggregate copy.
Definition: CGExprAgg.cpp:2056
clang::CodeGen::LValue::isGlobalReg
bool isGlobalReg() const
Definition: CGValue.h:269
FrontendDiagnostic.h
clang::CodeGen::AggValueSlot::getAddress
Address getAddress() const
Definition: CGValue.h:627
clang::CodeGen::LValue::getExtVectorAddress
Address getExtVectorAddress() const
Definition: CGValue.h:377
clang::Expr::getType
QualType getType() const
Definition: Expr.h:141
clang::CodeGen::RValue::getScalarVal
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
Definition: CGValue.h:61
clang::CodeGen::LValue::getAlignment
CharUnits getAlignment() const
Definition: CGValue.h:330
clang::CodeGen::CodeGenFunction::hasVolatileMember
bool hasVolatileMember(QualType T)
hasVolatileMember - returns true if aggregate type has a volatile member.
Definition: CodeGenFunction.h:2672
clang::AtomicExpr::getVal1
Expr * getVal1() const
Definition: Expr.h:6276
clang::CodeGen::CodeGenModule::getDiags
DiagnosticsEngine & getDiags() const
Definition: CodeGenModule.h:705
clang::CodeGen::RValue::isScalar
bool isScalar() const
Definition: CGValue.h:54
clang::CodeGen::CGBuilderTy::CreateAtomicRMW
llvm::AtomicRMWInst * CreateAtomicRMW(llvm::AtomicRMWInst::BinOp Op, llvm::Value *Ptr, llvm::Value *Val, llvm::AtomicOrdering Ordering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
Definition: CGBuilder.h:147
clang::ASTContext::getPointerType
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
Definition: ASTContext.cpp:3260
clang::CodeGen::TargetCodeGenInfo::performAddrSpaceCast
virtual llvm::Value * performAddrSpaceCast(CodeGen::CodeGenFunction &CGF, llvm::Value *V, LangAS SrcAddr, LangAS DestAddr, llvm::Type *DestTy, bool IsNonNull=false) const
Perform address space cast of an expression of pointer type.
Definition: TargetInfo.cpp:501
clang::CharUnits
CharUnits - This is an opaque type for sizes expressed in character units.
Definition: CharUnits.h:38
clang::ASTContext::getTypeInfoInChars
TypeInfoChars getTypeInfoInChars(const Type *T) const
Definition: ASTContext.cpp:1874
clang::AtomicType
Definition: Type.h:6318
true
#define true
Definition: stdbool.h:21
clang::CodeGen::CodeGenFunction::CreateMemTemp
Address CreateMemTemp(QualType T, const Twine &Name="tmp", Address *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
Definition: CGExpr.cpp:135
clang::CodeGen::LValue::getVectorPointer
llvm::Value * getVectorPointer() const
Definition: CGValue.h:355
clang::Expr
This represents one expression.
Definition: Expr.h:109
clang::CodeGen::AggValueSlot::IsNotDestructed
@ IsNotDestructed
Definition: CGValue.h:551
clang::CodeGen::CodeGenFunction::EmitPointerWithAlignment
Address EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
EmitPointerWithAlignment - Given an expression with a pointer type, emit the value and compute our be...
Definition: CGExpr.cpp:1072
clang::CodeGen::CodeGenTypeCache::Int8Ty
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
Definition: CodeGenTypeCache.h:37
clang::AtomicExpr::getWeak
Expr * getWeak() const
Definition: Expr.h:6292
clang::CodeGen::CallArgList
CallArgList - Type for representing both the value and type of arguments in a call.
Definition: CGCall.h:262
clang::LangAS::opencl_generic
@ opencl_generic
clang::CodeGen::CodeGenTypes::arrangeBuiltinFunctionCall
const CGFunctionInfo & arrangeBuiltinFunctionCall(QualType resultType, const CallArgList &args)
Definition: CGCall.cpp:653
clang::ASTContext::BoolTy
CanQualType BoolTy
Definition: ASTContext.h:1097
clang::CodeGen::CGBuilderTy::CreateMemSet
llvm::CallInst * CreateMemSet(Address Dest, llvm::Value *Value, llvm::Value *Size, bool IsVolatile=false)
Definition: CGBuilder.h:338
clang::CodeGen::AggValueSlot::IsNotZeroed
@ IsNotZeroed
Definition: CGValue.h:552
clang::CodeGen::LValue::isExtVectorElt
bool isExtVectorElt() const
Definition: CGValue.h:268
clang::CodeGen::CodeGenFunction::EmitBlock
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
Definition: CGStmt.cpp:544
clang::interp::Sub
bool Sub(InterpState &S, CodePtr OpPC)
Definition: Interp.h:142
clang::CodeGen::CodeGenModule::getLLVMContext
llvm::LLVMContext & getLLVMContext()
Definition: CodeGenModule.h:715
clang::CharUnits::getQuantity
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
Definition: CharUnits.h:179
clang::CodeGen::RValue::getAggregateAddress
Address getAggregateAddress() const
getAggregateAddr() - Return the Value* of the address of the aggregate.
Definition: CGValue.h:73
clang::DiagnosticsEngine::Report
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
Definition: Diagnostic.h:1536
clang::CodeGen::CGBuilderTy::CreateLoad
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
Definition: CGBuilder.h:69
clang::CodeGen::LValue::getBitFieldAddress
Address getBitFieldAddress() const
Definition: CGValue.h:390
clang::CodeGen::AggValueSlot::DoesNotOverlap
@ DoesNotOverlap
Definition: CGValue.h:553