21#include "llvm/ADT/DenseMap.h"
22#include "llvm/IR/DataLayout.h"
23#include "llvm/IR/Intrinsics.h"
24#include "llvm/IR/Operator.h"
27using namespace CodeGen;
44 : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
51 ValueTy = ATy->getValueType();
58 TypeInfo ValueTI =
C.getTypeInfo(ValueTy);
59 ValueSizeInBits = ValueTI.
Width;
60 ValueAlignInBits = ValueTI.
Align;
62 TypeInfo AtomicTI =
C.getTypeInfo(AtomicTy);
63 AtomicSizeInBits = AtomicTI.
Width;
64 AtomicAlignInBits = AtomicTI.
Align;
66 assert(ValueSizeInBits <= AtomicSizeInBits);
67 assert(ValueAlignInBits <= AtomicAlignInBits);
69 AtomicAlign =
C.toCharUnitsFromBits(AtomicAlignInBits);
70 ValueAlign =
C.toCharUnitsFromBits(ValueAlignInBits);
77 ValueSizeInBits =
C.getTypeSize(ValueTy);
79 auto Offset = OrigBFI.Offset %
C.toBits(lvalue.
getAlignment());
80 AtomicSizeInBits =
C.toBits(
81 C.toCharUnitsFromBits(Offset + OrigBFI.Size +
C.getCharWidth() - 1)
85 (
C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.
getAlignment()) *
87 llvm::Value *StoragePtr = CGF.
Builder.CreateConstGEP1_64(
88 CGF.
Int8Ty, BitFieldPtr, OffsetInChars.getQuantity());
90 StoragePtr, CGF.
UnqualPtrTy,
"atomic_bitfield_base");
95 llvm::Type *StorageTy = CGF.
Builder.getIntNTy(AtomicSizeInBits);
96 LVal = LValue::MakeBitfield(
99 AtomicTy =
C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
103 C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
104 AtomicTy =
C.getConstantArrayType(
C.CharTy, Size,
nullptr,
105 ArraySizeModifier::Normal,
111 ValueSizeInBits =
C.getTypeSize(ValueTy);
113 AtomicSizeInBits =
C.getTypeSize(AtomicTy);
119 ValueSizeInBits =
C.getTypeSize(ValueTy);
121 lvalue.
getType(), cast<llvm::FixedVectorType>(
124 AtomicSizeInBits =
C.getTypeSize(AtomicTy);
128 UseLibcall = !
C.getTargetInfo().hasBuiltinAtomic(
132 QualType getAtomicType()
const {
return AtomicTy; }
133 QualType getValueType()
const {
return ValueTy; }
134 CharUnits getAtomicAlignment()
const {
return AtomicAlign; }
135 uint64_t getAtomicSizeInBits()
const {
return AtomicSizeInBits; }
136 uint64_t getValueSizeInBits()
const {
return ValueSizeInBits; }
138 bool shouldUseLibcall()
const {
return UseLibcall; }
139 const LValue &getAtomicLValue()
const {
return LVal; }
140 llvm::Value *getAtomicPointer()
const {
150 Address getAtomicAddress()
const {
160 return Address(getAtomicPointer(), ElTy, getAtomicAlignment());
163 Address getAtomicAddressAsAtomicIntPointer()
const {
164 return castToAtomicIntPointer(getAtomicAddress());
173 bool hasPadding()
const {
174 return (ValueSizeInBits != AtomicSizeInBits);
177 bool emitMemSetZeroIfNecessary()
const;
179 llvm::Value *getAtomicSizeValue()
const {
197 llvm::Value *getScalarRValValueOrNull(
RValue RVal)
const;
200 llvm::Value *convertRValueToInt(
RValue RVal,
bool CastFP =
true)
const;
204 bool CastFP =
true)
const;
207 void emitCopyIntoMemory(
RValue rvalue)
const;
210 LValue projectValue()
const {
212 Address addr = getAtomicAddress();
216 return LValue::MakeAddr(addr, getValueType(), CGF.
getContext(),
223 bool AsValue, llvm::AtomicOrdering AO,
234 std::pair<RValue, llvm::Value *>
237 llvm::AtomicOrdering::SequentiallyConsistent,
238 llvm::AtomicOrdering Failure =
239 llvm::AtomicOrdering::SequentiallyConsistent,
240 bool IsWeak =
false);
245 void EmitAtomicUpdate(llvm::AtomicOrdering AO,
250 void EmitAtomicUpdate(llvm::AtomicOrdering AO,
RValue UpdateRVal,
257 Address CreateTempAlloca()
const;
259 bool requiresMemSetZero(llvm::Type *
type)
const;
263 void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
264 llvm::AtomicOrdering AO,
bool IsVolatile);
266 llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO,
bool IsVolatile,
269 llvm::Value *EmitAtomicCompareExchangeLibcall(
270 llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
272 llvm::AtomicOrdering::SequentiallyConsistent,
273 llvm::AtomicOrdering Failure =
274 llvm::AtomicOrdering::SequentiallyConsistent);
276 std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
277 llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
279 llvm::AtomicOrdering::SequentiallyConsistent,
280 llvm::AtomicOrdering Failure =
281 llvm::AtomicOrdering::SequentiallyConsistent,
282 bool IsWeak =
false);
285 EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
289 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
293 void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
RValue UpdateRVal,
296 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
RValue UpdateRal,
301Address AtomicInfo::CreateTempAlloca()
const {
303 (LVal.
isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
305 getAtomicAlignment(),
310 TempAlloca, getAtomicAddress().getType(),
311 getAtomicAddress().getElementType());
323 fnAttrB.addAttribute(llvm::Attribute::NoUnwind);
324 fnAttrB.addAttribute(llvm::Attribute::WillReturn);
325 llvm::AttributeList fnAttrs = llvm::AttributeList::get(
326 CGF.
getLLVMContext(), llvm::AttributeList::FunctionIndex, fnAttrB);
328 llvm::FunctionCallee fn =
336 uint64_t expectedSize) {
343bool AtomicInfo::requiresMemSetZero(llvm::Type *
type)
const {
345 if (hasPadding())
return true;
348 switch (getEvaluationKind()) {
355 AtomicSizeInBits / 2);
361 llvm_unreachable(
"bad evaluation kind");
364bool AtomicInfo::emitMemSetZeroIfNecessary()
const {
381 llvm::AtomicOrdering SuccessOrder,
382 llvm::AtomicOrdering FailureOrder,
383 llvm::SyncScope::ID
Scope) {
391 Pair->setWeak(IsWeak);
395 llvm::Value *Old = CGF.
Builder.CreateExtractValue(Pair, 0);
396 llvm::Value *Cmp = CGF.
Builder.CreateExtractValue(Pair, 1);
400 llvm::BasicBlock *StoreExpectedBB =
405 llvm::BasicBlock *ContinueBB =
410 CGF.
Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
412 CGF.
Builder.SetInsertPoint(StoreExpectedBB);
416 CGF.
Builder.CreateBr(ContinueBB);
418 CGF.
Builder.SetInsertPoint(ContinueBB);
429 llvm::Value *FailureOrderVal,
431 llvm::AtomicOrdering SuccessOrder,
432 llvm::SyncScope::ID
Scope) {
433 llvm::AtomicOrdering FailureOrder;
434 if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
435 auto FOS = FO->getSExtValue();
436 if (!llvm::isValidAtomicOrderingCABI(FOS))
437 FailureOrder = llvm::AtomicOrdering::Monotonic;
439 switch ((llvm::AtomicOrderingCABI)FOS) {
440 case llvm::AtomicOrderingCABI::relaxed:
443 case llvm::AtomicOrderingCABI::release:
444 case llvm::AtomicOrderingCABI::acq_rel:
445 FailureOrder = llvm::AtomicOrdering::Monotonic;
447 case llvm::AtomicOrderingCABI::consume:
448 case llvm::AtomicOrderingCABI::acquire:
449 FailureOrder = llvm::AtomicOrdering::Acquire;
451 case llvm::AtomicOrderingCABI::seq_cst:
452 FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent;
460 FailureOrder,
Scope);
473 llvm::SwitchInst *SI = CGF.
Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
475 SI->addCase(CGF.
Builder.getInt32((
int)llvm::AtomicOrderingCABI::consume),
477 SI->addCase(CGF.
Builder.getInt32((
int)llvm::AtomicOrderingCABI::acquire),
479 SI->addCase(CGF.
Builder.getInt32((
int)llvm::AtomicOrderingCABI::seq_cst),
483 CGF.
Builder.SetInsertPoint(MonotonicBB);
485 Size, SuccessOrder, llvm::AtomicOrdering::Monotonic,
Scope);
488 CGF.
Builder.SetInsertPoint(AcquireBB);
490 llvm::AtomicOrdering::Acquire,
Scope);
493 CGF.
Builder.SetInsertPoint(SeqCstBB);
495 llvm::AtomicOrdering::SequentiallyConsistent,
Scope);
498 CGF.
Builder.SetInsertPoint(ContBB);
508 llvm::CmpInst::Predicate Pred;
511 llvm_unreachable(
"Unexpected min/max operation");
512 case AtomicExpr::AO__atomic_max_fetch:
513 case AtomicExpr::AO__scoped_atomic_max_fetch:
514 Pred = IsSigned ? llvm::CmpInst::ICMP_SGT : llvm::CmpInst::ICMP_UGT;
516 case AtomicExpr::AO__atomic_min_fetch:
517 case AtomicExpr::AO__scoped_atomic_min_fetch:
518 Pred = IsSigned ? llvm::CmpInst::ICMP_SLT : llvm::CmpInst::ICMP_ULT;
521 llvm::Value *Cmp = Builder.CreateICmp(Pred, OldVal, RHS,
"tst");
522 return Builder.CreateSelect(Cmp, OldVal, RHS,
"newval");
527 llvm::Value *IsWeak, llvm::Value *FailureOrder,
528 uint64_t Size, llvm::AtomicOrdering Order,
529 llvm::SyncScope::ID
Scope) {
530 llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
531 bool PostOpMinMax =
false;
534 switch (E->
getOp()) {
535 case AtomicExpr::AO__c11_atomic_init:
536 case AtomicExpr::AO__opencl_atomic_init:
537 llvm_unreachable(
"Already handled!");
539 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
540 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
541 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
543 FailureOrder, Size, Order,
Scope);
545 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
546 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
547 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
549 FailureOrder, Size, Order,
Scope);
551 case AtomicExpr::AO__atomic_compare_exchange:
552 case AtomicExpr::AO__atomic_compare_exchange_n:
553 case AtomicExpr::AO__scoped_atomic_compare_exchange:
554 case AtomicExpr::AO__scoped_atomic_compare_exchange_n: {
555 if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
557 Val1, Val2, FailureOrder, Size, Order,
Scope);
560 llvm::BasicBlock *StrongBB =
563 llvm::BasicBlock *ContBB =
566 llvm::SwitchInst *SI = CGF.
Builder.CreateSwitch(IsWeak, WeakBB);
567 SI->addCase(CGF.
Builder.getInt1(
false), StrongBB);
569 CGF.
Builder.SetInsertPoint(StrongBB);
571 FailureOrder, Size, Order,
Scope);
574 CGF.
Builder.SetInsertPoint(WeakBB);
576 FailureOrder, Size, Order,
Scope);
579 CGF.
Builder.SetInsertPoint(ContBB);
583 case AtomicExpr::AO__c11_atomic_load:
584 case AtomicExpr::AO__opencl_atomic_load:
585 case AtomicExpr::AO__hip_atomic_load:
586 case AtomicExpr::AO__atomic_load_n:
587 case AtomicExpr::AO__atomic_load:
588 case AtomicExpr::AO__scoped_atomic_load_n:
589 case AtomicExpr::AO__scoped_atomic_load: {
591 Load->setAtomic(Order,
Scope);
597 case AtomicExpr::AO__c11_atomic_store:
598 case AtomicExpr::AO__opencl_atomic_store:
599 case AtomicExpr::AO__hip_atomic_store:
600 case AtomicExpr::AO__atomic_store:
601 case AtomicExpr::AO__atomic_store_n:
602 case AtomicExpr::AO__scoped_atomic_store:
603 case AtomicExpr::AO__scoped_atomic_store_n: {
606 Store->setAtomic(Order,
Scope);
611 case AtomicExpr::AO__c11_atomic_exchange:
612 case AtomicExpr::AO__hip_atomic_exchange:
613 case AtomicExpr::AO__opencl_atomic_exchange:
614 case AtomicExpr::AO__atomic_exchange_n:
615 case AtomicExpr::AO__atomic_exchange:
616 case AtomicExpr::AO__scoped_atomic_exchange_n:
617 case AtomicExpr::AO__scoped_atomic_exchange:
618 Op = llvm::AtomicRMWInst::Xchg;
621 case AtomicExpr::AO__atomic_add_fetch:
622 case AtomicExpr::AO__scoped_atomic_add_fetch:
624 : llvm::Instruction::Add;
626 case AtomicExpr::AO__c11_atomic_fetch_add:
627 case AtomicExpr::AO__hip_atomic_fetch_add:
628 case AtomicExpr::AO__opencl_atomic_fetch_add:
629 case AtomicExpr::AO__atomic_fetch_add:
630 case AtomicExpr::AO__scoped_atomic_fetch_add:
632 : llvm::AtomicRMWInst::Add;
635 case AtomicExpr::AO__atomic_sub_fetch:
636 case AtomicExpr::AO__scoped_atomic_sub_fetch:
638 : llvm::Instruction::Sub;
640 case AtomicExpr::AO__c11_atomic_fetch_sub:
641 case AtomicExpr::AO__hip_atomic_fetch_sub:
642 case AtomicExpr::AO__opencl_atomic_fetch_sub:
643 case AtomicExpr::AO__atomic_fetch_sub:
644 case AtomicExpr::AO__scoped_atomic_fetch_sub:
646 : llvm::AtomicRMWInst::Sub;
649 case AtomicExpr::AO__atomic_min_fetch:
650 case AtomicExpr::AO__scoped_atomic_min_fetch:
653 case AtomicExpr::AO__c11_atomic_fetch_min:
654 case AtomicExpr::AO__hip_atomic_fetch_min:
655 case AtomicExpr::AO__opencl_atomic_fetch_min:
656 case AtomicExpr::AO__atomic_fetch_min:
657 case AtomicExpr::AO__scoped_atomic_fetch_min:
659 ? llvm::AtomicRMWInst::FMin
661 ? llvm::AtomicRMWInst::Min
662 : llvm::AtomicRMWInst::UMin);
665 case AtomicExpr::AO__atomic_max_fetch:
666 case AtomicExpr::AO__scoped_atomic_max_fetch:
669 case AtomicExpr::AO__c11_atomic_fetch_max:
670 case AtomicExpr::AO__hip_atomic_fetch_max:
671 case AtomicExpr::AO__opencl_atomic_fetch_max:
672 case AtomicExpr::AO__atomic_fetch_max:
673 case AtomicExpr::AO__scoped_atomic_fetch_max:
675 ? llvm::AtomicRMWInst::FMax
677 ? llvm::AtomicRMWInst::Max
678 : llvm::AtomicRMWInst::UMax);
681 case AtomicExpr::AO__atomic_and_fetch:
682 case AtomicExpr::AO__scoped_atomic_and_fetch:
683 PostOp = llvm::Instruction::And;
685 case AtomicExpr::AO__c11_atomic_fetch_and:
686 case AtomicExpr::AO__hip_atomic_fetch_and:
687 case AtomicExpr::AO__opencl_atomic_fetch_and:
688 case AtomicExpr::AO__atomic_fetch_and:
689 case AtomicExpr::AO__scoped_atomic_fetch_and:
690 Op = llvm::AtomicRMWInst::And;
693 case AtomicExpr::AO__atomic_or_fetch:
694 case AtomicExpr::AO__scoped_atomic_or_fetch:
695 PostOp = llvm::Instruction::Or;
697 case AtomicExpr::AO__c11_atomic_fetch_or:
698 case AtomicExpr::AO__hip_atomic_fetch_or:
699 case AtomicExpr::AO__opencl_atomic_fetch_or:
700 case AtomicExpr::AO__atomic_fetch_or:
701 case AtomicExpr::AO__scoped_atomic_fetch_or:
702 Op = llvm::AtomicRMWInst::Or;
705 case AtomicExpr::AO__atomic_xor_fetch:
706 case AtomicExpr::AO__scoped_atomic_xor_fetch:
707 PostOp = llvm::Instruction::Xor;
709 case AtomicExpr::AO__c11_atomic_fetch_xor:
710 case AtomicExpr::AO__hip_atomic_fetch_xor:
711 case AtomicExpr::AO__opencl_atomic_fetch_xor:
712 case AtomicExpr::AO__atomic_fetch_xor:
713 case AtomicExpr::AO__scoped_atomic_fetch_xor:
714 Op = llvm::AtomicRMWInst::Xor;
717 case AtomicExpr::AO__atomic_nand_fetch:
718 case AtomicExpr::AO__scoped_atomic_nand_fetch:
719 PostOp = llvm::Instruction::And;
721 case AtomicExpr::AO__c11_atomic_fetch_nand:
722 case AtomicExpr::AO__atomic_fetch_nand:
723 case AtomicExpr::AO__scoped_atomic_fetch_nand:
724 Op = llvm::AtomicRMWInst::Nand;
729 llvm::AtomicRMWInst *RMWI =
735 llvm::Value *Result = RMWI;
741 Result = CGF.
Builder.CreateBinOp((llvm::Instruction::BinaryOps)PostOp, RMWI,
743 if (E->
getOp() == AtomicExpr::AO__atomic_nand_fetch ||
744 E->
getOp() == AtomicExpr::AO__scoped_atomic_nand_fetch)
745 Result = CGF.
Builder.CreateNot(Result);
761 llvm::Value *IsWeak, llvm::Value *FailureOrder,
762 uint64_t Size, llvm::AtomicOrdering Order,
763 llvm::Value *
Scope) {
764 auto ScopeModel =
Expr->getScopeModel();
775 if (
auto SC = dyn_cast<llvm::ConstantInt>(
Scope)) {
786 auto Scopes = ScopeModel->getRuntimeValues();
787 llvm::DenseMap<unsigned, llvm::BasicBlock *> BB;
788 for (
auto S : Scopes)
791 llvm::BasicBlock *ContBB =
794 auto *SC = Builder.CreateIntCast(
Scope, Builder.getInt32Ty(),
false);
797 auto FallBack = ScopeModel->getFallBackValue();
798 llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]);
799 for (
auto S : Scopes) {
802 SI->addCase(Builder.getInt32(S), B);
804 Builder.SetInsertPoint(B);
811 Builder.CreateBr(ContBB);
814 Builder.SetInsertPoint(ContBB);
821 MemTy = AT->getValueType();
822 llvm::Value *IsWeak =
nullptr, *OrderFail =
nullptr;
829 if (E->
getOp() == AtomicExpr::AO__c11_atomic_init ||
830 E->
getOp() == AtomicExpr::AO__opencl_atomic_init) {
843 bool Misaligned = (Ptr.
getAlignment() % TInfo.Width) != 0;
847 << (
int)TInfo.Width.getQuantity()
852 << (
int)TInfo.Width.getQuantity() << (
int)MaxInlineWidth.
getQuantity();
858 bool ShouldCastToIntPtrTy =
true;
860 switch (E->
getOp()) {
861 case AtomicExpr::AO__c11_atomic_init:
862 case AtomicExpr::AO__opencl_atomic_init:
863 llvm_unreachable(
"Already handled above with EmitAtomicInit!");
865 case AtomicExpr::AO__atomic_load_n:
866 case AtomicExpr::AO__scoped_atomic_load_n:
867 case AtomicExpr::AO__c11_atomic_load:
868 case AtomicExpr::AO__opencl_atomic_load:
869 case AtomicExpr::AO__hip_atomic_load:
872 case AtomicExpr::AO__atomic_load:
873 case AtomicExpr::AO__scoped_atomic_load:
877 case AtomicExpr::AO__atomic_store:
878 case AtomicExpr::AO__scoped_atomic_store:
882 case AtomicExpr::AO__atomic_exchange:
883 case AtomicExpr::AO__scoped_atomic_exchange:
888 case AtomicExpr::AO__atomic_compare_exchange:
889 case AtomicExpr::AO__atomic_compare_exchange_n:
890 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
891 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
892 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
893 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
894 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
895 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
896 case AtomicExpr::AO__scoped_atomic_compare_exchange:
897 case AtomicExpr::AO__scoped_atomic_compare_exchange_n:
899 if (E->
getOp() == AtomicExpr::AO__atomic_compare_exchange ||
900 E->
getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange)
905 if (E->
getOp() == AtomicExpr::AO__atomic_compare_exchange_n ||
906 E->
getOp() == AtomicExpr::AO__atomic_compare_exchange ||
907 E->
getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange_n ||
908 E->
getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange)
912 case AtomicExpr::AO__c11_atomic_fetch_add:
913 case AtomicExpr::AO__c11_atomic_fetch_sub:
914 case AtomicExpr::AO__hip_atomic_fetch_add:
915 case AtomicExpr::AO__hip_atomic_fetch_sub:
916 case AtomicExpr::AO__opencl_atomic_fetch_add:
917 case AtomicExpr::AO__opencl_atomic_fetch_sub:
934 case AtomicExpr::AO__atomic_fetch_add:
935 case AtomicExpr::AO__atomic_fetch_max:
936 case AtomicExpr::AO__atomic_fetch_min:
937 case AtomicExpr::AO__atomic_fetch_sub:
938 case AtomicExpr::AO__atomic_add_fetch:
939 case AtomicExpr::AO__atomic_max_fetch:
940 case AtomicExpr::AO__atomic_min_fetch:
941 case AtomicExpr::AO__atomic_sub_fetch:
942 case AtomicExpr::AO__c11_atomic_fetch_max:
943 case AtomicExpr::AO__c11_atomic_fetch_min:
944 case AtomicExpr::AO__opencl_atomic_fetch_max:
945 case AtomicExpr::AO__opencl_atomic_fetch_min:
946 case AtomicExpr::AO__hip_atomic_fetch_max:
947 case AtomicExpr::AO__hip_atomic_fetch_min:
948 case AtomicExpr::AO__scoped_atomic_fetch_add:
949 case AtomicExpr::AO__scoped_atomic_fetch_max:
950 case AtomicExpr::AO__scoped_atomic_fetch_min:
951 case AtomicExpr::AO__scoped_atomic_fetch_sub:
952 case AtomicExpr::AO__scoped_atomic_add_fetch:
953 case AtomicExpr::AO__scoped_atomic_max_fetch:
954 case AtomicExpr::AO__scoped_atomic_min_fetch:
955 case AtomicExpr::AO__scoped_atomic_sub_fetch:
959 case AtomicExpr::AO__atomic_fetch_and:
960 case AtomicExpr::AO__atomic_fetch_nand:
961 case AtomicExpr::AO__atomic_fetch_or:
962 case AtomicExpr::AO__atomic_fetch_xor:
963 case AtomicExpr::AO__atomic_and_fetch:
964 case AtomicExpr::AO__atomic_nand_fetch:
965 case AtomicExpr::AO__atomic_or_fetch:
966 case AtomicExpr::AO__atomic_xor_fetch:
967 case AtomicExpr::AO__atomic_store_n:
968 case AtomicExpr::AO__atomic_exchange_n:
969 case AtomicExpr::AO__c11_atomic_fetch_and:
970 case AtomicExpr::AO__c11_atomic_fetch_nand:
971 case AtomicExpr::AO__c11_atomic_fetch_or:
972 case AtomicExpr::AO__c11_atomic_fetch_xor:
973 case AtomicExpr::AO__c11_atomic_store:
974 case AtomicExpr::AO__c11_atomic_exchange:
975 case AtomicExpr::AO__hip_atomic_fetch_and:
976 case AtomicExpr::AO__hip_atomic_fetch_or:
977 case AtomicExpr::AO__hip_atomic_fetch_xor:
978 case AtomicExpr::AO__hip_atomic_store:
979 case AtomicExpr::AO__hip_atomic_exchange:
980 case AtomicExpr::AO__opencl_atomic_fetch_and:
981 case AtomicExpr::AO__opencl_atomic_fetch_or:
982 case AtomicExpr::AO__opencl_atomic_fetch_xor:
983 case AtomicExpr::AO__opencl_atomic_store:
984 case AtomicExpr::AO__opencl_atomic_exchange:
985 case AtomicExpr::AO__scoped_atomic_fetch_and:
986 case AtomicExpr::AO__scoped_atomic_fetch_nand:
987 case AtomicExpr::AO__scoped_atomic_fetch_or:
988 case AtomicExpr::AO__scoped_atomic_fetch_xor:
989 case AtomicExpr::AO__scoped_atomic_and_fetch:
990 case AtomicExpr::AO__scoped_atomic_nand_fetch:
991 case AtomicExpr::AO__scoped_atomic_or_fetch:
992 case AtomicExpr::AO__scoped_atomic_xor_fetch:
993 case AtomicExpr::AO__scoped_atomic_store_n:
994 case AtomicExpr::AO__scoped_atomic_exchange_n:
1005 AtomicInfo Atomics(*
this, AtomicVal);
1007 if (ShouldCastToIntPtrTy) {
1008 Ptr = Atomics.castToAtomicIntPointer(Ptr);
1010 Val1 = Atomics.convertToAtomicIntPointer(Val1);
1012 Val2 = Atomics.convertToAtomicIntPointer(Val2);
1015 if (ShouldCastToIntPtrTy)
1016 Dest = Atomics.castToAtomicIntPointer(Dest);
1020 Dest = Atomics.CreateTempAlloca();
1021 if (ShouldCastToIntPtrTy)
1022 Dest = Atomics.castToAtomicIntPointer(Dest);
1025 bool PowerOf2Size = (
Size & (
Size - 1)) == 0;
1026 bool UseLibcall = !PowerOf2Size || (
Size > 16);
1046 auto CastToGenericAddrSpace = [&](llvm::Value *
V,
QualType PT) {
1049 auto AS = PT->castAs<
PointerType>()->getPointeeType().getAddressSpace();
1053 auto *DestType = llvm::PointerType::get(
getLLVMContext(), DestAS);
1063 std::string LibCallName;
1065 bool HaveRetTy =
false;
1066 switch (E->
getOp()) {
1067 case AtomicExpr::AO__c11_atomic_init:
1068 case AtomicExpr::AO__opencl_atomic_init:
1069 llvm_unreachable(
"Already handled!");
1076 case AtomicExpr::AO__atomic_compare_exchange:
1077 case AtomicExpr::AO__atomic_compare_exchange_n:
1078 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1079 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1080 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
1081 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
1082 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1083 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1084 case AtomicExpr::AO__scoped_atomic_compare_exchange:
1085 case AtomicExpr::AO__scoped_atomic_compare_exchange_n:
1086 LibCallName =
"__atomic_compare_exchange";
1100 case AtomicExpr::AO__atomic_exchange:
1101 case AtomicExpr::AO__atomic_exchange_n:
1102 case AtomicExpr::AO__c11_atomic_exchange:
1103 case AtomicExpr::AO__hip_atomic_exchange:
1104 case AtomicExpr::AO__opencl_atomic_exchange:
1105 case AtomicExpr::AO__scoped_atomic_exchange:
1106 case AtomicExpr::AO__scoped_atomic_exchange_n:
1107 LibCallName =
"__atomic_exchange";
1113 case AtomicExpr::AO__atomic_store:
1114 case AtomicExpr::AO__atomic_store_n:
1115 case AtomicExpr::AO__c11_atomic_store:
1116 case AtomicExpr::AO__hip_atomic_store:
1117 case AtomicExpr::AO__opencl_atomic_store:
1118 case AtomicExpr::AO__scoped_atomic_store:
1119 case AtomicExpr::AO__scoped_atomic_store_n:
1120 LibCallName =
"__atomic_store";
1128 case AtomicExpr::AO__atomic_load:
1129 case AtomicExpr::AO__atomic_load_n:
1130 case AtomicExpr::AO__c11_atomic_load:
1131 case AtomicExpr::AO__hip_atomic_load:
1132 case AtomicExpr::AO__opencl_atomic_load:
1133 case AtomicExpr::AO__scoped_atomic_load:
1134 case AtomicExpr::AO__scoped_atomic_load_n:
1135 LibCallName =
"__atomic_load";
1137 case AtomicExpr::AO__atomic_add_fetch:
1138 case AtomicExpr::AO__scoped_atomic_add_fetch:
1139 case AtomicExpr::AO__atomic_fetch_add:
1140 case AtomicExpr::AO__c11_atomic_fetch_add:
1141 case AtomicExpr::AO__hip_atomic_fetch_add:
1142 case AtomicExpr::AO__opencl_atomic_fetch_add:
1143 case AtomicExpr::AO__scoped_atomic_fetch_add:
1144 case AtomicExpr::AO__atomic_and_fetch:
1145 case AtomicExpr::AO__scoped_atomic_and_fetch:
1146 case AtomicExpr::AO__atomic_fetch_and:
1147 case AtomicExpr::AO__c11_atomic_fetch_and:
1148 case AtomicExpr::AO__hip_atomic_fetch_and:
1149 case AtomicExpr::AO__opencl_atomic_fetch_and:
1150 case AtomicExpr::AO__scoped_atomic_fetch_and:
1151 case AtomicExpr::AO__atomic_or_fetch:
1152 case AtomicExpr::AO__scoped_atomic_or_fetch:
1153 case AtomicExpr::AO__atomic_fetch_or:
1154 case AtomicExpr::AO__c11_atomic_fetch_or:
1155 case AtomicExpr::AO__hip_atomic_fetch_or:
1156 case AtomicExpr::AO__opencl_atomic_fetch_or:
1157 case AtomicExpr::AO__scoped_atomic_fetch_or:
1158 case AtomicExpr::AO__atomic_sub_fetch:
1159 case AtomicExpr::AO__scoped_atomic_sub_fetch:
1160 case AtomicExpr::AO__atomic_fetch_sub:
1161 case AtomicExpr::AO__c11_atomic_fetch_sub:
1162 case AtomicExpr::AO__hip_atomic_fetch_sub:
1163 case AtomicExpr::AO__opencl_atomic_fetch_sub:
1164 case AtomicExpr::AO__scoped_atomic_fetch_sub:
1165 case AtomicExpr::AO__atomic_xor_fetch:
1166 case AtomicExpr::AO__scoped_atomic_xor_fetch:
1167 case AtomicExpr::AO__atomic_fetch_xor:
1168 case AtomicExpr::AO__c11_atomic_fetch_xor:
1169 case AtomicExpr::AO__hip_atomic_fetch_xor:
1170 case AtomicExpr::AO__opencl_atomic_fetch_xor:
1171 case AtomicExpr::AO__scoped_atomic_fetch_xor:
1172 case AtomicExpr::AO__atomic_nand_fetch:
1173 case AtomicExpr::AO__atomic_fetch_nand:
1174 case AtomicExpr::AO__c11_atomic_fetch_nand:
1175 case AtomicExpr::AO__scoped_atomic_fetch_nand:
1176 case AtomicExpr::AO__scoped_atomic_nand_fetch:
1177 case AtomicExpr::AO__atomic_min_fetch:
1178 case AtomicExpr::AO__atomic_fetch_min:
1179 case AtomicExpr::AO__c11_atomic_fetch_min:
1180 case AtomicExpr::AO__hip_atomic_fetch_min:
1181 case AtomicExpr::AO__opencl_atomic_fetch_min:
1182 case AtomicExpr::AO__scoped_atomic_fetch_min:
1183 case AtomicExpr::AO__scoped_atomic_min_fetch:
1184 case AtomicExpr::AO__atomic_max_fetch:
1185 case AtomicExpr::AO__atomic_fetch_max:
1186 case AtomicExpr::AO__c11_atomic_fetch_max:
1187 case AtomicExpr::AO__hip_atomic_fetch_max:
1188 case AtomicExpr::AO__opencl_atomic_fetch_max:
1189 case AtomicExpr::AO__scoped_atomic_fetch_max:
1190 case AtomicExpr::AO__scoped_atomic_max_fetch:
1191 llvm_unreachable(
"Integral atomic operations always become atomicrmw!");
1196 std::string(
"__opencl") + StringRef(LibCallName).drop_front(1).str();
1223 bool IsStore = E->
getOp() == AtomicExpr::AO__c11_atomic_store ||
1224 E->
getOp() == AtomicExpr::AO__opencl_atomic_store ||
1225 E->
getOp() == AtomicExpr::AO__hip_atomic_store ||
1226 E->
getOp() == AtomicExpr::AO__atomic_store ||
1227 E->
getOp() == AtomicExpr::AO__atomic_store_n ||
1228 E->
getOp() == AtomicExpr::AO__scoped_atomic_store ||
1229 E->
getOp() == AtomicExpr::AO__scoped_atomic_store_n;
1230 bool IsLoad = E->
getOp() == AtomicExpr::AO__c11_atomic_load ||
1231 E->
getOp() == AtomicExpr::AO__opencl_atomic_load ||
1232 E->
getOp() == AtomicExpr::AO__hip_atomic_load ||
1233 E->
getOp() == AtomicExpr::AO__atomic_load ||
1234 E->
getOp() == AtomicExpr::AO__atomic_load_n ||
1235 E->
getOp() == AtomicExpr::AO__scoped_atomic_load ||
1236 E->
getOp() == AtomicExpr::AO__scoped_atomic_load_n;
1238 if (isa<llvm::ConstantInt>(Order)) {
1239 auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1242 if (llvm::isValidAtomicOrderingCABI(ord))
1243 switch ((llvm::AtomicOrderingCABI)ord) {
1244 case llvm::AtomicOrderingCABI::relaxed:
1245 EmitAtomicOp(*
this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1246 llvm::AtomicOrdering::Monotonic,
Scope);
1248 case llvm::AtomicOrderingCABI::consume:
1249 case llvm::AtomicOrderingCABI::acquire:
1252 EmitAtomicOp(*
this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1253 llvm::AtomicOrdering::Acquire,
Scope);
1255 case llvm::AtomicOrderingCABI::release:
1258 EmitAtomicOp(*
this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1259 llvm::AtomicOrdering::Release,
Scope);
1261 case llvm::AtomicOrderingCABI::acq_rel:
1262 if (IsLoad || IsStore)
1264 EmitAtomicOp(*
this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1265 llvm::AtomicOrdering::AcquireRelease,
Scope);
1267 case llvm::AtomicOrderingCABI::seq_cst:
1268 EmitAtomicOp(*
this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1269 llvm::AtomicOrdering::SequentiallyConsistent,
Scope);
1282 llvm::BasicBlock *MonotonicBB =
nullptr, *AcquireBB =
nullptr,
1283 *ReleaseBB =
nullptr, *AcqRelBB =
nullptr,
1284 *SeqCstBB =
nullptr;
1290 if (!IsLoad && !IsStore)
1299 Order =
Builder.CreateIntCast(Order,
Builder.getInt32Ty(),
false);
1300 llvm::SwitchInst *SI =
Builder.CreateSwitch(Order, MonotonicBB);
1303 Builder.SetInsertPoint(MonotonicBB);
1304 EmitAtomicOp(*
this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1305 llvm::AtomicOrdering::Monotonic,
Scope);
1308 Builder.SetInsertPoint(AcquireBB);
1309 EmitAtomicOp(*
this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1310 llvm::AtomicOrdering::Acquire,
Scope);
1312 SI->addCase(
Builder.getInt32((
int)llvm::AtomicOrderingCABI::consume),
1314 SI->addCase(
Builder.getInt32((
int)llvm::AtomicOrderingCABI::acquire),
1318 Builder.SetInsertPoint(ReleaseBB);
1319 EmitAtomicOp(*
this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1320 llvm::AtomicOrdering::Release,
Scope);
1322 SI->addCase(
Builder.getInt32((
int)llvm::AtomicOrderingCABI::release),
1325 if (!IsLoad && !IsStore) {
1326 Builder.SetInsertPoint(AcqRelBB);
1327 EmitAtomicOp(*
this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1328 llvm::AtomicOrdering::AcquireRelease,
Scope);
1330 SI->addCase(
Builder.getInt32((
int)llvm::AtomicOrderingCABI::acq_rel),
1333 Builder.SetInsertPoint(SeqCstBB);
1334 EmitAtomicOp(*
this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1335 llvm::AtomicOrdering::SequentiallyConsistent,
Scope);
1337 SI->addCase(
Builder.getInt32((
int)llvm::AtomicOrderingCABI::seq_cst),
1341 Builder.SetInsertPoint(ContBB);
1345 assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
1351 llvm::IntegerType *ty =
1356Address AtomicInfo::convertToAtomicIntPointer(
Address Addr)
const {
1359 if (SourceSizeInBits != AtomicSizeInBits) {
1360 Address Tmp = CreateTempAlloca();
1362 std::min(AtomicSizeInBits, SourceSizeInBits) / 8);
1366 return castToAtomicIntPointer(Addr);
1372 bool asValue)
const {
1402RValue AtomicInfo::ConvertToValueOrAtomic(llvm::Value *Val,
1405 bool CastFP)
const {
1407 assert((Val->getType()->isIntegerTy() || Val->getType()->isIEEELikeFPTy()) &&
1408 "Expected integer or floating point value");
1414 auto *ValTy = AsValue
1416 : getAtomicAddress().getElementType();
1417 if (ValTy->isIntegerTy() || (!CastFP && ValTy->isIEEELikeFPTy())) {
1418 assert((!ValTy->isIntegerTy() || Val->getType() == ValTy) &&
1419 "Different integer types.");
1421 }
else if (ValTy->isPointerTy())
1423 else if (llvm::CastInst::isBitCastable(Val->getType(), ValTy))
1430 bool TempIsVolatile =
false;
1436 Temp = CreateTempAlloca();
1440 Address CastTemp = castToAtomicIntPointer(Temp);
1443 return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue);
1446void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1447 llvm::AtomicOrdering AO,
bool) {
1459llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1460 bool IsVolatile,
bool CastFP) {
1462 Address Addr = getAtomicAddress();
1464 Addr = castToAtomicIntPointer(Addr);
1466 Load->setAtomic(AO);
1470 Load->setVolatile(
true);
1480 AtomicInfo AI(*
this, LV);
1483 bool AtomicIsInline = !AI.shouldUseLibcall();
1488 return IsVolatile && AtomicIsInline;
1493 llvm::AtomicOrdering AO;
1496 AO = llvm::AtomicOrdering::SequentiallyConsistent;
1498 AO = llvm::AtomicOrdering::Acquire;
1505 bool AsValue, llvm::AtomicOrdering AO,
1508 if (shouldUseLibcall()) {
1514 TempAddr = CreateTempAlloca();
1516 EmitAtomicLoadLibcall(TempAddr.
getPointer(), AO, IsVolatile);
1520 return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
1524 auto *
Load = EmitAtomicLoadOp(AO, IsVolatile,
false);
1532 return ConvertToValueOrAtomic(Load, ResultSlot, Loc, AsValue,
1539 llvm::AtomicOrdering AO,
bool IsVolatile,
1541 AtomicInfo Atomics(*
this, src);
1542 return Atomics.EmitAtomicLoad(resultSlot, loc,
true, AO,
1548void AtomicInfo::emitCopyIntoMemory(
RValue rvalue)
const {
1567 emitMemSetZeroIfNecessary();
1570 LValue TempLVal = projectValue();
1591 AtomicInfo Atomics(CGF, TempLV);
1592 Atomics.emitCopyIntoMemory(rvalue);
1596llvm::Value *AtomicInfo::getScalarRValValueOrNull(
RValue RVal)
const {
1602llvm::Value *AtomicInfo::convertRValueToInt(
RValue RVal,
bool CastFP)
const {
1605 if (llvm::Value *
Value = getScalarRValValueOrNull(RVal)) {
1610 llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1612 LVal.
isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1615 else if (llvm::BitCastInst::isBitCastable(
Value->
getType(), InputIntTy))
1621 Address Addr = materializeRValue(RVal);
1624 Addr = castToAtomicIntPointer(Addr);
1628std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1629 llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1630 llvm::AtomicOrdering
Success, llvm::AtomicOrdering Failure,
bool IsWeak) {
1632 Address Addr = getAtomicAddressAsAtomicIntPointer();
1637 Inst->setWeak(IsWeak);
1640 auto *PreviousVal = CGF.
Builder.CreateExtractValue(Inst, 0);
1641 auto *SuccessFailureVal = CGF.
Builder.CreateExtractValue(Inst, 1);
1642 return std::make_pair(PreviousVal, SuccessFailureVal);
1646AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1647 llvm::Value *DesiredAddr,
1649 llvm::AtomicOrdering Failure) {
1658 llvm::ConstantInt::get(CGF.
IntTy, (
int)llvm::toCABI(
Success))),
1661 llvm::ConstantInt::get(CGF.
IntTy, (
int)llvm::toCABI(Failure))),
1666 return SuccessFailureRVal.getScalarVal();
1669std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1671 llvm::AtomicOrdering Failure,
bool IsWeak) {
1673 if (shouldUseLibcall()) {
1676 Address DesiredAddr = materializeRValue(Desired);
1677 auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.
getPointer(),
1680 return std::make_pair(
1688 auto *ExpectedVal = convertRValueToInt(
Expected);
1689 auto *DesiredVal = convertRValueToInt(Desired);
1690 auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal,
Success,
1692 return std::make_pair(
1703 LValue AtomicLVal = Atomics.getAtomicLValue();
1710 Address Ptr = Atomics.materializeRValue(OldRVal);
1743 RValue NewRVal = UpdateOp(UpRVal);
1753void AtomicInfo::EmitAtomicUpdateLibcall(
1754 llvm::AtomicOrdering AO,
const llvm::function_ref<
RValue(
RValue)> &UpdateOp,
1756 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1758 Address ExpectedAddr = CreateTempAlloca();
1760 EmitAtomicLoadLibcall(ExpectedAddr.
getPointer(), AO, IsVolatile);
1764 Address DesiredAddr = CreateTempAlloca();
1766 requiresMemSetZero(getAtomicAddress().getElementType())) {
1770 auto OldRVal = convertAtomicTempToRValue(ExpectedAddr,
1775 EmitAtomicCompareExchangeLibcall(ExpectedAddr.
getPointer(),
1778 CGF.
Builder.CreateCondBr(Res, ExitBB, ContBB);
1782void AtomicInfo::EmitAtomicUpdateOp(
1783 llvm::AtomicOrdering AO,
const llvm::function_ref<
RValue(
RValue)> &UpdateOp,
1785 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1788 auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile);
1792 auto *CurBB = CGF.
Builder.GetInsertBlock();
1794 llvm::PHINode *PHI = CGF.
Builder.CreatePHI(OldVal->getType(),
1796 PHI->addIncoming(OldVal, CurBB);
1797 Address NewAtomicAddr = CreateTempAlloca();
1798 Address NewAtomicIntAddr = castToAtomicIntPointer(NewAtomicAddr);
1800 requiresMemSetZero(getAtomicAddress().getElementType())) {
1808 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1809 PHI->addIncoming(Res.first, CGF.
Builder.GetInsertBlock());
1810 CGF.
Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1816 LValue AtomicLVal = Atomics.getAtomicLValue();
1840void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1841 RValue UpdateRVal,
bool IsVolatile) {
1842 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1844 Address ExpectedAddr = CreateTempAlloca();
1846 EmitAtomicLoadLibcall(ExpectedAddr.
getPointer(), AO, IsVolatile);
1850 Address DesiredAddr = CreateTempAlloca();
1852 requiresMemSetZero(getAtomicAddress().getElementType())) {
1858 EmitAtomicCompareExchangeLibcall(ExpectedAddr.
getPointer(),
1861 CGF.
Builder.CreateCondBr(Res, ExitBB, ContBB);
1865void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
RValue UpdateRVal,
1867 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1870 auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile);
1874 auto *CurBB = CGF.
Builder.GetInsertBlock();
1876 llvm::PHINode *PHI = CGF.
Builder.CreatePHI(OldVal->getType(),
1878 PHI->addIncoming(OldVal, CurBB);
1879 Address NewAtomicAddr = CreateTempAlloca();
1880 Address NewAtomicIntAddr = castToAtomicIntPointer(NewAtomicAddr);
1882 requiresMemSetZero(getAtomicAddress().getElementType())) {
1888 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1889 PHI->addIncoming(Res.first, CGF.
Builder.GetInsertBlock());
1890 CGF.
Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1894void AtomicInfo::EmitAtomicUpdate(
1895 llvm::AtomicOrdering AO,
const llvm::function_ref<
RValue(
RValue)> &UpdateOp,
1897 if (shouldUseLibcall()) {
1898 EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
1900 EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
1904void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO,
RValue UpdateRVal,
1906 if (shouldUseLibcall()) {
1907 EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
1909 EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
1916 llvm::AtomicOrdering AO;
1918 AO = llvm::AtomicOrdering::SequentiallyConsistent;
1920 AO = llvm::AtomicOrdering::Release;
1932 llvm::AtomicOrdering AO,
bool IsVolatile,
1940 AtomicInfo atomics(*
this, dest);
1941 LValue LVal = atomics.getAtomicLValue();
1946 atomics.emitCopyIntoMemory(rvalue);
1951 if (atomics.shouldUseLibcall()) {
1953 Address srcAddr = atomics.materializeRValue(rvalue);
1969 llvm::Value *ValToStore =
1970 atomics.convertRValueToInt(rvalue,
false);
1973 Address Addr = atomics.getAtomicAddress();
1974 bool ShouldCastToInt =
true;
1975 if (llvm::Value *
Value = atomics.getScalarRValValueOrNull(rvalue))
1978 ShouldCastToInt =
false;
1979 if (ShouldCastToInt) {
1980 Addr = atomics.castToAtomicIntPointer(Addr);
1986 if (AO == llvm::AtomicOrdering::Acquire)
1987 AO = llvm::AtomicOrdering::Monotonic;
1988 else if (AO == llvm::AtomicOrdering::AcquireRelease)
1989 AO = llvm::AtomicOrdering::Release;
1992 store->setAtomic(AO);
1996 store->setVolatile(
true);
2002 atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
2009 llvm::AtomicOrdering
Success, llvm::AtomicOrdering Failure,
bool IsWeak,
2014 Expected.getAggregateAddress().getElementType() ==
2019 AtomicInfo Atomics(*
this, Obj);
2021 return Atomics.EmitAtomicCompareExchange(
Expected, Desired,
Success, Failure,
2026 LValue LVal, llvm::AtomicOrdering AO,
2027 const llvm::function_ref<
RValue(
RValue)> &UpdateOp,
bool IsVolatile) {
2028 AtomicInfo Atomics(*
this, LVal);
2029 Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
2033 AtomicInfo atomics(*
this, dest);
2035 switch (atomics.getEvaluationKind()) {
2051 bool Zeroed =
false;
2053 Zeroed = atomics.emitMemSetZeroIfNecessary();
2054 dest = atomics.projectValue();
2068 llvm_unreachable(
"bad evaluation kind");
Defines the clang::ASTContext interface.
static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type, uint64_t expectedSize)
Does a store of the given IR type modify the full expected width?
static llvm::Value * EmitPostAtomicMinMax(CGBuilderTy &Builder, AtomicExpr::AtomicOp Op, bool IsSigned, llvm::Value *OldVal, llvm::Value *RHS)
Duplicate the atomic min/max operation in conventional IR for the builtin variants that return the ne...
static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal, const llvm::function_ref< RValue(RValue)> &UpdateOp, Address DesiredAddr)
static Address EmitValToTemp(CodeGenFunction &CGF, Expr *E)
static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, Address Ptr, Address Val1, Address Val2, llvm::Value *IsWeak, llvm::Value *FailureOrder, uint64_t Size, llvm::AtomicOrdering Order, llvm::SyncScope::ID Scope)
static RValue emitAtomicLibcall(CodeGenFunction &CGF, StringRef fnName, QualType resultType, CallArgList &args)
static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, Address Dest, Address Ptr, Address Val1, Address Val2, llvm::Value *FailureOrderVal, uint64_t Size, llvm::AtomicOrdering SuccessOrder, llvm::SyncScope::ID Scope)
Given an ordering required on success, emit all possible cmpxchg instructions to cope with the provid...
static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, Address Dest, Address Ptr, Address Val1, Address Val2, uint64_t Size, llvm::AtomicOrdering SuccessOrder, llvm::AtomicOrdering FailureOrder, llvm::SyncScope::ID Scope)
CodeGenFunction::ComplexPairTy ComplexPairTy
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
CanQualType getSizeType() const
Return the unique type for "size_t" (C99 7.17), defined in <stddef.h>.
TypeInfoChars getTypeInfoInChars(const Type *T) const
int64_t toBits(CharUnits CharSize) const
Convert a size in characters to a size in bits.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
QualType getExtVectorType(QualType VectorType, unsigned NumElts) const
Return the unique reference to an extended vector type of the specified element type and size.
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
unsigned getTargetAddressSpace(LangAS AS) const
AtomicExpr - Variadic atomic builtins: __atomic_exchange, __atomic_fetch_*, __atomic_load,...
static std::unique_ptr< AtomicScopeModel > getScopeModel(AtomicOp Op)
Get atomic scope model for the atomic op code.
QualType getValueType() const
SourceLocation getBeginLoc() const LLVM_READONLY
Expr * getOrderFail() const
CharUnits - This is an opaque type for sizes expressed in character units.
bool isZero() const
isZero - Test whether the quantity equals zero.
llvm::Align getAsAlign() const
getAsAlign - Returns Quantity as a valid llvm::Align, Beware llvm::Align assumes power of two 8-bit b...
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
CharUnits getAlignment() const
Return the alignment of this pointer.
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
llvm::Value * getPointer() const
static AggValueSlot ignored()
ignored - Returns an aggregate value slot indicating that the aggregate value is being ignored.
Address getAddress() const
static AggValueSlot forLValue(const LValue &LV, CodeGenFunction &CGF, IsDestructed_t isDestructed, NeedsGCBarriers_t needsGC, IsAliased_t isAliased, Overlap_t mayOverlap, IsZeroed_t isZeroed=IsNotZeroed, IsSanitizerChecked_t isChecked=IsNotSanitizerChecked)
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
llvm::AtomicRMWInst * CreateAtomicRMW(llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val, llvm::AtomicOrdering Ordering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
llvm::CallInst * CreateMemSet(Address Dest, llvm::Value *Value, llvm::Value *Size, bool IsVolatile=false)
Address CreateStructGEP(Address Addr, unsigned Index, const llvm::Twine &Name="")
llvm::AtomicCmpXchgInst * CreateAtomicCmpXchg(Address Addr, llvm::Value *Cmp, llvm::Value *New, llvm::AtomicOrdering SuccessOrdering, llvm::AtomicOrdering FailureOrdering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
llvm::CallInst * CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
Address CreateAddrSpaceCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
static CGCallee forDirect(llvm::Constant *functionPtr, const CGCalleeInfo &abstractInfo=CGCalleeInfo())
CGFunctionInfo - Class to encapsulate the information about a function definition.
CallArgList - Type for representing both the value and type of arguments in a call.
void add(RValue rvalue, QualType type)
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
llvm::Value * EmitFromMemory(llvm::Value *Value, QualType Ty)
EmitFromMemory - Change a scalar value from its memory representation to its value representation.
std::pair< RValue, llvm::Value * > EmitAtomicCompareExchange(LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, llvm::AtomicOrdering Success=llvm::AtomicOrdering::SequentiallyConsistent, llvm::AtomicOrdering Failure=llvm::AtomicOrdering::SequentiallyConsistent, bool IsWeak=false, AggValueSlot Slot=AggValueSlot::ignored())
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
RValue EmitAtomicLoad(LValue LV, SourceLocation SL, AggValueSlot Slot=AggValueSlot::ignored())
bool hasVolatileMember(QualType T)
hasVolatileMember - returns true if aggregate type has a volatile member.
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO, const llvm::function_ref< RValue(RValue)> &UpdateOp, bool IsVolatile)
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
ComplexPairTy EmitComplexExpr(const Expr *E, bool IgnoreReal=false, bool IgnoreImag=false)
EmitComplexExpr - Emit the computation of the specified expression of complex type,...
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
RValue convertTempToRValue(Address addr, QualType type, SourceLocation Loc)
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee, ReturnValueSlot ReturnValue, const CallArgList &Args, llvm::CallBase **callOrInvoke, bool IsMustTail, SourceLocation Loc)
EmitCall - Generate a call of the given function, expecting the given result type,...
llvm::Type * ConvertTypeForMem(QualType T)
void EmitAtomicInit(Expr *E, LValue lvalue)
const TargetInfo & getTarget() const
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
Address EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitPointerWithAlignment - Given an expression with a pointer type, emit the value and compute our be...
RValue EmitLoadOfExtVectorElementLValue(LValue V)
void EmitAggregateCopy(LValue Dest, LValue Src, QualType EltTy, AggValueSlot::Overlap_t MayOverlap, bool isVolatile=false)
EmitAggregateCopy - Emit an aggregate copy.
const TargetCodeGenInfo & getTargetHooks() const
void EmitAggExpr(const Expr *E, AggValueSlot AS)
EmitAggExpr - Emit the computation of the specified expression of aggregate type.
llvm::Value * EmitToMemory(llvm::Value *Value, QualType Ty)
EmitToMemory - Change a scalar value from its value representation to its in-memory representation.
ASTContext & getContext() const
Address CreateMemTemp(QualType T, const Twine &Name="tmp", Address *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit)
EmitStoreOfComplex - Store a complex number into the specified l-value.
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
llvm::LLVMContext & getLLVMContext()
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
bool LValueIsSuitableForInlineAtomic(LValue Src)
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
RValue EmitLoadOfBitfieldLValue(LValue LV, SourceLocation Loc)
This class organizes the cross-function state that is used while generating LLVM code.
llvm::FunctionCallee CreateRuntimeFunction(llvm::FunctionType *Ty, StringRef Name, llvm::AttributeList ExtraAttrs=llvm::AttributeList(), bool Local=false, bool AssumeConvergent=false)
Create or return a runtime function declaration with the specified type and name.
DiagnosticsEngine & getDiags() const
const LangOptions & getLangOpts() const
CodeGenTypes & getTypes()
const llvm::DataLayout & getDataLayout() const
void DecorateInstructionWithTBAA(llvm::Instruction *Inst, TBAAAccessInfo TBAAInfo)
DecorateInstructionWithTBAA - Decorate the instruction with a TBAA tag.
llvm::LLVMContext & getLLVMContext()
llvm::ConstantInt * getSize(CharUnits numChars)
Emit the given number of characters as a value of type size_t.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
const CGFunctionInfo & arrangeBuiltinFunctionCall(QualType resultType, const CallArgList &args)
LValue - This represents an lvalue references.
llvm::Constant * getExtVectorElts() const
static LValue MakeExtVectorElt(Address vecAddress, llvm::Constant *Elts, QualType type, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo)
void setAlignment(CharUnits A)
llvm::Value * getBitFieldPointer() const
bool isVolatileQualified() const
CharUnits getAlignment() const
Address getAddress(CodeGenFunction &CGF) const
llvm::Value * getVectorPointer() const
bool isExtVectorElt() const
llvm::Value * getVectorIdx() const
LValueBaseInfo getBaseInfo() const
llvm::Value * getPointer(CodeGenFunction &CGF) const
llvm::Value * getExtVectorPointer() const
const CGBitFieldInfo & getBitFieldInfo() const
TBAAAccessInfo getTBAAInfo() const
Address getVectorAddress() const
static LValue MakeBitfield(Address Addr, const CGBitFieldInfo &Info, QualType type, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo)
Create a new object to represent a bit-field access.
static LValue MakeVectorElt(Address vecAddress, llvm::Value *Idx, QualType type, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo)
Address getExtVectorAddress() const
Address getBitFieldAddress() const
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
static RValue get(llvm::Value *V)
static RValue getAggregate(Address addr, bool isVolatile=false)
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
Address getAggregateAddress() const
getAggregateAddr() - Return the Value* of the address of the aggregate.
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
bool isVolatileQualified() const
std::pair< llvm::Value *, llvm::Value * > getComplexVal() const
getComplexVal - Return the real/imag components of this complex value.
ReturnValueSlot - Contains the address where the return value of a function can be stored,...
virtual llvm::SyncScope::ID getLLVMSyncScopeID(const LangOptions &LangOpts, SyncScope Scope, llvm::AtomicOrdering Ordering, llvm::LLVMContext &Ctx) const
Get the syncscope used in LLVM IR.
virtual llvm::Value * performAddrSpaceCast(CodeGen::CodeGenFunction &CGF, llvm::Value *V, LangAS SrcAddr, LangAS DestAddr, llvm::Type *DestTy, bool IsNonNull=false) const
Perform address space cast of an expression of pointer type.
Concrete class used by the front-end to report problems and issues.
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
This represents one expression.
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
PointerType - C99 6.7.5.1 - Pointer Declarators.
A (possibly-)qualified type.
bool isNull() const
Return true if this QualType doesn't point to a type yet.
Qualifiers getQualifiers() const
Retrieve the set of qualifiers applied to this type.
QualType getUnqualifiedType() const
Retrieve the unqualified variant of the given type, removing as little sugar as possible.
Scope - A scope is a transient data structure that is used while parsing the program.
Encodes a location in the source.
unsigned getMaxAtomicInlineWidth() const
Return the maximum width lock-free atomic operation which can be inlined given the supported features...
bool isSignedIntegerType() const
Return true if this is an integer type that is signed, according to C99 6.2.5p4 [char,...
bool isPointerType() const
const T * castAs() const
Member-template castAs<specific type>.
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
bool isAtomicType() const
bool isFloatingType() const
const T * getAs() const
Member-template getAs<specific type>'.
Represents a GCC generic vector type.
TypeEvaluationKind
The kind of evaluation to perform on values of a particular type.
const internal::VariadicAllOfMatcher< Type > type
Matches Types in the clang AST.
bool Load(InterpState &S, CodePtr OpPC)
The JSON file list parser is used to communicate input to InstallAPI.
llvm::StringRef getAsString(SyncScope S)
@ Success
Template argument deduction was successful.
Structure with information about how a bitfield should be accessed.
CharUnits StorageOffset
The offset of the bitfield storage from the start of the struct.
unsigned Offset
The offset within a contiguous run of bitfields that are represented as a single "field" within the L...
unsigned Size
The total size of the bit-field, in bits.
unsigned StorageSize
The storage size in bits which should be used when accessing this bitfield.
llvm::PointerType * VoidPtrTy
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
llvm::IntegerType * SizeTy
llvm::IntegerType * IntTy
int
llvm::PointerType * UnqualPtrTy