32#include "llvm/ADT/APFloat.h"
33#include "llvm/ADT/APInt.h"
34#include "llvm/ADT/SmallPtrSet.h"
35#include "llvm/ADT/StringExtras.h"
36#include "llvm/Analysis/ValueTracking.h"
37#include "llvm/IR/DataLayout.h"
38#include "llvm/IR/InlineAsm.h"
39#include "llvm/IR/Intrinsics.h"
40#include "llvm/IR/IntrinsicsAArch64.h"
41#include "llvm/IR/IntrinsicsAMDGPU.h"
42#include "llvm/IR/IntrinsicsARM.h"
43#include "llvm/IR/IntrinsicsBPF.h"
44#include "llvm/IR/IntrinsicsHexagon.h"
45#include "llvm/IR/IntrinsicsLoongArch.h"
46#include "llvm/IR/IntrinsicsNVPTX.h"
47#include "llvm/IR/IntrinsicsPowerPC.h"
48#include "llvm/IR/IntrinsicsR600.h"
49#include "llvm/IR/IntrinsicsRISCV.h"
50#include "llvm/IR/IntrinsicsS390.h"
51#include "llvm/IR/IntrinsicsVE.h"
52#include "llvm/IR/IntrinsicsWebAssembly.h"
53#include "llvm/IR/IntrinsicsX86.h"
54#include "llvm/IR/MDBuilder.h"
55#include "llvm/IR/MatrixBuilder.h"
56#include "llvm/Support/ConvertUTF.h"
57#include "llvm/Support/ScopedPrinter.h"
58#include "llvm/TargetParser/AArch64TargetParser.h"
59#include "llvm/TargetParser/X86TargetParser.h"
64using namespace CodeGen;
68 Align AlignmentInBytes) {
70 switch (CGF.
getLangOpts().getTrivialAutoVarInit()) {
71 case LangOptions::TrivialAutoVarInitKind::Uninitialized:
74 case LangOptions::TrivialAutoVarInitKind::Zero:
75 Byte = CGF.
Builder.getInt8(0x00);
77 case LangOptions::TrivialAutoVarInitKind::Pattern: {
79 Byte = llvm::dyn_cast<llvm::ConstantInt>(
87 I->addAnnotationMetadata(
"auto-init");
102 static SmallDenseMap<unsigned, StringRef, 8> F128Builtins{
103 {Builtin::BI__builtin_printf,
"__printfieee128"},
104 {Builtin::BI__builtin_vsnprintf,
"__vsnprintfieee128"},
105 {Builtin::BI__builtin_vsprintf,
"__vsprintfieee128"},
106 {Builtin::BI__builtin_sprintf,
"__sprintfieee128"},
107 {Builtin::BI__builtin_snprintf,
"__snprintfieee128"},
108 {Builtin::BI__builtin_fprintf,
"__fprintfieee128"},
109 {Builtin::BI__builtin_nexttowardf128,
"__nexttowardieee128"},
115 static SmallDenseMap<unsigned, StringRef, 4> AIXLongDouble64Builtins{
116 {Builtin::BI__builtin_frexpl,
"frexp"},
117 {Builtin::BI__builtin_ldexpl,
"ldexp"},
118 {Builtin::BI__builtin_modfl,
"modf"},
124 if (FD->
hasAttr<AsmLabelAttr>())
130 &
getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad() &&
131 F128Builtins.find(BuiltinID) != F128Builtins.end())
132 Name = F128Builtins[BuiltinID];
135 &llvm::APFloat::IEEEdouble() &&
136 AIXLongDouble64Builtins.find(BuiltinID) !=
137 AIXLongDouble64Builtins.end())
138 Name = AIXLongDouble64Builtins[BuiltinID];
143 llvm::FunctionType *Ty =
146 return GetOrCreateLLVMFunction(Name, Ty, D,
false);
152 QualType T, llvm::IntegerType *IntType) {
155 if (
V->getType()->isPointerTy())
156 return CGF.
Builder.CreatePtrToInt(
V, IntType);
158 assert(
V->getType() == IntType);
163 QualType T, llvm::Type *ResultType) {
166 if (ResultType->isPointerTy())
167 return CGF.
Builder.CreateIntToPtr(
V, ResultType);
169 assert(
V->getType() == ResultType);
177 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
186 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
188 llvm::IntegerType *IntType =
191 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
193 llvm::Value *Args[2];
194 Args[0] = CGF.
Builder.CreateBitCast(DestPtr, IntPtrType);
196 llvm::Type *ValueType = Args[1]->getType();
197 Args[1] =
EmitToInt(CGF, Args[1], T, IntType);
200 Kind, Args[0], Args[1], Ordering);
210 unsigned SrcAddrSpace =
Address->
getType()->getPointerAddressSpace();
212 Address, llvm::PointerType::get(Val->getType(), SrcAddrSpace),
"cast");
234 if (Align % Bytes != 0) {
241 llvm::AtomicRMWInst::BinOp Kind,
251 llvm::AtomicRMWInst::BinOp Kind,
253 Instruction::BinaryOps Op,
254 bool Invert =
false) {
263 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
265 llvm::IntegerType *IntType =
268 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
270 llvm::Value *Args[2];
272 llvm::Type *ValueType = Args[1]->getType();
273 Args[1] =
EmitToInt(CGF, Args[1], T, IntType);
274 Args[0] = CGF.
Builder.CreateBitCast(DestPtr, IntPtrType);
277 Kind, Args[0], Args[1], llvm::AtomicOrdering::SequentiallyConsistent);
282 llvm::ConstantInt::getAllOnesValue(IntType));
306 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
308 llvm::IntegerType *IntType = llvm::IntegerType::get(
310 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
313 Args[0] = CGF.
Builder.CreateBitCast(DestPtr, IntPtrType);
315 llvm::Type *ValueType = Args[1]->getType();
316 Args[1] =
EmitToInt(CGF, Args[1], T, IntType);
320 Args[0], Args[1], Args[2], llvm::AtomicOrdering::SequentiallyConsistent,
321 llvm::AtomicOrdering::SequentiallyConsistent);
324 return CGF.
Builder.CreateZExt(CGF.
Builder.CreateExtractValue(Pair, 1),
347 AtomicOrdering SuccessOrdering = AtomicOrdering::SequentiallyConsistent) {
361 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release ?
362 AtomicOrdering::Monotonic :
370 Destination, Comparand, Exchange,
371 SuccessOrdering, FailureOrdering);
372 Result->setVolatile(
true);
386 AtomicOrdering SuccessOrdering) {
393 assert(Destination->getType()->isPointerTy());
394 assert(!ExchangeHigh->getType()->isPointerTy());
395 assert(!ExchangeLow->getType()->isPointerTy());
396 assert(ComparandPtr->getType()->isPointerTy());
399 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release
400 ? AtomicOrdering::Monotonic
404 llvm::Type *Int128Ty = llvm::IntegerType::get(CGF.
getLLVMContext(), 128);
405 llvm::Type *Int128PtrTy = Int128Ty->getPointerTo();
406 Destination = CGF.
Builder.CreateBitCast(Destination, Int128PtrTy);
407 Address ComparandResult(CGF.
Builder.CreateBitCast(ComparandPtr, Int128PtrTy),
411 ExchangeHigh = CGF.
Builder.CreateZExt(ExchangeHigh, Int128Ty);
412 ExchangeLow = CGF.
Builder.CreateZExt(ExchangeLow, Int128Ty);
414 CGF.
Builder.CreateShl(ExchangeHigh, llvm::ConstantInt::get(Int128Ty, 64));
415 llvm::Value *Exchange = CGF.
Builder.CreateOr(ExchangeHigh, ExchangeLow);
421 SuccessOrdering, FailureOrdering);
427 CXI->setVolatile(
true);
434 Value *Success = CGF.
Builder.CreateExtractValue(CXI, 1);
439 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
446 ConstantInt::get(IntTy, 1),
448 return CGF.
Builder.CreateAdd(
Result, ConstantInt::get(IntTy, 1));
452 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
459 ConstantInt::get(IntTy, 1),
461 return CGF.
Builder.CreateSub(
Result, ConstantInt::get(IntTy, 1));
471 Ptr = CGF.
Builder.CreateBitCast(Ptr, ITy->getPointerTo());
473 Load->setVolatile(
true);
485 Ptr = CGF.
Builder.CreateBitCast(Ptr, ITy->getPointerTo());
486 llvm::StoreInst *Store =
488 Store->setVolatile(
true);
496 const CallExpr *E,
unsigned IntrinsicID,
497 unsigned ConstrainedIntrinsicID) {
500 if (CGF.
Builder.getIsFPConstrained()) {
501 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
502 Function *F = CGF.
CGM.
getIntrinsic(ConstrainedIntrinsicID, Src0->getType());
503 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0 });
506 return CGF.
Builder.CreateCall(F, Src0);
513 const CallExpr *E,
unsigned IntrinsicID,
514 unsigned ConstrainedIntrinsicID) {
518 if (CGF.
Builder.getIsFPConstrained()) {
519 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
520 Function *F = CGF.
CGM.
getIntrinsic(ConstrainedIntrinsicID, Src0->getType());
521 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0, Src1 });
524 return CGF.
Builder.CreateCall(F, { Src0, Src1 });
531 const CallExpr *E,
unsigned IntrinsicID,
532 unsigned ConstrainedIntrinsicID) {
537 if (CGF.
Builder.getIsFPConstrained()) {
538 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
539 Function *F = CGF.
CGM.
getIntrinsic(ConstrainedIntrinsicID, Src0->getType());
540 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0, Src1, Src2 });
543 return CGF.
Builder.CreateCall(F, { Src0, Src1, Src2 });
550 unsigned IntrinsicID,
551 unsigned ConstrainedIntrinsicID,
555 if (CGF.
Builder.getIsFPConstrained())
560 if (CGF.
Builder.getIsFPConstrained())
561 return CGF.
Builder.CreateConstrainedFPCall(F, Args);
563 return CGF.
Builder.CreateCall(F, Args);
569 unsigned IntrinsicID,
570 llvm::StringRef Name =
"") {
574 return CGF.
Builder.CreateCall(F, Src0, Name);
580 unsigned IntrinsicID) {
585 return CGF.
Builder.CreateCall(F, { Src0, Src1 });
591 unsigned IntrinsicID) {
597 return CGF.
Builder.CreateCall(F, { Src0, Src1, Src2 });
603 unsigned IntrinsicID) {
608 return CGF.
Builder.CreateCall(F, {Src0, Src1});
614 unsigned IntrinsicID,
615 unsigned ConstrainedIntrinsicID) {
619 if (CGF.
Builder.getIsFPConstrained()) {
620 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
622 {ResultType, Src0->getType()});
623 return CGF.
Builder.CreateConstrainedFPCall(F, {Src0});
627 return CGF.
Builder.CreateCall(F, Src0);
634 llvm::CallInst *Call = CGF.
Builder.CreateCall(F,
V);
635 Call->setDoesNotAccessMemory();
644 llvm::Type *Ty =
V->getType();
645 int Width = Ty->getPrimitiveSizeInBits();
646 llvm::Type *IntTy = llvm::IntegerType::get(
C, Width);
648 if (Ty->isPPC_FP128Ty()) {
658 Value *ShiftCst = llvm::ConstantInt::get(IntTy, Width);
663 IntTy = llvm::IntegerType::get(
C, Width);
666 Value *Zero = llvm::Constant::getNullValue(IntTy);
667 return CGF.
Builder.CreateICmpSLT(
V, Zero);
671 const CallExpr *E, llvm::Constant *calleeValue) {
686 const llvm::Intrinsic::ID IntrinsicID,
687 llvm::Value *
X, llvm::Value *Y,
688 llvm::Value *&Carry) {
690 assert(
X->getType() == Y->getType() &&
691 "Arguments must be the same type. (Did you forget to make sure both "
692 "arguments have the same integer width?)");
695 llvm::Value *Tmp = CGF.
Builder.CreateCall(Callee, {
X, Y});
696 Carry = CGF.
Builder.CreateExtractValue(Tmp, 1);
697 return CGF.
Builder.CreateExtractValue(Tmp, 0);
701 unsigned IntrinsicID,
704 llvm::MDNode *RNode = MDHelper.createRange(APInt(32, low), APInt(32, high));
706 llvm::Instruction *Call = CGF.
Builder.CreateCall(F);
707 Call->setMetadata(llvm::LLVMContext::MD_range, RNode);
708 Call->setMetadata(llvm::LLVMContext::MD_noundef,
714 struct WidthAndSignedness {
720static WidthAndSignedness
734static struct WidthAndSignedness
736 assert(Types.size() > 0 &&
"Empty list of types.");
740 for (
const auto &
Type : Types) {
749 for (
const auto &
Type : Types) {
751 if (Width < MinWidth) {
761 if (ArgValue->getType() != DestType)
763 Builder.CreateBitCast(ArgValue, DestType, ArgValue->getName().data());
765 Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend;
775 return From == To || (From == 0 && To == 1) || (From == 3 && To == 2);
780 return ConstantInt::get(ResType, (
Type & 2) ? 0 : -1,
true);
784CodeGenFunction::evaluateOrEmitBuiltinObjectSize(
const Expr *E,
unsigned Type,
785 llvm::IntegerType *ResType,
786 llvm::Value *EmittedE,
790 return emitBuiltinObjectSize(E,
Type, ResType, EmittedE, IsDynamic);
791 return ConstantInt::get(ResType, ObjectSize,
true);
804CodeGenFunction::emitBuiltinObjectSize(
const Expr *E,
unsigned Type,
805 llvm::IntegerType *ResType,
806 llvm::Value *EmittedE,
bool IsDynamic) {
810 auto *Param = dyn_cast<ParmVarDecl>(D->getDecl());
811 auto *PS = D->getDecl()->getAttr<PassObjectSizeAttr>();
812 if (Param !=
nullptr && PS !=
nullptr &&
814 auto Iter = SizeArguments.find(Param);
815 assert(Iter != SizeArguments.end());
818 auto DIter = LocalDeclMap.find(D);
819 assert(DIter != LocalDeclMap.end());
833 assert(Ptr->getType()->isPointerTy() &&
834 "Non-pointer passed to __builtin_object_size?");
850 enum ActionKind : uint8_t { TestOnly, Complement, Reset, Set };
851 enum InterlockingKind : uint8_t {
860 InterlockingKind Interlocking;
863 static BitTest decodeBitTestBuiltin(
unsigned BuiltinID);
867BitTest BitTest::decodeBitTestBuiltin(
unsigned BuiltinID) {
870 case Builtin::BI_bittest:
871 return {TestOnly, Unlocked,
false};
872 case Builtin::BI_bittestandcomplement:
873 return {Complement, Unlocked,
false};
874 case Builtin::BI_bittestandreset:
875 return {Reset, Unlocked,
false};
876 case Builtin::BI_bittestandset:
877 return {Set, Unlocked,
false};
878 case Builtin::BI_interlockedbittestandreset:
879 return {Reset, Sequential,
false};
880 case Builtin::BI_interlockedbittestandset:
881 return {Set, Sequential,
false};
884 case Builtin::BI_bittest64:
885 return {TestOnly, Unlocked,
true};
886 case Builtin::BI_bittestandcomplement64:
887 return {Complement, Unlocked,
true};
888 case Builtin::BI_bittestandreset64:
889 return {Reset, Unlocked,
true};
890 case Builtin::BI_bittestandset64:
891 return {Set, Unlocked,
true};
892 case Builtin::BI_interlockedbittestandreset64:
893 return {Reset, Sequential,
true};
894 case Builtin::BI_interlockedbittestandset64:
895 return {Set, Sequential,
true};
898 case Builtin::BI_interlockedbittestandset_acq:
899 return {Set, Acquire,
false};
900 case Builtin::BI_interlockedbittestandset_rel:
901 return {Set, Release,
false};
902 case Builtin::BI_interlockedbittestandset_nf:
903 return {Set, NoFence,
false};
904 case Builtin::BI_interlockedbittestandreset_acq:
905 return {Reset, Acquire,
false};
906 case Builtin::BI_interlockedbittestandreset_rel:
907 return {Reset, Release,
false};
908 case Builtin::BI_interlockedbittestandreset_nf:
909 return {Reset, NoFence,
false};
911 llvm_unreachable(
"expected only bittest intrinsics");
916 case BitTest::TestOnly:
return '\0';
917 case BitTest::Complement:
return 'c';
918 case BitTest::Reset:
return 'r';
919 case BitTest::Set:
return 's';
921 llvm_unreachable(
"invalid action");
929 char SizeSuffix = BT.Is64Bit ?
'q' :
'l';
933 raw_svector_ostream AsmOS(
Asm);
934 if (BT.Interlocking != BitTest::Unlocked)
939 AsmOS << SizeSuffix <<
" $2, ($1)";
942 std::string Constraints =
"={@ccc},r,r,~{cc},~{memory}";
944 if (!MachineClobbers.empty()) {
946 Constraints += MachineClobbers;
948 llvm::IntegerType *IntType = llvm::IntegerType::get(
951 llvm::Type *IntPtrType = IntType->getPointerTo();
952 llvm::FunctionType *FTy =
953 llvm::FunctionType::get(CGF.
Int8Ty, {IntPtrType, IntType},
false);
955 llvm::InlineAsm *IA =
956 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
957 return CGF.
Builder.CreateCall(IA, {BitBase, BitPos});
960static llvm::AtomicOrdering
963 case BitTest::Unlocked:
return llvm::AtomicOrdering::NotAtomic;
964 case BitTest::Sequential:
return llvm::AtomicOrdering::SequentiallyConsistent;
965 case BitTest::Acquire:
return llvm::AtomicOrdering::Acquire;
966 case BitTest::Release:
return llvm::AtomicOrdering::Release;
967 case BitTest::NoFence:
return llvm::AtomicOrdering::Monotonic;
969 llvm_unreachable(
"invalid interlocking");
982 BitTest BT = BitTest::decodeBitTestBuiltin(BuiltinID);
994 BitPos, llvm::ConstantInt::get(BitPos->getType(), 3),
"bittest.byteidx");
997 ByteIndex,
"bittest.byteaddr"),
1001 llvm::ConstantInt::get(CGF.
Int8Ty, 0x7));
1004 Value *Mask =
nullptr;
1005 if (BT.Action != BitTest::TestOnly) {
1006 Mask = CGF.
Builder.CreateShl(llvm::ConstantInt::get(CGF.
Int8Ty, 1), PosLow,
1013 Value *OldByte =
nullptr;
1014 if (Ordering != llvm::AtomicOrdering::NotAtomic) {
1017 llvm::AtomicRMWInst::BinOp RMWOp = llvm::AtomicRMWInst::Or;
1018 if (BT.Action == BitTest::Reset) {
1019 Mask = CGF.
Builder.CreateNot(Mask);
1020 RMWOp = llvm::AtomicRMWInst::And;
1027 Value *NewByte =
nullptr;
1028 switch (BT.Action) {
1029 case BitTest::TestOnly:
1032 case BitTest::Complement:
1033 NewByte = CGF.
Builder.CreateXor(OldByte, Mask);
1035 case BitTest::Reset:
1036 NewByte = CGF.
Builder.CreateAnd(OldByte, CGF.
Builder.CreateNot(Mask));
1039 NewByte = CGF.
Builder.CreateOr(OldByte, Mask);
1048 Value *ShiftedByte = CGF.
Builder.CreateLShr(OldByte, PosLow,
"bittest.shr");
1050 ShiftedByte, llvm::ConstantInt::get(CGF.
Int8Ty, 1),
"bittest.res");
1059 raw_svector_ostream AsmOS(
Asm);
1060 llvm::IntegerType *RetType = CGF.
Int32Ty;
1062 switch (BuiltinID) {
1063 case clang::PPC::BI__builtin_ppc_ldarx:
1067 case clang::PPC::BI__builtin_ppc_lwarx:
1071 case clang::PPC::BI__builtin_ppc_lharx:
1075 case clang::PPC::BI__builtin_ppc_lbarx:
1080 llvm_unreachable(
"Expected only PowerPC load reserve intrinsics");
1083 AsmOS <<
"$0, ${1:y}";
1085 std::string Constraints =
"=r,*Z,~{memory}";
1087 if (!MachineClobbers.empty()) {
1089 Constraints += MachineClobbers;
1092 llvm::Type *IntPtrType = RetType->getPointerTo();
1093 llvm::FunctionType *FTy =
1094 llvm::FunctionType::get(RetType, {IntPtrType},
false);
1096 llvm::InlineAsm *IA =
1097 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
1098 llvm::CallInst *CI = CGF.
Builder.CreateCall(IA, {Addr});
1100 0, Attribute::get(CGF.
getLLVMContext(), Attribute::ElementType, RetType));
1105enum class MSVCSetJmpKind {
1117 llvm::Value *Arg1 =
nullptr;
1118 llvm::Type *Arg1Ty =
nullptr;
1120 bool IsVarArg =
false;
1121 if (SJKind == MSVCSetJmpKind::_setjmp3) {
1124 Arg1 = llvm::ConstantInt::get(CGF.
IntTy, 0);
1127 Name = SJKind == MSVCSetJmpKind::_setjmp ?
"_setjmp" :
"_setjmpex";
1130 Arg1 = CGF.
Builder.CreateCall(
1133 Arg1 = CGF.
Builder.CreateCall(
1135 llvm::ConstantInt::get(CGF.
Int32Ty, 0));
1139 llvm::Type *ArgTypes[2] = {CGF.
Int8PtrTy, Arg1Ty};
1140 llvm::AttributeList ReturnsTwiceAttr = llvm::AttributeList::get(
1142 llvm::Attribute::ReturnsTwice);
1144 llvm::FunctionType::get(CGF.
IntTy, ArgTypes, IsVarArg), Name,
1145 ReturnsTwiceAttr,
true);
1147 llvm::Value *Buf = CGF.
Builder.CreateBitOrPointerCast(
1149 llvm::Value *Args[] = {Buf, Arg1};
1151 CB->setAttributes(ReturnsTwiceAttr);
1199static std::optional<CodeGenFunction::MSVCIntrin>
1202 switch (BuiltinID) {
1204 return std::nullopt;
1205 case clang::ARM::BI_BitScanForward:
1206 case clang::ARM::BI_BitScanForward64:
1207 return MSVCIntrin::_BitScanForward;
1208 case clang::ARM::BI_BitScanReverse:
1209 case clang::ARM::BI_BitScanReverse64:
1210 return MSVCIntrin::_BitScanReverse;
1211 case clang::ARM::BI_InterlockedAnd64:
1212 return MSVCIntrin::_InterlockedAnd;
1213 case clang::ARM::BI_InterlockedExchange64:
1214 return MSVCIntrin::_InterlockedExchange;
1215 case clang::ARM::BI_InterlockedExchangeAdd64:
1216 return MSVCIntrin::_InterlockedExchangeAdd;
1217 case clang::ARM::BI_InterlockedExchangeSub64:
1218 return MSVCIntrin::_InterlockedExchangeSub;
1219 case clang::ARM::BI_InterlockedOr64:
1220 return MSVCIntrin::_InterlockedOr;
1221 case clang::ARM::BI_InterlockedXor64:
1222 return MSVCIntrin::_InterlockedXor;
1223 case clang::ARM::BI_InterlockedDecrement64:
1224 return MSVCIntrin::_InterlockedDecrement;
1225 case clang::ARM::BI_InterlockedIncrement64:
1226 return MSVCIntrin::_InterlockedIncrement;
1227 case clang::ARM::BI_InterlockedExchangeAdd8_acq:
1228 case clang::ARM::BI_InterlockedExchangeAdd16_acq:
1229 case clang::ARM::BI_InterlockedExchangeAdd_acq:
1230 case clang::ARM::BI_InterlockedExchangeAdd64_acq:
1231 return MSVCIntrin::_InterlockedExchangeAdd_acq;
1232 case clang::ARM::BI_InterlockedExchangeAdd8_rel:
1233 case clang::ARM::BI_InterlockedExchangeAdd16_rel:
1234 case clang::ARM::BI_InterlockedExchangeAdd_rel:
1235 case clang::ARM::BI_InterlockedExchangeAdd64_rel:
1236 return MSVCIntrin::_InterlockedExchangeAdd_rel;
1237 case clang::ARM::BI_InterlockedExchangeAdd8_nf:
1238 case clang::ARM::BI_InterlockedExchangeAdd16_nf:
1239 case clang::ARM::BI_InterlockedExchangeAdd_nf:
1240 case clang::ARM::BI_InterlockedExchangeAdd64_nf:
1241 return MSVCIntrin::_InterlockedExchangeAdd_nf;
1242 case clang::ARM::BI_InterlockedExchange8_acq:
1243 case clang::ARM::BI_InterlockedExchange16_acq:
1244 case clang::ARM::BI_InterlockedExchange_acq:
1245 case clang::ARM::BI_InterlockedExchange64_acq:
1246 return MSVCIntrin::_InterlockedExchange_acq;
1247 case clang::ARM::BI_InterlockedExchange8_rel:
1248 case clang::ARM::BI_InterlockedExchange16_rel:
1249 case clang::ARM::BI_InterlockedExchange_rel:
1250 case clang::ARM::BI_InterlockedExchange64_rel:
1251 return MSVCIntrin::_InterlockedExchange_rel;
1252 case clang::ARM::BI_InterlockedExchange8_nf:
1253 case clang::ARM::BI_InterlockedExchange16_nf:
1254 case clang::ARM::BI_InterlockedExchange_nf:
1255 case clang::ARM::BI_InterlockedExchange64_nf:
1256 return MSVCIntrin::_InterlockedExchange_nf;
1257 case clang::ARM::BI_InterlockedCompareExchange8_acq:
1258 case clang::ARM::BI_InterlockedCompareExchange16_acq:
1259 case clang::ARM::BI_InterlockedCompareExchange_acq:
1260 case clang::ARM::BI_InterlockedCompareExchange64_acq:
1261 return MSVCIntrin::_InterlockedCompareExchange_acq;
1262 case clang::ARM::BI_InterlockedCompareExchange8_rel:
1263 case clang::ARM::BI_InterlockedCompareExchange16_rel:
1264 case clang::ARM::BI_InterlockedCompareExchange_rel:
1265 case clang::ARM::BI_InterlockedCompareExchange64_rel:
1266 return MSVCIntrin::_InterlockedCompareExchange_rel;
1267 case clang::ARM::BI_InterlockedCompareExchange8_nf:
1268 case clang::ARM::BI_InterlockedCompareExchange16_nf:
1269 case clang::ARM::BI_InterlockedCompareExchange_nf:
1270 case clang::ARM::BI_InterlockedCompareExchange64_nf:
1271 return MSVCIntrin::_InterlockedCompareExchange_nf;
1272 case clang::ARM::BI_InterlockedOr8_acq:
1273 case clang::ARM::BI_InterlockedOr16_acq:
1274 case clang::ARM::BI_InterlockedOr_acq:
1275 case clang::ARM::BI_InterlockedOr64_acq:
1276 return MSVCIntrin::_InterlockedOr_acq;
1277 case clang::ARM::BI_InterlockedOr8_rel:
1278 case clang::ARM::BI_InterlockedOr16_rel:
1279 case clang::ARM::BI_InterlockedOr_rel:
1280 case clang::ARM::BI_InterlockedOr64_rel:
1281 return MSVCIntrin::_InterlockedOr_rel;
1282 case clang::ARM::BI_InterlockedOr8_nf:
1283 case clang::ARM::BI_InterlockedOr16_nf:
1284 case clang::ARM::BI_InterlockedOr_nf:
1285 case clang::ARM::BI_InterlockedOr64_nf:
1286 return MSVCIntrin::_InterlockedOr_nf;
1287 case clang::ARM::BI_InterlockedXor8_acq:
1288 case clang::ARM::BI_InterlockedXor16_acq:
1289 case clang::ARM::BI_InterlockedXor_acq:
1290 case clang::ARM::BI_InterlockedXor64_acq:
1291 return MSVCIntrin::_InterlockedXor_acq;
1292 case clang::ARM::BI_InterlockedXor8_rel:
1293 case clang::ARM::BI_InterlockedXor16_rel:
1294 case clang::ARM::BI_InterlockedXor_rel:
1295 case clang::ARM::BI_InterlockedXor64_rel:
1296 return MSVCIntrin::_InterlockedXor_rel;
1297 case clang::ARM::BI_InterlockedXor8_nf:
1298 case clang::ARM::BI_InterlockedXor16_nf:
1299 case clang::ARM::BI_InterlockedXor_nf:
1300 case clang::ARM::BI_InterlockedXor64_nf:
1301 return MSVCIntrin::_InterlockedXor_nf;
1302 case clang::ARM::BI_InterlockedAnd8_acq:
1303 case clang::ARM::BI_InterlockedAnd16_acq:
1304 case clang::ARM::BI_InterlockedAnd_acq:
1305 case clang::ARM::BI_InterlockedAnd64_acq:
1306 return MSVCIntrin::_InterlockedAnd_acq;
1307 case clang::ARM::BI_InterlockedAnd8_rel:
1308 case clang::ARM::BI_InterlockedAnd16_rel:
1309 case clang::ARM::BI_InterlockedAnd_rel:
1310 case clang::ARM::BI_InterlockedAnd64_rel:
1311 return MSVCIntrin::_InterlockedAnd_rel;
1312 case clang::ARM::BI_InterlockedAnd8_nf:
1313 case clang::ARM::BI_InterlockedAnd16_nf:
1314 case clang::ARM::BI_InterlockedAnd_nf:
1315 case clang::ARM::BI_InterlockedAnd64_nf:
1316 return MSVCIntrin::_InterlockedAnd_nf;
1317 case clang::ARM::BI_InterlockedIncrement16_acq:
1318 case clang::ARM::BI_InterlockedIncrement_acq:
1319 case clang::ARM::BI_InterlockedIncrement64_acq:
1320 return MSVCIntrin::_InterlockedIncrement_acq;
1321 case clang::ARM::BI_InterlockedIncrement16_rel:
1322 case clang::ARM::BI_InterlockedIncrement_rel:
1323 case clang::ARM::BI_InterlockedIncrement64_rel:
1324 return MSVCIntrin::_InterlockedIncrement_rel;
1325 case clang::ARM::BI_InterlockedIncrement16_nf:
1326 case clang::ARM::BI_InterlockedIncrement_nf:
1327 case clang::ARM::BI_InterlockedIncrement64_nf:
1328 return MSVCIntrin::_InterlockedIncrement_nf;
1329 case clang::ARM::BI_InterlockedDecrement16_acq:
1330 case clang::ARM::BI_InterlockedDecrement_acq:
1331 case clang::ARM::BI_InterlockedDecrement64_acq:
1332 return MSVCIntrin::_InterlockedDecrement_acq;
1333 case clang::ARM::BI_InterlockedDecrement16_rel:
1334 case clang::ARM::BI_InterlockedDecrement_rel:
1335 case clang::ARM::BI_InterlockedDecrement64_rel:
1336 return MSVCIntrin::_InterlockedDecrement_rel;
1337 case clang::ARM::BI_InterlockedDecrement16_nf:
1338 case clang::ARM::BI_InterlockedDecrement_nf:
1339 case clang::ARM::BI_InterlockedDecrement64_nf:
1340 return MSVCIntrin::_InterlockedDecrement_nf;
1342 llvm_unreachable(
"must return from switch");
1345static std::optional<CodeGenFunction::MSVCIntrin>
1348 switch (BuiltinID) {
1350 return std::nullopt;
1351 case clang::AArch64::BI_BitScanForward:
1352 case clang::AArch64::BI_BitScanForward64:
1353 return MSVCIntrin::_BitScanForward;
1354 case clang::AArch64::BI_BitScanReverse:
1355 case clang::AArch64::BI_BitScanReverse64:
1356 return MSVCIntrin::_BitScanReverse;
1357 case clang::AArch64::BI_InterlockedAnd64:
1358 return MSVCIntrin::_InterlockedAnd;
1359 case clang::AArch64::BI_InterlockedExchange64:
1360 return MSVCIntrin::_InterlockedExchange;
1361 case clang::AArch64::BI_InterlockedExchangeAdd64:
1362 return MSVCIntrin::_InterlockedExchangeAdd;
1363 case clang::AArch64::BI_InterlockedExchangeSub64:
1364 return MSVCIntrin::_InterlockedExchangeSub;
1365 case clang::AArch64::BI_InterlockedOr64:
1366 return MSVCIntrin::_InterlockedOr;
1367 case clang::AArch64::BI_InterlockedXor64:
1368 return MSVCIntrin::_InterlockedXor;
1369 case clang::AArch64::BI_InterlockedDecrement64:
1370 return MSVCIntrin::_InterlockedDecrement;
1371 case clang::AArch64::BI_InterlockedIncrement64:
1372 return MSVCIntrin::_InterlockedIncrement;
1373 case clang::AArch64::BI_InterlockedExchangeAdd8_acq:
1374 case clang::AArch64::BI_InterlockedExchangeAdd16_acq:
1375 case clang::AArch64::BI_InterlockedExchangeAdd_acq:
1376 case clang::AArch64::BI_InterlockedExchangeAdd64_acq:
1377 return MSVCIntrin::_InterlockedExchangeAdd_acq;
1378 case clang::AArch64::BI_InterlockedExchangeAdd8_rel:
1379 case clang::AArch64::BI_InterlockedExchangeAdd16_rel:
1380 case clang::AArch64::BI_InterlockedExchangeAdd_rel:
1381 case clang::AArch64::BI_InterlockedExchangeAdd64_rel:
1382 return MSVCIntrin::_InterlockedExchangeAdd_rel;
1383 case clang::AArch64::BI_InterlockedExchangeAdd8_nf:
1384 case clang::AArch64::BI_InterlockedExchangeAdd16_nf:
1385 case clang::AArch64::BI_InterlockedExchangeAdd_nf:
1386 case clang::AArch64::BI_InterlockedExchangeAdd64_nf:
1387 return MSVCIntrin::_InterlockedExchangeAdd_nf;
1388 case clang::AArch64::BI_InterlockedExchange8_acq:
1389 case clang::AArch64::BI_InterlockedExchange16_acq:
1390 case clang::AArch64::BI_InterlockedExchange_acq:
1391 case clang::AArch64::BI_InterlockedExchange64_acq:
1392 return MSVCIntrin::_InterlockedExchange_acq;
1393 case clang::AArch64::BI_InterlockedExchange8_rel:
1394 case clang::AArch64::BI_InterlockedExchange16_rel:
1395 case clang::AArch64::BI_InterlockedExchange_rel:
1396 case clang::AArch64::BI_InterlockedExchange64_rel:
1397 return MSVCIntrin::_InterlockedExchange_rel;
1398 case clang::AArch64::BI_InterlockedExchange8_nf:
1399 case clang::AArch64::BI_InterlockedExchange16_nf:
1400 case clang::AArch64::BI_InterlockedExchange_nf:
1401 case clang::AArch64::BI_InterlockedExchange64_nf:
1402 return MSVCIntrin::_InterlockedExchange_nf;
1403 case clang::AArch64::BI_InterlockedCompareExchange8_acq:
1404 case clang::AArch64::BI_InterlockedCompareExchange16_acq:
1405 case clang::AArch64::BI_InterlockedCompareExchange_acq:
1406 case clang::AArch64::BI_InterlockedCompareExchange64_acq:
1407 return MSVCIntrin::_InterlockedCompareExchange_acq;
1408 case clang::AArch64::BI_InterlockedCompareExchange8_rel:
1409 case clang::AArch64::BI_InterlockedCompareExchange16_rel:
1410 case clang::AArch64::BI_InterlockedCompareExchange_rel:
1411 case clang::AArch64::BI_InterlockedCompareExchange64_rel:
1412 return MSVCIntrin::_InterlockedCompareExchange_rel;
1413 case clang::AArch64::BI_InterlockedCompareExchange8_nf:
1414 case clang::AArch64::BI_InterlockedCompareExchange16_nf:
1415 case clang::AArch64::BI_InterlockedCompareExchange_nf:
1416 case clang::AArch64::BI_InterlockedCompareExchange64_nf:
1417 return MSVCIntrin::_InterlockedCompareExchange_nf;
1418 case clang::AArch64::BI_InterlockedCompareExchange128:
1419 return MSVCIntrin::_InterlockedCompareExchange128;
1420 case clang::AArch64::BI_InterlockedCompareExchange128_acq:
1421 return MSVCIntrin::_InterlockedCompareExchange128_acq;
1422 case clang::AArch64::BI_InterlockedCompareExchange128_nf:
1423 return MSVCIntrin::_InterlockedCompareExchange128_nf;
1424 case clang::AArch64::BI_InterlockedCompareExchange128_rel:
1425 return MSVCIntrin::_InterlockedCompareExchange128_rel;
1426 case clang::AArch64::BI_InterlockedOr8_acq:
1427 case clang::AArch64::BI_InterlockedOr16_acq:
1428 case clang::AArch64::BI_InterlockedOr_acq:
1429 case clang::AArch64::BI_InterlockedOr64_acq:
1430 return MSVCIntrin::_InterlockedOr_acq;
1431 case clang::AArch64::BI_InterlockedOr8_rel:
1432 case clang::AArch64::BI_InterlockedOr16_rel:
1433 case clang::AArch64::BI_InterlockedOr_rel:
1434 case clang::AArch64::BI_InterlockedOr64_rel:
1435 return MSVCIntrin::_InterlockedOr_rel;
1436 case clang::AArch64::BI_InterlockedOr8_nf:
1437 case clang::AArch64::BI_InterlockedOr16_nf:
1438 case clang::AArch64::BI_InterlockedOr_nf:
1439 case clang::AArch64::BI_InterlockedOr64_nf:
1440 return MSVCIntrin::_InterlockedOr_nf;
1441 case clang::AArch64::BI_InterlockedXor8_acq:
1442 case clang::AArch64::BI_InterlockedXor16_acq:
1443 case clang::AArch64::BI_InterlockedXor_acq:
1444 case clang::AArch64::BI_InterlockedXor64_acq:
1445 return MSVCIntrin::_InterlockedXor_acq;
1446 case clang::AArch64::BI_InterlockedXor8_rel:
1447 case clang::AArch64::BI_InterlockedXor16_rel:
1448 case clang::AArch64::BI_InterlockedXor_rel:
1449 case clang::AArch64::BI_InterlockedXor64_rel:
1450 return MSVCIntrin::_InterlockedXor_rel;
1451 case clang::AArch64::BI_InterlockedXor8_nf:
1452 case clang::AArch64::BI_InterlockedXor16_nf:
1453 case clang::AArch64::BI_InterlockedXor_nf:
1454 case clang::AArch64::BI_InterlockedXor64_nf:
1455 return MSVCIntrin::_InterlockedXor_nf;
1456 case clang::AArch64::BI_InterlockedAnd8_acq:
1457 case clang::AArch64::BI_InterlockedAnd16_acq:
1458 case clang::AArch64::BI_InterlockedAnd_acq:
1459 case clang::AArch64::BI_InterlockedAnd64_acq:
1460 return MSVCIntrin::_InterlockedAnd_acq;
1461 case clang::AArch64::BI_InterlockedAnd8_rel:
1462 case clang::AArch64::BI_InterlockedAnd16_rel:
1463 case clang::AArch64::BI_InterlockedAnd_rel:
1464 case clang::AArch64::BI_InterlockedAnd64_rel:
1465 return MSVCIntrin::_InterlockedAnd_rel;
1466 case clang::AArch64::BI_InterlockedAnd8_nf:
1467 case clang::AArch64::BI_InterlockedAnd16_nf:
1468 case clang::AArch64::BI_InterlockedAnd_nf:
1469 case clang::AArch64::BI_InterlockedAnd64_nf:
1470 return MSVCIntrin::_InterlockedAnd_nf;
1471 case clang::AArch64::BI_InterlockedIncrement16_acq:
1472 case clang::AArch64::BI_InterlockedIncrement_acq:
1473 case clang::AArch64::BI_InterlockedIncrement64_acq:
1474 return MSVCIntrin::_InterlockedIncrement_acq;
1475 case clang::AArch64::BI_InterlockedIncrement16_rel:
1476 case clang::AArch64::BI_InterlockedIncrement_rel:
1477 case clang::AArch64::BI_InterlockedIncrement64_rel:
1478 return MSVCIntrin::_InterlockedIncrement_rel;
1479 case clang::AArch64::BI_InterlockedIncrement16_nf:
1480 case clang::AArch64::BI_InterlockedIncrement_nf:
1481 case clang::AArch64::BI_InterlockedIncrement64_nf:
1482 return MSVCIntrin::_InterlockedIncrement_nf;
1483 case clang::AArch64::BI_InterlockedDecrement16_acq:
1484 case clang::AArch64::BI_InterlockedDecrement_acq:
1485 case clang::AArch64::BI_InterlockedDecrement64_acq:
1486 return MSVCIntrin::_InterlockedDecrement_acq;
1487 case clang::AArch64::BI_InterlockedDecrement16_rel:
1488 case clang::AArch64::BI_InterlockedDecrement_rel:
1489 case clang::AArch64::BI_InterlockedDecrement64_rel:
1490 return MSVCIntrin::_InterlockedDecrement_rel;
1491 case clang::AArch64::BI_InterlockedDecrement16_nf:
1492 case clang::AArch64::BI_InterlockedDecrement_nf:
1493 case clang::AArch64::BI_InterlockedDecrement64_nf:
1494 return MSVCIntrin::_InterlockedDecrement_nf;
1496 llvm_unreachable(
"must return from switch");
1499static std::optional<CodeGenFunction::MSVCIntrin>
1502 switch (BuiltinID) {
1504 return std::nullopt;
1505 case clang::X86::BI_BitScanForward:
1506 case clang::X86::BI_BitScanForward64:
1507 return MSVCIntrin::_BitScanForward;
1508 case clang::X86::BI_BitScanReverse:
1509 case clang::X86::BI_BitScanReverse64:
1510 return MSVCIntrin::_BitScanReverse;
1511 case clang::X86::BI_InterlockedAnd64:
1512 return MSVCIntrin::_InterlockedAnd;
1513 case clang::X86::BI_InterlockedCompareExchange128:
1514 return MSVCIntrin::_InterlockedCompareExchange128;
1515 case clang::X86::BI_InterlockedExchange64:
1516 return MSVCIntrin::_InterlockedExchange;
1517 case clang::X86::BI_InterlockedExchangeAdd64:
1518 return MSVCIntrin::_InterlockedExchangeAdd;
1519 case clang::X86::BI_InterlockedExchangeSub64:
1520 return MSVCIntrin::_InterlockedExchangeSub;
1521 case clang::X86::BI_InterlockedOr64:
1522 return MSVCIntrin::_InterlockedOr;
1523 case clang::X86::BI_InterlockedXor64:
1524 return MSVCIntrin::_InterlockedXor;
1525 case clang::X86::BI_InterlockedDecrement64:
1526 return MSVCIntrin::_InterlockedDecrement;
1527 case clang::X86::BI_InterlockedIncrement64:
1528 return MSVCIntrin::_InterlockedIncrement;
1530 llvm_unreachable(
"must return from switch");
1536 switch (BuiltinID) {
1537 case MSVCIntrin::_BitScanForward:
1538 case MSVCIntrin::_BitScanReverse: {
1542 llvm::Type *ArgType = ArgValue->getType();
1543 llvm::Type *IndexType = IndexAddress.getElementType();
1546 Value *ArgZero = llvm::Constant::getNullValue(ArgType);
1547 Value *ResZero = llvm::Constant::getNullValue(ResultType);
1548 Value *ResOne = llvm::ConstantInt::get(ResultType, 1);
1553 PHINode *
Result =
Builder.CreatePHI(ResultType, 2,
"bitscan_result");
1556 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, ArgZero);
1558 Builder.CreateCondBr(IsZero, End, NotZero);
1561 Builder.SetInsertPoint(NotZero);
1563 if (BuiltinID == MSVCIntrin::_BitScanForward) {
1566 ZeroCount =
Builder.CreateIntCast(ZeroCount, IndexType,
false);
1569 unsigned ArgWidth = cast<llvm::IntegerType>(ArgType)->getBitWidth();
1570 Value *ArgTypeLastIndex = llvm::ConstantInt::get(IndexType, ArgWidth - 1);
1574 ZeroCount =
Builder.CreateIntCast(ZeroCount, IndexType,
false);
1575 Value *Index =
Builder.CreateNSWSub(ArgTypeLastIndex, ZeroCount);
1579 Result->addIncoming(ResOne, NotZero);
1584 case MSVCIntrin::_InterlockedAnd:
1586 case MSVCIntrin::_InterlockedExchange:
1588 case MSVCIntrin::_InterlockedExchangeAdd:
1590 case MSVCIntrin::_InterlockedExchangeSub:
1592 case MSVCIntrin::_InterlockedOr:
1594 case MSVCIntrin::_InterlockedXor:
1596 case MSVCIntrin::_InterlockedExchangeAdd_acq:
1598 AtomicOrdering::Acquire);
1599 case MSVCIntrin::_InterlockedExchangeAdd_rel:
1601 AtomicOrdering::Release);
1602 case MSVCIntrin::_InterlockedExchangeAdd_nf:
1604 AtomicOrdering::Monotonic);
1605 case MSVCIntrin::_InterlockedExchange_acq:
1607 AtomicOrdering::Acquire);
1608 case MSVCIntrin::_InterlockedExchange_rel:
1610 AtomicOrdering::Release);
1611 case MSVCIntrin::_InterlockedExchange_nf:
1613 AtomicOrdering::Monotonic);
1614 case MSVCIntrin::_InterlockedCompareExchange_acq:
1616 case MSVCIntrin::_InterlockedCompareExchange_rel:
1618 case MSVCIntrin::_InterlockedCompareExchange_nf:
1620 case MSVCIntrin::_InterlockedCompareExchange128:
1622 *
this, E, AtomicOrdering::SequentiallyConsistent);
1623 case MSVCIntrin::_InterlockedCompareExchange128_acq:
1625 case MSVCIntrin::_InterlockedCompareExchange128_rel:
1627 case MSVCIntrin::_InterlockedCompareExchange128_nf:
1629 case MSVCIntrin::_InterlockedOr_acq:
1631 AtomicOrdering::Acquire);
1632 case MSVCIntrin::_InterlockedOr_rel:
1634 AtomicOrdering::Release);
1635 case MSVCIntrin::_InterlockedOr_nf:
1637 AtomicOrdering::Monotonic);
1638 case MSVCIntrin::_InterlockedXor_acq:
1640 AtomicOrdering::Acquire);
1641 case MSVCIntrin::_InterlockedXor_rel:
1643 AtomicOrdering::Release);
1644 case MSVCIntrin::_InterlockedXor_nf:
1646 AtomicOrdering::Monotonic);
1647 case MSVCIntrin::_InterlockedAnd_acq:
1649 AtomicOrdering::Acquire);
1650 case MSVCIntrin::_InterlockedAnd_rel:
1652 AtomicOrdering::Release);
1653 case MSVCIntrin::_InterlockedAnd_nf:
1655 AtomicOrdering::Monotonic);
1656 case MSVCIntrin::_InterlockedIncrement_acq:
1658 case MSVCIntrin::_InterlockedIncrement_rel:
1660 case MSVCIntrin::_InterlockedIncrement_nf:
1662 case MSVCIntrin::_InterlockedDecrement_acq:
1664 case MSVCIntrin::_InterlockedDecrement_rel:
1666 case MSVCIntrin::_InterlockedDecrement_nf:
1669 case MSVCIntrin::_InterlockedDecrement:
1671 case MSVCIntrin::_InterlockedIncrement:
1674 case MSVCIntrin::__fastfail: {
1679 StringRef
Asm, Constraints;
1684 case llvm::Triple::x86:
1685 case llvm::Triple::x86_64:
1687 Constraints =
"{cx}";
1689 case llvm::Triple::thumb:
1691 Constraints =
"{r0}";
1693 case llvm::Triple::aarch64:
1694 Asm =
"brk #0xF003";
1695 Constraints =
"{w0}";
1697 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy, {
Int32Ty},
false);
1698 llvm::InlineAsm *IA =
1699 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
1700 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
1702 llvm::Attribute::NoReturn);
1704 CI->setAttributes(NoReturnAttr);
1708 llvm_unreachable(
"Incorrect MSVC intrinsic!");
1714 CallObjCArcUse(llvm::Value *
object) :
object(
object) {}
1724 BuiltinCheckKind Kind) {
1726 &&
"Unsupported builtin check kind");
1732 SanitizerScope SanScope(
this);
1734 ArgValue, llvm::Constant::getNullValue(ArgValue->getType()));
1735 EmitCheck(std::make_pair(Cond, SanitizerKind::Builtin),
1736 SanitizerHandler::InvalidBuiltin,
1738 llvm::ConstantInt::get(
Builder.getInt8Ty(), Kind)},
1745 QualType UnsignedTy =
C.getIntTypeForBitwidth(Size * 8,
false);
1746 return C.getCanonicalType(UnsignedTy);
1756 raw_svector_ostream
OS(Name);
1757 OS <<
"__os_log_helper";
1761 for (
const auto &Item : Layout.
Items)
1762 OS <<
"_" <<
int(Item.getSizeByte()) <<
"_"
1763 <<
int(Item.getDescriptorByte());
1766 if (llvm::Function *F =
CGM.
getModule().getFunction(Name))
1776 for (
unsigned int I = 0, E = Layout.
Items.size(); I < E; ++I) {
1777 char Size = Layout.
Items[I].getSizeByte();
1784 &Ctx.
Idents.
get(std::string(
"arg") + llvm::to_string(I)), ArgTy,
1786 ArgTys.emplace_back(ArgTy);
1797 llvm::Function *Fn = llvm::Function::Create(
1798 FuncTy, llvm::GlobalValue::LinkOnceODRLinkage, Name, &
CGM.
getModule());
1799 Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);
1802 Fn->setDoesNotThrow();
1806 Fn->addFnAttr(llvm::Attribute::NoInline);
1824 for (
const auto &Item : Layout.
Items) {
1826 Builder.getInt8(Item.getDescriptorByte()),
1829 Builder.getInt8(Item.getSizeByte()),
1833 if (!
Size.getQuantity())
1852 "__builtin_os_log_format takes at least 2 arguments");
1863 for (
const auto &Item : Layout.
Items) {
1864 int Size = Item.getSizeByte();
1868 llvm::Value *ArgVal;
1872 for (
unsigned I = 0, E = Item.getMaskType().size(); I < E; ++I)
1873 Val |= ((
uint64_t)Item.getMaskType()[I]) << I * 8;
1874 ArgVal = llvm::Constant::getIntegerValue(
Int64Ty, llvm::APInt(64, Val));
1875 }
else if (
const Expr *TheExpr = Item.getExpr()) {
1881 auto LifetimeExtendObject = [&](
const Expr *E) {
1889 if (isa<CallExpr>(E) || isa<ObjCMessageExpr>(E))
1894 if (TheExpr->getType()->isObjCRetainableType() &&
1895 getLangOpts().ObjCAutoRefCount && LifetimeExtendObject(TheExpr)) {
1897 "Only scalar can be a ObjC retainable type");
1898 if (!isa<Constant>(ArgVal)) {
1912 pushCleanupAfterFullExpr<CallObjCArcUse>(Cleanup, ArgVal);
1916 ArgVal =
Builder.getInt32(Item.getConstValue().getQuantity());
1919 unsigned ArgValSize =
1923 ArgVal =
Builder.CreateBitOrPointerCast(ArgVal,
IntTy);
1939 unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info,
1940 WidthAndSignedness ResultInfo) {
1941 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
1942 Op1Info.Width == Op2Info.Width && Op2Info.Width == ResultInfo.Width &&
1943 !Op1Info.Signed && !Op2Info.Signed && ResultInfo.Signed;
1948 const clang::Expr *Op2, WidthAndSignedness Op2Info,
1950 WidthAndSignedness ResultInfo) {
1952 Builtin::BI__builtin_mul_overflow, Op1Info, Op2Info, ResultInfo) &&
1953 "Cannot specialize this multiply");
1958 llvm::Value *HasOverflow;
1960 CGF, llvm::Intrinsic::umul_with_overflow, V1, V2, HasOverflow);
1965 auto IntMax = llvm::APInt::getSignedMaxValue(ResultInfo.Width);
1966 llvm::Value *IntMaxValue = llvm::ConstantInt::get(
Result->getType(), IntMax);
1968 llvm::Value *IntMaxOverflow = CGF.
Builder.CreateICmpUGT(
Result, IntMaxValue);
1969 HasOverflow = CGF.
Builder.CreateOr(HasOverflow, IntMaxOverflow);
1981 WidthAndSignedness Op1Info,
1982 WidthAndSignedness Op2Info,
1983 WidthAndSignedness ResultInfo) {
1984 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
1985 std::max(Op1Info.Width, Op2Info.Width) >= ResultInfo.Width &&
1986 Op1Info.Signed != Op2Info.Signed;
1993 WidthAndSignedness Op1Info,
const clang::Expr *Op2,
1994 WidthAndSignedness Op2Info,
1996 WidthAndSignedness ResultInfo) {
1998 Op2Info, ResultInfo) &&
1999 "Not a mixed-sign multipliction we can specialize");
2002 const clang::Expr *SignedOp = Op1Info.Signed ? Op1 : Op2;
2003 const clang::Expr *UnsignedOp = Op1Info.Signed ? Op2 : Op1;
2006 unsigned SignedOpWidth = Op1Info.Signed ? Op1Info.Width : Op2Info.Width;
2007 unsigned UnsignedOpWidth = Op1Info.Signed ? Op2Info.Width : Op1Info.Width;
2010 if (SignedOpWidth < UnsignedOpWidth)
2012 if (UnsignedOpWidth < SignedOpWidth)
2015 llvm::Type *OpTy =
Signed->getType();
2016 llvm::Value *Zero = llvm::Constant::getNullValue(OpTy);
2019 unsigned OpWidth = std::max(Op1Info.Width, Op2Info.Width);
2022 llvm::Value *IsNegative = CGF.
Builder.CreateICmpSLT(
Signed, Zero);
2023 llvm::Value *AbsOfNegative = CGF.
Builder.CreateSub(Zero,
Signed);
2024 llvm::Value *AbsSigned =
2025 CGF.
Builder.CreateSelect(IsNegative, AbsOfNegative,
Signed);
2028 llvm::Value *UnsignedOverflow;
2029 llvm::Value *UnsignedResult =
2033 llvm::Value *Overflow, *
Result;
2034 if (ResultInfo.Signed) {
2038 llvm::APInt::getSignedMaxValue(ResultInfo.Width).zext(OpWidth);
2039 llvm::Value *MaxResult =
2040 CGF.
Builder.CreateAdd(llvm::ConstantInt::get(OpTy, IntMax),
2041 CGF.
Builder.CreateZExt(IsNegative, OpTy));
2042 llvm::Value *SignedOverflow =
2043 CGF.
Builder.CreateICmpUGT(UnsignedResult, MaxResult);
2044 Overflow = CGF.
Builder.CreateOr(UnsignedOverflow, SignedOverflow);
2047 llvm::Value *NegativeResult = CGF.
Builder.CreateNeg(UnsignedResult);
2048 llvm::Value *SignedResult =
2049 CGF.
Builder.CreateSelect(IsNegative, NegativeResult, UnsignedResult);
2053 llvm::Value *Underflow = CGF.
Builder.CreateAnd(
2054 IsNegative, CGF.
Builder.CreateIsNotNull(UnsignedResult));
2055 Overflow = CGF.
Builder.CreateOr(UnsignedOverflow, Underflow);
2056 if (ResultInfo.Width < OpWidth) {
2058 llvm::APInt::getMaxValue(ResultInfo.Width).zext(OpWidth);
2059 llvm::Value *TruncOverflow = CGF.
Builder.CreateICmpUGT(
2060 UnsignedResult, llvm::ConstantInt::get(OpTy, IntMax));
2061 Overflow = CGF.
Builder.CreateOr(Overflow, TruncOverflow);
2066 IsNegative, CGF.
Builder.CreateNeg(UnsignedResult), UnsignedResult);
2070 assert(Overflow &&
Result &&
"Missing overflow or result");
2081 llvm::SmallPtrSetImpl<const Decl *> &Seen) {
2090 if (!Seen.insert(Record).second)
2093 assert(Record->hasDefinition() &&
2094 "Incomplete types should already be diagnosed");
2096 if (Record->isDynamicClass())
2121 llvm::Type *Ty = Src->getType();
2122 ShiftAmt =
Builder.CreateIntCast(ShiftAmt, Ty,
false);
2125 unsigned IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2132 switch (BuiltinID) {
2133#define MUTATE_LDBL(func) \
2134 case Builtin::BI__builtin_##func##l: \
2135 return Builtin::BI__builtin_##func##f128;
2210 !
Result.hasSideEffects()) {
2214 if (
Result.Val.isFloat())
2223 if (
getTarget().getTriple().isPPC64() &&
2224 &
getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad())
2231 const unsigned BuiltinIDIfNoAsmLabel =
2232 FD->
hasAttr<AsmLabelAttr>() ? 0 : BuiltinID;
2241 bool ConstWithoutErrnoAndExceptions =
2243 bool ConstWithoutExceptions =
2245 if (FD->
hasAttr<ConstAttr>() ||
2246 ((ConstWithoutErrnoAndExceptions || ConstWithoutExceptions) &&
2247 (!ConstWithoutErrnoAndExceptions || (!
getLangOpts().MathErrno)))) {
2248 switch (BuiltinIDIfNoAsmLabel) {
2249 case Builtin::BIceil:
2250 case Builtin::BIceilf:
2251 case Builtin::BIceill:
2252 case Builtin::BI__builtin_ceil:
2253 case Builtin::BI__builtin_ceilf:
2254 case Builtin::BI__builtin_ceilf16:
2255 case Builtin::BI__builtin_ceill:
2256 case Builtin::BI__builtin_ceilf128:
2259 Intrinsic::experimental_constrained_ceil));
2261 case Builtin::BIcopysign:
2262 case Builtin::BIcopysignf:
2263 case Builtin::BIcopysignl:
2264 case Builtin::BI__builtin_copysign:
2265 case Builtin::BI__builtin_copysignf:
2266 case Builtin::BI__builtin_copysignf16:
2267 case Builtin::BI__builtin_copysignl:
2268 case Builtin::BI__builtin_copysignf128:
2271 case Builtin::BIcos:
2272 case Builtin::BIcosf:
2273 case Builtin::BIcosl:
2274 case Builtin::BI__builtin_cos:
2275 case Builtin::BI__builtin_cosf:
2276 case Builtin::BI__builtin_cosf16:
2277 case Builtin::BI__builtin_cosl:
2278 case Builtin::BI__builtin_cosf128:
2281 Intrinsic::experimental_constrained_cos));
2283 case Builtin::BIexp:
2284 case Builtin::BIexpf:
2285 case Builtin::BIexpl:
2286 case Builtin::BI__builtin_exp:
2287 case Builtin::BI__builtin_expf:
2288 case Builtin::BI__builtin_expf16:
2289 case Builtin::BI__builtin_expl:
2290 case Builtin::BI__builtin_expf128:
2293 Intrinsic::experimental_constrained_exp));
2295 case Builtin::BIexp2:
2296 case Builtin::BIexp2f:
2297 case Builtin::BIexp2l:
2298 case Builtin::BI__builtin_exp2:
2299 case Builtin::BI__builtin_exp2f:
2300 case Builtin::BI__builtin_exp2f16:
2301 case Builtin::BI__builtin_exp2l:
2302 case Builtin::BI__builtin_exp2f128:
2305 Intrinsic::experimental_constrained_exp2));
2307 case Builtin::BIfabs:
2308 case Builtin::BIfabsf:
2309 case Builtin::BIfabsl:
2310 case Builtin::BI__builtin_fabs:
2311 case Builtin::BI__builtin_fabsf:
2312 case Builtin::BI__builtin_fabsf16:
2313 case Builtin::BI__builtin_fabsl:
2314 case Builtin::BI__builtin_fabsf128:
2317 case Builtin::BIfloor:
2318 case Builtin::BIfloorf:
2319 case Builtin::BIfloorl:
2320 case Builtin::BI__builtin_floor:
2321 case Builtin::BI__builtin_floorf:
2322 case Builtin::BI__builtin_floorf16:
2323 case Builtin::BI__builtin_floorl:
2324 case Builtin::BI__builtin_floorf128:
2327 Intrinsic::experimental_constrained_floor));
2329 case Builtin::BIfma:
2330 case Builtin::BIfmaf:
2331 case Builtin::BIfmal:
2332 case Builtin::BI__builtin_fma:
2333 case Builtin::BI__builtin_fmaf:
2334 case Builtin::BI__builtin_fmaf16:
2335 case Builtin::BI__builtin_fmal:
2336 case Builtin::BI__builtin_fmaf128:
2339 Intrinsic::experimental_constrained_fma));
2341 case Builtin::BIfmax:
2342 case Builtin::BIfmaxf:
2343 case Builtin::BIfmaxl:
2344 case Builtin::BI__builtin_fmax:
2345 case Builtin::BI__builtin_fmaxf:
2346 case Builtin::BI__builtin_fmaxf16:
2347 case Builtin::BI__builtin_fmaxl:
2348 case Builtin::BI__builtin_fmaxf128:
2351 Intrinsic::experimental_constrained_maxnum));
2353 case Builtin::BIfmin:
2354 case Builtin::BIfminf:
2355 case Builtin::BIfminl:
2356 case Builtin::BI__builtin_fmin:
2357 case Builtin::BI__builtin_fminf:
2358 case Builtin::BI__builtin_fminf16:
2359 case Builtin::BI__builtin_fminl:
2360 case Builtin::BI__builtin_fminf128:
2363 Intrinsic::experimental_constrained_minnum));
2367 case Builtin::BIfmod:
2368 case Builtin::BIfmodf:
2369 case Builtin::BIfmodl:
2370 case Builtin::BI__builtin_fmod:
2371 case Builtin::BI__builtin_fmodf:
2372 case Builtin::BI__builtin_fmodf16:
2373 case Builtin::BI__builtin_fmodl:
2374 case Builtin::BI__builtin_fmodf128: {
2375 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
2381 case Builtin::BIlog:
2382 case Builtin::BIlogf:
2383 case Builtin::BIlogl:
2384 case Builtin::BI__builtin_log:
2385 case Builtin::BI__builtin_logf:
2386 case Builtin::BI__builtin_logf16:
2387 case Builtin::BI__builtin_logl:
2388 case Builtin::BI__builtin_logf128:
2391 Intrinsic::experimental_constrained_log));
2393 case Builtin::BIlog10:
2394 case Builtin::BIlog10f:
2395 case Builtin::BIlog10l:
2396 case Builtin::BI__builtin_log10:
2397 case Builtin::BI__builtin_log10f:
2398 case Builtin::BI__builtin_log10f16:
2399 case Builtin::BI__builtin_log10l:
2400 case Builtin::BI__builtin_log10f128:
2403 Intrinsic::experimental_constrained_log10));
2405 case Builtin::BIlog2:
2406 case Builtin::BIlog2f:
2407 case Builtin::BIlog2l:
2408 case Builtin::BI__builtin_log2:
2409 case Builtin::BI__builtin_log2f:
2410 case Builtin::BI__builtin_log2f16:
2411 case Builtin::BI__builtin_log2l:
2412 case Builtin::BI__builtin_log2f128:
2415 Intrinsic::experimental_constrained_log2));
2417 case Builtin::BInearbyint:
2418 case Builtin::BInearbyintf:
2419 case Builtin::BInearbyintl:
2420 case Builtin::BI__builtin_nearbyint:
2421 case Builtin::BI__builtin_nearbyintf:
2422 case Builtin::BI__builtin_nearbyintl:
2423 case Builtin::BI__builtin_nearbyintf128:
2425 Intrinsic::nearbyint,
2426 Intrinsic::experimental_constrained_nearbyint));
2428 case Builtin::BIpow:
2429 case Builtin::BIpowf:
2430 case Builtin::BIpowl:
2431 case Builtin::BI__builtin_pow:
2432 case Builtin::BI__builtin_powf:
2433 case Builtin::BI__builtin_powf16:
2434 case Builtin::BI__builtin_powl:
2435 case Builtin::BI__builtin_powf128:
2438 Intrinsic::experimental_constrained_pow));
2440 case Builtin::BIrint:
2441 case Builtin::BIrintf:
2442 case Builtin::BIrintl:
2443 case Builtin::BI__builtin_rint:
2444 case Builtin::BI__builtin_rintf:
2445 case Builtin::BI__builtin_rintf16:
2446 case Builtin::BI__builtin_rintl:
2447 case Builtin::BI__builtin_rintf128:
2450 Intrinsic::experimental_constrained_rint));
2452 case Builtin::BIround:
2453 case Builtin::BIroundf:
2454 case Builtin::BIroundl:
2455 case Builtin::BI__builtin_round:
2456 case Builtin::BI__builtin_roundf:
2457 case Builtin::BI__builtin_roundf16:
2458 case Builtin::BI__builtin_roundl:
2459 case Builtin::BI__builtin_roundf128:
2462 Intrinsic::experimental_constrained_round));
2464 case Builtin::BIroundeven:
2465 case Builtin::BIroundevenf:
2466 case Builtin::BIroundevenl:
2467 case Builtin::BI__builtin_roundeven:
2468 case Builtin::BI__builtin_roundevenf:
2469 case Builtin::BI__builtin_roundevenf16:
2470 case Builtin::BI__builtin_roundevenl:
2471 case Builtin::BI__builtin_roundevenf128:
2473 Intrinsic::roundeven,
2474 Intrinsic::experimental_constrained_roundeven));
2476 case Builtin::BIsin:
2477 case Builtin::BIsinf:
2478 case Builtin::BIsinl:
2479 case Builtin::BI__builtin_sin:
2480 case Builtin::BI__builtin_sinf:
2481 case Builtin::BI__builtin_sinf16:
2482 case Builtin::BI__builtin_sinl:
2483 case Builtin::BI__builtin_sinf128:
2486 Intrinsic::experimental_constrained_sin));
2488 case Builtin::BIsqrt:
2489 case Builtin::BIsqrtf:
2490 case Builtin::BIsqrtl:
2491 case Builtin::BI__builtin_sqrt:
2492 case Builtin::BI__builtin_sqrtf:
2493 case Builtin::BI__builtin_sqrtf16:
2494 case Builtin::BI__builtin_sqrtl:
2495 case Builtin::BI__builtin_sqrtf128:
2498 Intrinsic::experimental_constrained_sqrt));
2500 case Builtin::BItrunc:
2501 case Builtin::BItruncf:
2502 case Builtin::BItruncl:
2503 case Builtin::BI__builtin_trunc:
2504 case Builtin::BI__builtin_truncf:
2505 case Builtin::BI__builtin_truncf16:
2506 case Builtin::BI__builtin_truncl:
2507 case Builtin::BI__builtin_truncf128:
2510 Intrinsic::experimental_constrained_trunc));
2512 case Builtin::BIlround:
2513 case Builtin::BIlroundf:
2514 case Builtin::BIlroundl:
2515 case Builtin::BI__builtin_lround:
2516 case Builtin::BI__builtin_lroundf:
2517 case Builtin::BI__builtin_lroundl:
2518 case Builtin::BI__builtin_lroundf128:
2520 *
this, E, Intrinsic::lround,
2521 Intrinsic::experimental_constrained_lround));
2523 case Builtin::BIllround:
2524 case Builtin::BIllroundf:
2525 case Builtin::BIllroundl:
2526 case Builtin::BI__builtin_llround:
2527 case Builtin::BI__builtin_llroundf:
2528 case Builtin::BI__builtin_llroundl:
2529 case Builtin::BI__builtin_llroundf128:
2531 *
this, E, Intrinsic::llround,
2532 Intrinsic::experimental_constrained_llround));
2534 case Builtin::BIlrint:
2535 case Builtin::BIlrintf:
2536 case Builtin::BIlrintl:
2537 case Builtin::BI__builtin_lrint:
2538 case Builtin::BI__builtin_lrintf:
2539 case Builtin::BI__builtin_lrintl:
2540 case Builtin::BI__builtin_lrintf128:
2542 *
this, E, Intrinsic::lrint,
2543 Intrinsic::experimental_constrained_lrint));
2545 case Builtin::BIllrint:
2546 case Builtin::BIllrintf:
2547 case Builtin::BIllrintl:
2548 case Builtin::BI__builtin_llrint:
2549 case Builtin::BI__builtin_llrintf:
2550 case Builtin::BI__builtin_llrintl:
2551 case Builtin::BI__builtin_llrintf128:
2553 *
this, E, Intrinsic::llrint,
2554 Intrinsic::experimental_constrained_llrint));
2561 switch (BuiltinIDIfNoAsmLabel) {
2563 case Builtin::BI__builtin___CFStringMakeConstantString:
2564 case Builtin::BI__builtin___NSStringMakeConstantString:
2566 case Builtin::BI__builtin_stdarg_start:
2567 case Builtin::BI__builtin_va_start:
2568 case Builtin::BI__va_start:
2569 case Builtin::BI__builtin_va_end:
2573 BuiltinID != Builtin::BI__builtin_va_end);
2575 case Builtin::BI__builtin_va_copy: {
2586 case Builtin::BI__builtin_abs:
2587 case Builtin::BI__builtin_labs:
2588 case Builtin::BI__builtin_llabs: {
2593 Constant *
Zero = llvm::Constant::getNullValue(ArgValue->getType());
2594 Value *CmpResult =
Builder.CreateICmpSLT(ArgValue, Zero,
"abscond");
2598 case Builtin::BI__builtin_complex: {
2603 case Builtin::BI__builtin_conj:
2604 case Builtin::BI__builtin_conjf:
2605 case Builtin::BI__builtin_conjl:
2606 case Builtin::BIconj:
2607 case Builtin::BIconjf:
2608 case Builtin::BIconjl: {
2610 Value *Real = ComplexVal.first;
2611 Value *Imag = ComplexVal.second;
2612 Imag =
Builder.CreateFNeg(Imag,
"neg");
2615 case Builtin::BI__builtin_creal:
2616 case Builtin::BI__builtin_crealf:
2617 case Builtin::BI__builtin_creall:
2618 case Builtin::BIcreal:
2619 case Builtin::BIcrealf:
2620 case Builtin::BIcreall: {
2625 case Builtin::BI__builtin_preserve_access_index: {
2646 case Builtin::BI__builtin_cimag:
2647 case Builtin::BI__builtin_cimagf:
2648 case Builtin::BI__builtin_cimagl:
2649 case Builtin::BIcimag:
2650 case Builtin::BIcimagf:
2651 case Builtin::BIcimagl: {
2656 case Builtin::BI__builtin_clrsb:
2657 case Builtin::BI__builtin_clrsbl:
2658 case Builtin::BI__builtin_clrsbll: {
2662 llvm::Type *ArgType = ArgValue->getType();
2666 Value *
Zero = llvm::Constant::getNullValue(ArgType);
2667 Value *IsNeg =
Builder.CreateICmpSLT(ArgValue, Zero,
"isneg");
2669 Value *Tmp =
Builder.CreateSelect(IsNeg, Inverse, ArgValue);
2676 case Builtin::BI__builtin_ctzs:
2677 case Builtin::BI__builtin_ctz:
2678 case Builtin::BI__builtin_ctzl:
2679 case Builtin::BI__builtin_ctzll: {
2682 llvm::Type *ArgType = ArgValue->getType();
2688 if (
Result->getType() != ResultType)
2693 case Builtin::BI__builtin_clzs:
2694 case Builtin::BI__builtin_clz:
2695 case Builtin::BI__builtin_clzl:
2696 case Builtin::BI__builtin_clzll: {
2699 llvm::Type *ArgType = ArgValue->getType();
2705 if (
Result->getType() != ResultType)
2710 case Builtin::BI__builtin_ffs:
2711 case Builtin::BI__builtin_ffsl:
2712 case Builtin::BI__builtin_ffsll: {
2716 llvm::Type *ArgType = ArgValue->getType();
2721 Builder.CreateAdd(
Builder.CreateCall(F, {ArgValue, Builder.getTrue()}),
2722 llvm::ConstantInt::get(ArgType, 1));
2723 Value *
Zero = llvm::Constant::getNullValue(ArgType);
2724 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, Zero,
"iszero");
2726 if (
Result->getType() != ResultType)
2731 case Builtin::BI__builtin_parity:
2732 case Builtin::BI__builtin_parityl:
2733 case Builtin::BI__builtin_parityll: {
2737 llvm::Type *ArgType = ArgValue->getType();
2743 if (
Result->getType() != ResultType)
2748 case Builtin::BI__lzcnt16:
2749 case Builtin::BI__lzcnt:
2750 case Builtin::BI__lzcnt64: {
2753 llvm::Type *ArgType = ArgValue->getType();
2758 if (
Result->getType() != ResultType)
2763 case Builtin::BI__popcnt16:
2764 case Builtin::BI__popcnt:
2765 case Builtin::BI__popcnt64:
2766 case Builtin::BI__builtin_popcount:
2767 case Builtin::BI__builtin_popcountl:
2768 case Builtin::BI__builtin_popcountll: {
2771 llvm::Type *ArgType = ArgValue->getType();
2776 if (
Result->getType() != ResultType)
2781 case Builtin::BI__builtin_unpredictable: {
2787 case Builtin::BI__builtin_expect: {
2789 llvm::Type *ArgType = ArgValue->getType();
2800 Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue},
"expval");
2803 case Builtin::BI__builtin_expect_with_probability: {
2805 llvm::Type *ArgType = ArgValue->getType();
2808 llvm::APFloat Probability(0.0);
2811 assert(EvalSucceed &&
"probability should be able to evaluate as float");
2813 bool LoseInfo =
false;
2814 Probability.convert(llvm::APFloat::IEEEdouble(),
2815 llvm::RoundingMode::Dynamic, &LoseInfo);
2817 Constant *Confidence = ConstantFP::get(Ty, Probability);
2827 FnExpect, {ArgValue, ExpectedValue, Confidence},
"expval");
2830 case Builtin::BI__builtin_assume_aligned: {
2835 Value *OffsetValue =
2839 ConstantInt *AlignmentCI = cast<ConstantInt>(AlignmentValue);
2840 if (AlignmentCI->getValue().ugt(llvm::Value::MaximumAlignment))
2841 AlignmentCI = ConstantInt::get(AlignmentCI->getType(),
2842 llvm::Value::MaximumAlignment);
2846 AlignmentCI, OffsetValue);
2849 case Builtin::BI__assume:
2850 case Builtin::BI__builtin_assume: {
2856 Builder.CreateCall(FnAssume, ArgValue);
2859 case Builtin::BI__builtin_assume_separate_storage: {
2866 Value *Values[] = {Value0, Value1};
2867 OperandBundleDefT<Value *> OBD(
"separate_storage", Values);
2871 case Builtin::BI__arithmetic_fence: {
2874 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
2875 llvm::FastMathFlags FMF =
Builder.getFastMathFlags();
2876 bool isArithmeticFenceEnabled =
2877 FMF.allowReassoc() &&
2881 if (isArithmeticFenceEnabled) {
2884 Value *Real =
Builder.CreateArithmeticFence(ComplexVal.first,
2886 Value *Imag =
Builder.CreateArithmeticFence(ComplexVal.second,
2891 Value *Real = ComplexVal.first;
2892 Value *Imag = ComplexVal.second;
2896 if (isArithmeticFenceEnabled)
2901 case Builtin::BI__builtin_bswap16:
2902 case Builtin::BI__builtin_bswap32:
2903 case Builtin::BI__builtin_bswap64:
2904 case Builtin::BI_byteswap_ushort:
2905 case Builtin::BI_byteswap_ulong:
2906 case Builtin::BI_byteswap_uint64: {
2909 case Builtin::BI__builtin_bitreverse8:
2910 case Builtin::BI__builtin_bitreverse16:
2911 case Builtin::BI__builtin_bitreverse32:
2912 case Builtin::BI__builtin_bitreverse64: {
2915 case Builtin::BI__builtin_rotateleft8:
2916 case Builtin::BI__builtin_rotateleft16:
2917 case Builtin::BI__builtin_rotateleft32:
2918 case Builtin::BI__builtin_rotateleft64:
2919 case Builtin::BI_rotl8:
2920 case Builtin::BI_rotl16:
2921 case Builtin::BI_rotl:
2922 case Builtin::BI_lrotl:
2923 case Builtin::BI_rotl64:
2926 case Builtin::BI__builtin_rotateright8:
2927 case Builtin::BI__builtin_rotateright16:
2928 case Builtin::BI__builtin_rotateright32:
2929 case Builtin::BI__builtin_rotateright64:
2930 case Builtin::BI_rotr8:
2931 case Builtin::BI_rotr16:
2932 case Builtin::BI_rotr:
2933 case Builtin::BI_lrotr:
2934 case Builtin::BI_rotr64:
2937 case Builtin::BI__builtin_constant_p: {
2948 return RValue::get(ConstantInt::get(ResultType, 0));
2953 return RValue::get(ConstantInt::get(ResultType, 0));
2965 if (
Result->getType() != ResultType)
2969 case Builtin::BI__builtin_dynamic_object_size:
2970 case Builtin::BI__builtin_object_size: {
2977 bool IsDynamic = BuiltinID == Builtin::BI__builtin_dynamic_object_size;
2979 nullptr, IsDynamic));
2981 case Builtin::BI__builtin_prefetch: {
2985 llvm::ConstantInt::get(
Int32Ty, 0);
2987 llvm::ConstantInt::get(
Int32Ty, 3);
2993 case Builtin::BI__builtin_readcyclecounter: {
2997 case Builtin::BI__builtin___clear_cache: {
3003 case Builtin::BI__builtin_trap:
3006 case Builtin::BI__debugbreak:
3009 case Builtin::BI__builtin_unreachable: {
3018 case Builtin::BI__builtin_powi:
3019 case Builtin::BI__builtin_powif:
3020 case Builtin::BI__builtin_powil: {
3024 if (
Builder.getIsFPConstrained()) {
3025 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3032 { Src0->getType(), Src1->getType() });
3035 case Builtin::BI__builtin_isgreater:
3036 case Builtin::BI__builtin_isgreaterequal:
3037 case Builtin::BI__builtin_isless:
3038 case Builtin::BI__builtin_islessequal:
3039 case Builtin::BI__builtin_islessgreater:
3040 case Builtin::BI__builtin_isunordered: {
3043 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3047 switch (BuiltinID) {
3048 default: llvm_unreachable(
"Unknown ordered comparison");
3049 case Builtin::BI__builtin_isgreater:
3050 LHS =
Builder.CreateFCmpOGT(LHS, RHS,
"cmp");
3052 case Builtin::BI__builtin_isgreaterequal:
3053 LHS =
Builder.CreateFCmpOGE(LHS, RHS,
"cmp");
3055 case Builtin::BI__builtin_isless:
3056 LHS =
Builder.CreateFCmpOLT(LHS, RHS,
"cmp");
3058 case Builtin::BI__builtin_islessequal:
3059 LHS =
Builder.CreateFCmpOLE(LHS, RHS,
"cmp");
3061 case Builtin::BI__builtin_islessgreater:
3062 LHS =
Builder.CreateFCmpONE(LHS, RHS,
"cmp");
3064 case Builtin::BI__builtin_isunordered:
3065 LHS =
Builder.CreateFCmpUNO(LHS, RHS,
"cmp");
3071 case Builtin::BI__builtin_isnan: {
3072 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3074 llvm::Type *Ty =
V->getType();
3075 const llvm::fltSemantics &Semantics = Ty->getFltSemantics();
3076 if (!
Builder.getIsFPConstrained() ||
3077 Builder.getDefaultConstrainedExcept() == fp::ebIgnore ||
3088 unsigned bitsize = Ty->getScalarSizeInBits();
3091 APInt AndMask = APInt::getSignedMaxValue(bitsize);
3093 Builder.CreateAnd(IntV, llvm::ConstantInt::get(
IntTy, AndMask));
3094 APInt ExpMask = APFloat::getInf(Semantics).bitcastToAPInt();
3096 Builder.CreateSub(llvm::ConstantInt::get(
IntTy, ExpMask), AbsV);
3098 V =
Builder.CreateLShr(Sub, llvm::ConstantInt::get(
IntTy, bitsize - 1));
3104 case Builtin::BI__builtin_nondeterministic_value: {
3113 case Builtin::BI__builtin_elementwise_abs: {
3118 QT = VecTy->getElementType();
3122 Builder.getFalse(),
nullptr,
"elt.abs");
3129 case Builtin::BI__builtin_elementwise_ceil:
3132 case Builtin::BI__builtin_elementwise_exp:
3135 case Builtin::BI__builtin_elementwise_exp2:
3138 case Builtin::BI__builtin_elementwise_log:
3141 case Builtin::BI__builtin_elementwise_log2:
3144 case Builtin::BI__builtin_elementwise_log10:
3147 case Builtin::BI__builtin_elementwise_cos:
3150 case Builtin::BI__builtin_elementwise_floor:
3153 case Builtin::BI__builtin_elementwise_roundeven:
3156 case Builtin::BI__builtin_elementwise_sin:
3160 case Builtin::BI__builtin_elementwise_trunc:
3163 case Builtin::BI__builtin_elementwise_canonicalize:
3166 case Builtin::BI__builtin_elementwise_copysign:
3168 case Builtin::BI__builtin_elementwise_fma:
3170 case Builtin::BI__builtin_elementwise_add_sat:
3171 case Builtin::BI__builtin_elementwise_sub_sat: {
3175 assert(Op0->getType()->isIntOrIntVectorTy() &&
"integer type expected");
3178 Ty = VecTy->getElementType();
3181 if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_add_sat)
3182 Opc = IsSigned ? llvm::Intrinsic::sadd_sat : llvm::Intrinsic::uadd_sat;
3184 Opc = IsSigned ? llvm::Intrinsic::ssub_sat : llvm::Intrinsic::usub_sat;
3185 Result =
Builder.CreateBinaryIntrinsic(Opc, Op0, Op1,
nullptr,
"elt.sat");
3189 case Builtin::BI__builtin_elementwise_max: {
3193 if (Op0->getType()->isIntOrIntVectorTy()) {
3196 Ty = VecTy->getElementType();
3198 ? llvm::Intrinsic::smax
3199 : llvm::Intrinsic::umax,
3200 Op0, Op1,
nullptr,
"elt.max");
3205 case Builtin::BI__builtin_elementwise_min: {
3209 if (Op0->getType()->isIntOrIntVectorTy()) {
3212 Ty = VecTy->getElementType();
3214 ? llvm::Intrinsic::smin
3215 : llvm::Intrinsic::umin,
3216 Op0, Op1,
nullptr,
"elt.min");
3222 case Builtin::BI__builtin_reduce_max: {
3223 auto GetIntrinsicID = [](
QualType QT) {
3225 QT = VecTy->getElementType();
3227 return llvm::Intrinsic::vector_reduce_smax;
3229 return llvm::Intrinsic::vector_reduce_umax;
3231 return llvm::Intrinsic::vector_reduce_fmax;
3234 *
this, E, GetIntrinsicID(E->
getArg(0)->
getType()),
"rdx.min"));
3237 case Builtin::BI__builtin_reduce_min: {
3238 auto GetIntrinsicID = [](
QualType QT) {
3240 QT = VecTy->getElementType();
3242 return llvm::Intrinsic::vector_reduce_smin;
3244 return llvm::Intrinsic::vector_reduce_umin;
3246 return llvm::Intrinsic::vector_reduce_fmin;
3250 *
this, E, GetIntrinsicID(E->
getArg(0)->
getType()),
"rdx.min"));
3253 case Builtin::BI__builtin_reduce_add:
3255 *
this, E, llvm::Intrinsic::vector_reduce_add,
"rdx.add"));
3256 case Builtin::BI__builtin_reduce_mul:
3258 *
this, E, llvm::Intrinsic::vector_reduce_mul,
"rdx.mul"));
3259 case Builtin::BI__builtin_reduce_xor:
3261 *
this, E, llvm::Intrinsic::vector_reduce_xor,
"rdx.xor"));
3262 case Builtin::BI__builtin_reduce_or:
3264 *
this, E, llvm::Intrinsic::vector_reduce_or,
"rdx.or"));
3265 case Builtin::BI__builtin_reduce_and:
3267 *
this, E, llvm::Intrinsic::vector_reduce_and,
"rdx.and"));
3269 case Builtin::BI__builtin_matrix_transpose: {
3273 Value *
Result = MB.CreateMatrixTranspose(MatValue, MatrixTy->getNumRows(),
3274 MatrixTy->getNumColumns());
3278 case Builtin::BI__builtin_matrix_column_major_load: {
3284 assert(PtrTy &&
"arg0 must be of pointer type");
3293 ResultTy->getNumRows(), ResultTy->getNumColumns(),
3298 case Builtin::BI__builtin_matrix_column_major_store: {
3306 assert(PtrTy &&
"arg1 must be of pointer type");
3313 Stride, IsVolatile, MatrixTy->getNumRows(), MatrixTy->getNumColumns());
3317 case Builtin::BIfinite:
3318 case Builtin::BI__finite:
3319 case Builtin::BIfinitef:
3320 case Builtin::BI__finitef:
3321 case Builtin::BIfinitel:
3322 case Builtin::BI__finitel:
3323 case Builtin::BI__builtin_isinf:
3324 case Builtin::BI__builtin_isfinite: {
3328 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3330 llvm::Type *Ty =
V->getType();
3331 if (!
Builder.getIsFPConstrained() ||
3332 Builder.getDefaultConstrainedExcept() == fp::ebIgnore ||
3335 Constant *Infinity = ConstantFP::getInfinity(
V->getType());
3336 CmpInst::Predicate Pred = (BuiltinID == Builtin::BI__builtin_isinf)
3338 : CmpInst::FCMP_ONE;
3339 Value *FCmp =
Builder.CreateFCmp(Pred, Fabs, Infinity,
"cmpinf");
3349 unsigned bitsize = Ty->getScalarSizeInBits();
3353 const llvm::fltSemantics &Semantics = Ty->getFltSemantics();
3354 APInt ExpMask = APFloat::getInf(Semantics).bitcastToAPInt();
3355 Value *ExpMaskShl1 = llvm::ConstantInt::get(
IntTy, ExpMask.shl(1));
3356 if (BuiltinID == Builtin::BI__builtin_isinf)
3357 V =
Builder.CreateICmpEQ(Shl1, ExpMaskShl1);
3359 V =
Builder.CreateICmpULT(Shl1, ExpMaskShl1);
3363 case Builtin::BI__builtin_isinf_sign: {
3365 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3370 AbsArg, ConstantFP::getInfinity(Arg->getType()),
"isinf");
3376 Value *NegativeOne = ConstantInt::get(
IntTy, -1);
3377 Value *SignResult =
Builder.CreateSelect(IsNeg, NegativeOne, One);
3382 case Builtin::BI__builtin_isnormal: {
3384 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3390 Value *IsLessThanInf =
3391 Builder.CreateFCmpULT(Abs, ConstantFP::getInfinity(
V->getType()),
"isinf");
3392 APFloat Smallest = APFloat::getSmallestNormalized(
3395 Builder.CreateFCmpUGE(Abs, ConstantFP::get(
V->getContext(), Smallest),
3397 V =
Builder.CreateAnd(
Eq, IsLessThanInf,
"and");
3398 V =
Builder.CreateAnd(
V, IsNormal,
"and");
3402 case Builtin::BI__builtin_flt_rounds: {
3407 if (
Result->getType() != ResultType)
3413 case Builtin::BI__builtin_set_flt_rounds: {
3421 case Builtin::BI__builtin_fpclassify: {
3422 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3433 "fpclassify_result");
3437 Value *IsZero =
Builder.CreateFCmpOEQ(
V, Constant::getNullValue(Ty),
3441 Builder.CreateCondBr(IsZero, End, NotZero);
3445 Builder.SetInsertPoint(NotZero);
3449 Builder.CreateCondBr(IsNan, End, NotNan);
3450 Result->addIncoming(NanLiteral, NotZero);
3453 Builder.SetInsertPoint(NotNan);
3456 Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(
V->getType()),
3460 Builder.CreateCondBr(IsInf, End, NotInf);
3461 Result->addIncoming(InfLiteral, NotNan);
3464 Builder.SetInsertPoint(NotInf);
3465 APFloat Smallest = APFloat::getSmallestNormalized(
3468 Builder.CreateFCmpUGE(VAbs, ConstantFP::get(
V->getContext(), Smallest),
3470 Value *NormalResult =
3474 Result->addIncoming(NormalResult, NotInf);
3481 case Builtin::BIalloca:
3482 case Builtin::BI_alloca:
3483 case Builtin::BI__builtin_alloca_uninitialized:
3484 case Builtin::BI__builtin_alloca: {
3488 const Align SuitableAlignmentInBytes =
3492 AllocaInst *AI =
Builder.CreateAlloca(
Builder.getInt8Ty(), Size);
3493 AI->setAlignment(SuitableAlignmentInBytes);
3494 if (BuiltinID != Builtin::BI__builtin_alloca_uninitialized)
3499 case Builtin::BI__builtin_alloca_with_align_uninitialized:
3500 case Builtin::BI__builtin_alloca_with_align: {
3503 auto *AlignmentInBitsCI = cast<ConstantInt>(AlignmentInBitsValue);
3504 unsigned AlignmentInBits = AlignmentInBitsCI->getZExtValue();
3505 const Align AlignmentInBytes =
3507 AllocaInst *AI =
Builder.CreateAlloca(
Builder.getInt8Ty(), Size);
3508 AI->setAlignment(AlignmentInBytes);
3509 if (BuiltinID != Builtin::BI__builtin_alloca_with_align_uninitialized)
3514 case Builtin::BIbzero:
3515 case Builtin::BI__builtin_bzero: {
3523 case Builtin::BImemcpy:
3524 case Builtin::BI__builtin_memcpy:
3525 case Builtin::BImempcpy:
3526 case Builtin::BI__builtin_mempcpy: {
3535 if (BuiltinID == Builtin::BImempcpy ||
3536 BuiltinID == Builtin::BI__builtin_mempcpy)
3543 case Builtin::BI__builtin_memcpy_inline: {
3556 case Builtin::BI__builtin_char_memchr:
3557 BuiltinID = Builtin::BI__builtin_memchr;
3560 case Builtin::BI__builtin___memcpy_chk: {
3567 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
3568 if (
Size.ugt(DstSize))
3572 Value *SizeVal = llvm::ConstantInt::get(
Builder.getContext(), Size);
3577 case Builtin::BI__builtin_objc_memmove_collectable: {
3582 DestAddr, SrcAddr, SizeVal);
3586 case Builtin::BI__builtin___memmove_chk: {
3593 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
3594 if (
Size.ugt(DstSize))
3598 Value *SizeVal = llvm::ConstantInt::get(
Builder.getContext(), Size);
3603 case Builtin::BImemmove:
3604 case Builtin::BI__builtin_memmove: {
3615 case Builtin::BImemset:
3616 case Builtin::BI__builtin_memset: {
3626 case Builtin::BI__builtin_memset_inline: {
3637 case Builtin::BI__builtin___memset_chk: {
3644 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
3645 if (
Size.ugt(DstSize))
3650 Value *SizeVal = llvm::ConstantInt::get(
Builder.getContext(), Size);
3654 case Builtin::BI__builtin_wmemchr: {
3657 if (!
getTarget().getTriple().isOSMSVCRT())
3665 BasicBlock *Entry =
Builder.GetInsertBlock();
3670 Builder.CreateCondBr(SizeEq0, Exit, CmpEq);
3673 PHINode *StrPhi =
Builder.CreatePHI(Str->getType(), 2);
3674 StrPhi->addIncoming(Str, Entry);
3676 SizePhi->addIncoming(Size, Entry);
3680 Value *FoundChr =
Builder.CreateConstInBoundsGEP1_32(WCharTy, StrPhi, 0);
3682 Builder.CreateCondBr(StrEqChr, Exit, Next);
3685 Value *NextStr =
Builder.CreateConstInBoundsGEP1_32(WCharTy, StrPhi, 1);
3687 Value *NextSizeEq0 =
3688 Builder.CreateICmpEQ(NextSize, ConstantInt::get(
SizeTy, 0));
3689 Builder.CreateCondBr(NextSizeEq0, Exit, CmpEq);
3690 StrPhi->addIncoming(NextStr, Next);
3691 SizePhi->addIncoming(NextSize, Next);
3694 PHINode *
Ret =
Builder.CreatePHI(Str->getType(), 3);
3695 Ret->addIncoming(llvm::Constant::getNullValue(Str->getType()), Entry);
3696 Ret->addIncoming(llvm::Constant::getNullValue(Str->getType()), Next);
3697 Ret->addIncoming(FoundChr, CmpEq);
3700 case Builtin::BI__builtin_wmemcmp: {
3703 if (!
getTarget().getTriple().isOSMSVCRT())
3712 BasicBlock *Entry =
Builder.GetInsertBlock();
3718 Builder.CreateCondBr(SizeEq0, Exit, CmpGT);
3721 PHINode *DstPhi =
Builder.CreatePHI(Dst->getType(), 2);
3722 DstPhi->addIncoming(Dst, Entry);
3723 PHINode *SrcPhi =
Builder.CreatePHI(Src->getType(), 2);
3724 SrcPhi->addIncoming(Src, Entry);
3726 SizePhi->addIncoming(Size, Entry);
3732 Builder.CreateCondBr(DstGtSrc, Exit, CmpLT);
3736 Builder.CreateCondBr(DstLtSrc, Exit, Next);
3739 Value *NextDst =
Builder.CreateConstInBoundsGEP1_32(WCharTy, DstPhi, 1);
3740 Value *NextSrc =
Builder.CreateConstInBoundsGEP1_32(WCharTy, SrcPhi, 1);
3742 Value *NextSizeEq0 =
3743 Builder.CreateICmpEQ(NextSize, ConstantInt::get(
SizeTy, 0));
3744 Builder.CreateCondBr(NextSizeEq0, Exit, CmpGT);
3745 DstPhi->addIncoming(NextDst, Next);
3746 SrcPhi->addIncoming(NextSrc, Next);
3747 SizePhi->addIncoming(NextSize, Next);
3751 Ret->addIncoming(ConstantInt::get(
IntTy, 0), Entry);
3752 Ret->addIncoming(ConstantInt::get(
IntTy, 1), CmpGT);
3753 Ret->addIncoming(ConstantInt::get(
IntTy, -1), CmpLT);
3754 Ret->addIncoming(ConstantInt::get(
IntTy, 0), Next);
3757 case Builtin::BI__builtin_dwarf_cfa: {
3772 case Builtin::BI__builtin_return_address: {
3778 case Builtin::BI_ReturnAddress: {
3782 case Builtin::BI__builtin_frame_address: {
3788 case Builtin::BI__builtin_extract_return_addr: {
3793 case Builtin::BI__builtin_frob_return_addr: {
3798 case Builtin::BI__builtin_dwarf_sp_column: {
3799 llvm::IntegerType *Ty
3806 return RValue::get(llvm::ConstantInt::get(Ty, Column,
true));
3808 case Builtin::BI__builtin_init_dwarf_reg_size_table: {
3814 case Builtin::BI__builtin_eh_return: {
3818 llvm::IntegerType *
IntTy = cast<llvm::IntegerType>(Int->getType());
3819 assert((
IntTy->getBitWidth() == 32 ||
IntTy->getBitWidth() == 64) &&
3820 "LLVM's __builtin_eh_return only supports 32- and 64-bit variants");
3823 : Intrinsic::eh_return_i64);
3824 Builder.CreateCall(F, {Int, Ptr});
3832 case Builtin::BI__builtin_unwind_init: {
3837 case Builtin::BI__builtin_extend_pointer: {
3862 case Builtin::BI__builtin_setjmp: {
3869 ConstantInt::get(
Int32Ty, 0));
3883 case Builtin::BI__builtin_longjmp: {
3898 case Builtin::BI__builtin_launder: {
3907 case Builtin::BI__sync_fetch_and_add:
3908 case Builtin::BI__sync_fetch_and_sub:
3909 case Builtin::BI__sync_fetch_and_or:
3910 case Builtin::BI__sync_fetch_and_and:
3911 case Builtin::BI__sync_fetch_and_xor:
3912 case Builtin::BI__sync_fetch_and_nand:
3913 case Builtin::BI__sync_add_and_fetch:
3914 case Builtin::BI__sync_sub_and_fetch:
3915 case Builtin::BI__sync_and_and_fetch:
3916 case Builtin::BI__sync_or_and_fetch:
3917 case Builtin::BI__sync_xor_and_fetch:
3918 case Builtin::BI__sync_nand_and_fetch:
3919 case Builtin::BI__sync_val_compare_and_swap:
3920 case Builtin::BI__sync_bool_compare_and_swap:
3921 case Builtin::BI__sync_lock_test_and_set:
3922 case Builtin::BI__sync_lock_release:
3923 case Builtin::BI__sync_swap:
3924 llvm_unreachable(
"Shouldn't make it through sema");
3925 case Builtin::BI__sync_fetch_and_add_1:
3926 case Builtin::BI__sync_fetch_and_add_2:
3927 case Builtin::BI__sync_fetch_and_add_4:
3928 case Builtin::BI__sync_fetch_and_add_8:
3929 case Builtin::BI__sync_fetch_and_add_16:
3931 case Builtin::BI__sync_fetch_and_sub_1:
3932 case Builtin::BI__sync_fetch_and_sub_2:
3933 case Builtin::BI__sync_fetch_and_sub_4:
3934 case Builtin::BI__sync_fetch_and_sub_8:
3935 case Builtin::BI__sync_fetch_and_sub_16:
3937 case Builtin::BI__sync_fetch_and_or_1:
3938 case Builtin::BI__sync_fetch_and_or_2:
3939 case Builtin::BI__sync_fetch_and_or_4:
3940 case Builtin::BI__sync_fetch_and_or_8:
3941 case Builtin::BI__sync_fetch_and_or_16:
3943 case Builtin::BI__sync_fetch_and_and_1:
3944 case Builtin::BI__sync_fetch_and_and_2:
3945 case Builtin::BI__sync_fetch_and_and_4:
3946 case Builtin::BI__sync_fetch_and_and_8:
3947 case Builtin::BI__sync_fetch_and_and_16:
3949 case Builtin::BI__sync_fetch_and_xor_1:
3950 case Builtin::BI__sync_fetch_and_xor_2:
3951 case Builtin::BI__sync_fetch_and_xor_4:
3952 case Builtin::BI__sync_fetch_and_xor_8:
3953 case Builtin::BI__sync_fetch_and_xor_16:
3955 case Builtin::BI__sync_fetch_and_nand_1:
3956 case Builtin::BI__sync_fetch_and_nand_2:
3957 case Builtin::BI__sync_fetch_and_nand_4:
3958 case Builtin::BI__sync_fetch_and_nand_8:
3959 case Builtin::BI__sync_fetch_and_nand_16:
3963 case Builtin::BI__sync_fetch_and_min:
3965 case Builtin::BI__sync_fetch_and_max:
3967 case Builtin::BI__sync_fetch_and_umin:
3969 case Builtin::BI__sync_fetch_and_umax:
3972 case Builtin::BI__sync_add_and_fetch_1:
3973 case Builtin::BI__sync_add_and_fetch_2:
3974 case Builtin::BI__sync_add_and_fetch_4:
3975 case Builtin::BI__sync_add_and_fetch_8:
3976 case Builtin::BI__sync_add_and_fetch_16:
3978 llvm::Instruction::Add);
3979 case Builtin::BI__sync_sub_and_fetch_1:
3980 case Builtin::BI__sync_sub_and_fetch_2:
3981 case Builtin::BI__sync_sub_and_fetch_4:
3982 case Builtin::BI__sync_sub_and_fetch_8:
3983 case Builtin::BI__sync_sub_and_fetch_16:
3985 llvm::Instruction::Sub);
3986 case Builtin::BI__sync_and_and_fetch_1:
3987 case Builtin::BI__sync_and_and_fetch_2:
3988 case Builtin::BI__sync_and_and_fetch_4:
3989 case Builtin::BI__sync_and_and_fetch_8:
3990 case Builtin::BI__sync_and_and_fetch_16:
3992 llvm::Instruction::And);
3993 case Builtin::BI__sync_or_and_fetch_1:
3994 case Builtin::BI__sync_or_and_fetch_2:
3995 case Builtin::BI__sync_or_and_fetch_4:
3996 case Builtin::BI__sync_or_and_fetch_8:
3997 case Builtin::BI__sync_or_and_fetch_16:
3999 llvm::Instruction::Or);
4000 case Builtin::BI__sync_xor_and_fetch_1:
4001 case Builtin::BI__sync_xor_and_fetch_2:
4002 case Builtin::BI__sync_xor_and_fetch_4:
4003 case Builtin::BI__sync_xor_and_fetch_8:
4004 case Builtin::BI__sync_xor_and_fetch_16:
4006 llvm::Instruction::Xor);
4007 case Builtin::BI__sync_nand_and_fetch_1:
4008 case Builtin::BI__sync_nand_and_fetch_2:
4009 case Builtin::BI__sync_nand_and_fetch_4:
4010 case Builtin::BI__sync_nand_and_fetch_8:
4011 case Builtin::BI__sync_nand_and_fetch_16:
4013 llvm::Instruction::And,
true);
4015 case Builtin::BI__sync_val_compare_and_swap_1:
4016 case Builtin::BI__sync_val_compare_and_swap_2:
4017 case Builtin::BI__sync_val_compare_and_swap_4:
4018 case Builtin::BI__sync_val_compare_and_swap_8:
4019 case Builtin::BI__sync_val_compare_and_swap_16:
4022 case Builtin::BI__sync_bool_compare_and_swap_1:
4023 case Builtin::BI__sync_bool_compare_and_swap_2:
4024 case Builtin::BI__sync_bool_compare_and_swap_4:
4025 case Builtin::BI__sync_bool_compare_and_swap_8:
4026 case Builtin::BI__sync_bool_compare_and_swap_16:
4029 case Builtin::BI__sync_swap_1:
4030 case Builtin::BI__sync_swap_2:
4031 case Builtin::BI__sync_swap_4:
4032 case Builtin::BI__sync_swap_8:
4033 case Builtin::BI__sync_swap_16:
4036 case Builtin::BI__sync_lock_test_and_set_1:
4037 case Builtin::BI__sync_lock_test_and_set_2:
4038 case Builtin::BI__sync_lock_test_and_set_4:
4039 case Builtin::BI__sync_lock_test_and_set_8:
4040 case Builtin::BI__sync_lock_test_and_set_16:
4043 case Builtin::BI__sync_lock_release_1:
4044 case Builtin::BI__sync_lock_release_2:
4045 case Builtin::BI__sync_lock_release_4:
4046 case Builtin::BI__sync_lock_release_8:
4047 case Builtin::BI__sync_lock_release_16: {
4054 Ptr =
Builder.CreateBitCast(Ptr, ITy->getPointerTo());
4055 llvm::StoreInst *
Store =
4058 Store->setAtomic(llvm::AtomicOrdering::Release);
4062 case Builtin::BI__sync_synchronize: {
4070 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent);
4074 case Builtin::BI__builtin_nontemporal_load:
4076 case Builtin::BI__builtin_nontemporal_store:
4078 case Builtin::BI__c11_atomic_is_lock_free:
4079 case Builtin::BI__atomic_is_lock_free: {
4083 const char *LibCallName =
"__atomic_is_lock_free";
4087 if (BuiltinID == Builtin::BI__atomic_is_lock_free)
4101 case Builtin::BI__atomic_test_and_set: {
4109 unsigned AddrSpace = Ptr->getType()->getPointerAddressSpace();
4110 Ptr =
Builder.CreateBitCast(Ptr,
Int8Ty->getPointerTo(AddrSpace));
4113 if (isa<llvm::ConstantInt>(Order)) {
4114 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
4115 AtomicRMWInst *
Result =
nullptr;
4120 llvm::AtomicOrdering::Monotonic);
4125 llvm::AtomicOrdering::Acquire);
4129 llvm::AtomicOrdering::Release);
4134 llvm::AtomicOrdering::AcquireRelease);
4138 llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
4139 llvm::AtomicOrdering::SequentiallyConsistent);
4142 Result->setVolatile(Volatile);
4148 llvm::BasicBlock *BBs[5] = {
4155 llvm::AtomicOrdering Orders[5] = {
4156 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Acquire,
4157 llvm::AtomicOrdering::Release, llvm::AtomicOrdering::AcquireRelease,
4158 llvm::AtomicOrdering::SequentiallyConsistent};
4160 Order =
Builder.CreateIntCast(Order,
Builder.getInt32Ty(),
false);
4161 llvm::SwitchInst *SI =
Builder.CreateSwitch(Order, BBs[0]);
4163 Builder.SetInsertPoint(ContBB);
4166 for (
unsigned i = 0; i < 5; ++i) {
4167 Builder.SetInsertPoint(BBs[i]);
4169 Ptr, NewVal, Orders[i]);
4170 RMW->setVolatile(Volatile);
4171 Result->addIncoming(RMW, BBs[i]);
4175 SI->addCase(
Builder.getInt32(0), BBs[0]);
4176 SI->addCase(
Builder.getInt32(1), BBs[1]);
4177 SI->addCase(
Builder.getInt32(2), BBs[1]);
4178 SI->addCase(
Builder.getInt32(3), BBs[2]);
4179 SI->addCase(
Builder.getInt32(4), BBs[3]);
4180 SI->addCase(
Builder.getInt32(5), BBs[4]);
4182 Builder.SetInsertPoint(ContBB);
4186 case Builtin::BI__atomic_clear: {
4195 if (isa<llvm::ConstantInt>(Order)) {
4196 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
4201 Store->setOrdering(llvm::AtomicOrdering::Monotonic);
4204 Store->setOrdering(llvm::AtomicOrdering::Release);
4207 Store->setOrdering(llvm::AtomicOrdering::SequentiallyConsistent);
4215 llvm::BasicBlock *BBs[3] = {
4220 llvm::AtomicOrdering Orders[3] = {
4221 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Release,
4222 llvm::AtomicOrdering::SequentiallyConsistent};
4224 Order =
Builder.CreateIntCast(Order,
Builder.getInt32Ty(),
false);
4225 llvm::SwitchInst *SI =
Builder.CreateSwitch(Order, BBs[0]);
4227 for (
unsigned i = 0; i < 3; ++i) {
4228 Builder.SetInsertPoint(BBs[i]);
4230 Store->setOrdering(Orders[i]);
4234 SI->addCase(
Builder.getInt32(0), BBs[0]);
4235 SI->addCase(
Builder.getInt32(3), BBs[1]);
4236 SI->addCase(
Builder.getInt32(5), BBs[2]);
4238 Builder.SetInsertPoint(ContBB);
4242 case Builtin::BI__atomic_thread_fence:
4243 case Builtin::BI__atomic_signal_fence:
4244 case Builtin::BI__c11_atomic_thread_fence:
4245 case Builtin::BI__c11_atomic_signal_fence: {
4246 llvm::SyncScope::ID SSID;
4247 if (BuiltinID == Builtin::BI__atomic_signal_fence ||
4248 BuiltinID == Builtin::BI__c11_atomic_signal_fence)
4249 SSID = llvm::SyncScope::SingleThread;
4251 SSID = llvm::SyncScope::System;
4253 if (isa<llvm::ConstantInt>(Order)) {
4254 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
4261 Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
4264 Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
4267 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
4270 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
4276 llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB;
4283 Order =
Builder.CreateIntCast(Order,
Builder.getInt32Ty(),
false);
4284 llvm::SwitchInst *SI =
Builder.CreateSwitch(Order, ContBB);
4286 Builder.SetInsertPoint(AcquireBB);
4287 Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
4289 SI->addCase(
Builder.getInt32(1), AcquireBB);
4290 SI->addCase(
Builder.getInt32(2), AcquireBB);
4292 Builder.SetInsertPoint(ReleaseBB);
4293 Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
4295 SI->addCase(
Builder.getInt32(3), ReleaseBB);
4297 Builder.SetInsertPoint(AcqRelBB);
4298 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
4300 SI->addCase(
Builder.getInt32(4), AcqRelBB);
4302 Builder.SetInsertPoint(SeqCstBB);
4303 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
4305 SI->addCase(
Builder.getInt32(5), SeqCstBB);
4307 Builder.SetInsertPoint(ContBB);
4311 case Builtin::BI__builtin_signbit:
4312 case Builtin::BI__builtin_signbitf:
4313 case Builtin::BI__builtin_signbitl: {
4318 case Builtin::BI__warn_memset_zero_len:
4320 case Builtin::BI__annotation: {
4325 assert(Str->getCharByteWidth() == 2);
4326 StringRef WideBytes = Str->getBytes();
4327 std::string StrUtf8;
4328 if (!convertUTF16ToUTF8String(
4329 ArrayRef(WideBytes.data(), WideBytes.size()), StrUtf8)) {
4333 Strings.push_back(llvm::MDString::get(
getLLVMContext(), StrUtf8));
4343 case Builtin::BI__builtin_annotation: {
4352 StringRef Str = cast<StringLiteral>(AnnotationStrExpr)->getString();
4356 case Builtin::BI__builtin_addcb:
4357 case Builtin::BI__builtin_addcs:
4358 case Builtin::BI__builtin_addc:
4359 case Builtin::BI__builtin_addcl:
4360 case Builtin::BI__builtin_addcll:
4361 case Builtin::BI__builtin_subcb:
4362 case Builtin::BI__builtin_subcs:
4363 case Builtin::BI__builtin_subc:
4364 case Builtin::BI__builtin_subcl:
4365 case Builtin::BI__builtin_subcll: {
4391 llvm::Intrinsic::ID IntrinsicId;
4392 switch (BuiltinID) {
4393 default: llvm_unreachable(
"Unknown multiprecision builtin id.");
4394 case Builtin::BI__builtin_addcb:
4395 case Builtin::BI__builtin_addcs:
4396 case Builtin::BI__builtin_addc:
4397 case Builtin::BI__builtin_addcl:
4398 case Builtin::BI__builtin_addcll:
4399 IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
4401 case Builtin::BI__builtin_subcb:
4402 case Builtin::BI__builtin_subcs:
4403 case Builtin::BI__builtin_subc:
4404 case Builtin::BI__builtin_subcl:
4405 case Builtin::BI__builtin_subcll:
4406 IntrinsicId = llvm::Intrinsic::usub_with_overflow;
4411 llvm::Value *Carry1;
4414 llvm::Value *Carry2;
4416 Sum1, Carryin, Carry2);
4417 llvm::Value *CarryOut =
Builder.CreateZExt(
Builder.CreateOr(Carry1, Carry2),
4423 case Builtin::BI__builtin_add_overflow:
4424 case Builtin::BI__builtin_sub_overflow:
4425 case Builtin::BI__builtin_mul_overflow: {
4433 WidthAndSignedness LeftInfo =
4435 WidthAndSignedness RightInfo =
4437 WidthAndSignedness ResultInfo =
4444 RightInfo, ResultArg, ResultQTy,
4450 *
this, LeftArg, LeftInfo, RightArg, RightInfo, ResultArg, ResultQTy,
4453 WidthAndSignedness EncompassingInfo =
4456 llvm::Type *EncompassingLLVMTy =
4461 llvm::Intrinsic::ID IntrinsicId;
4462 switch (BuiltinID) {
4464 llvm_unreachable(
"Unknown overflow builtin id.");
4465 case Builtin::BI__builtin_add_overflow:
4466 IntrinsicId = EncompassingInfo.Signed
4467 ? llvm::Intrinsic::sadd_with_overflow
4468 : llvm::Intrinsic::uadd_with_overflow;
4470 case Builtin::BI__builtin_sub_overflow:
4471 IntrinsicId = EncompassingInfo.Signed
4472 ? llvm::Intrinsic::ssub_with_overflow
4473 : llvm::Intrinsic::usub_with_overflow;
4475 case Builtin::BI__builtin_mul_overflow:
4476 IntrinsicId = EncompassingInfo.Signed
4477 ? llvm::Intrinsic::smul_with_overflow
4478 : llvm::Intrinsic::umul_with_overflow;