34#include "llvm/ADT/APFloat.h"
35#include "llvm/ADT/APInt.h"
36#include "llvm/ADT/FloatingPointMode.h"
37#include "llvm/ADT/SmallPtrSet.h"
38#include "llvm/ADT/StringExtras.h"
39#include "llvm/Analysis/ValueTracking.h"
40#include "llvm/IR/DataLayout.h"
41#include "llvm/IR/InlineAsm.h"
42#include "llvm/IR/Intrinsics.h"
43#include "llvm/IR/IntrinsicsAArch64.h"
44#include "llvm/IR/IntrinsicsAMDGPU.h"
45#include "llvm/IR/IntrinsicsARM.h"
46#include "llvm/IR/IntrinsicsBPF.h"
47#include "llvm/IR/IntrinsicsHexagon.h"
48#include "llvm/IR/IntrinsicsNVPTX.h"
49#include "llvm/IR/IntrinsicsPowerPC.h"
50#include "llvm/IR/IntrinsicsR600.h"
51#include "llvm/IR/IntrinsicsRISCV.h"
52#include "llvm/IR/IntrinsicsS390.h"
53#include "llvm/IR/IntrinsicsVE.h"
54#include "llvm/IR/IntrinsicsWebAssembly.h"
55#include "llvm/IR/IntrinsicsX86.h"
56#include "llvm/IR/MDBuilder.h"
57#include "llvm/IR/MatrixBuilder.h"
58#include "llvm/Support/ConvertUTF.h"
59#include "llvm/Support/MathExtras.h"
60#include "llvm/Support/ScopedPrinter.h"
61#include "llvm/TargetParser/AArch64TargetParser.h"
62#include "llvm/TargetParser/X86TargetParser.h"
67using namespace CodeGen;
71 Align AlignmentInBytes) {
73 switch (CGF.
getLangOpts().getTrivialAutoVarInit()) {
74 case LangOptions::TrivialAutoVarInitKind::Uninitialized:
77 case LangOptions::TrivialAutoVarInitKind::Zero:
78 Byte = CGF.
Builder.getInt8(0x00);
80 case LangOptions::TrivialAutoVarInitKind::Pattern: {
82 Byte = llvm::dyn_cast<llvm::ConstantInt>(
90 I->addAnnotationMetadata(
"auto-init");
105 static SmallDenseMap<unsigned, StringRef, 64> F128Builtins{
106 {Builtin::BI__builtin___fprintf_chk,
"__fprintf_chkieee128"},
107 {Builtin::BI__builtin___printf_chk,
"__printf_chkieee128"},
108 {Builtin::BI__builtin___snprintf_chk,
"__snprintf_chkieee128"},
109 {Builtin::BI__builtin___sprintf_chk,
"__sprintf_chkieee128"},
110 {Builtin::BI__builtin___vfprintf_chk,
"__vfprintf_chkieee128"},
111 {Builtin::BI__builtin___vprintf_chk,
"__vprintf_chkieee128"},
112 {Builtin::BI__builtin___vsnprintf_chk,
"__vsnprintf_chkieee128"},
113 {Builtin::BI__builtin___vsprintf_chk,
"__vsprintf_chkieee128"},
114 {Builtin::BI__builtin_fprintf,
"__fprintfieee128"},
115 {Builtin::BI__builtin_printf,
"__printfieee128"},
116 {Builtin::BI__builtin_snprintf,
"__snprintfieee128"},
117 {Builtin::BI__builtin_sprintf,
"__sprintfieee128"},
118 {Builtin::BI__builtin_vfprintf,
"__vfprintfieee128"},
119 {Builtin::BI__builtin_vprintf,
"__vprintfieee128"},
120 {Builtin::BI__builtin_vsnprintf,
"__vsnprintfieee128"},
121 {Builtin::BI__builtin_vsprintf,
"__vsprintfieee128"},
122 {Builtin::BI__builtin_fscanf,
"__fscanfieee128"},
123 {Builtin::BI__builtin_scanf,
"__scanfieee128"},
124 {Builtin::BI__builtin_sscanf,
"__sscanfieee128"},
125 {Builtin::BI__builtin_vfscanf,
"__vfscanfieee128"},
126 {Builtin::BI__builtin_vscanf,
"__vscanfieee128"},
127 {Builtin::BI__builtin_vsscanf,
"__vsscanfieee128"},
128 {Builtin::BI__builtin_nexttowardf128,
"__nexttowardieee128"},
134 static SmallDenseMap<unsigned, StringRef, 4> AIXLongDouble64Builtins{
135 {Builtin::BI__builtin_frexpl,
"frexp"},
136 {Builtin::BI__builtin_ldexpl,
"ldexp"},
137 {Builtin::BI__builtin_modfl,
"modf"},
143 if (FD->
hasAttr<AsmLabelAttr>())
149 &
getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad() &&
150 F128Builtins.contains(BuiltinID))
151 Name = F128Builtins[BuiltinID];
154 &llvm::APFloat::IEEEdouble() &&
155 AIXLongDouble64Builtins.contains(BuiltinID))
156 Name = AIXLongDouble64Builtins[BuiltinID];
161 llvm::FunctionType *Ty =
164 return GetOrCreateLLVMFunction(Name, Ty, D,
false);
170 QualType T, llvm::IntegerType *IntType) {
173 if (
V->getType()->isPointerTy())
174 return CGF.
Builder.CreatePtrToInt(
V, IntType);
176 assert(
V->getType() == IntType);
181 QualType T, llvm::Type *ResultType) {
184 if (ResultType->isPointerTy())
185 return CGF.
Builder.CreateIntToPtr(
V, ResultType);
187 assert(
V->getType() == ResultType);
198 if (Align % Bytes != 0) {
211 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
221 llvm::IntegerType *IntType = llvm::IntegerType::get(
225 llvm::Type *ValueType = Val->getType();
253 llvm::AtomicRMWInst::BinOp Kind,
262 llvm::AtomicRMWInst::BinOp Kind,
264 Instruction::BinaryOps Op,
265 bool Invert =
false) {
274 llvm::IntegerType *IntType = llvm::IntegerType::get(
278 llvm::Type *ValueType = Val->getType();
282 Kind, DestAddr, Val, llvm::AtomicOrdering::SequentiallyConsistent);
287 llvm::ConstantInt::getAllOnesValue(IntType));
311 llvm::IntegerType *IntType = llvm::IntegerType::get(
315 llvm::Type *ValueType = Cmp->getType();
320 DestAddr, Cmp, New, llvm::AtomicOrdering::SequentiallyConsistent,
321 llvm::AtomicOrdering::SequentiallyConsistent);
324 return CGF.
Builder.CreateZExt(CGF.
Builder.CreateExtractValue(Pair, 1),
347 AtomicOrdering SuccessOrdering = AtomicOrdering::SequentiallyConsistent) {
362 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release ?
363 AtomicOrdering::Monotonic :
371 DestAddr, Comparand, Exchange, SuccessOrdering, FailureOrdering);
372 Result->setVolatile(
true);
390 AtomicOrdering SuccessOrdering) {
397 assert(DestPtr->getType()->isPointerTy());
398 assert(!ExchangeHigh->getType()->isPointerTy());
399 assert(!ExchangeLow->getType()->isPointerTy());
402 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release
403 ? AtomicOrdering::Monotonic
408 llvm::Type *Int128Ty = llvm::IntegerType::get(CGF.
getLLVMContext(), 128);
409 Address DestAddr(DestPtr, Int128Ty,
414 ExchangeHigh = CGF.
Builder.CreateZExt(ExchangeHigh, Int128Ty);
415 ExchangeLow = CGF.
Builder.CreateZExt(ExchangeLow, Int128Ty);
417 CGF.
Builder.CreateShl(ExchangeHigh, llvm::ConstantInt::get(Int128Ty, 64));
418 llvm::Value *Exchange = CGF.
Builder.CreateOr(ExchangeHigh, ExchangeLow);
424 SuccessOrdering, FailureOrdering);
430 CXI->setVolatile(
true);
437 Value *Success = CGF.
Builder.CreateExtractValue(CXI, 1);
442 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
448 AtomicRMWInst::Add, DestAddr, ConstantInt::get(IntTy, 1), Ordering);
449 return CGF.
Builder.CreateAdd(
Result, ConstantInt::get(IntTy, 1));
454 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
460 AtomicRMWInst::Sub, DestAddr, ConstantInt::get(IntTy, 1), Ordering);
461 return CGF.
Builder.CreateSub(
Result, ConstantInt::get(IntTy, 1));
472 Load->setVolatile(
true);
482 llvm::StoreInst *Store =
484 Store->setVolatile(
true);
492 const CallExpr *E,
unsigned IntrinsicID,
493 unsigned ConstrainedIntrinsicID) {
496 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
497 if (CGF.
Builder.getIsFPConstrained()) {
499 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0 });
502 return CGF.
Builder.CreateCall(F, Src0);
509 const CallExpr *E,
unsigned IntrinsicID,
510 unsigned ConstrainedIntrinsicID) {
514 if (CGF.
Builder.getIsFPConstrained()) {
515 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
517 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0, Src1 });
520 return CGF.
Builder.CreateCall(F, { Src0, Src1 });
527 llvm::Intrinsic::ID ConstrainedIntrinsicID) {
531 if (CGF.
Builder.getIsFPConstrained()) {
532 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
534 {Src0->getType(), Src1->getType()});
535 return CGF.
Builder.CreateConstrainedFPCall(F, {Src0, Src1});
540 return CGF.
Builder.CreateCall(F, {Src0, Src1});
546 const CallExpr *E,
unsigned IntrinsicID,
547 unsigned ConstrainedIntrinsicID) {
552 if (CGF.
Builder.getIsFPConstrained()) {
553 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
555 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0, Src1, Src2 });
558 return CGF.
Builder.CreateCall(F, { Src0, Src1, Src2 });
565 unsigned IntrinsicID,
566 unsigned ConstrainedIntrinsicID,
570 if (CGF.
Builder.getIsFPConstrained())
575 if (CGF.
Builder.getIsFPConstrained())
576 return CGF.
Builder.CreateConstrainedFPCall(F, Args);
578 return CGF.
Builder.CreateCall(F, Args);
584 unsigned IntrinsicID,
585 llvm::StringRef Name =
"") {
589 return CGF.
Builder.CreateCall(F, Src0, Name);
595 unsigned IntrinsicID) {
600 return CGF.
Builder.CreateCall(F, { Src0, Src1 });
606 unsigned IntrinsicID) {
612 return CGF.
Builder.CreateCall(F, { Src0, Src1, Src2 });
618 unsigned IntrinsicID) {
623 return CGF.
Builder.CreateCall(F, {Src0, Src1});
629 unsigned IntrinsicID,
630 unsigned ConstrainedIntrinsicID) {
634 if (CGF.
Builder.getIsFPConstrained()) {
635 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
637 {ResultType, Src0->getType()});
638 return CGF.
Builder.CreateConstrainedFPCall(F, {Src0});
642 return CGF.
Builder.CreateCall(F, Src0);
647 llvm::Intrinsic::ID IntrinsicID) {
655 llvm::Value *
Call = CGF.
Builder.CreateCall(F, Src0);
657 llvm::Value *Exp = CGF.
Builder.CreateExtractValue(
Call, 1);
668 Call->setDoesNotAccessMemory();
677 llvm::Type *Ty =
V->getType();
678 int Width = Ty->getPrimitiveSizeInBits();
679 llvm::Type *IntTy = llvm::IntegerType::get(
C, Width);
681 if (Ty->isPPC_FP128Ty()) {
691 Value *ShiftCst = llvm::ConstantInt::get(IntTy, Width);
696 IntTy = llvm::IntegerType::get(
C, Width);
699 Value *Zero = llvm::Constant::getNullValue(IntTy);
700 return CGF.
Builder.CreateICmpSLT(
V, Zero);
704 const CallExpr *E, llvm::Constant *calleeValue) {
719 const llvm::Intrinsic::ID IntrinsicID,
720 llvm::Value *
X, llvm::Value *Y,
721 llvm::Value *&Carry) {
723 assert(
X->getType() == Y->getType() &&
724 "Arguments must be the same type. (Did you forget to make sure both "
725 "arguments have the same integer width?)");
728 llvm::Value *Tmp = CGF.
Builder.CreateCall(Callee, {
X, Y});
729 Carry = CGF.
Builder.CreateExtractValue(Tmp, 1);
730 return CGF.
Builder.CreateExtractValue(Tmp, 0);
734 unsigned IntrinsicID,
737 llvm::MDNode *RNode = MDHelper.createRange(APInt(32, low), APInt(32, high));
739 llvm::Instruction *
Call = CGF.
Builder.CreateCall(F);
740 Call->setMetadata(llvm::LLVMContext::MD_range, RNode);
741 Call->setMetadata(llvm::LLVMContext::MD_noundef,
747 struct WidthAndSignedness {
753static WidthAndSignedness
767static struct WidthAndSignedness
769 assert(Types.size() > 0 &&
"Empty list of types.");
773 for (
const auto &
Type : Types) {
782 for (
const auto &
Type : Types) {
784 if (Width < MinWidth) {
793 Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend;
803 return From == To || (From == 0 && To == 1) || (From == 3 && To == 2);
808 return ConstantInt::get(ResType, (
Type & 2) ? 0 : -1,
true);
812CodeGenFunction::evaluateOrEmitBuiltinObjectSize(
const Expr *E,
unsigned Type,
813 llvm::IntegerType *ResType,
814 llvm::Value *EmittedE,
818 return emitBuiltinObjectSize(E,
Type, ResType, EmittedE, IsDynamic);
819 return ConstantInt::get(ResType, ObjectSize,
true);
823CodeGenFunction::emitFlexibleArrayMemberSize(
const Expr *E,
unsigned Type,
824 llvm::IntegerType *ResType) {
853 const Expr *Idx =
nullptr;
855 if (
const auto *UO = dyn_cast<UnaryOperator>(
Base);
856 UO && UO->getOpcode() == UO_AddrOf) {
858 if (
const auto *ASE = dyn_cast<ArraySubscriptExpr>(SubExpr)) {
859 Base = ASE->getBase()->IgnoreParenImpCasts();
862 if (
const auto *IL = dyn_cast<IntegerLiteral>(Idx)) {
863 int64_t Val = IL->getValue().getSExtValue();
880 if (
const auto *ME = dyn_cast<MemberExpr>(
Base)) {
882 if (
const ValueDecl *MD = ME->getMemberDecl()) {
886 Ctx, MD, MD->getType(), StrictFlexArraysLevel,
892 }
else if (
const auto *DRE = dyn_cast<DeclRefExpr>(
Base)) {
894 QualType Ty = DRE->getDecl()->getType();
904 if (!FAMDecl || !FAMDecl->
hasAttr<CountedByAttr>())
918 llvm::Type *CountedByTy = CountedByInst->
getType();
921 Value *IdxInst =
nullptr;
925 IdxInst = IdxSigned ?
Builder.CreateSExtOrTrunc(IdxInst, CountedByTy)
926 :
Builder.CreateZExtOrTrunc(IdxInst, CountedByTy);
931 Builder.CreateSub(CountedByInst, IdxInst,
"", !IsSigned, IsSigned);
937 llvm::Constant *ElemSize =
938 llvm::ConstantInt::get(CountedByTy,
Size.getQuantity(), IsSigned);
940 Builder.CreateMul(CountedByInst, ElemSize,
"", !IsSigned, IsSigned);
941 FAMSize = IsSigned ?
Builder.CreateSExtOrTrunc(FAMSize, ResType)
942 :
Builder.CreateZExtOrTrunc(FAMSize, ResType);
943 Value *Res = FAMSize;
945 if (
const auto *DRE = dyn_cast<DeclRefExpr>(
Base)) {
953 llvm::Constant *FAMOffset =
954 ConstantInt::get(ResType, Offset.getQuantity(), IsSigned);
955 Value *OffsetAndFAMSize =
956 Builder.CreateAdd(FAMOffset, Res,
"", !IsSigned, IsSigned);
959 llvm::Constant *SizeofStruct =
965 ?
Builder.CreateBinaryIntrinsic(llvm::Intrinsic::smax,
966 OffsetAndFAMSize, SizeofStruct)
967 :
Builder.CreateBinaryIntrinsic(llvm::Intrinsic::umax,
968 OffsetAndFAMSize, SizeofStruct);
978 return Builder.CreateSelect(Cmp, Res, ConstantInt::get(ResType, 0, IsSigned));
991CodeGenFunction::emitBuiltinObjectSize(
const Expr *E,
unsigned Type,
992 llvm::IntegerType *ResType,
993 llvm::Value *EmittedE,
bool IsDynamic) {
997 auto *Param = dyn_cast<ParmVarDecl>(D->getDecl());
998 auto *PS = D->getDecl()->getAttr<PassObjectSizeAttr>();
999 if (Param !=
nullptr && PS !=
nullptr &&
1001 auto Iter = SizeArguments.find(Param);
1002 assert(
Iter != SizeArguments.end());
1005 auto DIter = LocalDeclMap.find(D);
1006 assert(DIter != LocalDeclMap.end());
1016 if (
Value *
V = emitFlexibleArrayMemberSize(E,
Type, ResType))
1027 assert(Ptr->
getType()->isPointerTy() &&
1028 "Non-pointer passed to __builtin_object_size?");
1038 return Builder.CreateCall(F, {Ptr, Min, NullIsUnknown,
Dynamic});
1044 enum ActionKind : uint8_t { TestOnly, Complement, Reset,
Set };
1045 enum InterlockingKind : uint8_t {
1054 InterlockingKind Interlocking;
1057 static BitTest decodeBitTestBuiltin(
unsigned BuiltinID);
1061BitTest BitTest::decodeBitTestBuiltin(
unsigned BuiltinID) {
1062 switch (BuiltinID) {
1064 case Builtin::BI_bittest:
1065 return {TestOnly, Unlocked,
false};
1066 case Builtin::BI_bittestandcomplement:
1067 return {Complement, Unlocked,
false};
1068 case Builtin::BI_bittestandreset:
1069 return {Reset, Unlocked,
false};
1070 case Builtin::BI_bittestandset:
1071 return {
Set, Unlocked,
false};
1072 case Builtin::BI_interlockedbittestandreset:
1073 return {Reset, Sequential,
false};
1074 case Builtin::BI_interlockedbittestandset:
1075 return {
Set, Sequential,
false};
1078 case Builtin::BI_bittest64:
1079 return {TestOnly, Unlocked,
true};
1080 case Builtin::BI_bittestandcomplement64:
1081 return {Complement, Unlocked,
true};
1082 case Builtin::BI_bittestandreset64:
1083 return {Reset, Unlocked,
true};
1084 case Builtin::BI_bittestandset64:
1085 return {
Set, Unlocked,
true};
1086 case Builtin::BI_interlockedbittestandreset64:
1087 return {Reset, Sequential,
true};
1088 case Builtin::BI_interlockedbittestandset64:
1089 return {
Set, Sequential,
true};
1092 case Builtin::BI_interlockedbittestandset_acq:
1093 return {
Set, Acquire,
false};
1094 case Builtin::BI_interlockedbittestandset_rel:
1095 return {
Set, Release,
false};
1096 case Builtin::BI_interlockedbittestandset_nf:
1097 return {
Set, NoFence,
false};
1098 case Builtin::BI_interlockedbittestandreset_acq:
1099 return {Reset, Acquire,
false};
1100 case Builtin::BI_interlockedbittestandreset_rel:
1101 return {Reset, Release,
false};
1102 case Builtin::BI_interlockedbittestandreset_nf:
1103 return {Reset, NoFence,
false};
1105 llvm_unreachable(
"expected only bittest intrinsics");
1110 case BitTest::TestOnly:
return '\0';
1111 case BitTest::Complement:
return 'c';
1112 case BitTest::Reset:
return 'r';
1113 case BitTest::Set:
return 's';
1115 llvm_unreachable(
"invalid action");
1123 char SizeSuffix = BT.Is64Bit ?
'q' :
'l';
1127 raw_svector_ostream AsmOS(
Asm);
1128 if (BT.Interlocking != BitTest::Unlocked)
1133 AsmOS << SizeSuffix <<
" $2, ($1)";
1136 std::string Constraints =
"={@ccc},r,r,~{cc},~{memory}";
1138 if (!MachineClobbers.empty()) {
1140 Constraints += MachineClobbers;
1142 llvm::IntegerType *IntType = llvm::IntegerType::get(
1145 llvm::FunctionType *FTy =
1146 llvm::FunctionType::get(CGF.
Int8Ty, {CGF.UnqualPtrTy, IntType},
false);
1148 llvm::InlineAsm *IA =
1149 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
1150 return CGF.
Builder.CreateCall(IA, {BitBase, BitPos});
1153static llvm::AtomicOrdering
1156 case BitTest::Unlocked:
return llvm::AtomicOrdering::NotAtomic;
1157 case BitTest::Sequential:
return llvm::AtomicOrdering::SequentiallyConsistent;
1158 case BitTest::Acquire:
return llvm::AtomicOrdering::Acquire;
1159 case BitTest::Release:
return llvm::AtomicOrdering::Release;
1160 case BitTest::NoFence:
return llvm::AtomicOrdering::Monotonic;
1162 llvm_unreachable(
"invalid interlocking");
1175 BitTest BT = BitTest::decodeBitTestBuiltin(BuiltinID);
1187 BitPos, llvm::ConstantInt::get(BitPos->
getType(), 3),
"bittest.byteidx");
1190 ByteIndex,
"bittest.byteaddr"),
1194 llvm::ConstantInt::get(CGF.
Int8Ty, 0x7));
1197 Value *Mask =
nullptr;
1198 if (BT.Action != BitTest::TestOnly) {
1199 Mask = CGF.
Builder.CreateShl(llvm::ConstantInt::get(CGF.
Int8Ty, 1), PosLow,
1206 Value *OldByte =
nullptr;
1207 if (Ordering != llvm::AtomicOrdering::NotAtomic) {
1210 llvm::AtomicRMWInst::BinOp RMWOp = llvm::AtomicRMWInst::Or;
1211 if (BT.Action == BitTest::Reset) {
1212 Mask = CGF.
Builder.CreateNot(Mask);
1213 RMWOp = llvm::AtomicRMWInst::And;
1219 Value *NewByte =
nullptr;
1220 switch (BT.Action) {
1221 case BitTest::TestOnly:
1224 case BitTest::Complement:
1225 NewByte = CGF.
Builder.CreateXor(OldByte, Mask);
1227 case BitTest::Reset:
1228 NewByte = CGF.
Builder.CreateAnd(OldByte, CGF.
Builder.CreateNot(Mask));
1231 NewByte = CGF.
Builder.CreateOr(OldByte, Mask);
1240 Value *ShiftedByte = CGF.
Builder.CreateLShr(OldByte, PosLow,
"bittest.shr");
1242 ShiftedByte, llvm::ConstantInt::get(CGF.
Int8Ty, 1),
"bittest.res");
1251 raw_svector_ostream AsmOS(
Asm);
1252 llvm::IntegerType *RetType = CGF.
Int32Ty;
1254 switch (BuiltinID) {
1255 case clang::PPC::BI__builtin_ppc_ldarx:
1259 case clang::PPC::BI__builtin_ppc_lwarx:
1263 case clang::PPC::BI__builtin_ppc_lharx:
1267 case clang::PPC::BI__builtin_ppc_lbarx:
1272 llvm_unreachable(
"Expected only PowerPC load reserve intrinsics");
1275 AsmOS <<
"$0, ${1:y}";
1277 std::string Constraints =
"=r,*Z,~{memory}";
1279 if (!MachineClobbers.empty()) {
1281 Constraints += MachineClobbers;
1285 llvm::FunctionType *FTy = llvm::FunctionType::get(RetType, {PtrType},
false);
1287 llvm::InlineAsm *IA =
1288 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
1289 llvm::CallInst *CI = CGF.
Builder.CreateCall(IA, {Addr});
1291 0, Attribute::get(CGF.
getLLVMContext(), Attribute::ElementType, RetType));
1296enum class MSVCSetJmpKind {
1308 llvm::Value *Arg1 =
nullptr;
1309 llvm::Type *Arg1Ty =
nullptr;
1311 bool IsVarArg =
false;
1312 if (SJKind == MSVCSetJmpKind::_setjmp3) {
1315 Arg1 = llvm::ConstantInt::get(CGF.
IntTy, 0);
1318 Name = SJKind == MSVCSetJmpKind::_setjmp ?
"_setjmp" :
"_setjmpex";
1321 Arg1 = CGF.
Builder.CreateCall(
1324 Arg1 = CGF.
Builder.CreateCall(
1326 llvm::ConstantInt::get(CGF.
Int32Ty, 0));
1330 llvm::Type *ArgTypes[2] = {CGF.
Int8PtrTy, Arg1Ty};
1331 llvm::AttributeList ReturnsTwiceAttr = llvm::AttributeList::get(
1333 llvm::Attribute::ReturnsTwice);
1335 llvm::FunctionType::get(CGF.
IntTy, ArgTypes, IsVarArg), Name,
1336 ReturnsTwiceAttr,
true);
1338 llvm::Value *Buf = CGF.
Builder.CreateBitOrPointerCast(
1340 llvm::Value *Args[] = {Buf, Arg1};
1342 CB->setAttributes(ReturnsTwiceAttr);
1390static std::optional<CodeGenFunction::MSVCIntrin>
1393 switch (BuiltinID) {
1395 return std::nullopt;
1396 case clang::ARM::BI_BitScanForward:
1397 case clang::ARM::BI_BitScanForward64:
1398 return MSVCIntrin::_BitScanForward;
1399 case clang::ARM::BI_BitScanReverse:
1400 case clang::ARM::BI_BitScanReverse64:
1401 return MSVCIntrin::_BitScanReverse;
1402 case clang::ARM::BI_InterlockedAnd64:
1403 return MSVCIntrin::_InterlockedAnd;
1404 case clang::ARM::BI_InterlockedExchange64:
1405 return MSVCIntrin::_InterlockedExchange;
1406 case clang::ARM::BI_InterlockedExchangeAdd64:
1407 return MSVCIntrin::_InterlockedExchangeAdd;
1408 case clang::ARM::BI_InterlockedExchangeSub64:
1409 return MSVCIntrin::_InterlockedExchangeSub;
1410 case clang::ARM::BI_InterlockedOr64:
1411 return MSVCIntrin::_InterlockedOr;
1412 case clang::ARM::BI_InterlockedXor64:
1413 return MSVCIntrin::_InterlockedXor;
1414 case clang::ARM::BI_InterlockedDecrement64:
1415 return MSVCIntrin::_InterlockedDecrement;
1416 case clang::ARM::BI_InterlockedIncrement64:
1417 return MSVCIntrin::_InterlockedIncrement;
1418 case clang::ARM::BI_InterlockedExchangeAdd8_acq:
1419 case clang::ARM::BI_InterlockedExchangeAdd16_acq:
1420 case clang::ARM::BI_InterlockedExchangeAdd_acq:
1421 case clang::ARM::BI_InterlockedExchangeAdd64_acq:
1422 return MSVCIntrin::_InterlockedExchangeAdd_acq;
1423 case clang::ARM::BI_InterlockedExchangeAdd8_rel:
1424 case clang::ARM::BI_InterlockedExchangeAdd16_rel:
1425 case clang::ARM::BI_InterlockedExchangeAdd_rel:
1426 case clang::ARM::BI_InterlockedExchangeAdd64_rel:
1427 return MSVCIntrin::_InterlockedExchangeAdd_rel;
1428 case clang::ARM::BI_InterlockedExchangeAdd8_nf:
1429 case clang::ARM::BI_InterlockedExchangeAdd16_nf:
1430 case clang::ARM::BI_InterlockedExchangeAdd_nf:
1431 case clang::ARM::BI_InterlockedExchangeAdd64_nf:
1432 return MSVCIntrin::_InterlockedExchangeAdd_nf;
1433 case clang::ARM::BI_InterlockedExchange8_acq:
1434 case clang::ARM::BI_InterlockedExchange16_acq:
1435 case clang::ARM::BI_InterlockedExchange_acq:
1436 case clang::ARM::BI_InterlockedExchange64_acq:
1437 return MSVCIntrin::_InterlockedExchange_acq;
1438 case clang::ARM::BI_InterlockedExchange8_rel:
1439 case clang::ARM::BI_InterlockedExchange16_rel:
1440 case clang::ARM::BI_InterlockedExchange_rel:
1441 case clang::ARM::BI_InterlockedExchange64_rel:
1442 return MSVCIntrin::_InterlockedExchange_rel;
1443 case clang::ARM::BI_InterlockedExchange8_nf:
1444 case clang::ARM::BI_InterlockedExchange16_nf:
1445 case clang::ARM::BI_InterlockedExchange_nf:
1446 case clang::ARM::BI_InterlockedExchange64_nf:
1447 return MSVCIntrin::_InterlockedExchange_nf;
1448 case clang::ARM::BI_InterlockedCompareExchange8_acq:
1449 case clang::ARM::BI_InterlockedCompareExchange16_acq:
1450 case clang::ARM::BI_InterlockedCompareExchange_acq:
1451 case clang::ARM::BI_InterlockedCompareExchange64_acq:
1452 return MSVCIntrin::_InterlockedCompareExchange_acq;
1453 case clang::ARM::BI_InterlockedCompareExchange8_rel:
1454 case clang::ARM::BI_InterlockedCompareExchange16_rel:
1455 case clang::ARM::BI_InterlockedCompareExchange_rel:
1456 case clang::ARM::BI_InterlockedCompareExchange64_rel:
1457 return MSVCIntrin::_InterlockedCompareExchange_rel;
1458 case clang::ARM::BI_InterlockedCompareExchange8_nf:
1459 case clang::ARM::BI_InterlockedCompareExchange16_nf:
1460 case clang::ARM::BI_InterlockedCompareExchange_nf:
1461 case clang::ARM::BI_InterlockedCompareExchange64_nf:
1462 return MSVCIntrin::_InterlockedCompareExchange_nf;
1463 case clang::ARM::BI_InterlockedOr8_acq:
1464 case clang::ARM::BI_InterlockedOr16_acq:
1465 case clang::ARM::BI_InterlockedOr_acq:
1466 case clang::ARM::BI_InterlockedOr64_acq:
1467 return MSVCIntrin::_InterlockedOr_acq;
1468 case clang::ARM::BI_InterlockedOr8_rel:
1469 case clang::ARM::BI_InterlockedOr16_rel:
1470 case clang::ARM::BI_InterlockedOr_rel:
1471 case clang::ARM::BI_InterlockedOr64_rel:
1472 return MSVCIntrin::_InterlockedOr_rel;
1473 case clang::ARM::BI_InterlockedOr8_nf:
1474 case clang::ARM::BI_InterlockedOr16_nf:
1475 case clang::ARM::BI_InterlockedOr_nf:
1476 case clang::ARM::BI_InterlockedOr64_nf:
1477 return MSVCIntrin::_InterlockedOr_nf;
1478 case clang::ARM::BI_InterlockedXor8_acq:
1479 case clang::ARM::BI_InterlockedXor16_acq:
1480 case clang::ARM::BI_InterlockedXor_acq:
1481 case clang::ARM::BI_InterlockedXor64_acq:
1482 return MSVCIntrin::_InterlockedXor_acq;
1483 case clang::ARM::BI_InterlockedXor8_rel:
1484 case clang::ARM::BI_InterlockedXor16_rel:
1485 case clang::ARM::BI_InterlockedXor_rel:
1486 case clang::ARM::BI_InterlockedXor64_rel:
1487 return MSVCIntrin::_InterlockedXor_rel;
1488 case clang::ARM::BI_InterlockedXor8_nf:
1489 case clang::ARM::BI_InterlockedXor16_nf:
1490 case clang::ARM::BI_InterlockedXor_nf:
1491 case clang::ARM::BI_InterlockedXor64_nf:
1492 return MSVCIntrin::_InterlockedXor_nf;
1493 case clang::ARM::BI_InterlockedAnd8_acq:
1494 case clang::ARM::BI_InterlockedAnd16_acq:
1495 case clang::ARM::BI_InterlockedAnd_acq:
1496 case clang::ARM::BI_InterlockedAnd64_acq:
1497 return MSVCIntrin::_InterlockedAnd_acq;
1498 case clang::ARM::BI_InterlockedAnd8_rel:
1499 case clang::ARM::BI_InterlockedAnd16_rel:
1500 case clang::ARM::BI_InterlockedAnd_rel:
1501 case clang::ARM::BI_InterlockedAnd64_rel:
1502 return MSVCIntrin::_InterlockedAnd_rel;
1503 case clang::ARM::BI_InterlockedAnd8_nf:
1504 case clang::ARM::BI_InterlockedAnd16_nf:
1505 case clang::ARM::BI_InterlockedAnd_nf:
1506 case clang::ARM::BI_InterlockedAnd64_nf:
1507 return MSVCIntrin::_InterlockedAnd_nf;
1508 case clang::ARM::BI_InterlockedIncrement16_acq:
1509 case clang::ARM::BI_InterlockedIncrement_acq:
1510 case clang::ARM::BI_InterlockedIncrement64_acq:
1511 return MSVCIntrin::_InterlockedIncrement_acq;
1512 case clang::ARM::BI_InterlockedIncrement16_rel:
1513 case clang::ARM::BI_InterlockedIncrement_rel:
1514 case clang::ARM::BI_InterlockedIncrement64_rel:
1515 return MSVCIntrin::_InterlockedIncrement_rel;
1516 case clang::ARM::BI_InterlockedIncrement16_nf:
1517 case clang::ARM::BI_InterlockedIncrement_nf:
1518 case clang::ARM::BI_InterlockedIncrement64_nf:
1519 return MSVCIntrin::_InterlockedIncrement_nf;
1520 case clang::ARM::BI_InterlockedDecrement16_acq:
1521 case clang::ARM::BI_InterlockedDecrement_acq:
1522 case clang::ARM::BI_InterlockedDecrement64_acq:
1523 return MSVCIntrin::_InterlockedDecrement_acq;
1524 case clang::ARM::BI_InterlockedDecrement16_rel:
1525 case clang::ARM::BI_InterlockedDecrement_rel:
1526 case clang::ARM::BI_InterlockedDecrement64_rel:
1527 return MSVCIntrin::_InterlockedDecrement_rel;
1528 case clang::ARM::BI_InterlockedDecrement16_nf:
1529 case clang::ARM::BI_InterlockedDecrement_nf:
1530 case clang::ARM::BI_InterlockedDecrement64_nf:
1531 return MSVCIntrin::_InterlockedDecrement_nf;
1533 llvm_unreachable(
"must return from switch");
1536static std::optional<CodeGenFunction::MSVCIntrin>
1539 switch (BuiltinID) {
1541 return std::nullopt;
1542 case clang::AArch64::BI_BitScanForward:
1543 case clang::AArch64::BI_BitScanForward64:
1544 return MSVCIntrin::_BitScanForward;
1545 case clang::AArch64::BI_BitScanReverse:
1546 case clang::AArch64::BI_BitScanReverse64:
1547 return MSVCIntrin::_BitScanReverse;
1548 case clang::AArch64::BI_InterlockedAnd64:
1549 return MSVCIntrin::_InterlockedAnd;
1550 case clang::AArch64::BI_InterlockedExchange64:
1551 return MSVCIntrin::_InterlockedExchange;
1552 case clang::AArch64::BI_InterlockedExchangeAdd64:
1553 return MSVCIntrin::_InterlockedExchangeAdd;
1554 case clang::AArch64::BI_InterlockedExchangeSub64:
1555 return MSVCIntrin::_InterlockedExchangeSub;
1556 case clang::AArch64::BI_InterlockedOr64:
1557 return MSVCIntrin::_InterlockedOr;
1558 case clang::AArch64::BI_InterlockedXor64:
1559 return MSVCIntrin::_InterlockedXor;
1560 case clang::AArch64::BI_InterlockedDecrement64:
1561 return MSVCIntrin::_InterlockedDecrement;
1562 case clang::AArch64::BI_InterlockedIncrement64:
1563 return MSVCIntrin::_InterlockedIncrement;
1564 case clang::AArch64::BI_InterlockedExchangeAdd8_acq:
1565 case clang::AArch64::BI_InterlockedExchangeAdd16_acq:
1566 case clang::AArch64::BI_InterlockedExchangeAdd_acq:
1567 case clang::AArch64::BI_InterlockedExchangeAdd64_acq:
1568 return MSVCIntrin::_InterlockedExchangeAdd_acq;
1569 case clang::AArch64::BI_InterlockedExchangeAdd8_rel:
1570 case clang::AArch64::BI_InterlockedExchangeAdd16_rel:
1571 case clang::AArch64::BI_InterlockedExchangeAdd_rel:
1572 case clang::AArch64::BI_InterlockedExchangeAdd64_rel:
1573 return MSVCIntrin::_InterlockedExchangeAdd_rel;
1574 case clang::AArch64::BI_InterlockedExchangeAdd8_nf:
1575 case clang::AArch64::BI_InterlockedExchangeAdd16_nf:
1576 case clang::AArch64::BI_InterlockedExchangeAdd_nf:
1577 case clang::AArch64::BI_InterlockedExchangeAdd64_nf:
1578 return MSVCIntrin::_InterlockedExchangeAdd_nf;
1579 case clang::AArch64::BI_InterlockedExchange8_acq:
1580 case clang::AArch64::BI_InterlockedExchange16_acq:
1581 case clang::AArch64::BI_InterlockedExchange_acq:
1582 case clang::AArch64::BI_InterlockedExchange64_acq:
1583 return MSVCIntrin::_InterlockedExchange_acq;
1584 case clang::AArch64::BI_InterlockedExchange8_rel:
1585 case clang::AArch64::BI_InterlockedExchange16_rel:
1586 case clang::AArch64::BI_InterlockedExchange_rel:
1587 case clang::AArch64::BI_InterlockedExchange64_rel:
1588 return MSVCIntrin::_InterlockedExchange_rel;
1589 case clang::AArch64::BI_InterlockedExchange8_nf:
1590 case clang::AArch64::BI_InterlockedExchange16_nf:
1591 case clang::AArch64::BI_InterlockedExchange_nf:
1592 case clang::AArch64::BI_InterlockedExchange64_nf:
1593 return MSVCIntrin::_InterlockedExchange_nf;
1594 case clang::AArch64::BI_InterlockedCompareExchange8_acq:
1595 case clang::AArch64::BI_InterlockedCompareExchange16_acq:
1596 case clang::AArch64::BI_InterlockedCompareExchange_acq:
1597 case clang::AArch64::BI_InterlockedCompareExchange64_acq:
1598 return MSVCIntrin::_InterlockedCompareExchange_acq;
1599 case clang::AArch64::BI_InterlockedCompareExchange8_rel:
1600 case clang::AArch64::BI_InterlockedCompareExchange16_rel:
1601 case clang::AArch64::BI_InterlockedCompareExchange_rel:
1602 case clang::AArch64::BI_InterlockedCompareExchange64_rel:
1603 return MSVCIntrin::_InterlockedCompareExchange_rel;
1604 case clang::AArch64::BI_InterlockedCompareExchange8_nf:
1605 case clang::AArch64::BI_InterlockedCompareExchange16_nf:
1606 case clang::AArch64::BI_InterlockedCompareExchange_nf:
1607 case clang::AArch64::BI_InterlockedCompareExchange64_nf:
1608 return MSVCIntrin::_InterlockedCompareExchange_nf;
1609 case clang::AArch64::BI_InterlockedCompareExchange128:
1610 return MSVCIntrin::_InterlockedCompareExchange128;
1611 case clang::AArch64::BI_InterlockedCompareExchange128_acq:
1612 return MSVCIntrin::_InterlockedCompareExchange128_acq;
1613 case clang::AArch64::BI_InterlockedCompareExchange128_nf:
1614 return MSVCIntrin::_InterlockedCompareExchange128_nf;
1615 case clang::AArch64::BI_InterlockedCompareExchange128_rel:
1616 return MSVCIntrin::_InterlockedCompareExchange128_rel;
1617 case clang::AArch64::BI_InterlockedOr8_acq:
1618 case clang::AArch64::BI_InterlockedOr16_acq:
1619 case clang::AArch64::BI_InterlockedOr_acq:
1620 case clang::AArch64::BI_InterlockedOr64_acq:
1621 return MSVCIntrin::_InterlockedOr_acq;
1622 case clang::AArch64::BI_InterlockedOr8_rel:
1623 case clang::AArch64::BI_InterlockedOr16_rel:
1624 case clang::AArch64::BI_InterlockedOr_rel:
1625 case clang::AArch64::BI_InterlockedOr64_rel:
1626 return MSVCIntrin::_InterlockedOr_rel;
1627 case clang::AArch64::BI_InterlockedOr8_nf:
1628 case clang::AArch64::BI_InterlockedOr16_nf:
1629 case clang::AArch64::BI_InterlockedOr_nf:
1630 case clang::AArch64::BI_InterlockedOr64_nf:
1631 return MSVCIntrin::_InterlockedOr_nf;
1632 case clang::AArch64::BI_InterlockedXor8_acq:
1633 case clang::AArch64::BI_InterlockedXor16_acq:
1634 case clang::AArch64::BI_InterlockedXor_acq:
1635 case clang::AArch64::BI_InterlockedXor64_acq:
1636 return MSVCIntrin::_InterlockedXor_acq;
1637 case clang::AArch64::BI_InterlockedXor8_rel:
1638 case clang::AArch64::BI_InterlockedXor16_rel:
1639 case clang::AArch64::BI_InterlockedXor_rel:
1640 case clang::AArch64::BI_InterlockedXor64_rel:
1641 return MSVCIntrin::_InterlockedXor_rel;
1642 case clang::AArch64::BI_InterlockedXor8_nf:
1643 case clang::AArch64::BI_InterlockedXor16_nf:
1644 case clang::AArch64::BI_InterlockedXor_nf:
1645 case clang::AArch64::BI_InterlockedXor64_nf:
1646 return MSVCIntrin::_InterlockedXor_nf;
1647 case clang::AArch64::BI_InterlockedAnd8_acq:
1648 case clang::AArch64::BI_InterlockedAnd16_acq:
1649 case clang::AArch64::BI_InterlockedAnd_acq:
1650 case clang::AArch64::BI_InterlockedAnd64_acq:
1651 return MSVCIntrin::_InterlockedAnd_acq;
1652 case clang::AArch64::BI_InterlockedAnd8_rel:
1653 case clang::AArch64::BI_InterlockedAnd16_rel:
1654 case clang::AArch64::BI_InterlockedAnd_rel:
1655 case clang::AArch64::BI_InterlockedAnd64_rel:
1656 return MSVCIntrin::_InterlockedAnd_rel;
1657 case clang::AArch64::BI_InterlockedAnd8_nf:
1658 case clang::AArch64::BI_InterlockedAnd16_nf:
1659 case clang::AArch64::BI_InterlockedAnd_nf:
1660 case clang::AArch64::BI_InterlockedAnd64_nf:
1661 return MSVCIntrin::_InterlockedAnd_nf;
1662 case clang::AArch64::BI_InterlockedIncrement16_acq:
1663 case clang::AArch64::BI_InterlockedIncrement_acq:
1664 case clang::AArch64::BI_InterlockedIncrement64_acq:
1665 return MSVCIntrin::_InterlockedIncrement_acq;
1666 case clang::AArch64::BI_InterlockedIncrement16_rel:
1667 case clang::AArch64::BI_InterlockedIncrement_rel:
1668 case clang::AArch64::BI_InterlockedIncrement64_rel:
1669 return MSVCIntrin::_InterlockedIncrement_rel;
1670 case clang::AArch64::BI_InterlockedIncrement16_nf:
1671 case clang::AArch64::BI_InterlockedIncrement_nf:
1672 case clang::AArch64::BI_InterlockedIncrement64_nf:
1673 return MSVCIntrin::_InterlockedIncrement_nf;
1674 case clang::AArch64::BI_InterlockedDecrement16_acq:
1675 case clang::AArch64::BI_InterlockedDecrement_acq:
1676 case clang::AArch64::BI_InterlockedDecrement64_acq:
1677 return MSVCIntrin::_InterlockedDecrement_acq;
1678 case clang::AArch64::BI_InterlockedDecrement16_rel:
1679 case clang::AArch64::BI_InterlockedDecrement_rel:
1680 case clang::AArch64::BI_InterlockedDecrement64_rel:
1681 return MSVCIntrin::_InterlockedDecrement_rel;
1682 case clang::AArch64::BI_InterlockedDecrement16_nf:
1683 case clang::AArch64::BI_InterlockedDecrement_nf:
1684 case clang::AArch64::BI_InterlockedDecrement64_nf:
1685 return MSVCIntrin::_InterlockedDecrement_nf;
1687 llvm_unreachable(
"must return from switch");
1690static std::optional<CodeGenFunction::MSVCIntrin>
1693 switch (BuiltinID) {
1695 return std::nullopt;
1696 case clang::X86::BI_BitScanForward:
1697 case clang::X86::BI_BitScanForward64:
1698 return MSVCIntrin::_BitScanForward;
1699 case clang::X86::BI_BitScanReverse:
1700 case clang::X86::BI_BitScanReverse64:
1701 return MSVCIntrin::_BitScanReverse;
1702 case clang::X86::BI_InterlockedAnd64:
1703 return MSVCIntrin::_InterlockedAnd;
1704 case clang::X86::BI_InterlockedCompareExchange128:
1705 return MSVCIntrin::_InterlockedCompareExchange128;
1706 case clang::X86::BI_InterlockedExchange64:
1707 return MSVCIntrin::_InterlockedExchange;
1708 case clang::X86::BI_InterlockedExchangeAdd64:
1709 return MSVCIntrin::_InterlockedExchangeAdd;
1710 case clang::X86::BI_InterlockedExchangeSub64:
1711 return MSVCIntrin::_InterlockedExchangeSub;
1712 case clang::X86::BI_InterlockedOr64:
1713 return MSVCIntrin::_InterlockedOr;
1714 case clang::X86::BI_InterlockedXor64:
1715 return MSVCIntrin::_InterlockedXor;
1716 case clang::X86::BI_InterlockedDecrement64:
1717 return MSVCIntrin::_InterlockedDecrement;
1718 case clang::X86::BI_InterlockedIncrement64:
1719 return MSVCIntrin::_InterlockedIncrement;
1721 llvm_unreachable(
"must return from switch");
1727 switch (BuiltinID) {
1728 case MSVCIntrin::_BitScanForward:
1729 case MSVCIntrin::_BitScanReverse: {
1733 llvm::Type *ArgType = ArgValue->
getType();
1734 llvm::Type *IndexType = IndexAddress.getElementType();
1737 Value *ArgZero = llvm::Constant::getNullValue(ArgType);
1738 Value *ResZero = llvm::Constant::getNullValue(ResultType);
1739 Value *ResOne = llvm::ConstantInt::get(ResultType, 1);
1744 PHINode *
Result =
Builder.CreatePHI(ResultType, 2,
"bitscan_result");
1747 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, ArgZero);
1749 Builder.CreateCondBr(IsZero, End, NotZero);
1752 Builder.SetInsertPoint(NotZero);
1754 if (BuiltinID == MSVCIntrin::_BitScanForward) {
1757 ZeroCount =
Builder.CreateIntCast(ZeroCount, IndexType,
false);
1760 unsigned ArgWidth = cast<llvm::IntegerType>(ArgType)->getBitWidth();
1761 Value *ArgTypeLastIndex = llvm::ConstantInt::get(IndexType, ArgWidth - 1);
1765 ZeroCount =
Builder.CreateIntCast(ZeroCount, IndexType,
false);
1766 Value *Index =
Builder.CreateNSWSub(ArgTypeLastIndex, ZeroCount);
1770 Result->addIncoming(ResOne, NotZero);
1775 case MSVCIntrin::_InterlockedAnd:
1777 case MSVCIntrin::_InterlockedExchange:
1779 case MSVCIntrin::_InterlockedExchangeAdd:
1781 case MSVCIntrin::_InterlockedExchangeSub:
1783 case MSVCIntrin::_InterlockedOr:
1785 case MSVCIntrin::_InterlockedXor:
1787 case MSVCIntrin::_InterlockedExchangeAdd_acq:
1789 AtomicOrdering::Acquire);
1790 case MSVCIntrin::_InterlockedExchangeAdd_rel:
1792 AtomicOrdering::Release);
1793 case MSVCIntrin::_InterlockedExchangeAdd_nf:
1795 AtomicOrdering::Monotonic);
1796 case MSVCIntrin::_InterlockedExchange_acq:
1798 AtomicOrdering::Acquire);
1799 case MSVCIntrin::_InterlockedExchange_rel:
1801 AtomicOrdering::Release);
1802 case MSVCIntrin::_InterlockedExchange_nf:
1804 AtomicOrdering::Monotonic);
1805 case MSVCIntrin::_InterlockedCompareExchange_acq:
1807 case MSVCIntrin::_InterlockedCompareExchange_rel:
1809 case MSVCIntrin::_InterlockedCompareExchange_nf:
1811 case MSVCIntrin::_InterlockedCompareExchange128:
1813 *
this, E, AtomicOrdering::SequentiallyConsistent);
1814 case MSVCIntrin::_InterlockedCompareExchange128_acq:
1816 case MSVCIntrin::_InterlockedCompareExchange128_rel:
1818 case MSVCIntrin::_InterlockedCompareExchange128_nf:
1820 case MSVCIntrin::_InterlockedOr_acq:
1822 AtomicOrdering::Acquire);
1823 case MSVCIntrin::_InterlockedOr_rel:
1825 AtomicOrdering::Release);
1826 case MSVCIntrin::_InterlockedOr_nf:
1828 AtomicOrdering::Monotonic);
1829 case MSVCIntrin::_InterlockedXor_acq:
1831 AtomicOrdering::Acquire);
1832 case MSVCIntrin::_InterlockedXor_rel:
1834 AtomicOrdering::Release);
1835 case MSVCIntrin::_InterlockedXor_nf:
1837 AtomicOrdering::Monotonic);
1838 case MSVCIntrin::_InterlockedAnd_acq:
1840 AtomicOrdering::Acquire);
1841 case MSVCIntrin::_InterlockedAnd_rel:
1843 AtomicOrdering::Release);
1844 case MSVCIntrin::_InterlockedAnd_nf:
1846 AtomicOrdering::Monotonic);
1847 case MSVCIntrin::_InterlockedIncrement_acq:
1849 case MSVCIntrin::_InterlockedIncrement_rel:
1851 case MSVCIntrin::_InterlockedIncrement_nf:
1853 case MSVCIntrin::_InterlockedDecrement_acq:
1855 case MSVCIntrin::_InterlockedDecrement_rel:
1857 case MSVCIntrin::_InterlockedDecrement_nf:
1860 case MSVCIntrin::_InterlockedDecrement:
1862 case MSVCIntrin::_InterlockedIncrement:
1865 case MSVCIntrin::__fastfail: {
1870 StringRef
Asm, Constraints;
1875 case llvm::Triple::x86:
1876 case llvm::Triple::x86_64:
1878 Constraints =
"{cx}";
1880 case llvm::Triple::thumb:
1882 Constraints =
"{r0}";
1884 case llvm::Triple::aarch64:
1885 Asm =
"brk #0xF003";
1886 Constraints =
"{w0}";
1888 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy, {
Int32Ty},
false);
1889 llvm::InlineAsm *IA =
1890 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
1891 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
1893 llvm::Attribute::NoReturn);
1895 CI->setAttributes(NoReturnAttr);
1899 llvm_unreachable(
"Incorrect MSVC intrinsic!");
1905 CallObjCArcUse(llvm::Value *
object) :
object(
object) {}
1915 BuiltinCheckKind Kind) {
1917 &&
"Unsupported builtin check kind");
1923 SanitizerScope SanScope(
this);
1925 ArgValue, llvm::Constant::getNullValue(ArgValue->
getType()));
1926 EmitCheck(std::make_pair(Cond, SanitizerKind::Builtin),
1927 SanitizerHandler::InvalidBuiltin,
1929 llvm::ConstantInt::get(
Builder.getInt8Ty(), Kind)},
1935 return CGF.
Builder.CreateBinaryIntrinsic(
1936 Intrinsic::abs, ArgValue,
1937 ConstantInt::get(CGF.
Builder.getInt1Ty(), HasNSW));
1941 bool SanitizeOverflow) {
1945 if (
const auto *VCI = dyn_cast<llvm::ConstantInt>(ArgValue)) {
1946 if (!VCI->isMinSignedValue())
1947 return EmitAbs(CGF, ArgValue,
true);
1950 CodeGenFunction::SanitizerScope SanScope(&CGF);
1952 Constant *Zero = Constant::getNullValue(ArgValue->
getType());
1953 Value *ResultAndOverflow = CGF.
Builder.CreateBinaryIntrinsic(
1954 Intrinsic::ssub_with_overflow, Zero, ArgValue);
1957 CGF.
Builder.CreateExtractValue(ResultAndOverflow, 1));
1960 if (SanitizeOverflow) {
1961 CGF.
EmitCheck({{NotOverflow, SanitizerKind::SignedIntegerOverflow}},
1962 SanitizerHandler::NegateOverflow,
1967 CGF.
EmitTrapCheck(NotOverflow, SanitizerHandler::SubOverflow);
1969 Value *CmpResult = CGF.
Builder.CreateICmpSLT(ArgValue, Zero,
"abscond");
1970 return CGF.
Builder.CreateSelect(CmpResult,
Result, ArgValue,
"abs");
1975 QualType UnsignedTy =
C.getIntTypeForBitwidth(Size * 8,
false);
1976 return C.getCanonicalType(UnsignedTy);
1986 raw_svector_ostream OS(Name);
1987 OS <<
"__os_log_helper";
1991 for (
const auto &Item : Layout.
Items)
1992 OS <<
"_" <<
int(Item.getSizeByte()) <<
"_"
1993 <<
int(Item.getDescriptorByte());
1996 if (llvm::Function *F =
CGM.
getModule().getFunction(Name))
2006 for (
unsigned int I = 0, E = Layout.
Items.size(); I < E; ++I) {
2007 char Size = Layout.
Items[I].getSizeByte();
2014 &Ctx.
Idents.
get(std::string(
"arg") + llvm::to_string(I)), ArgTy,
2016 ArgTys.emplace_back(ArgTy);
2027 llvm::Function *Fn = llvm::Function::Create(
2028 FuncTy, llvm::GlobalValue::LinkOnceODRLinkage, Name, &
CGM.
getModule());
2029 Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);
2032 Fn->setDoesNotThrow();
2036 Fn->addFnAttr(llvm::Attribute::NoInline);
2054 for (
const auto &Item : Layout.
Items) {
2056 Builder.getInt8(Item.getDescriptorByte()),
2059 Builder.getInt8(Item.getSizeByte()),
2063 if (!
Size.getQuantity())
2081 "__builtin_os_log_format takes at least 2 arguments");
2092 for (
const auto &Item : Layout.
Items) {
2093 int Size = Item.getSizeByte();
2097 llvm::Value *ArgVal;
2101 for (
unsigned I = 0, E = Item.getMaskType().size(); I < E; ++I)
2102 Val |= ((
uint64_t)Item.getMaskType()[I]) << I * 8;
2103 ArgVal = llvm::Constant::getIntegerValue(
Int64Ty, llvm::APInt(64, Val));
2104 }
else if (
const Expr *TheExpr = Item.getExpr()) {
2110 auto LifetimeExtendObject = [&](
const Expr *E) {
2118 if (isa<CallExpr>(E) || isa<ObjCMessageExpr>(E))
2123 if (TheExpr->getType()->isObjCRetainableType() &&
2124 getLangOpts().ObjCAutoRefCount && LifetimeExtendObject(TheExpr)) {
2126 "Only scalar can be a ObjC retainable type");
2127 if (!isa<Constant>(ArgVal)) {
2141 pushCleanupAfterFullExpr<CallObjCArcUse>(Cleanup, ArgVal);
2145 ArgVal =
Builder.getInt32(Item.getConstValue().getQuantity());
2148 unsigned ArgValSize =
2152 ArgVal =
Builder.CreateBitOrPointerCast(ArgVal,
IntTy);
2168 unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info,
2169 WidthAndSignedness ResultInfo) {
2170 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2171 Op1Info.Width == Op2Info.Width && Op2Info.Width == ResultInfo.Width &&
2172 !Op1Info.Signed && !Op2Info.Signed && ResultInfo.Signed;
2177 const clang::Expr *Op2, WidthAndSignedness Op2Info,
2179 WidthAndSignedness ResultInfo) {
2181 Builtin::BI__builtin_mul_overflow, Op1Info, Op2Info, ResultInfo) &&
2182 "Cannot specialize this multiply");
2187 llvm::Value *HasOverflow;
2189 CGF, llvm::Intrinsic::umul_with_overflow, V1, V2, HasOverflow);
2194 auto IntMax = llvm::APInt::getSignedMaxValue(ResultInfo.Width);
2195 llvm::Value *IntMaxValue = llvm::ConstantInt::get(
Result->getType(), IntMax);
2197 llvm::Value *IntMaxOverflow = CGF.
Builder.CreateICmpUGT(
Result, IntMaxValue);
2198 HasOverflow = CGF.
Builder.CreateOr(HasOverflow, IntMaxOverflow);
2210 WidthAndSignedness Op1Info,
2211 WidthAndSignedness Op2Info,
2212 WidthAndSignedness ResultInfo) {
2213 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2214 std::max(Op1Info.Width, Op2Info.Width) >= ResultInfo.Width &&
2215 Op1Info.Signed != Op2Info.Signed;
2222 WidthAndSignedness Op1Info,
const clang::Expr *Op2,
2223 WidthAndSignedness Op2Info,
2225 WidthAndSignedness ResultInfo) {
2227 Op2Info, ResultInfo) &&
2228 "Not a mixed-sign multipliction we can specialize");
2231 const clang::Expr *SignedOp = Op1Info.Signed ? Op1 : Op2;
2232 const clang::Expr *UnsignedOp = Op1Info.Signed ? Op2 : Op1;
2235 unsigned SignedOpWidth = Op1Info.Signed ? Op1Info.Width : Op2Info.Width;
2236 unsigned UnsignedOpWidth = Op1Info.Signed ? Op2Info.Width : Op1Info.Width;
2239 if (SignedOpWidth < UnsignedOpWidth)
2241 if (UnsignedOpWidth < SignedOpWidth)
2244 llvm::Type *OpTy =
Signed->getType();
2245 llvm::Value *Zero = llvm::Constant::getNullValue(OpTy);
2248 unsigned OpWidth = std::max(Op1Info.Width, Op2Info.Width);
2251 llvm::Value *IsNegative = CGF.
Builder.CreateICmpSLT(
Signed, Zero);
2252 llvm::Value *AbsOfNegative = CGF.
Builder.CreateSub(Zero,
Signed);
2253 llvm::Value *AbsSigned =
2254 CGF.
Builder.CreateSelect(IsNegative, AbsOfNegative,
Signed);
2257 llvm::Value *UnsignedOverflow;
2258 llvm::Value *UnsignedResult =
2262 llvm::Value *Overflow, *
Result;
2263 if (ResultInfo.Signed) {
2267 llvm::APInt::getSignedMaxValue(ResultInfo.Width).zext(OpWidth);
2268 llvm::Value *MaxResult =
2269 CGF.
Builder.CreateAdd(llvm::ConstantInt::get(OpTy, IntMax),
2270 CGF.
Builder.CreateZExt(IsNegative, OpTy));
2271 llvm::Value *SignedOverflow =
2272 CGF.
Builder.CreateICmpUGT(UnsignedResult, MaxResult);
2273 Overflow = CGF.
Builder.CreateOr(UnsignedOverflow, SignedOverflow);
2276 llvm::Value *NegativeResult = CGF.
Builder.CreateNeg(UnsignedResult);
2277 llvm::Value *SignedResult =
2278 CGF.
Builder.CreateSelect(IsNegative, NegativeResult, UnsignedResult);
2282 llvm::Value *Underflow = CGF.
Builder.CreateAnd(
2283 IsNegative, CGF.
Builder.CreateIsNotNull(UnsignedResult));
2284 Overflow = CGF.
Builder.CreateOr(UnsignedOverflow, Underflow);
2285 if (ResultInfo.Width < OpWidth) {
2287 llvm::APInt::getMaxValue(ResultInfo.Width).zext(OpWidth);
2288 llvm::Value *TruncOverflow = CGF.
Builder.CreateICmpUGT(
2289 UnsignedResult, llvm::ConstantInt::get(OpTy, IntMax));
2290 Overflow = CGF.
Builder.CreateOr(Overflow, TruncOverflow);
2295 IsNegative, CGF.
Builder.CreateNeg(UnsignedResult), UnsignedResult);
2299 assert(Overflow &&
Result &&
"Missing overflow or result");
2310 llvm::SmallPtrSetImpl<const Decl *> &Seen) {
2319 if (!Seen.insert(Record).second)
2322 assert(Record->hasDefinition() &&
2323 "Incomplete types should already be diagnosed");
2325 if (Record->isDynamicClass())
2350 llvm::Type *Ty = Src->getType();
2351 ShiftAmt =
Builder.CreateIntCast(ShiftAmt, Ty,
false);
2354 unsigned IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2361 switch (BuiltinID) {
2362#define MUTATE_LDBL(func) \
2363 case Builtin::BI__builtin_##func##l: \
2364 return Builtin::BI__builtin_##func##f128;
2433 if (CGF.
Builder.getIsFPConstrained() &&
2434 CGF.
Builder.getDefaultConstrainedExcept() != fp::ebIgnore) {
2446 auto UBF = CGF->
CGM.
getModule().getOrInsertFunction(Name, FnTy);
2449 for (
auto &&FormalTy : FnTy->params())
2450 Args.push_back(llvm::PoisonValue::get(FormalTy));
2463 !
Result.hasSideEffects()) {
2467 if (
Result.Val.isFloat())
2476 if (
getTarget().getTriple().isPPC64() &&
2477 &
getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad())
2484 const unsigned BuiltinIDIfNoAsmLabel =
2485 FD->
hasAttr<AsmLabelAttr>() ? 0 : BuiltinID;
2487 std::optional<bool> ErrnoOverriden;
2493 if (OP.hasMathErrnoOverride())
2494 ErrnoOverriden = OP.getMathErrnoOverride();
2503 bool ErrnoOverridenToFalseWithOpt =
2504 ErrnoOverriden.has_value() && !ErrnoOverriden.value() && !OptNone &&
2522 switch (BuiltinID) {
2523 case Builtin::BI__builtin_fma:
2524 case Builtin::BI__builtin_fmaf:
2525 case Builtin::BI__builtin_fmal:
2526 case Builtin::BIfma:
2527 case Builtin::BIfmaf:
2528 case Builtin::BIfmal: {
2530 if (Trip.isGNUEnvironment() || Trip.isOSMSVCRT())
2538 bool ConstWithoutErrnoAndExceptions =
2540 bool ConstWithoutExceptions =
2558 bool ConstWithoutErrnoOrExceptions =
2559 ConstWithoutErrnoAndExceptions || ConstWithoutExceptions;
2560 bool GenerateIntrinsics =
2561 (ConstAlways && !OptNone) ||
2563 !(ErrnoOverriden.has_value() && ErrnoOverriden.value()) && !OptNone);
2564 if (!GenerateIntrinsics) {
2565 GenerateIntrinsics =
2566 ConstWithoutErrnoOrExceptions && !ConstWithoutErrnoAndExceptions;
2567 if (!GenerateIntrinsics)
2568 GenerateIntrinsics =
2569 ConstWithoutErrnoOrExceptions &&
2571 !(ErrnoOverriden.has_value() && ErrnoOverriden.value()) && !OptNone);
2572 if (!GenerateIntrinsics)
2573 GenerateIntrinsics =
2574 ConstWithoutErrnoOrExceptions && ErrnoOverridenToFalseWithOpt;
2576 if (GenerateIntrinsics) {
2577 switch (BuiltinIDIfNoAsmLabel) {
2578 case Builtin::BIceil:
2579 case Builtin::BIceilf:
2580 case Builtin::BIceill:
2581 case Builtin::BI__builtin_ceil:
2582 case Builtin::BI__builtin_ceilf:
2583 case Builtin::BI__builtin_ceilf16:
2584 case Builtin::BI__builtin_ceill:
2585 case Builtin::BI__builtin_ceilf128:
2588 Intrinsic::experimental_constrained_ceil));
2590 case Builtin::BIcopysign:
2591 case Builtin::BIcopysignf:
2592 case Builtin::BIcopysignl:
2593 case Builtin::BI__builtin_copysign:
2594 case Builtin::BI__builtin_copysignf:
2595 case Builtin::BI__builtin_copysignf16:
2596 case Builtin::BI__builtin_copysignl:
2597 case Builtin::BI__builtin_copysignf128:
2600 case Builtin::BIcos:
2601 case Builtin::BIcosf:
2602 case Builtin::BIcosl:
2603 case Builtin::BI__builtin_cos:
2604 case Builtin::BI__builtin_cosf:
2605 case Builtin::BI__builtin_cosf16:
2606 case Builtin::BI__builtin_cosl:
2607 case Builtin::BI__builtin_cosf128:
2610 Intrinsic::experimental_constrained_cos));
2612 case Builtin::BIexp:
2613 case Builtin::BIexpf:
2614 case Builtin::BIexpl:
2615 case Builtin::BI__builtin_exp:
2616 case Builtin::BI__builtin_expf:
2617 case Builtin::BI__builtin_expf16:
2618 case Builtin::BI__builtin_expl:
2619 case Builtin::BI__builtin_expf128:
2622 Intrinsic::experimental_constrained_exp));
2624 case Builtin::BIexp2:
2625 case Builtin::BIexp2f:
2626 case Builtin::BIexp2l:
2627 case Builtin::BI__builtin_exp2:
2628 case Builtin::BI__builtin_exp2f:
2629 case Builtin::BI__builtin_exp2f16:
2630 case Builtin::BI__builtin_exp2l:
2631 case Builtin::BI__builtin_exp2f128:
2634 Intrinsic::experimental_constrained_exp2));
2635 case Builtin::BI__builtin_exp10:
2636 case Builtin::BI__builtin_exp10f:
2637 case Builtin::BI__builtin_exp10f16:
2638 case Builtin::BI__builtin_exp10l:
2639 case Builtin::BI__builtin_exp10f128: {
2641 if (
Builder.getIsFPConstrained())
2645 case Builtin::BIfabs:
2646 case Builtin::BIfabsf:
2647 case Builtin::BIfabsl:
2648 case Builtin::BI__builtin_fabs:
2649 case Builtin::BI__builtin_fabsf:
2650 case Builtin::BI__builtin_fabsf16:
2651 case Builtin::BI__builtin_fabsl:
2652 case Builtin::BI__builtin_fabsf128:
2655 case Builtin::BIfloor:
2656 case Builtin::BIfloorf:
2657 case Builtin::BIfloorl:
2658 case Builtin::BI__builtin_floor:
2659 case Builtin::BI__builtin_floorf:
2660 case Builtin::BI__builtin_floorf16:
2661 case Builtin::BI__builtin_floorl:
2662 case Builtin::BI__builtin_floorf128:
2665 Intrinsic::experimental_constrained_floor));
2667 case Builtin::BIfma:
2668 case Builtin::BIfmaf:
2669 case Builtin::BIfmal:
2670 case Builtin::BI__builtin_fma:
2671 case Builtin::BI__builtin_fmaf:
2672 case Builtin::BI__builtin_fmaf16:
2673 case Builtin::BI__builtin_fmal:
2674 case Builtin::BI__builtin_fmaf128:
2677 Intrinsic::experimental_constrained_fma));
2679 case Builtin::BIfmax:
2680 case Builtin::BIfmaxf:
2681 case Builtin::BIfmaxl:
2682 case Builtin::BI__builtin_fmax:
2683 case Builtin::BI__builtin_fmaxf:
2684 case Builtin::BI__builtin_fmaxf16:
2685 case Builtin::BI__builtin_fmaxl:
2686 case Builtin::BI__builtin_fmaxf128:
2689 Intrinsic::experimental_constrained_maxnum));
2691 case Builtin::BIfmin:
2692 case Builtin::BIfminf:
2693 case Builtin::BIfminl:
2694 case Builtin::BI__builtin_fmin:
2695 case Builtin::BI__builtin_fminf:
2696 case Builtin::BI__builtin_fminf16:
2697 case Builtin::BI__builtin_fminl:
2698 case Builtin::BI__builtin_fminf128:
2701 Intrinsic::experimental_constrained_minnum));
2705 case Builtin::BIfmod:
2706 case Builtin::BIfmodf:
2707 case Builtin::BIfmodl:
2708 case Builtin::BI__builtin_fmod:
2709 case Builtin::BI__builtin_fmodf:
2710 case Builtin::BI__builtin_fmodf16:
2711 case Builtin::BI__builtin_fmodl:
2712 case Builtin::BI__builtin_fmodf128: {
2713 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
2719 case Builtin::BIlog:
2720 case Builtin::BIlogf:
2721 case Builtin::BIlogl:
2722 case Builtin::BI__builtin_log:
2723 case Builtin::BI__builtin_logf:
2724 case Builtin::BI__builtin_logf16:
2725 case Builtin::BI__builtin_logl:
2726 case Builtin::BI__builtin_logf128:
2729 Intrinsic::experimental_constrained_log));
2731 case Builtin::BIlog10:
2732 case Builtin::BIlog10f:
2733 case Builtin::BIlog10l:
2734 case Builtin::BI__builtin_log10:
2735 case Builtin::BI__builtin_log10f:
2736 case Builtin::BI__builtin_log10f16:
2737 case Builtin::BI__builtin_log10l:
2738 case Builtin::BI__builtin_log10f128:
2741 Intrinsic::experimental_constrained_log10));
2743 case Builtin::BIlog2:
2744 case Builtin::BIlog2f:
2745 case Builtin::BIlog2l:
2746 case Builtin::BI__builtin_log2:
2747 case Builtin::BI__builtin_log2f:
2748 case Builtin::BI__builtin_log2f16:
2749 case Builtin::BI__builtin_log2l:
2750 case Builtin::BI__builtin_log2f128:
2753 Intrinsic::experimental_constrained_log2));
2755 case Builtin::BInearbyint:
2756 case Builtin::BInearbyintf:
2757 case Builtin::BInearbyintl:
2758 case Builtin::BI__builtin_nearbyint:
2759 case Builtin::BI__builtin_nearbyintf:
2760 case Builtin::BI__builtin_nearbyintl:
2761 case Builtin::BI__builtin_nearbyintf128:
2763 Intrinsic::nearbyint,
2764 Intrinsic::experimental_constrained_nearbyint));
2766 case Builtin::BIpow:
2767 case Builtin::BIpowf:
2768 case Builtin::BIpowl:
2769 case Builtin::BI__builtin_pow:
2770 case Builtin::BI__builtin_powf:
2771 case Builtin::BI__builtin_powf16:
2772 case Builtin::BI__builtin_powl:
2773 case Builtin::BI__builtin_powf128:
2776 Intrinsic::experimental_constrained_pow));
2778 case Builtin::BIrint:
2779 case Builtin::BIrintf:
2780 case Builtin::BIrintl:
2781 case Builtin::BI__builtin_rint:
2782 case Builtin::BI__builtin_rintf:
2783 case Builtin::BI__builtin_rintf16:
2784 case Builtin::BI__builtin_rintl:
2785 case Builtin::BI__builtin_rintf128:
2788 Intrinsic::experimental_constrained_rint));
2790 case Builtin::BIround:
2791 case Builtin::BIroundf:
2792 case Builtin::BIroundl:
2793 case Builtin::BI__builtin_round:
2794 case Builtin::BI__builtin_roundf:
2795 case Builtin::BI__builtin_roundf16:
2796 case Builtin::BI__builtin_roundl:
2797 case Builtin::BI__builtin_roundf128:
2800 Intrinsic::experimental_constrained_round));
2802 case Builtin::BIroundeven:
2803 case Builtin::BIroundevenf:
2804 case Builtin::BIroundevenl:
2805 case Builtin::BI__builtin_roundeven:
2806 case Builtin::BI__builtin_roundevenf:
2807 case Builtin::BI__builtin_roundevenf16:
2808 case Builtin::BI__builtin_roundevenl:
2809 case Builtin::BI__builtin_roundevenf128:
2811 Intrinsic::roundeven,
2812 Intrinsic::experimental_constrained_roundeven));
2814 case Builtin::BIsin:
2815 case Builtin::BIsinf:
2816 case Builtin::BIsinl:
2817 case Builtin::BI__builtin_sin:
2818 case Builtin::BI__builtin_sinf:
2819 case Builtin::BI__builtin_sinf16:
2820 case Builtin::BI__builtin_sinl:
2821 case Builtin::BI__builtin_sinf128:
2824 Intrinsic::experimental_constrained_sin));
2826 case Builtin::BIsqrt:
2827 case Builtin::BIsqrtf:
2828 case Builtin::BIsqrtl:
2829 case Builtin::BI__builtin_sqrt:
2830 case Builtin::BI__builtin_sqrtf:
2831 case Builtin::BI__builtin_sqrtf16:
2832 case Builtin::BI__builtin_sqrtl:
2833 case Builtin::BI__builtin_sqrtf128:
2834 case Builtin::BI__builtin_elementwise_sqrt: {
2836 *
this, E, Intrinsic::sqrt, Intrinsic::experimental_constrained_sqrt);
2840 case Builtin::BItrunc:
2841 case Builtin::BItruncf:
2842 case Builtin::BItruncl:
2843 case Builtin::BI__builtin_trunc:
2844 case Builtin::BI__builtin_truncf:
2845 case Builtin::BI__builtin_truncf16:
2846 case Builtin::BI__builtin_truncl:
2847 case Builtin::BI__builtin_truncf128:
2850 Intrinsic::experimental_constrained_trunc));
2852 case Builtin::BIlround:
2853 case Builtin::BIlroundf:
2854 case Builtin::BIlroundl:
2855 case Builtin::BI__builtin_lround:
2856 case Builtin::BI__builtin_lroundf:
2857 case Builtin::BI__builtin_lroundl:
2858 case Builtin::BI__builtin_lroundf128:
2860 *
this, E, Intrinsic::lround,
2861 Intrinsic::experimental_constrained_lround));
2863 case Builtin::BIllround:
2864 case Builtin::BIllroundf:
2865 case Builtin::BIllroundl:
2866 case Builtin::BI__builtin_llround:
2867 case Builtin::BI__builtin_llroundf:
2868 case Builtin::BI__builtin_llroundl:
2869 case Builtin::BI__builtin_llroundf128:
2871 *
this, E, Intrinsic::llround,
2872 Intrinsic::experimental_constrained_llround));
2874 case Builtin::BIlrint:
2875 case Builtin::BIlrintf:
2876 case Builtin::BIlrintl:
2877 case Builtin::BI__builtin_lrint:
2878 case Builtin::BI__builtin_lrintf:
2879 case Builtin::BI__builtin_lrintl:
2880 case Builtin::BI__builtin_lrintf128:
2882 *
this, E, Intrinsic::lrint,
2883 Intrinsic::experimental_constrained_lrint));
2885 case Builtin::BIllrint:
2886 case Builtin::BIllrintf:
2887 case Builtin::BIllrintl:
2888 case Builtin::BI__builtin_llrint:
2889 case Builtin::BI__builtin_llrintf:
2890 case Builtin::BI__builtin_llrintl:
2891 case Builtin::BI__builtin_llrintf128:
2893 *
this, E, Intrinsic::llrint,
2894 Intrinsic::experimental_constrained_llrint));
2895 case Builtin::BI__builtin_ldexp:
2896 case Builtin::BI__builtin_ldexpf:
2897 case Builtin::BI__builtin_ldexpl:
2898 case Builtin::BI__builtin_ldexpf16:
2899 case Builtin::BI__builtin_ldexpf128: {
2901 *
this, E, Intrinsic::ldexp,
2902 Intrinsic::experimental_constrained_ldexp));
2912 Value *Val = A.getPointer();
2918 SkippedChecks.
set(SanitizerKind::All);
2919 SkippedChecks.
clear(SanitizerKind::Alignment);
2922 if (
auto *CE = dyn_cast<ImplicitCastExpr>(Arg))
2923 if (CE->getCastKind() == CK_BitCast)
2924 Arg = CE->getSubExpr();
2925 EmitTypeCheck(Kind, Loc, Val, Arg->getType(), A.getAlignment(),
2930 switch (BuiltinIDIfNoAsmLabel) {
2932 case Builtin::BI__builtin___CFStringMakeConstantString:
2933 case Builtin::BI__builtin___NSStringMakeConstantString:
2935 case Builtin::BI__builtin_stdarg_start:
2936 case Builtin::BI__builtin_va_start:
2937 case Builtin::BI__va_start:
2938 case Builtin::BI__builtin_va_end:
2942 BuiltinID != Builtin::BI__builtin_va_end);
2944 case Builtin::BI__builtin_va_copy: {
2950 case Builtin::BIabs:
2951 case Builtin::BIlabs:
2952 case Builtin::BIllabs:
2953 case Builtin::BI__builtin_abs:
2954 case Builtin::BI__builtin_labs:
2955 case Builtin::BI__builtin_llabs: {
2956 bool SanitizeOverflow =
SanOpts.
has(SanitizerKind::SignedIntegerOverflow);
2959 switch (
getLangOpts().getSignedOverflowBehavior()) {
2964 if (!SanitizeOverflow) {
2976 case Builtin::BI__builtin_complex: {
2981 case Builtin::BI__builtin_conj:
2982 case Builtin::BI__builtin_conjf:
2983 case Builtin::BI__builtin_conjl:
2984 case Builtin::BIconj:
2985 case Builtin::BIconjf:
2986 case Builtin::BIconjl: {
2988 Value *Real = ComplexVal.first;
2989 Value *Imag = ComplexVal.second;
2990 Imag =
Builder.CreateFNeg(Imag,
"neg");
2993 case Builtin::BI__builtin_creal:
2994 case Builtin::BI__builtin_crealf:
2995 case Builtin::BI__builtin_creall:
2996 case Builtin::BIcreal:
2997 case Builtin::BIcrealf:
2998 case Builtin::BIcreall: {
3003 case Builtin::BI__builtin_preserve_access_index: {
3024 case Builtin::BI__builtin_cimag:
3025 case Builtin::BI__builtin_cimagf:
3026 case Builtin::BI__builtin_cimagl:
3027 case Builtin::BIcimag:
3028 case Builtin::BIcimagf:
3029 case Builtin::BIcimagl: {
3034 case Builtin::BI__builtin_clrsb:
3035 case Builtin::BI__builtin_clrsbl:
3036 case Builtin::BI__builtin_clrsbll: {
3040 llvm::Type *ArgType = ArgValue->
getType();
3044 Value *
Zero = llvm::Constant::getNullValue(ArgType);
3045 Value *IsNeg =
Builder.CreateICmpSLT(ArgValue, Zero,
"isneg");
3047 Value *Tmp =
Builder.CreateSelect(IsNeg, Inverse, ArgValue);
3054 case Builtin::BI__builtin_ctzs:
3055 case Builtin::BI__builtin_ctz:
3056 case Builtin::BI__builtin_ctzl:
3057 case Builtin::BI__builtin_ctzll: {
3060 llvm::Type *ArgType = ArgValue->
getType();
3066 if (
Result->getType() != ResultType)
3071 case Builtin::BI__builtin_clzs:
3072 case Builtin::BI__builtin_clz:
3073 case Builtin::BI__builtin_clzl:
3074 case Builtin::BI__builtin_clzll: {
3077 llvm::Type *ArgType = ArgValue->
getType();
3083 if (
Result->getType() != ResultType)
3088 case Builtin::BI__builtin_ffs:
3089 case Builtin::BI__builtin_ffsl:
3090 case Builtin::BI__builtin_ffsll: {
3094 llvm::Type *ArgType = ArgValue->
getType();
3099 Builder.CreateAdd(
Builder.CreateCall(F, {ArgValue, Builder.getTrue()}),
3100 llvm::ConstantInt::get(ArgType, 1));
3101 Value *
Zero = llvm::Constant::getNullValue(ArgType);
3102 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, Zero,
"iszero");
3104 if (
Result->getType() != ResultType)
3109 case Builtin::BI__builtin_parity:
3110 case Builtin::BI__builtin_parityl:
3111 case Builtin::BI__builtin_parityll: {
3115 llvm::Type *ArgType = ArgValue->
getType();
3121 if (
Result->getType() != ResultType)
3126 case Builtin::BI__lzcnt16:
3127 case Builtin::BI__lzcnt:
3128 case Builtin::BI__lzcnt64: {
3131 llvm::Type *ArgType = ArgValue->
getType();
3136 if (
Result->getType() != ResultType)
3141 case Builtin::BI__popcnt16:
3142 case Builtin::BI__popcnt:
3143 case Builtin::BI__popcnt64:
3144 case Builtin::BI__builtin_popcount:
3145 case Builtin::BI__builtin_popcountl:
3146 case Builtin::BI__builtin_popcountll: {
3149 llvm::Type *ArgType = ArgValue->
getType();
3154 if (
Result->getType() != ResultType)
3159 case Builtin::BI__builtin_unpredictable: {
3165 case Builtin::BI__builtin_expect: {
3167 llvm::Type *ArgType = ArgValue->
getType();
3178 Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue},
"expval");
3181 case Builtin::BI__builtin_expect_with_probability: {
3183 llvm::Type *ArgType = ArgValue->
getType();
3186 llvm::APFloat Probability(0.0);
3189 assert(EvalSucceed &&
"probability should be able to evaluate as float");
3191 bool LoseInfo =
false;
3192 Probability.convert(llvm::APFloat::IEEEdouble(),
3193 llvm::RoundingMode::Dynamic, &LoseInfo);
3195 Constant *Confidence = ConstantFP::get(Ty, Probability);
3205 FnExpect, {ArgValue, ExpectedValue, Confidence},
"expval");
3208 case Builtin::BI__builtin_assume_aligned: {
3211 Value *OffsetValue =
3215 ConstantInt *AlignmentCI = cast<ConstantInt>(AlignmentValue);
3216 if (AlignmentCI->getValue().ugt(llvm::Value::MaximumAlignment))
3217 AlignmentCI = ConstantInt::get(AlignmentCI->getType(),
3218 llvm::Value::MaximumAlignment);
3222 AlignmentCI, OffsetValue);
3225 case Builtin::BI__assume:
3226 case Builtin::BI__builtin_assume: {
3232 Builder.CreateCall(FnAssume, ArgValue);
3235 case Builtin::BI__builtin_assume_separate_storage: {
3242 Value *Values[] = {Value0, Value1};
3243 OperandBundleDefT<Value *> OBD(
"separate_storage", Values);
3247 case Builtin::BI__arithmetic_fence: {
3250 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3251 llvm::FastMathFlags FMF =
Builder.getFastMathFlags();
3252 bool isArithmeticFenceEnabled =
3253 FMF.allowReassoc() &&
3257 if (isArithmeticFenceEnabled) {
3260 Value *Real =
Builder.CreateArithmeticFence(ComplexVal.first,
3262 Value *Imag =
Builder.CreateArithmeticFence(ComplexVal.second,
3267 Value *Real = ComplexVal.first;
3268 Value *Imag = ComplexVal.second;
3272 if (isArithmeticFenceEnabled)
3277 case Builtin::BI__builtin_bswap16:
3278 case Builtin::BI__builtin_bswap32:
3279 case Builtin::BI__builtin_bswap64:
3280 case Builtin::BI_byteswap_ushort:
3281 case Builtin::BI_byteswap_ulong:
3282 case Builtin::BI_byteswap_uint64: {
3285 case Builtin::BI__builtin_bitreverse8:
3286 case Builtin::BI__builtin_bitreverse16:
3287 case Builtin::BI__builtin_bitreverse32:
3288 case Builtin::BI__builtin_bitreverse64: {
3291 case Builtin::BI__builtin_rotateleft8:
3292 case Builtin::BI__builtin_rotateleft16:
3293 case Builtin::BI__builtin_rotateleft32:
3294 case Builtin::BI__builtin_rotateleft64:
3295 case Builtin::BI_rotl8:
3296 case Builtin::BI_rotl16:
3297 case Builtin::BI_rotl:
3298 case Builtin::BI_lrotl:
3299 case Builtin::BI_rotl64:
3302 case Builtin::BI__builtin_rotateright8:
3303 case Builtin::BI__builtin_rotateright16:
3304 case Builtin::BI__builtin_rotateright32:
3305 case Builtin::BI__builtin_rotateright64:
3306 case Builtin::BI_rotr8:
3307 case Builtin::BI_rotr16:
3308 case Builtin::BI_rotr:
3309 case Builtin::BI_lrotr:
3310 case Builtin::BI_rotr64:
3313 case Builtin::BI__builtin_constant_p: {
3324 return RValue::get(ConstantInt::get(ResultType, 0));
3329 return RValue::get(ConstantInt::get(ResultType, 0));
3341 if (
Result->getType() != ResultType)
3345 case Builtin::BI__builtin_dynamic_object_size:
3346 case Builtin::BI__builtin_object_size: {
3353 bool IsDynamic = BuiltinID == Builtin::BI__builtin_dynamic_object_size;
3355 nullptr, IsDynamic));
3357 case Builtin::BI__builtin_prefetch: {
3361 llvm::ConstantInt::get(
Int32Ty, 0);
3363 llvm::ConstantInt::get(
Int32Ty, 3);
3369 case Builtin::BI__builtin_readcyclecounter: {
3373 case Builtin::BI__builtin___clear_cache: {
3379 case Builtin::BI__builtin_trap:
3382 case Builtin::BI__debugbreak:
3385 case Builtin::BI__builtin_unreachable: {
3394 case Builtin::BI__builtin_powi:
3395 case Builtin::BI__builtin_powif:
3396 case Builtin::BI__builtin_powil: {
3400 if (
Builder.getIsFPConstrained()) {
3403 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3410 { Src0->getType(), Src1->getType() });
3413 case Builtin::BI__builtin_frexp:
3414 case Builtin::BI__builtin_frexpf:
3415 case Builtin::BI__builtin_frexpl:
3416 case Builtin::BI__builtin_frexpf128:
3417 case Builtin::BI__builtin_frexpf16:
3419 case Builtin::BI__builtin_isgreater:
3420 case Builtin::BI__builtin_isgreaterequal:
3421 case Builtin::BI__builtin_isless:
3422 case Builtin::BI__builtin_islessequal:
3423 case Builtin::BI__builtin_islessgreater:
3424 case Builtin::BI__builtin_isunordered: {
3427 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3431 switch (BuiltinID) {
3432 default: llvm_unreachable(
"Unknown ordered comparison");
3433 case Builtin::BI__builtin_isgreater:
3434 LHS =
Builder.CreateFCmpOGT(LHS, RHS,
"cmp");
3436 case Builtin::BI__builtin_isgreaterequal:
3437 LHS =
Builder.CreateFCmpOGE(LHS, RHS,
"cmp");
3439 case Builtin::BI__builtin_isless:
3440 LHS =
Builder.CreateFCmpOLT(LHS, RHS,
"cmp");
3442 case Builtin::BI__builtin_islessequal:
3443 LHS =
Builder.CreateFCmpOLE(LHS, RHS,
"cmp");
3445 case Builtin::BI__builtin_islessgreater:
3446 LHS =
Builder.CreateFCmpONE(LHS, RHS,
"cmp");
3448 case Builtin::BI__builtin_isunordered:
3449 LHS =
Builder.CreateFCmpUNO(LHS, RHS,
"cmp");
3456 case Builtin::BI__builtin_isnan: {
3457 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3466 case Builtin::BI__builtin_issignaling: {
3467 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3474 case Builtin::BI__builtin_isinf: {
3475 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3484 case Builtin::BIfinite:
3485 case Builtin::BI__finite:
3486 case Builtin::BIfinitef:
3487 case Builtin::BI__finitef:
3488 case Builtin::BIfinitel:
3489 case Builtin::BI__finitel:
3490 case Builtin::BI__builtin_isfinite: {
3491 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3500 case Builtin::BI__builtin_isnormal: {
3501 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3508 case Builtin::BI__builtin_issubnormal: {
3509 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3512 Builder.CreateZExt(
Builder.createIsFPClass(
V, FPClassTest::fcSubnormal),
3516 case Builtin::BI__builtin_iszero: {
3517 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3524 case Builtin::BI__builtin_isfpclass: {
3529 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3535 case Builtin::BI__builtin_nondeterministic_value: {
3544 case Builtin::BI__builtin_elementwise_abs: {
3549 QT = VecTy->getElementType();
3553 Builder.getFalse(),
nullptr,
"elt.abs");
3560 case Builtin::BI__builtin_elementwise_ceil:
3563 case Builtin::BI__builtin_elementwise_exp:
3566 case Builtin::BI__builtin_elementwise_exp2:
3569 case Builtin::BI__builtin_elementwise_log:
3572 case Builtin::BI__builtin_elementwise_log2:
3575 case Builtin::BI__builtin_elementwise_log10:
3578 case Builtin::BI__builtin_elementwise_pow: {
3581 case Builtin::BI__builtin_elementwise_bitreverse:
3584 case Builtin::BI__builtin_elementwise_cos:
3587 case Builtin::BI__builtin_elementwise_floor:
3590 case Builtin::BI__builtin_elementwise_roundeven:
3593 case Builtin::BI__builtin_elementwise_round:
3596 case Builtin::BI__builtin_elementwise_rint:
3599 case Builtin::BI__builtin_elementwise_nearbyint:
3602 case Builtin::BI__builtin_elementwise_sin:
3606 case Builtin::BI__builtin_elementwise_trunc:
3609 case Builtin::BI__builtin_elementwise_canonicalize:
3611 emitUnaryBuiltin(*
this, E, llvm::Intrinsic::canonicalize,
"elt.canonicalize"));
3612 case Builtin::BI__builtin_elementwise_copysign:
3614 case Builtin::BI__builtin_elementwise_fma:
3616 case Builtin::BI__builtin_elementwise_add_sat:
3617 case Builtin::BI__builtin_elementwise_sub_sat: {
3621 assert(Op0->
getType()->isIntOrIntVectorTy() &&
"integer type expected");
3624 Ty = VecTy->getElementType();
3627 if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_add_sat)
3628 Opc = IsSigned ? llvm::Intrinsic::sadd_sat : llvm::Intrinsic::uadd_sat;
3630 Opc = IsSigned ? llvm::Intrinsic::ssub_sat : llvm::Intrinsic::usub_sat;
3631 Result =
Builder.CreateBinaryIntrinsic(Opc, Op0, Op1,
nullptr,
"elt.sat");
3635 case Builtin::BI__builtin_elementwise_max: {
3639 if (Op0->
getType()->isIntOrIntVectorTy()) {
3642 Ty = VecTy->getElementType();
3644 ? llvm::Intrinsic::smax
3645 : llvm::Intrinsic::umax,
3646 Op0, Op1,
nullptr,
"elt.max");
3651 case Builtin::BI__builtin_elementwise_min: {
3655 if (Op0->
getType()->isIntOrIntVectorTy()) {
3658 Ty = VecTy->getElementType();
3660 ? llvm::Intrinsic::smin
3661 : llvm::Intrinsic::umin,
3662 Op0, Op1,
nullptr,
"elt.min");
3668 case Builtin::BI__builtin_reduce_max: {
3669 auto GetIntrinsicID = [](
QualType QT) {
3671 QT = VecTy->getElementType();
3673 return llvm::Intrinsic::vector_reduce_smax;
3675 return llvm::Intrinsic::vector_reduce_umax;
3677 return llvm::Intrinsic::vector_reduce_fmax;
3680 *
this, E, GetIntrinsicID(E->
getArg(0)->
getType()),
"rdx.min"));
3683 case Builtin::BI__builtin_reduce_min: {
3684 auto GetIntrinsicID = [](
QualType QT) {
3686 QT = VecTy->getElementType();
3688 return llvm::Intrinsic::vector_reduce_smin;
3690 return llvm::Intrinsic::vector_reduce_umin;
3692 return llvm::Intrinsic::vector_reduce_fmin;
3696 *
this, E, GetIntrinsicID(E->
getArg(0)->
getType()),
"rdx.min"));
3699 case Builtin::BI__builtin_reduce_add:
3701 *
this, E, llvm::Intrinsic::vector_reduce_add,
"rdx.add"));
3702 case Builtin::BI__builtin_reduce_mul:
3704 *
this, E, llvm::Intrinsic::vector_reduce_mul,
"rdx.mul"));
3705 case Builtin::BI__builtin_reduce_xor:
3707 *
this, E, llvm::Intrinsic::vector_reduce_xor,
"rdx.xor"));
3708 case Builtin::BI__builtin_reduce_or:
3710 *
this, E, llvm::Intrinsic::vector_reduce_or,
"rdx.or"));
3711 case Builtin::BI__builtin_reduce_and:
3713 *
this, E, llvm::Intrinsic::vector_reduce_and,
"rdx.and"));
3715 case Builtin::BI__builtin_matrix_transpose: {
3719 Value *
Result = MB.CreateMatrixTranspose(MatValue, MatrixTy->getNumRows(),
3720 MatrixTy->getNumColumns());
3724 case Builtin::BI__builtin_matrix_column_major_load: {
3730 assert(PtrTy &&
"arg0 must be of pointer type");
3739 ResultTy->getNumRows(), ResultTy->getNumColumns(),
3744 case Builtin::BI__builtin_matrix_column_major_store: {
3752 assert(PtrTy &&
"arg1 must be of pointer type");
3759 Stride, IsVolatile, MatrixTy->getNumRows(), MatrixTy->getNumColumns());
3763 case Builtin::BI__builtin_isinf_sign: {
3765 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3770 AbsArg, ConstantFP::getInfinity(Arg->
getType()),
"isinf");
3776 Value *NegativeOne = ConstantInt::get(
IntTy, -1);
3777 Value *SignResult =
Builder.CreateSelect(IsNeg, NegativeOne, One);
3782 case Builtin::BI__builtin_flt_rounds: {
3787 if (
Result->getType() != ResultType)
3793 case Builtin::BI__builtin_set_flt_rounds: {
3801 case Builtin::BI__builtin_fpclassify: {
3802 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3813 "fpclassify_result");
3817 Value *IsZero =
Builder.CreateFCmpOEQ(
V, Constant::getNullValue(Ty),
3821 Builder.CreateCondBr(IsZero, End, NotZero);
3825 Builder.SetInsertPoint(NotZero);
3829 Builder.CreateCondBr(IsNan, End, NotNan);
3830 Result->addIncoming(NanLiteral, NotZero);
3833 Builder.SetInsertPoint(NotNan);
3836 Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(
V->getType()),
3840 Builder.CreateCondBr(IsInf, End, NotInf);
3841 Result->addIncoming(InfLiteral, NotNan);
3844 Builder.SetInsertPoint(NotInf);
3845 APFloat Smallest = APFloat::getSmallestNormalized(
3848 Builder.CreateFCmpUGE(VAbs, ConstantFP::get(
V->getContext(), Smallest),
3850 Value *NormalResult =
3854 Result->addIncoming(NormalResult, NotInf);
3867 case Builtin::BIalloca:
3868 case Builtin::BI_alloca:
3869 case Builtin::BI__builtin_alloca_uninitialized:
3870 case Builtin::BI__builtin_alloca: {
3874 const Align SuitableAlignmentInBytes =
3878 AllocaInst *AI =
Builder.CreateAlloca(
Builder.getInt8Ty(), Size);
3879 AI->setAlignment(SuitableAlignmentInBytes);
3880 if (BuiltinID != Builtin::BI__builtin_alloca_uninitialized)
3892 case Builtin::BI__builtin_alloca_with_align_uninitialized:
3893 case Builtin::BI__builtin_alloca_with_align: {
3896 auto *AlignmentInBitsCI = cast<ConstantInt>(AlignmentInBitsValue);
3897 unsigned AlignmentInBits = AlignmentInBitsCI->getZExtValue();
3898 const Align AlignmentInBytes =
3900 AllocaInst *AI =
Builder.CreateAlloca(
Builder.getInt8Ty(), Size);
3901 AI->setAlignment(AlignmentInBytes);
3902 if (BuiltinID != Builtin::BI__builtin_alloca_with_align_uninitialized)
3914 case Builtin::BIbzero:
3915 case Builtin::BI__builtin_bzero: {
3924 case Builtin::BIbcopy:
3925 case Builtin::BI__builtin_bcopy: {
3937 case Builtin::BImemcpy:
3938 case Builtin::BI__builtin_memcpy:
3939 case Builtin::BImempcpy:
3940 case Builtin::BI__builtin_mempcpy: {
3947 if (BuiltinID == Builtin::BImempcpy ||
3948 BuiltinID == Builtin::BI__builtin_mempcpy)
3955 case Builtin::BI__builtin_memcpy_inline: {
3966 case Builtin::BI__builtin_char_memchr:
3967 BuiltinID = Builtin::BI__builtin_memchr;
3970 case Builtin::BI__builtin___memcpy_chk: {
3977 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
3978 if (
Size.ugt(DstSize))
3982 Value *SizeVal = llvm::ConstantInt::get(
Builder.getContext(), Size);
3987 case Builtin::BI__builtin_objc_memmove_collectable: {
3992 DestAddr, SrcAddr, SizeVal);
3996 case Builtin::BI__builtin___memmove_chk: {
4003 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
4004 if (
Size.ugt(DstSize))
4008 Value *SizeVal = llvm::ConstantInt::get(
Builder.getContext(), Size);
4013 case Builtin::BImemmove:
4014 case Builtin::BI__builtin_memmove: {
4023 case Builtin::BImemset:
4024 case Builtin::BI__builtin_memset: {
4034 case Builtin::BI__builtin_memset_inline: {
4045 case Builtin::BI__builtin___memset_chk: {
4052 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
4053 if (
Size.ugt(DstSize))
4058 Value *SizeVal = llvm::ConstantInt::get(
Builder.getContext(), Size);
4062 case Builtin::BI__builtin_wmemchr: {
4065 if (!
getTarget().getTriple().isOSMSVCRT())
4073 BasicBlock *Entry =
Builder.GetInsertBlock();
4078 Builder.CreateCondBr(SizeEq0, Exit, CmpEq);
4082 StrPhi->addIncoming(Str, Entry);
4084 SizePhi->addIncoming(Size, Entry);
4088 Value *FoundChr =
Builder.CreateConstInBoundsGEP1_32(WCharTy, StrPhi, 0);
4090 Builder.CreateCondBr(StrEqChr, Exit, Next);
4093 Value *NextStr =
Builder.CreateConstInBoundsGEP1_32(WCharTy, StrPhi, 1);
4095 Value *NextSizeEq0 =
4096 Builder.CreateICmpEQ(NextSize, ConstantInt::get(
SizeTy, 0));
4097 Builder.CreateCondBr(NextSizeEq0, Exit, CmpEq);
4098 StrPhi->addIncoming(NextStr, Next);
4099 SizePhi->addIncoming(NextSize, Next);
4103 Ret->addIncoming(llvm::Constant::getNullValue(Str->
getType()), Entry);
4104 Ret->addIncoming(llvm::Constant::getNullValue(Str->
getType()), Next);
4105 Ret->addIncoming(FoundChr, CmpEq);
4108 case Builtin::BI__builtin_wmemcmp: {
4111 if (!
getTarget().getTriple().isOSMSVCRT())
4120 BasicBlock *Entry =
Builder.GetInsertBlock();
4126 Builder.CreateCondBr(SizeEq0, Exit, CmpGT);
4130 DstPhi->addIncoming(Dst, Entry);
4132 SrcPhi->addIncoming(Src, Entry);
4134 SizePhi->addIncoming(Size, Entry);
4140 Builder.CreateCondBr(DstGtSrc, Exit, CmpLT);
4144 Builder.CreateCondBr(DstLtSrc, Exit, Next);
4147 Value *NextDst =
Builder.CreateConstInBoundsGEP1_32(WCharTy, DstPhi, 1);
4148 Value *NextSrc =
Builder.CreateConstInBoundsGEP1_32(WCharTy, SrcPhi, 1);
4150 Value *NextSizeEq0 =
4151 Builder.CreateICmpEQ(NextSize, ConstantInt::get(
SizeTy, 0));
4152 Builder.CreateCondBr(NextSizeEq0, Exit, CmpGT);
4153 DstPhi->addIncoming(NextDst, Next);
4154 SrcPhi->addIncoming(NextSrc, Next);
4155 SizePhi->addIncoming(NextSize, Next);
4159 Ret->addIncoming(ConstantInt::get(
IntTy, 0), Entry);
4160 Ret->addIncoming(ConstantInt::get(
IntTy, 1), CmpGT);
4161 Ret->addIncoming(ConstantInt::get(
IntTy, -1), CmpLT);
4162 Ret->addIncoming(ConstantInt::get(
IntTy, 0), Next);
4165 case Builtin::BI__builtin_dwarf_cfa: {
4178 llvm::ConstantInt::get(
Int32Ty, Offset)));
4180 case Builtin::BI__builtin_return_address: {
4186 case Builtin::BI_ReturnAddress: {
4190 case Builtin::BI__builtin_frame_address: {
4196 case Builtin::BI__builtin_extract_return_addr: {
4201 case Builtin::BI__builtin_frob_return_addr: {
4206 case Builtin::BI__builtin_dwarf_sp_column: {
4207 llvm::IntegerType *Ty
4216 case Builtin::BI__builtin_init_dwarf_reg_size_table: {
4222 case Builtin::BI__builtin_eh_return: {
4226 llvm::IntegerType *
IntTy = cast<llvm::IntegerType>(Int->
getType());
4227 assert((
IntTy->getBitWidth() == 32 ||
IntTy->getBitWidth() == 64) &&
4228 "LLVM's __builtin_eh_return only supports 32- and 64-bit variants");
4231 : Intrinsic::eh_return_i64);
4232 Builder.CreateCall(F, {Int, Ptr});
4240 case Builtin::BI__builtin_unwind_init: {
4245 case Builtin::BI__builtin_extend_pointer: {
4270 case Builtin::BI__builtin_setjmp: {
4277 ConstantInt::get(
Int32Ty, 0));
4291 case Builtin::BI__builtin_longjmp: {
4305 case Builtin::BI__builtin_launder: {
4314 case Builtin::BI__sync_fetch_and_add:
4315 case Builtin::BI__sync_fetch_and_sub:
4316 case Builtin::BI__sync_fetch_and_or:
4317 case Builtin::BI__sync_fetch_and_and:
4318 case Builtin::BI__sync_fetch_and_xor:
4319 case Builtin::BI__sync_fetch_and_nand:
4320 case Builtin::BI__sync_add_and_fetch:
4321 case Builtin::BI__sync_sub_and_fetch:
4322 case Builtin::BI__sync_and_and_fetch:
4323 case Builtin::BI__sync_or_and_fetch:
4324 case Builtin::BI__sync_xor_and_fetch:
4325 case Builtin::BI__sync_nand_and_fetch:
4326 case Builtin::BI__sync_val_compare_and_swap:
4327 case Builtin::BI__sync_bool_compare_and_swap:
4328 case Builtin::BI__sync_lock_test_and_set:
4329 case Builtin::BI__sync_lock_release:
4330 case Builtin::BI__sync_swap:
4331 llvm_unreachable(
"Shouldn't make it through sema");
4332 case Builtin::BI__sync_fetch_and_add_1:
4333 case Builtin::BI__sync_fetch_and_add_2:
4334 case Builtin::BI__sync_fetch_and_add_4:
4335 case Builtin::BI__sync_fetch_and_add_8:
4336 case Builtin::BI__sync_fetch_and_add_16:
4338 case Builtin::BI__sync_fetch_and_sub_1:
4339 case Builtin::BI__sync_fetch_and_sub_2:
4340 case Builtin::BI__sync_fetch_and_sub_4:
4341 case Builtin::BI__sync_fetch_and_sub_8:
4342 case Builtin::BI__sync_fetch_and_sub_16:
4344 case Builtin::BI__sync_fetch_and_or_1:
4345 case Builtin::BI__sync_fetch_and_or_2:
4346 case Builtin::BI__sync_fetch_and_or_4:
4347 case Builtin::BI__sync_fetch_and_or_8:
4348 case Builtin::BI__sync_fetch_and_or_16:
4350 case Builtin::BI__sync_fetch_and_and_1:
4351 case Builtin::BI__sync_fetch_and_and_2:
4352 case Builtin::BI__sync_fetch_and_and_4:
4353 case Builtin::BI__sync_fetch_and_and_8:
4354 case Builtin::BI__sync_fetch_and_and_16:
4356 case Builtin::BI__sync_fetch_and_xor_1:
4357 case Builtin::BI__sync_fetch_and_xor_2:
4358 case Builtin::BI__sync_fetch_and_xor_4:
4359 case Builtin::BI__sync_fetch_and_xor_8:
4360 case Builtin::BI__sync_fetch_and_xor_16:
4362 case Builtin::BI__sync_fetch_and_nand_1:
4363 case Builtin::BI__sync_fetch_and_nand_2:
4364 case Builtin::BI__sync_fetch_and_nand_4:
4365 case Builtin::BI__sync_fetch_and_nand_8:
4366 case Builtin::BI__sync_fetch_and_nand_16:
4370 case Builtin::BI__sync_fetch_and_min:
4372 case Builtin::BI__sync_fetch_and_max:
4374 case Builtin::BI__sync_fetch_and_umin:
4376 case Builtin::BI__sync_fetch_and_umax:
4379 case Builtin::BI__sync_add_and_fetch_1:
4380 case Builtin::BI__sync_add_and_fetch_2:
4381 case Builtin::BI__sync_add_and_fetch_4:
4382 case Builtin::BI__sync_add_and_fetch_8:
4383 case Builtin::BI__sync_add_and_fetch_16:
4385 llvm::Instruction::Add);
4386 case Builtin::BI__sync_sub_and_fetch_1:
4387 case Builtin::BI__sync_sub_and_fetch_2:
4388 case Builtin::BI__sync_sub_and_fetch_4:
4389 case Builtin::BI__sync_sub_and_fetch_8:
4390 case Builtin::BI__sync_sub_and_fetch_16:
4392 llvm::Instruction::Sub);
4393 case Builtin::BI__sync_and_and_fetch_1:
4394 case Builtin::BI__sync_and_and_fetch_2:
4395 case Builtin::BI__sync_and_and_fetch_4:
4396 case Builtin::BI__sync_and_and_fetch_8:
4397 case Builtin::BI__sync_and_and_fetch_16:
4399 llvm::Instruction::And);
4400 case Builtin::BI__sync_or_and_fetch_1:
4401 case Builtin::BI__sync_or_and_fetch_2:
4402 case Builtin::BI__sync_or_and_fetch_4:
4403 case Builtin::BI__sync_or_and_fetch_8:
4404 case Builtin::BI__sync_or_and_fetch_16:
4406 llvm::Instruction::Or);
4407 case Builtin::BI__sync_xor_and_fetch_1:
4408 case Builtin::BI__sync_xor_and_fetch_2:
4409 case Builtin::BI__sync_xor_and_fetch_4:
4410 case Builtin::BI__sync_xor_and_fetch_8:
4411 case Builtin::BI__sync_xor_and_fetch_16:
4413 llvm::Instruction::Xor);
4414 case Builtin::BI__sync_nand_and_fetch_1:
4415 case Builtin::BI__sync_nand_and_fetch_2:
4416 case Builtin::BI__sync_nand_and_fetch_4:
4417 case Builtin::BI__sync_nand_and_fetch_8:
4418 case Builtin::BI__sync_nand_and_fetch_16:
4420 llvm::Instruction::And,
true);
4422 case Builtin::BI__sync_val_compare_and_swap_1:
4423 case Builtin::BI__sync_val_compare_and_swap_2:
4424 case Builtin::BI__sync_val_compare_and_swap_4:
4425 case Builtin::BI__sync_val_compare_and_swap_8:
4426 case Builtin::BI__sync_val_compare_and_swap_16:
4429 case Builtin::BI__sync_bool_compare_and_swap_1:
4430 case Builtin::BI__sync_bool_compare_and_swap_2:
4431 case Builtin::BI__sync_bool_compare_and_swap_4:
4432 case Builtin::BI__sync_bool_compare_and_swap_8:
4433 case Builtin::BI__sync_bool_compare_and_swap_16:
4436 case Builtin::BI__sync_swap_1:
4437 case Builtin::BI__sync_swap_2:
4438 case Builtin::BI__sync_swap_4:
4439 case Builtin::BI__sync_swap_8:
4440 case Builtin::BI__sync_swap_16:
4443 case Builtin::BI__sync_lock_test_and_set_1:
4444 case Builtin::BI__sync_lock_test_and_set_2:
4445 case Builtin::BI__sync_lock_test_and_set_4:
4446 case Builtin::BI__sync_lock_test_and_set_8:
4447 case Builtin::BI__sync_lock_test_and_set_16:
4450 case Builtin::BI__sync_lock_release_1:
4451 case Builtin::BI__sync_lock_release_2:
4452 case Builtin::BI__sync_lock_release_4:
4453 case Builtin::BI__sync_lock_release_8:
4454 case Builtin::BI__sync_lock_release_16: {
4460 llvm::StoreInst *
Store =
4462 Store->setAtomic(llvm::AtomicOrdering::Release);
4466 case Builtin::BI__sync_synchronize: {
4474 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent);
4478 case Builtin::BI__builtin_nontemporal_load:
4480 case Builtin::BI__builtin_nontemporal_store:
4482 case Builtin::BI__c11_atomic_is_lock_free:
4483 case Builtin::BI__atomic_is_lock_free: {
4487 const char *LibCallName =
"__atomic_is_lock_free";
4491 if (BuiltinID == Builtin::BI__atomic_is_lock_free)
4505 case Builtin::BI__atomic_test_and_set: {
4517 if (isa<llvm::ConstantInt>(Order)) {
4518 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
4519 AtomicRMWInst *
Result =
nullptr;
4524 llvm::AtomicOrdering::Monotonic);