38#include "llvm/ADT/APFloat.h"
39#include "llvm/ADT/APInt.h"
40#include "llvm/ADT/FloatingPointMode.h"
41#include "llvm/ADT/SmallPtrSet.h"
42#include "llvm/ADT/StringExtras.h"
43#include "llvm/Analysis/ValueTracking.h"
44#include "llvm/IR/DataLayout.h"
45#include "llvm/IR/InlineAsm.h"
46#include "llvm/IR/Intrinsics.h"
47#include "llvm/IR/IntrinsicsAArch64.h"
48#include "llvm/IR/IntrinsicsAMDGPU.h"
49#include "llvm/IR/IntrinsicsARM.h"
50#include "llvm/IR/IntrinsicsBPF.h"
51#include "llvm/IR/IntrinsicsDirectX.h"
52#include "llvm/IR/IntrinsicsHexagon.h"
53#include "llvm/IR/IntrinsicsNVPTX.h"
54#include "llvm/IR/IntrinsicsPowerPC.h"
55#include "llvm/IR/IntrinsicsR600.h"
56#include "llvm/IR/IntrinsicsRISCV.h"
57#include "llvm/IR/IntrinsicsS390.h"
58#include "llvm/IR/IntrinsicsWebAssembly.h"
59#include "llvm/IR/IntrinsicsX86.h"
60#include "llvm/IR/MDBuilder.h"
61#include "llvm/IR/MatrixBuilder.h"
62#include "llvm/IR/MemoryModelRelaxationAnnotations.h"
63#include "llvm/Support/AMDGPUAddrSpace.h"
64#include "llvm/Support/ConvertUTF.h"
65#include "llvm/Support/MathExtras.h"
66#include "llvm/Support/ScopedPrinter.h"
67#include "llvm/TargetParser/AArch64TargetParser.h"
68#include "llvm/TargetParser/RISCVISAInfo.h"
69#include "llvm/TargetParser/RISCVTargetParser.h"
70#include "llvm/TargetParser/X86TargetParser.h"
75using namespace CodeGen;
79 Align AlignmentInBytes) {
81 switch (CGF.
getLangOpts().getTrivialAutoVarInit()) {
82 case LangOptions::TrivialAutoVarInitKind::Uninitialized:
85 case LangOptions::TrivialAutoVarInitKind::Zero:
86 Byte = CGF.
Builder.getInt8(0x00);
88 case LangOptions::TrivialAutoVarInitKind::Pattern: {
90 Byte = llvm::dyn_cast<llvm::ConstantInt>(
98 I->addAnnotationMetadata(
"auto-init");
104 Constant *FZeroConst = ConstantFP::getZero(CGF->
FloatTy);
109 FZeroConst = ConstantVector::getSplat(
110 ElementCount::getFixed(VecTy->getNumElements()), FZeroConst);
111 auto *FCompInst = CGF->
Builder.CreateFCmpOLT(Op0, FZeroConst);
112 CMP = CGF->
Builder.CreateIntrinsic(
114 {FCompInst},
nullptr);
116 CMP = CGF->
Builder.CreateFCmpOLT(Op0, FZeroConst);
119 LastInstr = CGF->
Builder.CreateIntrinsic(
120 CGF->
VoidTy, llvm::Intrinsic::dx_discard, {CMP},
nullptr);
125 CGF->
Builder.CreateCondBr(CMP, LT0, End);
127 CGF->
Builder.SetInsertPoint(LT0);
129 CGF->
Builder.CreateIntrinsic(CGF->
VoidTy, llvm::Intrinsic::spv_discard, {},
132 LastInstr = CGF->
Builder.CreateBr(End);
134 CGF->
Builder.SetInsertPoint(End);
136 llvm_unreachable(
"Backend Codegen not supported.");
144 const auto *OutArg1 = dyn_cast<HLSLOutArgExpr>(
E->getArg(1));
145 const auto *OutArg2 = dyn_cast<HLSLOutArgExpr>(
E->getArg(2));
156 Value *LowBits =
nullptr;
157 Value *HighBits =
nullptr;
161 llvm::Type *RetElementTy = CGF->
Int32Ty;
163 RetElementTy = llvm::VectorType::get(
164 CGF->
Int32Ty, ElementCount::getFixed(Op0VecTy->getNumElements()));
165 auto *RetTy = llvm::StructType::get(RetElementTy, RetElementTy);
167 CallInst *CI = CGF->
Builder.CreateIntrinsic(
168 RetTy, Intrinsic::dx_splitdouble, {Op0},
nullptr,
"hlsl.splitdouble");
170 LowBits = CGF->
Builder.CreateExtractValue(CI, 0);
171 HighBits = CGF->
Builder.CreateExtractValue(CI, 1);
176 if (!Op0->
getType()->isVectorTy()) {
177 FixedVectorType *DestTy = FixedVectorType::get(CGF->
Int32Ty, 2);
178 Value *Bitcast = CGF->
Builder.CreateBitCast(Op0, DestTy);
180 LowBits = CGF->
Builder.CreateExtractElement(Bitcast, (uint64_t)0);
181 HighBits = CGF->
Builder.CreateExtractElement(Bitcast, 1);
184 if (
const auto *VecTy =
186 NumElements = VecTy->getNumElements();
188 FixedVectorType *Uint32VecTy =
189 FixedVectorType::get(CGF->
Int32Ty, NumElements * 2);
190 Value *Uint32Vec = CGF->
Builder.CreateBitCast(Op0, Uint32VecTy);
191 if (NumElements == 1) {
192 LowBits = CGF->
Builder.CreateExtractElement(Uint32Vec, (uint64_t)0);
193 HighBits = CGF->
Builder.CreateExtractElement(Uint32Vec, 1);
196 for (
int I = 0,
E = NumElements; I !=
E; ++I) {
197 EvenMask.push_back(I * 2);
198 OddMask.push_back(I * 2 + 1);
200 LowBits = CGF->
Builder.CreateShuffleVector(Uint32Vec, EvenMask);
201 HighBits = CGF->
Builder.CreateShuffleVector(Uint32Vec, OddMask);
215 "asdouble operands types mismatch");
219 llvm::Type *ResultType = CGF.
DoubleTy;
222 N = VTy->getNumElements();
223 ResultType = llvm::FixedVectorType::get(CGF.
DoubleTy, N);
227 return CGF.
Builder.CreateIntrinsic(
228 ResultType, Intrinsic::dx_asdouble,
232 OpLowBits = CGF.
Builder.CreateVectorSplat(1, OpLowBits);
233 OpHighBits = CGF.
Builder.CreateVectorSplat(1, OpHighBits);
237 for (
int i = 0; i < N; i++) {
239 Mask.push_back(i + N);
242 Value *BitVec = CGF.
Builder.CreateShuffleVector(OpLowBits, OpHighBits, Mask);
244 return CGF.
Builder.CreateBitCast(BitVec, ResultType);
251 llvm::Metadata *Ops[] = {llvm::MDString::get(Context,
"x18")};
252 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
253 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
256 llvm::Value *X18 = CGF.
Builder.CreateCall(F, Metadata);
263 unsigned BuiltinID) {
272 static SmallDenseMap<unsigned, StringRef, 64> F128Builtins{
273 {Builtin::BI__builtin___fprintf_chk,
"__fprintf_chkieee128"},
274 {Builtin::BI__builtin___printf_chk,
"__printf_chkieee128"},
275 {Builtin::BI__builtin___snprintf_chk,
"__snprintf_chkieee128"},
276 {Builtin::BI__builtin___sprintf_chk,
"__sprintf_chkieee128"},
277 {Builtin::BI__builtin___vfprintf_chk,
"__vfprintf_chkieee128"},
278 {Builtin::BI__builtin___vprintf_chk,
"__vprintf_chkieee128"},
279 {Builtin::BI__builtin___vsnprintf_chk,
"__vsnprintf_chkieee128"},
280 {Builtin::BI__builtin___vsprintf_chk,
"__vsprintf_chkieee128"},
281 {Builtin::BI__builtin_fprintf,
"__fprintfieee128"},
282 {Builtin::BI__builtin_printf,
"__printfieee128"},
283 {Builtin::BI__builtin_snprintf,
"__snprintfieee128"},
284 {Builtin::BI__builtin_sprintf,
"__sprintfieee128"},
285 {Builtin::BI__builtin_vfprintf,
"__vfprintfieee128"},
286 {Builtin::BI__builtin_vprintf,
"__vprintfieee128"},
287 {Builtin::BI__builtin_vsnprintf,
"__vsnprintfieee128"},
288 {Builtin::BI__builtin_vsprintf,
"__vsprintfieee128"},
289 {Builtin::BI__builtin_fscanf,
"__fscanfieee128"},
290 {Builtin::BI__builtin_scanf,
"__scanfieee128"},
291 {Builtin::BI__builtin_sscanf,
"__sscanfieee128"},
292 {Builtin::BI__builtin_vfscanf,
"__vfscanfieee128"},
293 {Builtin::BI__builtin_vscanf,
"__vscanfieee128"},
294 {Builtin::BI__builtin_vsscanf,
"__vsscanfieee128"},
295 {Builtin::BI__builtin_nexttowardf128,
"__nexttowardieee128"},
301 static SmallDenseMap<unsigned, StringRef, 4> AIXLongDouble64Builtins{
302 {Builtin::BI__builtin_frexpl,
"frexp"},
303 {Builtin::BI__builtin_ldexpl,
"ldexp"},
304 {Builtin::BI__builtin_modfl,
"modf"},
310 if (FD->
hasAttr<AsmLabelAttr>())
316 &
getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad() &&
317 F128Builtins.contains(BuiltinID))
318 Name = F128Builtins[BuiltinID];
321 &llvm::APFloat::IEEEdouble() &&
322 AIXLongDouble64Builtins.contains(BuiltinID))
323 Name = AIXLongDouble64Builtins[BuiltinID];
328 llvm::FunctionType *Ty =
331 return GetOrCreateLLVMFunction(Name, Ty,
D,
false);
337 QualType T, llvm::IntegerType *IntType) {
340 if (
V->getType()->isPointerTy())
341 return CGF.
Builder.CreatePtrToInt(
V, IntType);
343 assert(
V->getType() == IntType);
351 if (ResultType->isPointerTy())
352 return CGF.
Builder.CreateIntToPtr(
V, ResultType);
354 assert(
V->getType() == ResultType);
365 if (Align % Bytes != 0) {
378 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
388 llvm::IntegerType *IntType = llvm::IntegerType::get(
392 llvm::Type *ValueType = Val->getType();
420 llvm::AtomicRMWInst::BinOp Kind,
429 llvm::AtomicRMWInst::BinOp Kind,
431 Instruction::BinaryOps Op,
432 bool Invert =
false) {
441 llvm::IntegerType *IntType = llvm::IntegerType::get(
445 llvm::Type *ValueType = Val->getType();
449 Kind, DestAddr, Val, llvm::AtomicOrdering::SequentiallyConsistent);
454 llvm::ConstantInt::getAllOnesValue(IntType));
478 llvm::IntegerType *IntType = llvm::IntegerType::get(
482 llvm::Type *ValueType = Cmp->getType();
487 DestAddr, Cmp, New, llvm::AtomicOrdering::SequentiallyConsistent,
488 llvm::AtomicOrdering::SequentiallyConsistent);
491 return CGF.
Builder.CreateZExt(CGF.
Builder.CreateExtractValue(Pair, 1),
514 AtomicOrdering SuccessOrdering = AtomicOrdering::SequentiallyConsistent) {
526 auto *RTy = Exchange->getType();
530 if (RTy->isPointerTy()) {
536 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release ?
537 AtomicOrdering::Monotonic :
545 DestAddr, Comparand, Exchange, SuccessOrdering, FailureOrdering);
546 CmpXchg->setVolatile(
true);
549 if (RTy->isPointerTy()) {
570 AtomicOrdering SuccessOrdering) {
571 assert(
E->getNumArgs() == 4);
577 assert(DestPtr->getType()->isPointerTy());
578 assert(!ExchangeHigh->getType()->isPointerTy());
579 assert(!ExchangeLow->getType()->isPointerTy());
582 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release
583 ? AtomicOrdering::Monotonic
588 llvm::Type *Int128Ty = llvm::IntegerType::get(CGF.
getLLVMContext(), 128);
589 Address DestAddr(DestPtr, Int128Ty,
594 ExchangeHigh = CGF.
Builder.CreateZExt(ExchangeHigh, Int128Ty);
595 ExchangeLow = CGF.
Builder.CreateZExt(ExchangeLow, Int128Ty);
597 CGF.
Builder.CreateShl(ExchangeHigh, llvm::ConstantInt::get(Int128Ty, 64));
598 llvm::Value *Exchange = CGF.
Builder.CreateOr(ExchangeHigh, ExchangeLow);
604 SuccessOrdering, FailureOrdering);
610 CXI->setVolatile(
true);
622 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
628 AtomicRMWInst::Add, DestAddr, ConstantInt::get(IntTy, 1), Ordering);
629 return CGF.
Builder.CreateAdd(
Result, ConstantInt::get(IntTy, 1));
634 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
640 AtomicRMWInst::Sub, DestAddr, ConstantInt::get(IntTy, 1), Ordering);
641 return CGF.
Builder.CreateSub(
Result, ConstantInt::get(IntTy, 1));
652 Load->setVolatile(
true);
662 llvm::StoreInst *Store =
664 Store->setVolatile(
true);
673 unsigned ConstrainedIntrinsicID) {
676 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
677 if (CGF.
Builder.getIsFPConstrained()) {
679 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0 });
682 return CGF.
Builder.CreateCall(F, Src0);
690 unsigned ConstrainedIntrinsicID) {
694 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
695 if (CGF.
Builder.getIsFPConstrained()) {
697 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0, Src1 });
700 return CGF.
Builder.CreateCall(F, { Src0, Src1 });
707 llvm::Intrinsic::ID ConstrainedIntrinsicID) {
711 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
712 if (CGF.
Builder.getIsFPConstrained()) {
714 {Src0->getType(), Src1->getType()});
715 return CGF.
Builder.CreateConstrainedFPCall(F, {Src0, Src1});
720 return CGF.
Builder.CreateCall(F, {Src0, Src1});
727 unsigned ConstrainedIntrinsicID) {
732 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
733 if (CGF.
Builder.getIsFPConstrained()) {
735 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0, Src1, Src2 });
738 return CGF.
Builder.CreateCall(F, { Src0, Src1, Src2 });
745 unsigned IntrinsicID,
746 unsigned ConstrainedIntrinsicID,
750 if (CGF.
Builder.getIsFPConstrained())
755 if (CGF.
Builder.getIsFPConstrained())
756 return CGF.
Builder.CreateConstrainedFPCall(F, Args);
758 return CGF.
Builder.CreateCall(F, Args);
767 unsigned IntrinsicID,
768 llvm::StringRef Name =
"") {
769 static_assert(N,
"expect non-empty argument");
771 for (
unsigned I = 0; I < N; ++I)
774 return CGF.
Builder.CreateCall(F, Args, Name);
779 unsigned IntrinsicID) {
786 return CGF.
Builder.CreateCall(F, {Src0, Src1, Src2, Src3});
792 unsigned IntrinsicID) {
797 return CGF.
Builder.CreateCall(F, {Src0, Src1});
803 unsigned IntrinsicID,
804 unsigned ConstrainedIntrinsicID) {
808 if (CGF.
Builder.getIsFPConstrained()) {
809 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
811 {ResultType, Src0->getType()});
812 return CGF.
Builder.CreateConstrainedFPCall(F, {Src0});
816 return CGF.
Builder.CreateCall(F, Src0);
821 llvm::Intrinsic::ID IntrinsicID) {
829 llvm::Value *
Call = CGF.
Builder.CreateCall(F, Src0);
831 llvm::Value *Exp = CGF.
Builder.CreateExtractValue(
Call, 1);
839 llvm::Intrinsic::ID IntrinsicID) {
844 llvm::Function *F = CGF.
CGM.
getIntrinsic(IntrinsicID, {Val->getType()});
845 llvm::Value *
Call = CGF.
Builder.CreateCall(F, Val);
847 llvm::Value *SinResult = CGF.
Builder.CreateExtractValue(
Call, 0);
848 llvm::Value *CosResult = CGF.
Builder.CreateExtractValue(
Call, 1);
854 llvm::StoreInst *StoreSin =
856 llvm::StoreInst *StoreCos =
863 MDNode *
Domain = MDHelper.createAnonymousAliasScopeDomain();
864 MDNode *AliasScope = MDHelper.createAnonymousAliasScope(
Domain);
865 MDNode *AliasScopeList = MDNode::get(
Call->getContext(), AliasScope);
866 StoreSin->setMetadata(LLVMContext::MD_alias_scope, AliasScopeList);
867 StoreCos->setMetadata(LLVMContext::MD_noalias, AliasScopeList);
874 Call->setDoesNotAccessMemory();
883 llvm::Type *Ty =
V->getType();
884 int Width = Ty->getPrimitiveSizeInBits();
885 llvm::Type *IntTy = llvm::IntegerType::get(
C, Width);
887 if (Ty->isPPC_FP128Ty()) {
897 Value *ShiftCst = llvm::ConstantInt::get(IntTy, Width);
902 IntTy = llvm::IntegerType::get(
C, Width);
905 Value *Zero = llvm::Constant::getNullValue(IntTy);
906 return CGF.
Builder.CreateICmpSLT(
V, Zero);
915 auto IsIndirect = [&](
ABIArgInfo const &info) {
916 return info.isIndirect() || info.isIndirectAliased() || info.isInAlloca();
921 return IsIndirect(ArgInfo.info);
926 const CallExpr *
E, llvm::Constant *calleeValue) {
927 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
929 llvm::CallBase *callOrInvoke =
nullptr;
933 nullptr, &callOrInvoke, &FnInfo);
938 bool ConstWithoutErrnoAndExceptions =
942 if (ConstWithoutErrnoAndExceptions && CGF.
CGM.
getLangOpts().MathErrno &&
943 !CGF.
Builder.getIsFPConstrained() &&
Call.isScalar() &&
964 const llvm::Intrinsic::ID IntrinsicID,
965 llvm::Value *
X, llvm::Value *Y,
966 llvm::Value *&Carry) {
968 assert(
X->getType() == Y->getType() &&
969 "Arguments must be the same type. (Did you forget to make sure both "
970 "arguments have the same integer width?)");
973 llvm::Value *Tmp = CGF.
Builder.CreateCall(Callee, {
X, Y});
974 Carry = CGF.
Builder.CreateExtractValue(Tmp, 1);
975 return CGF.
Builder.CreateExtractValue(Tmp, 0);
982 llvm::ConstantRange CR(APInt(32, low), APInt(32, high));
983 Call->addRangeRetAttr(CR);
984 Call->addRetAttr(llvm::Attribute::AttrKind::NoUndef);
989 struct WidthAndSignedness {
995static WidthAndSignedness
1007static struct WidthAndSignedness
1009 assert(Types.size() > 0 &&
"Empty list of types.");
1013 for (
const auto &
Type : Types) {
1022 for (
const auto &
Type : Types) {
1024 if (Width < MinWidth) {
1033 Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend;
1044 return From == To || (From == 0 && To == 1) || (From == 3 && To == 2);
1049 return ConstantInt::get(ResType, (
Type & 2) ? 0 : -1,
true);
1053CodeGenFunction::evaluateOrEmitBuiltinObjectSize(
const Expr *
E,
unsigned Type,
1054 llvm::IntegerType *ResType,
1055 llvm::Value *EmittedE,
1059 return emitBuiltinObjectSize(
E,
Type, ResType, EmittedE, IsDynamic);
1060 return ConstantInt::get(ResType, ObjectSize,
true);
1074 if ((!FAMDecl || FD == FAMDecl) &&
1076 Ctx, FD, FD->getType(), StrictFlexArraysLevel,
1104 if (FD->getType()->isCountAttributedType())
1116CodeGenFunction::emitFlexibleArrayMemberSize(
const Expr *
E,
unsigned Type,
1117 llvm::IntegerType *ResType) {
1146 const Expr *Idx =
nullptr;
1148 if (
const auto *UO = dyn_cast<UnaryOperator>(
Base);
1149 UO && UO->getOpcode() == UO_AddrOf) {
1151 if (
const auto *ASE = dyn_cast<ArraySubscriptExpr>(SubExpr)) {
1152 Base = ASE->getBase()->IgnoreParenImpCasts();
1155 if (
const auto *IL = dyn_cast<IntegerLiteral>(Idx)) {
1156 int64_t Val = IL->getValue().getSExtValue();
1173 if (
const auto *ME = dyn_cast<MemberExpr>(
Base)) {
1175 const ValueDecl *VD = ME->getMemberDecl();
1177 FAMDecl = dyn_cast<FieldDecl>(VD);
1180 }
else if (
const auto *DRE = dyn_cast<DeclRefExpr>(
Base)) {
1182 QualType Ty = DRE->getDecl()->getType();
1235 if (isa<DeclRefExpr>(
Base))
1259 CountedByInst =
Builder.CreateIntCast(CountedByInst, ResType, IsSigned);
1262 Value *IdxInst =
nullptr;
1270 IdxInst =
Builder.CreateIntCast(IdxInst, ResType, IdxSigned);
1275 Builder.CreateSub(CountedByInst, IdxInst,
"", !IsSigned, IsSigned);
1281 llvm::Constant *ElemSize =
1282 llvm::ConstantInt::get(ResType,
Size.getQuantity(), IsSigned);
1284 Builder.CreateMul(CountedByInst, ElemSize,
"", !IsSigned, IsSigned);
1285 Res =
Builder.CreateIntCast(Res, ResType, IsSigned);
1294 return Builder.CreateSelect(Cmp, Res, ConstantInt::get(ResType, 0, IsSigned));
1307CodeGenFunction::emitBuiltinObjectSize(
const Expr *
E,
unsigned Type,
1308 llvm::IntegerType *ResType,
1309 llvm::Value *EmittedE,
bool IsDynamic) {
1313 auto *Param = dyn_cast<ParmVarDecl>(
D->getDecl());
1314 auto *PS =
D->getDecl()->
getAttr<PassObjectSizeAttr>();
1315 if (Param !=
nullptr && PS !=
nullptr &&
1317 auto Iter = SizeArguments.find(Param);
1318 assert(
Iter != SizeArguments.end());
1321 auto DIter = LocalDeclMap.find(
D);
1322 assert(DIter != LocalDeclMap.end());
1332 if (
Value *
V = emitFlexibleArrayMemberSize(
E,
Type, ResType))
1343 assert(Ptr->
getType()->isPointerTy() &&
1344 "Non-pointer passed to __builtin_object_size?");
1360 enum ActionKind : uint8_t { TestOnly, Complement, Reset,
Set };
1361 enum InterlockingKind : uint8_t {
1370 InterlockingKind Interlocking;
1373 static BitTest decodeBitTestBuiltin(
unsigned BuiltinID);
1378BitTest BitTest::decodeBitTestBuiltin(
unsigned BuiltinID) {
1379 switch (BuiltinID) {
1381 case Builtin::BI_bittest:
1382 return {TestOnly, Unlocked,
false};
1383 case Builtin::BI_bittestandcomplement:
1384 return {Complement, Unlocked,
false};
1385 case Builtin::BI_bittestandreset:
1386 return {Reset, Unlocked,
false};
1387 case Builtin::BI_bittestandset:
1388 return {
Set, Unlocked,
false};
1389 case Builtin::BI_interlockedbittestandreset:
1390 return {Reset, Sequential,
false};
1391 case Builtin::BI_interlockedbittestandset:
1392 return {
Set, Sequential,
false};
1395 case Builtin::BI_bittest64:
1396 return {TestOnly, Unlocked,
true};
1397 case Builtin::BI_bittestandcomplement64:
1398 return {Complement, Unlocked,
true};
1399 case Builtin::BI_bittestandreset64:
1400 return {Reset, Unlocked,
true};
1401 case Builtin::BI_bittestandset64:
1402 return {
Set, Unlocked,
true};
1403 case Builtin::BI_interlockedbittestandreset64:
1404 return {Reset, Sequential,
true};
1405 case Builtin::BI_interlockedbittestandset64:
1406 return {
Set, Sequential,
true};
1409 case Builtin::BI_interlockedbittestandset_acq:
1410 return {
Set, Acquire,
false};
1411 case Builtin::BI_interlockedbittestandset_rel:
1412 return {
Set, Release,
false};
1413 case Builtin::BI_interlockedbittestandset_nf:
1414 return {
Set, NoFence,
false};
1415 case Builtin::BI_interlockedbittestandreset_acq:
1416 return {Reset, Acquire,
false};
1417 case Builtin::BI_interlockedbittestandreset_rel:
1418 return {Reset, Release,
false};
1419 case Builtin::BI_interlockedbittestandreset_nf:
1420 return {Reset, NoFence,
false};
1422 llvm_unreachable(
"expected only bittest intrinsics");
1427 case BitTest::TestOnly:
return '\0';
1428 case BitTest::Complement:
return 'c';
1429 case BitTest::Reset:
return 'r';
1430 case BitTest::Set:
return 's';
1432 llvm_unreachable(
"invalid action");
1440 char SizeSuffix = BT.Is64Bit ?
'q' :
'l';
1444 raw_svector_ostream AsmOS(
Asm);
1445 if (BT.Interlocking != BitTest::Unlocked)
1450 AsmOS << SizeSuffix <<
" $2, ($1)";
1453 std::string Constraints =
"={@ccc},r,r,~{cc},~{memory}";
1455 if (!MachineClobbers.empty()) {
1457 Constraints += MachineClobbers;
1459 llvm::IntegerType *IntType = llvm::IntegerType::get(
1462 llvm::FunctionType *FTy =
1463 llvm::FunctionType::get(CGF.
Int8Ty, {CGF.UnqualPtrTy, IntType},
false);
1465 llvm::InlineAsm *IA =
1466 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
1467 return CGF.
Builder.CreateCall(IA, {BitBase, BitPos});
1470static llvm::AtomicOrdering
1473 case BitTest::Unlocked:
return llvm::AtomicOrdering::NotAtomic;
1474 case BitTest::Sequential:
return llvm::AtomicOrdering::SequentiallyConsistent;
1475 case BitTest::Acquire:
return llvm::AtomicOrdering::Acquire;
1476 case BitTest::Release:
return llvm::AtomicOrdering::Release;
1477 case BitTest::NoFence:
return llvm::AtomicOrdering::Monotonic;
1479 llvm_unreachable(
"invalid interlocking");
1492 BitTest BT = BitTest::decodeBitTestBuiltin(BuiltinID);
1504 BitPos, llvm::ConstantInt::get(BitPos->
getType(), 3),
"bittest.byteidx");
1506 "bittest.byteaddr"),
1510 llvm::ConstantInt::get(CGF.
Int8Ty, 0x7));
1513 Value *Mask =
nullptr;
1514 if (BT.Action != BitTest::TestOnly) {
1515 Mask = CGF.
Builder.CreateShl(llvm::ConstantInt::get(CGF.
Int8Ty, 1), PosLow,
1522 Value *OldByte =
nullptr;
1523 if (Ordering != llvm::AtomicOrdering::NotAtomic) {
1526 llvm::AtomicRMWInst::BinOp RMWOp = llvm::AtomicRMWInst::Or;
1527 if (BT.Action == BitTest::Reset) {
1528 Mask = CGF.
Builder.CreateNot(Mask);
1529 RMWOp = llvm::AtomicRMWInst::And;
1535 Value *NewByte =
nullptr;
1536 switch (BT.Action) {
1537 case BitTest::TestOnly:
1540 case BitTest::Complement:
1541 NewByte = CGF.
Builder.CreateXor(OldByte, Mask);
1543 case BitTest::Reset:
1544 NewByte = CGF.
Builder.CreateAnd(OldByte, CGF.
Builder.CreateNot(Mask));
1547 NewByte = CGF.
Builder.CreateOr(OldByte, Mask);
1556 Value *ShiftedByte = CGF.
Builder.CreateLShr(OldByte, PosLow,
"bittest.shr");
1558 ShiftedByte, llvm::ConstantInt::get(CGF.
Int8Ty, 1),
"bittest.res");
1567 raw_svector_ostream AsmOS(
Asm);
1568 llvm::IntegerType *RetType = CGF.
Int32Ty;
1570 switch (BuiltinID) {
1571 case clang::PPC::BI__builtin_ppc_ldarx:
1575 case clang::PPC::BI__builtin_ppc_lwarx:
1579 case clang::PPC::BI__builtin_ppc_lharx:
1583 case clang::PPC::BI__builtin_ppc_lbarx:
1588 llvm_unreachable(
"Expected only PowerPC load reserve intrinsics");
1591 AsmOS <<
"$0, ${1:y}";
1593 std::string Constraints =
"=r,*Z,~{memory}";
1595 if (!MachineClobbers.empty()) {
1597 Constraints += MachineClobbers;
1601 llvm::FunctionType *FTy = llvm::FunctionType::get(RetType, {PtrType},
false);
1603 llvm::InlineAsm *IA =
1604 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
1605 llvm::CallInst *CI = CGF.
Builder.CreateCall(IA, {Addr});
1607 0, Attribute::get(CGF.
getLLVMContext(), Attribute::ElementType, RetType));
1612enum class MSVCSetJmpKind {
1624 llvm::Value *Arg1 =
nullptr;
1625 llvm::Type *Arg1Ty =
nullptr;
1627 bool IsVarArg =
false;
1628 if (SJKind == MSVCSetJmpKind::_setjmp3) {
1631 Arg1 = llvm::ConstantInt::get(CGF.
IntTy, 0);
1634 Name = SJKind == MSVCSetJmpKind::_setjmp ?
"_setjmp" :
"_setjmpex";
1637 Arg1 = CGF.
Builder.CreateCall(
1640 Arg1 = CGF.
Builder.CreateCall(
1642 llvm::ConstantInt::get(CGF.
Int32Ty, 0));
1646 llvm::Type *ArgTypes[2] = {CGF.
Int8PtrTy, Arg1Ty};
1647 llvm::AttributeList ReturnsTwiceAttr = llvm::AttributeList::get(
1649 llvm::Attribute::ReturnsTwice);
1651 llvm::FunctionType::get(CGF.
IntTy, ArgTypes, IsVarArg), Name,
1652 ReturnsTwiceAttr,
true);
1654 llvm::Value *Buf = CGF.
Builder.CreateBitOrPointerCast(
1656 llvm::Value *Args[] = {Buf, Arg1};
1658 CB->setAttributes(ReturnsTwiceAttr);
1707static std::optional<CodeGenFunction::MSVCIntrin>
1710 switch (BuiltinID) {
1712 return std::nullopt;
1713 case clang::ARM::BI_BitScanForward:
1714 case clang::ARM::BI_BitScanForward64:
1715 return MSVCIntrin::_BitScanForward;
1716 case clang::ARM::BI_BitScanReverse:
1717 case clang::ARM::BI_BitScanReverse64:
1718 return MSVCIntrin::_BitScanReverse;
1719 case clang::ARM::BI_InterlockedAnd64:
1720 return MSVCIntrin::_InterlockedAnd;
1721 case clang::ARM::BI_InterlockedExchange64:
1722 return MSVCIntrin::_InterlockedExchange;
1723 case clang::ARM::BI_InterlockedExchangeAdd64:
1724 return MSVCIntrin::_InterlockedExchangeAdd;
1725 case clang::ARM::BI_InterlockedExchangeSub64:
1726 return MSVCIntrin::_InterlockedExchangeSub;
1727 case clang::ARM::BI_InterlockedOr64:
1728 return MSVCIntrin::_InterlockedOr;
1729 case clang::ARM::BI_InterlockedXor64:
1730 return MSVCIntrin::_InterlockedXor;
1731 case clang::ARM::BI_InterlockedDecrement64:
1732 return MSVCIntrin::_InterlockedDecrement;
1733 case clang::ARM::BI_InterlockedIncrement64:
1734 return MSVCIntrin::_InterlockedIncrement;
1735 case clang::ARM::BI_InterlockedExchangeAdd8_acq:
1736 case clang::ARM::BI_InterlockedExchangeAdd16_acq:
1737 case clang::ARM::BI_InterlockedExchangeAdd_acq:
1738 case clang::ARM::BI_InterlockedExchangeAdd64_acq:
1739 return MSVCIntrin::_InterlockedExchangeAdd_acq;
1740 case clang::ARM::BI_InterlockedExchangeAdd8_rel:
1741 case clang::ARM::BI_InterlockedExchangeAdd16_rel:
1742 case clang::ARM::BI_InterlockedExchangeAdd_rel:
1743 case clang::ARM::BI_InterlockedExchangeAdd64_rel:
1744 return MSVCIntrin::_InterlockedExchangeAdd_rel;
1745 case clang::ARM::BI_InterlockedExchangeAdd8_nf:
1746 case clang::ARM::BI_InterlockedExchangeAdd16_nf:
1747 case clang::ARM::BI_InterlockedExchangeAdd_nf:
1748 case clang::ARM::BI_InterlockedExchangeAdd64_nf:
1749 return MSVCIntrin::_InterlockedExchangeAdd_nf;
1750 case clang::ARM::BI_InterlockedExchange8_acq:
1751 case clang::ARM::BI_InterlockedExchange16_acq:
1752 case clang::ARM::BI_InterlockedExchange_acq:
1753 case clang::ARM::BI_InterlockedExchange64_acq:
1754 case clang::ARM::BI_InterlockedExchangePointer_acq:
1755 return MSVCIntrin::_InterlockedExchange_acq;
1756 case clang::ARM::BI_InterlockedExchange8_rel:
1757 case clang::ARM::BI_InterlockedExchange16_rel:
1758 case clang::ARM::BI_InterlockedExchange_rel:
1759 case clang::ARM::BI_InterlockedExchange64_rel:
1760 case clang::ARM::BI_InterlockedExchangePointer_rel:
1761 return MSVCIntrin::_InterlockedExchange_rel;
1762 case clang::ARM::BI_InterlockedExchange8_nf:
1763 case clang::ARM::BI_InterlockedExchange16_nf:
1764 case clang::ARM::BI_InterlockedExchange_nf:
1765 case clang::ARM::BI_InterlockedExchange64_nf:
1766 case clang::ARM::BI_InterlockedExchangePointer_nf:
1767 return MSVCIntrin::_InterlockedExchange_nf;
1768 case clang::ARM::BI_InterlockedCompareExchange8_acq:
1769 case clang::ARM::BI_InterlockedCompareExchange16_acq:
1770 case clang::ARM::BI_InterlockedCompareExchange_acq:
1771 case clang::ARM::BI_InterlockedCompareExchange64_acq:
1772 case clang::ARM::BI_InterlockedCompareExchangePointer_acq:
1773 return MSVCIntrin::_InterlockedCompareExchange_acq;
1774 case clang::ARM::BI_InterlockedCompareExchange8_rel:
1775 case clang::ARM::BI_InterlockedCompareExchange16_rel:
1776 case clang::ARM::BI_InterlockedCompareExchange_rel:
1777 case clang::ARM::BI_InterlockedCompareExchange64_rel:
1778 case clang::ARM::BI_InterlockedCompareExchangePointer_rel:
1779 return MSVCIntrin::_InterlockedCompareExchange_rel;
1780 case clang::ARM::BI_InterlockedCompareExchange8_nf:
1781 case clang::ARM::BI_InterlockedCompareExchange16_nf:
1782 case clang::ARM::BI_InterlockedCompareExchange_nf:
1783 case clang::ARM::BI_InterlockedCompareExchange64_nf:
1784 return MSVCIntrin::_InterlockedCompareExchange_nf;
1785 case clang::ARM::BI_InterlockedOr8_acq:
1786 case clang::ARM::BI_InterlockedOr16_acq:
1787 case clang::ARM::BI_InterlockedOr_acq:
1788 case clang::ARM::BI_InterlockedOr64_acq:
1789 return MSVCIntrin::_InterlockedOr_acq;
1790 case clang::ARM::BI_InterlockedOr8_rel:
1791 case clang::ARM::BI_InterlockedOr16_rel:
1792 case clang::ARM::BI_InterlockedOr_rel:
1793 case clang::ARM::BI_InterlockedOr64_rel:
1794 return MSVCIntrin::_InterlockedOr_rel;
1795 case clang::ARM::BI_InterlockedOr8_nf:
1796 case clang::ARM::BI_InterlockedOr16_nf:
1797 case clang::ARM::BI_InterlockedOr_nf:
1798 case clang::ARM::BI_InterlockedOr64_nf:
1799 return MSVCIntrin::_InterlockedOr_nf;
1800 case clang::ARM::BI_InterlockedXor8_acq:
1801 case clang::ARM::BI_InterlockedXor16_acq:
1802 case clang::ARM::BI_InterlockedXor_acq:
1803 case clang::ARM::BI_InterlockedXor64_acq:
1804 return MSVCIntrin::_InterlockedXor_acq;
1805 case clang::ARM::BI_InterlockedXor8_rel:
1806 case clang::ARM::BI_InterlockedXor16_rel:
1807 case clang::ARM::BI_InterlockedXor_rel:
1808 case clang::ARM::BI_InterlockedXor64_rel:
1809 return MSVCIntrin::_InterlockedXor_rel;
1810 case clang::ARM::BI_InterlockedXor8_nf:
1811 case clang::ARM::BI_InterlockedXor16_nf:
1812 case clang::ARM::BI_InterlockedXor_nf:
1813 case clang::ARM::BI_InterlockedXor64_nf:
1814 return MSVCIntrin::_InterlockedXor_nf;
1815 case clang::ARM::BI_InterlockedAnd8_acq:
1816 case clang::ARM::BI_InterlockedAnd16_acq:
1817 case clang::ARM::BI_InterlockedAnd_acq:
1818 case clang::ARM::BI_InterlockedAnd64_acq:
1819 return MSVCIntrin::_InterlockedAnd_acq;
1820 case clang::ARM::BI_InterlockedAnd8_rel:
1821 case clang::ARM::BI_InterlockedAnd16_rel:
1822 case clang::ARM::BI_InterlockedAnd_rel:
1823 case clang::ARM::BI_InterlockedAnd64_rel:
1824 return MSVCIntrin::_InterlockedAnd_rel;
1825 case clang::ARM::BI_InterlockedAnd8_nf:
1826 case clang::ARM::BI_InterlockedAnd16_nf:
1827 case clang::ARM::BI_InterlockedAnd_nf:
1828 case clang::ARM::BI_InterlockedAnd64_nf:
1829 return MSVCIntrin::_InterlockedAnd_nf;
1830 case clang::ARM::BI_InterlockedIncrement16_acq:
1831 case clang::ARM::BI_InterlockedIncrement_acq:
1832 case clang::ARM::BI_InterlockedIncrement64_acq:
1833 return MSVCIntrin::_InterlockedIncrement_acq;
1834 case clang::ARM::BI_InterlockedIncrement16_rel:
1835 case clang::ARM::BI_InterlockedIncrement_rel:
1836 case clang::ARM::BI_InterlockedIncrement64_rel:
1837 return MSVCIntrin::_InterlockedIncrement_rel;
1838 case clang::ARM::BI_InterlockedIncrement16_nf:
1839 case clang::ARM::BI_InterlockedIncrement_nf:
1840 case clang::ARM::BI_InterlockedIncrement64_nf:
1841 return MSVCIntrin::_InterlockedIncrement_nf;
1842 case clang::ARM::BI_InterlockedDecrement16_acq:
1843 case clang::ARM::BI_InterlockedDecrement_acq:
1844 case clang::ARM::BI_InterlockedDecrement64_acq:
1845 return MSVCIntrin::_InterlockedDecrement_acq;
1846 case clang::ARM::BI_InterlockedDecrement16_rel:
1847 case clang::ARM::BI_InterlockedDecrement_rel:
1848 case clang::ARM::BI_InterlockedDecrement64_rel:
1849 return MSVCIntrin::_InterlockedDecrement_rel;
1850 case clang::ARM::BI_InterlockedDecrement16_nf:
1851 case clang::ARM::BI_InterlockedDecrement_nf:
1852 case clang::ARM::BI_InterlockedDecrement64_nf:
1853 return MSVCIntrin::_InterlockedDecrement_nf;
1855 llvm_unreachable(
"must return from switch");
1858static std::optional<CodeGenFunction::MSVCIntrin>
1861 switch (BuiltinID) {
1863 return std::nullopt;
1864 case clang::AArch64::BI_BitScanForward:
1865 case clang::AArch64::BI_BitScanForward64:
1866 return MSVCIntrin::_BitScanForward;
1867 case clang::AArch64::BI_BitScanReverse:
1868 case clang::AArch64::BI_BitScanReverse64:
1869 return MSVCIntrin::_BitScanReverse;
1870 case clang::AArch64::BI_InterlockedAnd64:
1871 return MSVCIntrin::_InterlockedAnd;
1872 case clang::AArch64::BI_InterlockedExchange64:
1873 return MSVCIntrin::_InterlockedExchange;
1874 case clang::AArch64::BI_InterlockedExchangeAdd64:
1875 return MSVCIntrin::_InterlockedExchangeAdd;
1876 case clang::AArch64::BI_InterlockedExchangeSub64:
1877 return MSVCIntrin::_InterlockedExchangeSub;
1878 case clang::AArch64::BI_InterlockedOr64:
1879 return MSVCIntrin::_InterlockedOr;
1880 case clang::AArch64::BI_InterlockedXor64:
1881 return MSVCIntrin::_InterlockedXor;
1882 case clang::AArch64::BI_InterlockedDecrement64:
1883 return MSVCIntrin::_InterlockedDecrement;
1884 case clang::AArch64::BI_InterlockedIncrement64:
1885 return MSVCIntrin::_InterlockedIncrement;
1886 case clang::AArch64::BI_InterlockedExchangeAdd8_acq:
1887 case clang::AArch64::BI_InterlockedExchangeAdd16_acq:
1888 case clang::AArch64::BI_InterlockedExchangeAdd_acq:
1889 case clang::AArch64::BI_InterlockedExchangeAdd64_acq:
1890 return MSVCIntrin::_InterlockedExchangeAdd_acq;
1891 case clang::AArch64::BI_InterlockedExchangeAdd8_rel:
1892 case clang::AArch64::BI_InterlockedExchangeAdd16_rel:
1893 case clang::AArch64::BI_InterlockedExchangeAdd_rel:
1894 case clang::AArch64::BI_InterlockedExchangeAdd64_rel:
1895 return MSVCIntrin::_InterlockedExchangeAdd_rel;
1896 case clang::AArch64::BI_InterlockedExchangeAdd8_nf:
1897 case clang::AArch64::BI_InterlockedExchangeAdd16_nf:
1898 case clang::AArch64::BI_InterlockedExchangeAdd_nf:
1899 case clang::AArch64::BI_InterlockedExchangeAdd64_nf:
1900 return MSVCIntrin::_InterlockedExchangeAdd_nf;
1901 case clang::AArch64::BI_InterlockedExchange8_acq:
1902 case clang::AArch64::BI_InterlockedExchange16_acq:
1903 case clang::AArch64::BI_InterlockedExchange_acq:
1904 case clang::AArch64::BI_InterlockedExchange64_acq:
1905 case clang::AArch64::BI_InterlockedExchangePointer_acq:
1906 return MSVCIntrin::_InterlockedExchange_acq;
1907 case clang::AArch64::BI_InterlockedExchange8_rel:
1908 case clang::AArch64::BI_InterlockedExchange16_rel:
1909 case clang::AArch64::BI_InterlockedExchange_rel:
1910 case clang::AArch64::BI_InterlockedExchange64_rel:
1911 case clang::AArch64::BI_InterlockedExchangePointer_rel:
1912 return MSVCIntrin::_InterlockedExchange_rel;
1913 case clang::AArch64::BI_InterlockedExchange8_nf:
1914 case clang::AArch64::BI_InterlockedExchange16_nf:
1915 case clang::AArch64::BI_InterlockedExchange_nf:
1916 case clang::AArch64::BI_InterlockedExchange64_nf:
1917 case clang::AArch64::BI_InterlockedExchangePointer_nf:
1918 return MSVCIntrin::_InterlockedExchange_nf;
1919 case clang::AArch64::BI_InterlockedCompareExchange8_acq:
1920 case clang::AArch64::BI_InterlockedCompareExchange16_acq:
1921 case clang::AArch64::BI_InterlockedCompareExchange_acq:
1922 case clang::AArch64::BI_InterlockedCompareExchange64_acq:
1923 case clang::AArch64::BI_InterlockedCompareExchangePointer_acq:
1924 return MSVCIntrin::_InterlockedCompareExchange_acq;
1925 case clang::AArch64::BI_InterlockedCompareExchange8_rel:
1926 case clang::AArch64::BI_InterlockedCompareExchange16_rel:
1927 case clang::AArch64::BI_InterlockedCompareExchange_rel:
1928 case clang::AArch64::BI_InterlockedCompareExchange64_rel:
1929 case clang::AArch64::BI_InterlockedCompareExchangePointer_rel:
1930 return MSVCIntrin::_InterlockedCompareExchange_rel;
1931 case clang::AArch64::BI_InterlockedCompareExchange8_nf:
1932 case clang::AArch64::BI_InterlockedCompareExchange16_nf:
1933 case clang::AArch64::BI_InterlockedCompareExchange_nf:
1934 case clang::AArch64::BI_InterlockedCompareExchange64_nf:
1935 return MSVCIntrin::_InterlockedCompareExchange_nf;
1936 case clang::AArch64::BI_InterlockedCompareExchange128:
1937 return MSVCIntrin::_InterlockedCompareExchange128;
1938 case clang::AArch64::BI_InterlockedCompareExchange128_acq:
1939 return MSVCIntrin::_InterlockedCompareExchange128_acq;
1940 case clang::AArch64::BI_InterlockedCompareExchange128_nf:
1941 return MSVCIntrin::_InterlockedCompareExchange128_nf;
1942 case clang::AArch64::BI_InterlockedCompareExchange128_rel:
1943 return MSVCIntrin::_InterlockedCompareExchange128_rel;
1944 case clang::AArch64::BI_InterlockedOr8_acq:
1945 case clang::AArch64::BI_InterlockedOr16_acq:
1946 case clang::AArch64::BI_InterlockedOr_acq:
1947 case clang::AArch64::BI_InterlockedOr64_acq:
1948 return MSVCIntrin::_InterlockedOr_acq;
1949 case clang::AArch64::BI_InterlockedOr8_rel:
1950 case clang::AArch64::BI_InterlockedOr16_rel:
1951 case clang::AArch64::BI_InterlockedOr_rel:
1952 case clang::AArch64::BI_InterlockedOr64_rel:
1953 return MSVCIntrin::_InterlockedOr_rel;
1954 case clang::AArch64::BI_InterlockedOr8_nf:
1955 case clang::AArch64::BI_InterlockedOr16_nf:
1956 case clang::AArch64::BI_InterlockedOr_nf:
1957 case clang::AArch64::BI_InterlockedOr64_nf:
1958 return MSVCIntrin::_InterlockedOr_nf;
1959 case clang::AArch64::BI_InterlockedXor8_acq:
1960 case clang::AArch64::BI_InterlockedXor16_acq:
1961 case clang::AArch64::BI_InterlockedXor_acq:
1962 case clang::AArch64::BI_InterlockedXor64_acq:
1963 return MSVCIntrin::_InterlockedXor_acq;
1964 case clang::AArch64::BI_InterlockedXor8_rel:
1965 case clang::AArch64::BI_InterlockedXor16_rel:
1966 case clang::AArch64::BI_InterlockedXor_rel:
1967 case clang::AArch64::BI_InterlockedXor64_rel:
1968 return MSVCIntrin::_InterlockedXor_rel;
1969 case clang::AArch64::BI_InterlockedXor8_nf:
1970 case clang::AArch64::BI_InterlockedXor16_nf:
1971 case clang::AArch64::BI_InterlockedXor_nf:
1972 case clang::AArch64::BI_InterlockedXor64_nf:
1973 return MSVCIntrin::_InterlockedXor_nf;
1974 case clang::AArch64::BI_InterlockedAnd8_acq:
1975 case clang::AArch64::BI_InterlockedAnd16_acq:
1976 case clang::AArch64::BI_InterlockedAnd_acq:
1977 case clang::AArch64::BI_InterlockedAnd64_acq:
1978 return MSVCIntrin::_InterlockedAnd_acq;
1979 case clang::AArch64::BI_InterlockedAnd8_rel:
1980 case clang::AArch64::BI_InterlockedAnd16_rel:
1981 case clang::AArch64::BI_InterlockedAnd_rel:
1982 case clang::AArch64::BI_InterlockedAnd64_rel:
1983 return MSVCIntrin::_InterlockedAnd_rel;
1984 case clang::AArch64::BI_InterlockedAnd8_nf:
1985 case clang::AArch64::BI_InterlockedAnd16_nf:
1986 case clang::AArch64::BI_InterlockedAnd_nf:
1987 case clang::AArch64::BI_InterlockedAnd64_nf:
1988 return MSVCIntrin::_InterlockedAnd_nf;
1989 case clang::AArch64::BI_InterlockedIncrement16_acq:
1990 case clang::AArch64::BI_InterlockedIncrement_acq:
1991 case clang::AArch64::BI_InterlockedIncrement64_acq:
1992 return MSVCIntrin::_InterlockedIncrement_acq;
1993 case clang::AArch64::BI_InterlockedIncrement16_rel:
1994 case clang::AArch64::BI_InterlockedIncrement_rel:
1995 case clang::AArch64::BI_InterlockedIncrement64_rel:
1996 return MSVCIntrin::_InterlockedIncrement_rel;
1997 case clang::AArch64::BI_InterlockedIncrement16_nf:
1998 case clang::AArch64::BI_InterlockedIncrement_nf:
1999 case clang::AArch64::BI_InterlockedIncrement64_nf:
2000 return MSVCIntrin::_InterlockedIncrement_nf;
2001 case clang::AArch64::BI_InterlockedDecrement16_acq:
2002 case clang::AArch64::BI_InterlockedDecrement_acq:
2003 case clang::AArch64::BI_InterlockedDecrement64_acq:
2004 return MSVCIntrin::_InterlockedDecrement_acq;
2005 case clang::AArch64::BI_InterlockedDecrement16_rel:
2006 case clang::AArch64::BI_InterlockedDecrement_rel:
2007 case clang::AArch64::BI_InterlockedDecrement64_rel:
2008 return MSVCIntrin::_InterlockedDecrement_rel;
2009 case clang::AArch64::BI_InterlockedDecrement16_nf:
2010 case clang::AArch64::BI_InterlockedDecrement_nf:
2011 case clang::AArch64::BI_InterlockedDecrement64_nf:
2012 return MSVCIntrin::_InterlockedDecrement_nf;
2014 llvm_unreachable(
"must return from switch");
2017static std::optional<CodeGenFunction::MSVCIntrin>
2020 switch (BuiltinID) {
2022 return std::nullopt;
2023 case clang::X86::BI_BitScanForward:
2024 case clang::X86::BI_BitScanForward64:
2025 return MSVCIntrin::_BitScanForward;
2026 case clang::X86::BI_BitScanReverse:
2027 case clang::X86::BI_BitScanReverse64:
2028 return MSVCIntrin::_BitScanReverse;
2029 case clang::X86::BI_InterlockedAnd64:
2030 return MSVCIntrin::_InterlockedAnd;
2031 case clang::X86::BI_InterlockedCompareExchange128:
2032 return MSVCIntrin::_InterlockedCompareExchange128;
2033 case clang::X86::BI_InterlockedExchange64:
2034 return MSVCIntrin::_InterlockedExchange;
2035 case clang::X86::BI_InterlockedExchangeAdd64:
2036 return MSVCIntrin::_InterlockedExchangeAdd;
2037 case clang::X86::BI_InterlockedExchangeSub64:
2038 return MSVCIntrin::_InterlockedExchangeSub;
2039 case clang::X86::BI_InterlockedOr64:
2040 return MSVCIntrin::_InterlockedOr;
2041 case clang::X86::BI_InterlockedXor64:
2042 return MSVCIntrin::_InterlockedXor;
2043 case clang::X86::BI_InterlockedDecrement64:
2044 return MSVCIntrin::_InterlockedDecrement;
2045 case clang::X86::BI_InterlockedIncrement64:
2046 return MSVCIntrin::_InterlockedIncrement;
2048 llvm_unreachable(
"must return from switch");
2054 switch (BuiltinID) {
2055 case MSVCIntrin::_BitScanForward:
2056 case MSVCIntrin::_BitScanReverse: {
2060 llvm::Type *ArgType = ArgValue->
getType();
2061 llvm::Type *IndexType = IndexAddress.getElementType();
2064 Value *ArgZero = llvm::Constant::getNullValue(ArgType);
2065 Value *ResZero = llvm::Constant::getNullValue(ResultType);
2066 Value *ResOne = llvm::ConstantInt::get(ResultType, 1);
2071 PHINode *
Result =
Builder.CreatePHI(ResultType, 2,
"bitscan_result");
2074 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, ArgZero);
2076 Builder.CreateCondBr(IsZero, End, NotZero);
2079 Builder.SetInsertPoint(NotZero);
2081 if (BuiltinID == MSVCIntrin::_BitScanForward) {
2084 ZeroCount =
Builder.CreateIntCast(ZeroCount, IndexType,
false);
2087 unsigned ArgWidth = cast<llvm::IntegerType>(ArgType)->getBitWidth();
2088 Value *ArgTypeLastIndex = llvm::ConstantInt::get(IndexType, ArgWidth - 1);
2092 ZeroCount =
Builder.CreateIntCast(ZeroCount, IndexType,
false);
2093 Value *Index =
Builder.CreateNSWSub(ArgTypeLastIndex, ZeroCount);
2097 Result->addIncoming(ResOne, NotZero);
2102 case MSVCIntrin::_InterlockedAnd:
2104 case MSVCIntrin::_InterlockedExchange:
2106 case MSVCIntrin::_InterlockedExchangeAdd:
2108 case MSVCIntrin::_InterlockedExchangeSub:
2110 case MSVCIntrin::_InterlockedOr:
2112 case MSVCIntrin::_InterlockedXor:
2114 case MSVCIntrin::_InterlockedExchangeAdd_acq:
2116 AtomicOrdering::Acquire);
2117 case MSVCIntrin::_InterlockedExchangeAdd_rel:
2119 AtomicOrdering::Release);
2120 case MSVCIntrin::_InterlockedExchangeAdd_nf:
2122 AtomicOrdering::Monotonic);
2123 case MSVCIntrin::_InterlockedExchange_acq:
2125 AtomicOrdering::Acquire);
2126 case MSVCIntrin::_InterlockedExchange_rel:
2128 AtomicOrdering::Release);
2129 case MSVCIntrin::_InterlockedExchange_nf:
2131 AtomicOrdering::Monotonic);
2132 case MSVCIntrin::_InterlockedCompareExchange:
2134 case MSVCIntrin::_InterlockedCompareExchange_acq:
2136 case MSVCIntrin::_InterlockedCompareExchange_rel:
2138 case MSVCIntrin::_InterlockedCompareExchange_nf:
2140 case MSVCIntrin::_InterlockedCompareExchange128:
2142 *
this,
E, AtomicOrdering::SequentiallyConsistent);
2143 case MSVCIntrin::_InterlockedCompareExchange128_acq:
2145 case MSVCIntrin::_InterlockedCompareExchange128_rel:
2147 case MSVCIntrin::_InterlockedCompareExchange128_nf:
2149 case MSVCIntrin::_InterlockedOr_acq:
2151 AtomicOrdering::Acquire);
2152 case MSVCIntrin::_InterlockedOr_rel:
2154 AtomicOrdering::Release);
2155 case MSVCIntrin::_InterlockedOr_nf:
2157 AtomicOrdering::Monotonic);
2158 case MSVCIntrin::_InterlockedXor_acq:
2160 AtomicOrdering::Acquire);
2161 case MSVCIntrin::_InterlockedXor_rel:
2163 AtomicOrdering::Release);
2164 case MSVCIntrin::_InterlockedXor_nf:
2166 AtomicOrdering::Monotonic);
2167 case MSVCIntrin::_InterlockedAnd_acq:
2169 AtomicOrdering::Acquire);
2170 case MSVCIntrin::_InterlockedAnd_rel:
2172 AtomicOrdering::Release);
2173 case MSVCIntrin::_InterlockedAnd_nf:
2175 AtomicOrdering::Monotonic);
2176 case MSVCIntrin::_InterlockedIncrement_acq:
2178 case MSVCIntrin::_InterlockedIncrement_rel:
2180 case MSVCIntrin::_InterlockedIncrement_nf:
2182 case MSVCIntrin::_InterlockedDecrement_acq:
2184 case MSVCIntrin::_InterlockedDecrement_rel:
2186 case MSVCIntrin::_InterlockedDecrement_nf:
2189 case MSVCIntrin::_InterlockedDecrement:
2191 case MSVCIntrin::_InterlockedIncrement:
2194 case MSVCIntrin::__fastfail: {
2199 StringRef
Asm, Constraints;
2204 case llvm::Triple::x86:
2205 case llvm::Triple::x86_64:
2207 Constraints =
"{cx}";
2209 case llvm::Triple::thumb:
2211 Constraints =
"{r0}";
2213 case llvm::Triple::aarch64:
2214 Asm =
"brk #0xF003";
2215 Constraints =
"{w0}";
2217 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy, {
Int32Ty},
false);
2218 llvm::InlineAsm *IA =
2219 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
2220 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
2222 llvm::Attribute::NoReturn);
2224 CI->setAttributes(NoReturnAttr);
2228 llvm_unreachable(
"Incorrect MSVC intrinsic!");
2234 CallObjCArcUse(llvm::Value *
object) : object(object) {}
2235 llvm::Value *object;
2244 BuiltinCheckKind Kind) {
2246 "Unsupported builtin check kind");
2252 SanitizerScope SanScope(
this);
2254 ArgValue, llvm::Constant::getNullValue(ArgValue->
getType()));
2255 EmitCheck(std::make_pair(Cond, SanitizerKind::Builtin),
2256 SanitizerHandler::InvalidBuiltin,
2258 llvm::ConstantInt::get(
Builder.getInt8Ty(), Kind)},
2268 SanitizerScope SanScope(
this);
2270 std::make_pair(ArgValue, SanitizerKind::Builtin),
2271 SanitizerHandler::InvalidBuiltin,
2279 return CGF.
Builder.CreateBinaryIntrinsic(
2280 Intrinsic::abs, ArgValue,
2281 ConstantInt::get(CGF.
Builder.getInt1Ty(), HasNSW));
2285 bool SanitizeOverflow) {
2289 if (
const auto *VCI = dyn_cast<llvm::ConstantInt>(ArgValue)) {
2290 if (!VCI->isMinSignedValue())
2291 return EmitAbs(CGF, ArgValue,
true);
2294 CodeGenFunction::SanitizerScope SanScope(&CGF);
2296 Constant *Zero = Constant::getNullValue(ArgValue->
getType());
2297 Value *ResultAndOverflow = CGF.
Builder.CreateBinaryIntrinsic(
2298 Intrinsic::ssub_with_overflow, Zero, ArgValue);
2301 CGF.
Builder.CreateExtractValue(ResultAndOverflow, 1));
2304 if (SanitizeOverflow) {
2305 CGF.
EmitCheck({{NotOverflow, SanitizerKind::SignedIntegerOverflow}},
2306 SanitizerHandler::NegateOverflow,
2311 CGF.
EmitTrapCheck(NotOverflow, SanitizerHandler::SubOverflow);
2313 Value *CmpResult = CGF.
Builder.CreateICmpSLT(ArgValue, Zero,
"abscond");
2314 return CGF.
Builder.CreateSelect(CmpResult,
Result, ArgValue,
"abs");
2319 QualType UnsignedTy =
C.getIntTypeForBitwidth(Size * 8,
false);
2320 return C.getCanonicalType(UnsignedTy);
2330 raw_svector_ostream OS(Name);
2331 OS <<
"__os_log_helper";
2335 for (
const auto &Item : Layout.
Items)
2336 OS <<
"_" <<
int(Item.getSizeByte()) <<
"_"
2337 <<
int(Item.getDescriptorByte());
2340 if (llvm::Function *F =
CGM.
getModule().getFunction(Name))
2350 for (
unsigned int I = 0,
E = Layout.
Items.size(); I <
E; ++I) {
2351 char Size = Layout.
Items[I].getSizeByte();
2358 &Ctx.
Idents.
get(std::string(
"arg") + llvm::to_string(I)), ArgTy,
2360 ArgTys.emplace_back(ArgTy);
2371 llvm::Function *
Fn = llvm::Function::Create(
2372 FuncTy, llvm::GlobalValue::LinkOnceODRLinkage, Name, &
CGM.
getModule());
2373 Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);
2376 Fn->setDoesNotThrow();
2380 Fn->addFnAttr(llvm::Attribute::NoInline);
2398 for (
const auto &Item : Layout.
Items) {
2400 Builder.getInt8(Item.getDescriptorByte()),
2403 Builder.getInt8(Item.getSizeByte()),
2407 if (!
Size.getQuantity())
2424 assert(
E.getNumArgs() >= 2 &&
2425 "__builtin_os_log_format takes at least 2 arguments");
2436 for (
const auto &Item : Layout.
Items) {
2437 int Size = Item.getSizeByte();
2441 llvm::Value *ArgVal;
2445 for (
unsigned I = 0,
E = Item.getMaskType().size(); I <
E; ++I)
2446 Val |= ((
uint64_t)Item.getMaskType()[I]) << I * 8;
2447 ArgVal = llvm::Constant::getIntegerValue(
Int64Ty, llvm::APInt(64, Val));
2448 }
else if (
const Expr *TheExpr = Item.getExpr()) {
2454 auto LifetimeExtendObject = [&](
const Expr *
E) {
2462 if (isa<CallExpr>(
E) || isa<ObjCMessageExpr>(
E))
2467 if (TheExpr->getType()->isObjCRetainableType() &&
2468 getLangOpts().ObjCAutoRefCount && LifetimeExtendObject(TheExpr)) {
2470 "Only scalar can be a ObjC retainable type");
2471 if (!isa<Constant>(ArgVal)) {
2485 pushCleanupAfterFullExpr<CallObjCArcUse>(Cleanup, ArgVal);
2489 ArgVal =
Builder.getInt32(Item.getConstValue().getQuantity());
2492 unsigned ArgValSize =
2496 ArgVal =
Builder.CreateBitOrPointerCast(ArgVal,
IntTy);
2512 unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info,
2513 WidthAndSignedness ResultInfo) {
2514 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2515 Op1Info.Width == Op2Info.Width && Op2Info.Width == ResultInfo.Width &&
2516 !Op1Info.Signed && !Op2Info.Signed && ResultInfo.Signed;
2521 const clang::Expr *Op2, WidthAndSignedness Op2Info,
2523 WidthAndSignedness ResultInfo) {
2525 Builtin::BI__builtin_mul_overflow, Op1Info, Op2Info, ResultInfo) &&
2526 "Cannot specialize this multiply");
2531 llvm::Value *HasOverflow;
2533 CGF, llvm::Intrinsic::umul_with_overflow, V1, V2, HasOverflow);
2538 auto IntMax = llvm::APInt::getSignedMaxValue(ResultInfo.Width);
2539 llvm::Value *IntMaxValue = llvm::ConstantInt::get(
Result->getType(), IntMax);
2541 llvm::Value *IntMaxOverflow = CGF.
Builder.CreateICmpUGT(
Result, IntMaxValue);
2542 HasOverflow = CGF.
Builder.CreateOr(HasOverflow, IntMaxOverflow);
2554 WidthAndSignedness Op1Info,
2555 WidthAndSignedness Op2Info,
2556 WidthAndSignedness ResultInfo) {
2557 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2558 std::max(Op1Info.Width, Op2Info.Width) >= ResultInfo.Width &&
2559 Op1Info.Signed != Op2Info.Signed;
2566 WidthAndSignedness Op1Info,
const clang::Expr *Op2,
2567 WidthAndSignedness Op2Info,
2569 WidthAndSignedness ResultInfo) {
2571 Op2Info, ResultInfo) &&
2572 "Not a mixed-sign multipliction we can specialize");
2575 const clang::Expr *SignedOp = Op1Info.Signed ? Op1 : Op2;
2576 const clang::Expr *UnsignedOp = Op1Info.Signed ? Op2 : Op1;
2579 unsigned SignedOpWidth = Op1Info.Signed ? Op1Info.Width : Op2Info.Width;
2580 unsigned UnsignedOpWidth = Op1Info.Signed ? Op2Info.Width : Op1Info.Width;
2583 if (SignedOpWidth < UnsignedOpWidth)
2585 if (UnsignedOpWidth < SignedOpWidth)
2588 llvm::Type *OpTy =
Signed->getType();
2589 llvm::Value *Zero = llvm::Constant::getNullValue(OpTy);
2592 unsigned OpWidth = std::max(Op1Info.Width, Op2Info.Width);
2595 llvm::Value *IsNegative = CGF.
Builder.CreateICmpSLT(
Signed, Zero);
2596 llvm::Value *AbsOfNegative = CGF.
Builder.CreateSub(Zero,
Signed);
2597 llvm::Value *AbsSigned =
2598 CGF.
Builder.CreateSelect(IsNegative, AbsOfNegative,
Signed);
2601 llvm::Value *UnsignedOverflow;
2602 llvm::Value *UnsignedResult =
2606 llvm::Value *Overflow, *
Result;
2607 if (ResultInfo.Signed) {
2611 llvm::APInt::getSignedMaxValue(ResultInfo.Width).zext(OpWidth);
2612 llvm::Value *MaxResult =
2613 CGF.
Builder.CreateAdd(llvm::ConstantInt::get(OpTy, IntMax),
2614 CGF.
Builder.CreateZExt(IsNegative, OpTy));
2615 llvm::Value *SignedOverflow =
2616 CGF.
Builder.CreateICmpUGT(UnsignedResult, MaxResult);
2617 Overflow = CGF.
Builder.CreateOr(UnsignedOverflow, SignedOverflow);
2620 llvm::Value *NegativeResult = CGF.
Builder.CreateNeg(UnsignedResult);
2621 llvm::Value *SignedResult =
2622 CGF.
Builder.CreateSelect(IsNegative, NegativeResult, UnsignedResult);
2626 llvm::Value *Underflow = CGF.
Builder.CreateAnd(
2627 IsNegative, CGF.
Builder.CreateIsNotNull(UnsignedResult));
2628 Overflow = CGF.
Builder.CreateOr(UnsignedOverflow, Underflow);
2629 if (ResultInfo.Width < OpWidth) {
2631 llvm::APInt::getMaxValue(ResultInfo.Width).zext(OpWidth);
2632 llvm::Value *TruncOverflow = CGF.
Builder.CreateICmpUGT(
2633 UnsignedResult, llvm::ConstantInt::get(OpTy, IntMax));
2634 Overflow = CGF.
Builder.CreateOr(Overflow, TruncOverflow);
2639 IsNegative, CGF.
Builder.CreateNeg(UnsignedResult), UnsignedResult);
2643 assert(Overflow &&
Result &&
"Missing overflow or result");
2654 llvm::SmallPtrSetImpl<const Decl *> &Seen) {
2663 if (!Seen.insert(
Record).second)
2666 assert(
Record->hasDefinition() &&
2667 "Incomplete types should already be diagnosed");
2669 if (
Record->isDynamicClass())
2694 llvm::Type *Ty = Src->getType();
2695 ShiftAmt =
Builder.CreateIntCast(ShiftAmt, Ty,
false);
2698 unsigned IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2705 switch (BuiltinID) {
2706#define MUTATE_LDBL(func) \
2707 case Builtin::BI__builtin_##func##l: \
2708 return Builtin::BI__builtin_##func##f128;
2777 if (CGF.
Builder.getIsFPConstrained() &&
2778 CGF.
Builder.getDefaultConstrainedExcept() != fp::ebIgnore) {
2790 auto UBF = CGF->
CGM.
getModule().getOrInsertFunction(Name, FnTy);
2793 for (
auto &&FormalTy : FnTy->params())
2794 Args.push_back(llvm::PoisonValue::get(FormalTy));
2803 "Should not codegen for consteval builtins");
2810 !
Result.hasSideEffects()) {
2814 if (
Result.Val.isFloat())
2823 if (
getTarget().getTriple().isPPC64() &&
2824 &
getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad())
2831 const unsigned BuiltinIDIfNoAsmLabel =
2832 FD->
hasAttr<AsmLabelAttr>() ? 0 : BuiltinID;
2834 std::optional<bool> ErrnoOverriden;
2838 if (
E->hasStoredFPFeatures()) {
2840 if (OP.hasMathErrnoOverride())
2841 ErrnoOverriden = OP.getMathErrnoOverride();
2850 bool ErrnoOverridenToFalseWithOpt =
2851 ErrnoOverriden.has_value() && !ErrnoOverriden.value() && !OptNone &&
2869 switch (BuiltinID) {
2870 case Builtin::BI__builtin_fma:
2871 case Builtin::BI__builtin_fmaf:
2872 case Builtin::BI__builtin_fmal:
2873 case Builtin::BI__builtin_fmaf16:
2874 case Builtin::BIfma:
2875 case Builtin::BIfmaf:
2876 case Builtin::BIfmal: {
2878 if (Trip.isGNUEnvironment() || Trip.isOSMSVCRT())
2886 bool ConstWithoutErrnoAndExceptions =
2888 bool ConstWithoutExceptions =
2906 bool ConstWithoutErrnoOrExceptions =
2907 ConstWithoutErrnoAndExceptions || ConstWithoutExceptions;
2908 bool GenerateIntrinsics =
2909 (ConstAlways && !OptNone) ||
2911 !(ErrnoOverriden.has_value() && ErrnoOverriden.value()) && !OptNone);
2912 if (!GenerateIntrinsics) {
2913 GenerateIntrinsics =
2914 ConstWithoutErrnoOrExceptions && !ConstWithoutErrnoAndExceptions;
2915 if (!GenerateIntrinsics)
2916 GenerateIntrinsics =
2917 ConstWithoutErrnoOrExceptions &&
2919 !(ErrnoOverriden.has_value() && ErrnoOverriden.value()) && !OptNone);
2920 if (!GenerateIntrinsics)
2921 GenerateIntrinsics =
2922 ConstWithoutErrnoOrExceptions && ErrnoOverridenToFalseWithOpt;
2924 if (GenerateIntrinsics) {
2925 switch (BuiltinIDIfNoAsmLabel) {
2926 case Builtin::BIacos:
2927 case Builtin::BIacosf:
2928 case Builtin::BIacosl:
2929 case Builtin::BI__builtin_acos:
2930 case Builtin::BI__builtin_acosf:
2931 case Builtin::BI__builtin_acosf16:
2932 case Builtin::BI__builtin_acosl:
2933 case Builtin::BI__builtin_acosf128:
2935 *
this,
E, Intrinsic::acos, Intrinsic::experimental_constrained_acos));
2937 case Builtin::BIasin:
2938 case Builtin::BIasinf:
2939 case Builtin::BIasinl:
2940 case Builtin::BI__builtin_asin:
2941 case Builtin::BI__builtin_asinf:
2942 case Builtin::BI__builtin_asinf16:
2943 case Builtin::BI__builtin_asinl:
2944 case Builtin::BI__builtin_asinf128:
2946 *
this,
E, Intrinsic::asin, Intrinsic::experimental_constrained_asin));
2948 case Builtin::BIatan:
2949 case Builtin::BIatanf:
2950 case Builtin::BIatanl:
2951 case Builtin::BI__builtin_atan:
2952 case Builtin::BI__builtin_atanf:
2953 case Builtin::BI__builtin_atanf16:
2954 case Builtin::BI__builtin_atanl:
2955 case Builtin::BI__builtin_atanf128:
2957 *
this,
E, Intrinsic::atan, Intrinsic::experimental_constrained_atan));
2959 case Builtin::BIatan2:
2960 case Builtin::BIatan2f:
2961 case Builtin::BIatan2l:
2962 case Builtin::BI__builtin_atan2:
2963 case Builtin::BI__builtin_atan2f:
2964 case Builtin::BI__builtin_atan2f16:
2965 case Builtin::BI__builtin_atan2l:
2966 case Builtin::BI__builtin_atan2f128:
2968 *
this,
E, Intrinsic::atan2,
2969 Intrinsic::experimental_constrained_atan2));
2971 case Builtin::BIceil:
2972 case Builtin::BIceilf:
2973 case Builtin::BIceill:
2974 case Builtin::BI__builtin_ceil:
2975 case Builtin::BI__builtin_ceilf:
2976 case Builtin::BI__builtin_ceilf16:
2977 case Builtin::BI__builtin_ceill:
2978 case Builtin::BI__builtin_ceilf128:
2981 Intrinsic::experimental_constrained_ceil));
2983 case Builtin::BIcopysign:
2984 case Builtin::BIcopysignf:
2985 case Builtin::BIcopysignl:
2986 case Builtin::BI__builtin_copysign:
2987 case Builtin::BI__builtin_copysignf:
2988 case Builtin::BI__builtin_copysignf16:
2989 case Builtin::BI__builtin_copysignl:
2990 case Builtin::BI__builtin_copysignf128:
2992 emitBuiltinWithOneOverloadedType<2>(*
this,
E, Intrinsic::copysign));
2994 case Builtin::BIcos:
2995 case Builtin::BIcosf:
2996 case Builtin::BIcosl:
2997 case Builtin::BI__builtin_cos:
2998 case Builtin::BI__builtin_cosf:
2999 case Builtin::BI__builtin_cosf16:
3000 case Builtin::BI__builtin_cosl:
3001 case Builtin::BI__builtin_cosf128:
3004 Intrinsic::experimental_constrained_cos));
3006 case Builtin::BIcosh:
3007 case Builtin::BIcoshf:
3008 case Builtin::BIcoshl:
3009 case Builtin::BI__builtin_cosh:
3010 case Builtin::BI__builtin_coshf:
3011 case Builtin::BI__builtin_coshf16:
3012 case Builtin::BI__builtin_coshl:
3013 case Builtin::BI__builtin_coshf128:
3015 *
this,
E, Intrinsic::cosh, Intrinsic::experimental_constrained_cosh));
3017 case Builtin::BIexp:
3018 case Builtin::BIexpf:
3019 case Builtin::BIexpl:
3020 case Builtin::BI__builtin_exp:
3021 case Builtin::BI__builtin_expf:
3022 case Builtin::BI__builtin_expf16:
3023 case Builtin::BI__builtin_expl:
3024 case Builtin::BI__builtin_expf128:
3027 Intrinsic::experimental_constrained_exp));
3029 case Builtin::BIexp2:
3030 case Builtin::BIexp2f:
3031 case Builtin::BIexp2l:
3032 case Builtin::BI__builtin_exp2:
3033 case Builtin::BI__builtin_exp2f:
3034 case Builtin::BI__builtin_exp2f16:
3035 case Builtin::BI__builtin_exp2l:
3036 case Builtin::BI__builtin_exp2f128:
3039 Intrinsic::experimental_constrained_exp2));
3040 case Builtin::BI__builtin_exp10:
3041 case Builtin::BI__builtin_exp10f:
3042 case Builtin::BI__builtin_exp10f16:
3043 case Builtin::BI__builtin_exp10l:
3044 case Builtin::BI__builtin_exp10f128: {
3046 if (
Builder.getIsFPConstrained())
3049 emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::exp10));
3051 case Builtin::BIfabs:
3052 case Builtin::BIfabsf:
3053 case Builtin::BIfabsl:
3054 case Builtin::BI__builtin_fabs:
3055 case Builtin::BI__builtin_fabsf:
3056 case Builtin::BI__builtin_fabsf16:
3057 case Builtin::BI__builtin_fabsl:
3058 case Builtin::BI__builtin_fabsf128:
3060 emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::fabs));
3062 case Builtin::BIfloor:
3063 case Builtin::BIfloorf:
3064 case Builtin::BIfloorl:
3065 case Builtin::BI__builtin_floor:
3066 case Builtin::BI__builtin_floorf:
3067 case Builtin::BI__builtin_floorf16:
3068 case Builtin::BI__builtin_floorl:
3069 case Builtin::BI__builtin_floorf128:
3072 Intrinsic::experimental_constrained_floor));
3074 case Builtin::BIfma:
3075 case Builtin::BIfmaf:
3076 case Builtin::BIfmal:
3077 case Builtin::BI__builtin_fma:
3078 case Builtin::BI__builtin_fmaf:
3079 case Builtin::BI__builtin_fmaf16:
3080 case Builtin::BI__builtin_fmal:
3081 case Builtin::BI__builtin_fmaf128:
3084 Intrinsic::experimental_constrained_fma));
3086 case Builtin::BIfmax:
3087 case Builtin::BIfmaxf:
3088 case Builtin::BIfmaxl:
3089 case Builtin::BI__builtin_fmax:
3090 case Builtin::BI__builtin_fmaxf:
3091 case Builtin::BI__builtin_fmaxf16:
3092 case Builtin::BI__builtin_fmaxl:
3093 case Builtin::BI__builtin_fmaxf128:
3096 Intrinsic::experimental_constrained_maxnum));
3098 case Builtin::BIfmin:
3099 case Builtin::BIfminf:
3100 case Builtin::BIfminl:
3101 case Builtin::BI__builtin_fmin:
3102 case Builtin::BI__builtin_fminf:
3103 case Builtin::BI__builtin_fminf16:
3104 case Builtin::BI__builtin_fminl:
3105 case Builtin::BI__builtin_fminf128:
3108 Intrinsic::experimental_constrained_minnum));
3110 case Builtin::BIfmaximum_num:
3111 case Builtin::BIfmaximum_numf:
3112 case Builtin::BIfmaximum_numl:
3113 case Builtin::BI__builtin_fmaximum_num:
3114 case Builtin::BI__builtin_fmaximum_numf:
3115 case Builtin::BI__builtin_fmaximum_numf16:
3116 case Builtin::BI__builtin_fmaximum_numl:
3117 case Builtin::BI__builtin_fmaximum_numf128:
3119 emitBuiltinWithOneOverloadedType<2>(*
this,
E, Intrinsic::maximumnum));
3121 case Builtin::BIfminimum_num:
3122 case Builtin::BIfminimum_numf:
3123 case Builtin::BIfminimum_numl:
3124 case Builtin::BI__builtin_fminimum_num:
3125 case Builtin::BI__builtin_fminimum_numf:
3126 case Builtin::BI__builtin_fminimum_numf16:
3127 case Builtin::BI__builtin_fminimum_numl:
3128 case Builtin::BI__builtin_fminimum_numf128:
3130 emitBuiltinWithOneOverloadedType<2>(*
this,
E, Intrinsic::minimumnum));
3134 case Builtin::BIfmod:
3135 case Builtin::BIfmodf:
3136 case Builtin::BIfmodl:
3137 case Builtin::BI__builtin_fmod:
3138 case Builtin::BI__builtin_fmodf:
3139 case Builtin::BI__builtin_fmodf16:
3140 case Builtin::BI__builtin_fmodl:
3141 case Builtin::BI__builtin_fmodf128:
3142 case Builtin::BI__builtin_elementwise_fmod: {
3143 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
3149 case Builtin::BIlog:
3150 case Builtin::BIlogf:
3151 case Builtin::BIlogl:
3152 case Builtin::BI__builtin_log:
3153 case Builtin::BI__builtin_logf:
3154 case Builtin::BI__builtin_logf16:
3155 case Builtin::BI__builtin_logl:
3156 case Builtin::BI__builtin_logf128:
3159 Intrinsic::experimental_constrained_log));
3161 case Builtin::BIlog10:
3162 case Builtin::BIlog10f:
3163 case Builtin::BIlog10l:
3164 case Builtin::BI__builtin_log10:
3165 case Builtin::BI__builtin_log10f:
3166 case Builtin::BI__builtin_log10f16:
3167 case Builtin::BI__builtin_log10l:
3168 case Builtin::BI__builtin_log10f128:
3171 Intrinsic::experimental_constrained_log10));
3173 case Builtin::BIlog2:
3174 case Builtin::BIlog2f:
3175 case Builtin::BIlog2l:
3176 case Builtin::BI__builtin_log2:
3177 case Builtin::BI__builtin_log2f:
3178 case Builtin::BI__builtin_log2f16:
3179 case Builtin::BI__builtin_log2l:
3180 case Builtin::BI__builtin_log2f128:
3183 Intrinsic::experimental_constrained_log2));
3185 case Builtin::BInearbyint:
3186 case Builtin::BInearbyintf:
3187 case Builtin::BInearbyintl:
3188 case Builtin::BI__builtin_nearbyint:
3189 case Builtin::BI__builtin_nearbyintf:
3190 case Builtin::BI__builtin_nearbyintl:
3191 case Builtin::BI__builtin_nearbyintf128:
3193 Intrinsic::nearbyint,
3194 Intrinsic::experimental_constrained_nearbyint));
3196 case Builtin::BIpow:
3197 case Builtin::BIpowf:
3198 case Builtin::BIpowl:
3199 case Builtin::BI__builtin_pow:
3200 case Builtin::BI__builtin_powf:
3201 case Builtin::BI__builtin_powf16:
3202 case Builtin::BI__builtin_powl:
3203 case Builtin::BI__builtin_powf128:
3206 Intrinsic::experimental_constrained_pow));
3208 case Builtin::BIrint:
3209 case Builtin::BIrintf:
3210 case Builtin::BIrintl:
3211 case Builtin::BI__builtin_rint:
3212 case Builtin::BI__builtin_rintf:
3213 case Builtin::BI__builtin_rintf16:
3214 case Builtin::BI__builtin_rintl:
3215 case Builtin::BI__builtin_rintf128:
3218 Intrinsic::experimental_constrained_rint));
3220 case Builtin::BIround:
3221 case Builtin::BIroundf:
3222 case Builtin::BIroundl:
3223 case Builtin::BI__builtin_round:
3224 case Builtin::BI__builtin_roundf:
3225 case Builtin::BI__builtin_roundf16:
3226 case Builtin::BI__builtin_roundl:
3227 case Builtin::BI__builtin_roundf128:
3230 Intrinsic::experimental_constrained_round));
3232 case Builtin::BIroundeven:
3233 case Builtin::BIroundevenf:
3234 case Builtin::BIroundevenl:
3235 case Builtin::BI__builtin_roundeven:
3236 case Builtin::BI__builtin_roundevenf:
3237 case Builtin::BI__builtin_roundevenf16:
3238 case Builtin::BI__builtin_roundevenl:
3239 case Builtin::BI__builtin_roundevenf128:
3241 Intrinsic::roundeven,
3242 Intrinsic::experimental_constrained_roundeven));
3244 case Builtin::BIsin:
3245 case Builtin::BIsinf:
3246 case Builtin::BIsinl:
3247 case Builtin::BI__builtin_sin:
3248 case Builtin::BI__builtin_sinf:
3249 case Builtin::BI__builtin_sinf16:
3250 case Builtin::BI__builtin_sinl:
3251 case Builtin::BI__builtin_sinf128:
3254 Intrinsic::experimental_constrained_sin));
3256 case Builtin::BIsinh:
3257 case Builtin::BIsinhf:
3258 case Builtin::BIsinhl:
3259 case Builtin::BI__builtin_sinh:
3260 case Builtin::BI__builtin_sinhf:
3261 case Builtin::BI__builtin_sinhf16:
3262 case Builtin::BI__builtin_sinhl:
3263 case Builtin::BI__builtin_sinhf128:
3265 *
this,
E, Intrinsic::sinh, Intrinsic::experimental_constrained_sinh));
3267 case Builtin::BI__builtin_sincos:
3268 case Builtin::BI__builtin_sincosf:
3269 case Builtin::BI__builtin_sincosf16:
3270 case Builtin::BI__builtin_sincosl:
3271 case Builtin::BI__builtin_sincosf128:
3275 case Builtin::BIsqrt:
3276 case Builtin::BIsqrtf:
3277 case Builtin::BIsqrtl:
3278 case Builtin::BI__builtin_sqrt:
3279 case Builtin::BI__builtin_sqrtf:
3280 case Builtin::BI__builtin_sqrtf16:
3281 case Builtin::BI__builtin_sqrtl:
3282 case Builtin::BI__builtin_sqrtf128:
3283 case Builtin::BI__builtin_elementwise_sqrt: {
3285 *
this,
E, Intrinsic::sqrt, Intrinsic::experimental_constrained_sqrt);
3290 case Builtin::BItan:
3291 case Builtin::BItanf:
3292 case Builtin::BItanl:
3293 case Builtin::BI__builtin_tan:
3294 case Builtin::BI__builtin_tanf:
3295 case Builtin::BI__builtin_tanf16:
3296 case Builtin::BI__builtin_tanl:
3297 case Builtin::BI__builtin_tanf128:
3299 *
this,
E, Intrinsic::tan, Intrinsic::experimental_constrained_tan));
3301 case Builtin::BItanh:
3302 case Builtin::BItanhf:
3303 case Builtin::BItanhl:
3304 case Builtin::BI__builtin_tanh:
3305 case Builtin::BI__builtin_tanhf:
3306 case Builtin::BI__builtin_tanhf16:
3307 case Builtin::BI__builtin_tanhl:
3308 case Builtin::BI__builtin_tanhf128:
3310 *
this,
E, Intrinsic::tanh, Intrinsic::experimental_constrained_tanh));
3312 case Builtin::BItrunc:
3313 case Builtin::BItruncf:
3314 case Builtin::BItruncl:
3315 case Builtin::BI__builtin_trunc:
3316 case Builtin::BI__builtin_truncf:
3317 case Builtin::BI__builtin_truncf16:
3318 case Builtin::BI__builtin_truncl:
3319 case Builtin::BI__builtin_truncf128:
3322 Intrinsic::experimental_constrained_trunc));
3324 case Builtin::BIlround:
3325 case Builtin::BIlroundf:
3326 case Builtin::BIlroundl:
3327 case Builtin::BI__builtin_lround:
3328 case Builtin::BI__builtin_lroundf:
3329 case Builtin::BI__builtin_lroundl:
3330 case Builtin::BI__builtin_lroundf128:
3332 *
this,
E, Intrinsic::lround,
3333 Intrinsic::experimental_constrained_lround));
3335 case Builtin::BIllround:
3336 case Builtin::BIllroundf:
3337 case Builtin::BIllroundl:
3338 case Builtin::BI__builtin_llround:
3339 case Builtin::BI__builtin_llroundf:
3340 case Builtin::BI__builtin_llroundl:
3341 case Builtin::BI__builtin_llroundf128:
3343 *
this,
E, Intrinsic::llround,
3344 Intrinsic::experimental_constrained_llround));
3346 case Builtin::BIlrint:
3347 case Builtin::BIlrintf:
3348 case Builtin::BIlrintl:
3349 case Builtin::BI__builtin_lrint:
3350 case Builtin::BI__builtin_lrintf:
3351 case Builtin::BI__builtin_lrintl:
3352 case Builtin::BI__builtin_lrintf128:
3354 *
this,
E, Intrinsic::lrint,
3355 Intrinsic::experimental_constrained_lrint));
3357 case Builtin::BIllrint:
3358 case Builtin::BIllrintf:
3359 case Builtin::BIllrintl:
3360 case Builtin::BI__builtin_llrint:
3361 case Builtin::BI__builtin_llrintf:
3362 case Builtin::BI__builtin_llrintl:
3363 case Builtin::BI__builtin_llrintf128:
3365 *
this,
E, Intrinsic::llrint,
3366 Intrinsic::experimental_constrained_llrint));
3367 case Builtin::BI__builtin_ldexp:
3368 case Builtin::BI__builtin_ldexpf:
3369 case Builtin::BI__builtin_ldexpl:
3370 case Builtin::BI__builtin_ldexpf16:
3371 case Builtin::BI__builtin_ldexpf128: {
3373 *
this,
E, Intrinsic::ldexp,
3374 Intrinsic::experimental_constrained_ldexp));
3384 Value *Val = A.emitRawPointer(*
this);
3390 SkippedChecks.
set(SanitizerKind::All);
3391 SkippedChecks.
clear(SanitizerKind::Alignment);
3394 if (
auto *CE = dyn_cast<ImplicitCastExpr>(Arg))
3395 if (CE->getCastKind() == CK_BitCast)
3396 Arg = CE->getSubExpr();
3402 switch (BuiltinIDIfNoAsmLabel) {
3404 case Builtin::BI__builtin___CFStringMakeConstantString:
3405 case Builtin::BI__builtin___NSStringMakeConstantString:
3407 case Builtin::BI__builtin_stdarg_start:
3408 case Builtin::BI__builtin_va_start:
3409 case Builtin::BI__va_start:
3410 case Builtin::BI__builtin_va_end:
3414 BuiltinID != Builtin::BI__builtin_va_end);
3416 case Builtin::BI__builtin_va_copy: {
3423 case Builtin::BIabs:
3424 case Builtin::BIlabs:
3425 case Builtin::BIllabs:
3426 case Builtin::BI__builtin_abs:
3427 case Builtin::BI__builtin_labs:
3428 case Builtin::BI__builtin_llabs: {
3429 bool SanitizeOverflow =
SanOpts.
has(SanitizerKind::SignedIntegerOverflow);
3432 switch (
getLangOpts().getSignedOverflowBehavior()) {
3437 if (!SanitizeOverflow) {
3449 case Builtin::BI__builtin_complex: {
3454 case Builtin::BI__builtin_conj:
3455 case Builtin::BI__builtin_conjf:
3456 case Builtin::BI__builtin_conjl:
3457 case Builtin::BIconj:
3458 case Builtin::BIconjf:
3459 case Builtin::BIconjl: {
3461 Value *Real = ComplexVal.first;
3462 Value *Imag = ComplexVal.second;
3463 Imag =
Builder.CreateFNeg(Imag,
"neg");
3466 case Builtin::BI__builtin_creal:
3467 case Builtin::BI__builtin_crealf:
3468 case Builtin::BI__builtin_creall:
3469 case Builtin::BIcreal:
3470 case Builtin::BIcrealf:
3471 case Builtin::BIcreall: {
3476 case Builtin::BI__builtin_preserve_access_index: {
3497 case Builtin::BI__builtin_cimag:
3498 case Builtin::BI__builtin_cimagf:
3499 case Builtin::BI__builtin_cimagl:
3500 case Builtin::BIcimag:
3501 case Builtin::BIcimagf:
3502 case Builtin::BIcimagl: {
3507 case Builtin::BI__builtin_clrsb:
3508 case Builtin::BI__builtin_clrsbl:
3509 case Builtin::BI__builtin_clrsbll: {
3513 llvm::Type *ArgType = ArgValue->
getType();
3517 Value *
Zero = llvm::Constant::getNullValue(ArgType);
3518 Value *IsNeg =
Builder.CreateICmpSLT(ArgValue, Zero,
"isneg");
3520 Value *Tmp =
Builder.CreateSelect(IsNeg, Inverse, ArgValue);
3527 case Builtin::BI__builtin_ctzs:
3528 case Builtin::BI__builtin_ctz:
3529 case Builtin::BI__builtin_ctzl:
3530 case Builtin::BI__builtin_ctzll:
3531 case Builtin::BI__builtin_ctzg: {
3532 bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_ctzg &&
3533 E->getNumArgs() > 1;
3539 llvm::Type *ArgType = ArgValue->
getType();
3546 if (
Result->getType() != ResultType)
3552 Value *
Zero = Constant::getNullValue(ArgType);
3553 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, Zero,
"iszero");
3555 Value *ResultOrFallback =
3556 Builder.CreateSelect(IsZero, FallbackValue,
Result,
"ctzg");
3559 case Builtin::BI__builtin_clzs:
3560 case Builtin::BI__builtin_clz:
3561 case Builtin::BI__builtin_clzl:
3562 case Builtin::BI__builtin_clzll:
3563 case Builtin::BI__builtin_clzg: {
3564 bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_clzg &&
3565 E->getNumArgs() > 1;
3571 llvm::Type *ArgType = ArgValue->
getType();
3578 if (
Result->getType() != ResultType)
3584 Value *
Zero = Constant::getNullValue(ArgType);
3585 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, Zero,
"iszero");
3587 Value *ResultOrFallback =
3588 Builder.CreateSelect(IsZero, FallbackValue,
Result,
"clzg");
3591 case Builtin::BI__builtin_ffs:
3592 case Builtin::BI__builtin_ffsl:
3593 case Builtin::BI__builtin_ffsll: {
3597 llvm::Type *ArgType = ArgValue->
getType();
3602 Builder.CreateAdd(
Builder.CreateCall(F, {ArgValue, Builder.getTrue()}),
3603 llvm::ConstantInt::get(ArgType, 1));
3604 Value *
Zero = llvm::Constant::getNullValue(ArgType);
3605 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, Zero,
"iszero");
3607 if (
Result->getType() != ResultType)
3612 case Builtin::BI__builtin_parity:
3613 case Builtin::BI__builtin_parityl:
3614 case Builtin::BI__builtin_parityll: {
3618 llvm::Type *ArgType = ArgValue->
getType();
3624 if (
Result->getType() != ResultType)
3629 case Builtin::BI__lzcnt16:
3630 case Builtin::BI__lzcnt:
3631 case Builtin::BI__lzcnt64: {
3634 llvm::Type *ArgType = ArgValue->
getType();
3639 if (
Result->getType() != ResultType)
3644 case Builtin::BI__popcnt16:
3645 case Builtin::BI__popcnt:
3646 case Builtin::BI__popcnt64:
3647 case Builtin::BI__builtin_popcount:
3648 case Builtin::BI__builtin_popcountl:
3649 case Builtin::BI__builtin_popcountll:
3650 case Builtin::BI__builtin_popcountg: {
3653 llvm::Type *ArgType = ArgValue->
getType();
3658 if (
Result->getType() != ResultType)
3663 case Builtin::BI__builtin_unpredictable: {
3669 case Builtin::BI__builtin_expect: {
3671 llvm::Type *ArgType = ArgValue->
getType();
3682 Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue},
"expval");
3685 case Builtin::BI__builtin_expect_with_probability: {
3687 llvm::Type *ArgType = ArgValue->
getType();
3690 llvm::APFloat Probability(0.0);
3691 const Expr *ProbArg =
E->getArg(2);
3693 assert(EvalSucceed &&
"probability should be able to evaluate as float");
3695 bool LoseInfo =
false;
3696 Probability.convert(llvm::APFloat::IEEEdouble(),
3697 llvm::RoundingMode::Dynamic, &LoseInfo);
3699 Constant *Confidence = ConstantFP::get(Ty, Probability);
3709 FnExpect, {ArgValue, ExpectedValue, Confidence},
"expval");
3712 case Builtin::BI__builtin_assume_aligned: {
3713 const Expr *Ptr =
E->getArg(0);
3715 Value *OffsetValue =
3719 ConstantInt *AlignmentCI = cast<ConstantInt>(AlignmentValue);
3720 if (AlignmentCI->getValue().ugt(llvm::Value::MaximumAlignment))
3721 AlignmentCI = ConstantInt::get(AlignmentCI->getIntegerType(),
3722 llvm::Value::MaximumAlignment);
3726 AlignmentCI, OffsetValue);
3729 case Builtin::BI__assume:
3730 case Builtin::BI__builtin_assume: {
3736 Builder.CreateCall(FnAssume, ArgValue);
3739 case Builtin::BI__builtin_assume_separate_storage: {
3740 const Expr *Arg0 =
E->getArg(0);
3741 const Expr *Arg1 =
E->getArg(1);
3746 Value *Values[] = {Value0, Value1};
3747 OperandBundleDefT<Value *> OBD(
"separate_storage", Values);
3751 case Builtin::BI__builtin_allow_runtime_check: {
3755 llvm::Value *Allow =
Builder.CreateCall(
3757 llvm::MetadataAsValue::get(Ctx, llvm::MDString::get(Ctx, Kind)));
3760 case Builtin::BI__arithmetic_fence: {
3763 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
3764 llvm::FastMathFlags FMF =
Builder.getFastMathFlags();
3765 bool isArithmeticFenceEnabled =
3766 FMF.allowReassoc() &&
3770 if (isArithmeticFenceEnabled) {
3773 Value *Real =
Builder.CreateArithmeticFence(ComplexVal.first,
3775 Value *Imag =
Builder.CreateArithmeticFence(ComplexVal.second,
3780 Value *Real = ComplexVal.first;
3781 Value *Imag = ComplexVal.second;
3785 if (isArithmeticFenceEnabled)
3790 case Builtin::BI__builtin_bswap16:
3791 case Builtin::BI__builtin_bswap32:
3792 case Builtin::BI__builtin_bswap64:
3793 case Builtin::BI_byteswap_ushort:
3794 case Builtin::BI_byteswap_ulong:
3795 case Builtin::BI_byteswap_uint64: {
3797 emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::bswap));
3799 case Builtin::BI__builtin_bitreverse8:
3800 case Builtin::BI__builtin_bitreverse16:
3801 case Builtin::BI__builtin_bitreverse32:
3802 case Builtin::BI__builtin_bitreverse64: {
3804 emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::bitreverse));
3806 case Builtin::BI__builtin_rotateleft8:
3807 case Builtin::BI__builtin_rotateleft16:
3808 case Builtin::BI__builtin_rotateleft32:
3809 case Builtin::BI__builtin_rotateleft64:
3810 case Builtin::BI_rotl8:
3811 case Builtin::BI_rotl16:
3812 case Builtin::BI_rotl:
3813 case Builtin::BI_lrotl:
3814 case Builtin::BI_rotl64:
3817 case Builtin::BI__builtin_rotateright8:
3818 case Builtin::BI__builtin_rotateright16:
3819 case Builtin::BI__builtin_rotateright32:
3820 case Builtin::BI__builtin_rotateright64:
3821 case Builtin::BI_rotr8:
3822 case Builtin::BI_rotr16:
3823 case Builtin::BI_rotr:
3824 case Builtin::BI_lrotr:
3825 case Builtin::BI_rotr64:
3828 case Builtin::BI__builtin_constant_p: {
3831 const Expr *Arg =
E->getArg(0);
3839 return RValue::get(ConstantInt::get(ResultType, 0));
3844 return RValue::get(ConstantInt::get(ResultType, 0));
3856 if (
Result->getType() != ResultType)
3860 case Builtin::BI__builtin_dynamic_object_size:
3861 case Builtin::BI__builtin_object_size: {
3868 bool IsDynamic = BuiltinID == Builtin::BI__builtin_dynamic_object_size;
3870 nullptr, IsDynamic));
3872 case Builtin::BI__builtin_counted_by_ref: {
3874 llvm::Value *
Result = llvm::ConstantPointerNull::get(
3879 if (
auto *UO = dyn_cast<UnaryOperator>(Arg);
3880 UO && UO->getOpcode() == UO_AddrOf) {
3883 if (
auto *ASE = dyn_cast<ArraySubscriptExpr>(Arg))
3887 if (
const MemberExpr *ME = dyn_cast_if_present<MemberExpr>(Arg)) {
3891 const auto *FAMDecl = cast<FieldDecl>(ME->getMemberDecl());
3895 llvm::report_fatal_error(
"Cannot find the counted_by 'count' field");
3901 case Builtin::BI__builtin_prefetch: {
3905 llvm::ConstantInt::get(
Int32Ty, 0);
3907 llvm::ConstantInt::get(
Int32Ty, 3);
3913 case Builtin::BI__builtin_readcyclecounter: {
3917 case Builtin::BI__builtin_readsteadycounter: {
3921 case Builtin::BI__builtin___clear_cache: {
3927 case Builtin::BI__builtin_trap:
3930 case Builtin::BI__builtin_verbose_trap: {
3931 llvm::DILocation *TrapLocation =
Builder.getCurrentDebugLocation();
3942 case Builtin::BI__debugbreak:
3945 case Builtin::BI__builtin_unreachable: {
3954 case Builtin::BI__builtin_powi:
3955 case Builtin::BI__builtin_powif:
3956 case Builtin::BI__builtin_powil: {
3960 if (
Builder.getIsFPConstrained()) {
3963 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
3970 { Src0->getType(), Src1->getType() });
3973 case Builtin::BI__builtin_frexpl: {
3977 if (&
getTarget().getLongDoubleFormat() == &llvm::APFloat::PPCDoubleDouble())
3981 case Builtin::BI__builtin_frexp:
3982 case Builtin::BI__builtin_frexpf:
3983 case Builtin::BI__builtin_frexpf128:
3984 case Builtin::BI__builtin_frexpf16:
3986 case Builtin::BI__builtin_isgreater:
3987 case Builtin::BI__builtin_isgreaterequal:
3988 case Builtin::BI__builtin_isless:
3989 case Builtin::BI__builtin_islessequal:
3990 case Builtin::BI__builtin_islessgreater:
3991 case Builtin::BI__builtin_isunordered: {
3994 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
3998 switch (BuiltinID) {
3999 default: llvm_unreachable(
"Unknown ordered comparison");
4000 case Builtin::BI__builtin_isgreater:
4001 LHS =
Builder.CreateFCmpOGT(LHS, RHS,
"cmp");
4003 case Builtin::BI__builtin_isgreaterequal:
4004 LHS =
Builder.CreateFCmpOGE(LHS, RHS,
"cmp");
4006 case Builtin::BI__builtin_isless:
4007 LHS =
Builder.CreateFCmpOLT(LHS, RHS,
"cmp");
4009 case Builtin::BI__builtin_islessequal:
4010 LHS =
Builder.CreateFCmpOLE(LHS, RHS,
"cmp");
4012 case Builtin::BI__builtin_islessgreater:
4013 LHS =
Builder.CreateFCmpONE(LHS, RHS,
"cmp");
4015 case Builtin::BI__builtin_isunordered:
4016 LHS =
Builder.CreateFCmpUNO(LHS, RHS,
"cmp");
4023 case Builtin::BI__builtin_isnan: {
4024 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4033 case Builtin::BI__builtin_issignaling: {
4034 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4041 case Builtin::BI__builtin_isinf: {
4042 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4051 case Builtin::BIfinite:
4052 case Builtin::BI__finite:
4053 case Builtin::BIfinitef:
4054 case Builtin::BI__finitef:
4055 case Builtin::BIfinitel:
4056 case Builtin::BI__finitel:
4057 case Builtin::BI__builtin_isfinite: {
4058 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4067 case Builtin::BI__builtin_isnormal: {
4068 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4075 case Builtin::BI__builtin_issubnormal: {
4076 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4079 Builder.CreateZExt(
Builder.createIsFPClass(
V, FPClassTest::fcSubnormal),
4083 case Builtin::BI__builtin_iszero: {
4084 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4091 case Builtin::BI__builtin_isfpclass: {
4096 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4102 case Builtin::BI__builtin_nondeterministic_value: {
4111 case Builtin::BI__builtin_elementwise_abs: {
4116 QT = VecTy->getElementType();
4120 Builder.getFalse(),
nullptr,
"elt.abs");
4122 Result = emitBuiltinWithOneOverloadedType<1>(
4123 *
this,
E, llvm::Intrinsic::fabs,
"elt.abs");
4127 case Builtin::BI__builtin_elementwise_acos:
4128 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4129 *
this,
E, llvm::Intrinsic::acos,
"elt.acos"));
4130 case Builtin::BI__builtin_elementwise_asin:
4131 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4132 *
this,
E, llvm::Intrinsic::asin,
"elt.asin"));
4133 case Builtin::BI__builtin_elementwise_atan:
4134 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4135 *
this,
E, llvm::Intrinsic::atan,
"elt.atan"));
4136 case Builtin::BI__builtin_elementwise_atan2:
4137 return RValue::get(emitBuiltinWithOneOverloadedType<2>(
4138 *
this,
E, llvm::Intrinsic::atan2,
"elt.atan2"));
4139 case Builtin::BI__builtin_elementwise_ceil:
4140 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4141 *
this,
E, llvm::Intrinsic::ceil,
"elt.ceil"));
4142 case Builtin::BI__builtin_elementwise_exp:
4143 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4144 *
this,
E, llvm::Intrinsic::exp,
"elt.exp"));
4145 case Builtin::BI__builtin_elementwise_exp2:
4146 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4147 *
this,
E, llvm::Intrinsic::exp2,
"elt.exp2"));
4148 case Builtin::BI__builtin_elementwise_log:
4149 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4150 *
this,
E, llvm::Intrinsic::log,
"elt.log"));
4151 case Builtin::BI__builtin_elementwise_log2:
4152 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4153 *
this,
E, llvm::Intrinsic::log2,
"elt.log2"));
4154 case Builtin::BI__builtin_elementwise_log10:
4155 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4156 *
this,
E, llvm::Intrinsic::log10,
"elt.log10"));
4157 case Builtin::BI__builtin_elementwise_pow: {
4159 emitBuiltinWithOneOverloadedType<2>(*
this,
E, llvm::Intrinsic::pow));
4161 case Builtin::BI__builtin_elementwise_bitreverse:
4162 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4163 *
this,
E, llvm::Intrinsic::bitreverse,
"elt.bitreverse"));
4164 case Builtin::BI__builtin_elementwise_cos:
4165 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4166 *
this,
E, llvm::Intrinsic::cos,
"elt.cos"));
4167 case Builtin::BI__builtin_elementwise_cosh:
4168 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4169 *
this,
E, llvm::Intrinsic::cosh,
"elt.cosh"));
4170 case Builtin::BI__builtin_elementwise_floor:
4171 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4172 *
this,
E, llvm::Intrinsic::floor,
"elt.floor"));
4173 case Builtin::BI__builtin_elementwise_popcount:
4174 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4175 *
this,
E, llvm::Intrinsic::ctpop,
"elt.ctpop"));
4176 case Builtin::BI__builtin_elementwise_roundeven:
4177 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4178 *
this,
E, llvm::Intrinsic::roundeven,
"elt.roundeven"));
4179 case Builtin::BI__builtin_elementwise_round:
4180 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4181 *
this,
E, llvm::Intrinsic::round,
"elt.round"));
4182 case Builtin::BI__builtin_elementwise_rint:
4183 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4184 *
this,
E, llvm::Intrinsic::rint,
"elt.rint"));
4185 case Builtin::BI__builtin_elementwise_nearbyint:
4186 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4187 *
this,
E, llvm::Intrinsic::nearbyint,
"elt.nearbyint"));
4188 case Builtin::BI__builtin_elementwise_sin:
4189 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4190 *
this,
E, llvm::Intrinsic::sin,
"elt.sin"));
4191 case Builtin::BI__builtin_elementwise_sinh:
4192 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4193 *
this,
E, llvm::Intrinsic::sinh,
"elt.sinh"));
4194 case Builtin::BI__builtin_elementwise_tan:
4195 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4196 *
this,
E, llvm::Intrinsic::tan,
"elt.tan"));
4197 case Builtin::BI__builtin_elementwise_tanh:
4198 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4199 *
this,
E, llvm::Intrinsic::tanh,
"elt.tanh"));
4200 case Builtin::BI__builtin_elementwise_trunc:
4201 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4202 *
this,
E, llvm::Intrinsic::trunc,
"elt.trunc"));
4203 case Builtin::BI__builtin_elementwise_canonicalize:
4204 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4205 *
this,
E, llvm::Intrinsic::canonicalize,
"elt.canonicalize"));
4206 case Builtin::BI__builtin_elementwise_copysign:
4207 return RValue::get(emitBuiltinWithOneOverloadedType<2>(
4208 *
this,
E, llvm::Intrinsic::copysign));
4209 case Builtin::BI__builtin_elementwise_fma:
4211 emitBuiltinWithOneOverloadedType<3>(*
this,
E, llvm::Intrinsic::fma));
4212 case Builtin::BI__builtin_elementwise_add_sat:
4213 case Builtin::BI__builtin_elementwise_sub_sat: {
4217 assert(Op0->
getType()->isIntOrIntVectorTy() &&
"integer type expected");
4220 Ty = VecTy->getElementType();
4223 if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_add_sat)
4224 Opc = IsSigned ? llvm::Intrinsic::sadd_sat : llvm::Intrinsic::uadd_sat;
4226 Opc = IsSigned ? llvm::Intrinsic::ssub_sat : llvm::Intrinsic::usub_sat;
4227 Result =
Builder.CreateBinaryIntrinsic(Opc, Op0, Op1,
nullptr,
"elt.sat");
4231 case Builtin::BI__builtin_elementwise_max: {
4235 if (Op0->
getType()->isIntOrIntVectorTy()) {
4238 Ty = VecTy->getElementType();
4240 ? llvm::Intrinsic::smax
4241 : llvm::Intrinsic::umax,
4242 Op0, Op1,
nullptr,
"elt.max");
4247 case Builtin::BI__builtin_elementwise_min: {
4251 if (Op0->
getType()->isIntOrIntVectorTy()) {
4254 Ty = VecTy->getElementType();
4256 ? llvm::Intrinsic::smin
4257 : llvm::Intrinsic::umin,
4258 Op0, Op1,
nullptr,
"elt.min");
4264 case Builtin::BI__builtin_elementwise_maximum: {
4268 Op1,
nullptr,
"elt.maximum");
4272 case Builtin::BI__builtin_elementwise_minimum: {
4276 Op1,
nullptr,
"elt.minimum");
4280 case Builtin::BI__builtin_reduce_max: {
4281 auto GetIntrinsicID = [
this](
QualType QT) {
4283 QT = VecTy->getElementType();
4288 return llvm::Intrinsic::vector_reduce_smax;
4290 return llvm::Intrinsic::vector_reduce_umax;
4292 return llvm::Intrinsic::vector_reduce_fmax;
4294 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4295 *
this,
E, GetIntrinsicID(
E->getArg(0)->
getType()),
"rdx.min"));
4298 case Builtin::BI__builtin_reduce_min: {
4299 auto GetIntrinsicID = [
this](
QualType QT) {
4301 QT = VecTy->getElementType();
4306 return llvm::Intrinsic::vector_reduce_smin;
4308 return llvm::Intrinsic::vector_reduce_umin;
4310 return llvm::Intrinsic::vector_reduce_fmin;
4313 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4314 *
this,
E, GetIntrinsicID(
E->getArg(0)->
getType()),
"rdx.min"));
4317 case Builtin::BI__builtin_reduce_add:
4318 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4319 *
this,
E, llvm::Intrinsic::vector_reduce_add,
"rdx.add"));
4320 case Builtin::BI__builtin_reduce_mul:
4321 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4322 *
this,
E, llvm::Intrinsic::vector_reduce_mul,
"rdx.mul"));
4323 case Builtin::BI__builtin_reduce_xor:
4324 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4325 *
this,
E, llvm::Intrinsic::vector_reduce_xor,
"rdx.xor"));
4326 case Builtin::BI__builtin_reduce_or:
4327 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4328 *
this,
E, llvm::Intrinsic::vector_reduce_or,
"rdx.or"));
4329 case Builtin::BI__builtin_reduce_and:
4330 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4331 *
this,
E, llvm::Intrinsic::vector_reduce_and,
"rdx.and"));
4332 case Builtin::BI__builtin_reduce_maximum:
4333 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4334 *
this,
E, llvm::Intrinsic::vector_reduce_fmaximum,
"rdx.maximum"));
4335 case Builtin::BI__builtin_reduce_minimum:
4336 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4337 *
this,
E, llvm::Intrinsic::vector_reduce_fminimum,
"rdx.minimum"));
4339 case Builtin::BI__builtin_matrix_transpose: {
4343 Value *
Result = MB.CreateMatrixTranspose(MatValue, MatrixTy->getNumRows(),
4344 MatrixTy->getNumColumns());
4348 case Builtin::BI__builtin_matrix_column_major_load: {
4354 assert(PtrTy &&
"arg0 must be of pointer type");
4364 ResultTy->getNumRows(), ResultTy->getNumColumns(),
"matrix");
4368 case Builtin::BI__builtin_matrix_column_major_store: {
4376 assert(PtrTy &&
"arg1 must be of pointer type");
4385 MatrixTy->getNumRows(), MatrixTy->getNumColumns());
4389 case Builtin::BI__builtin_isinf_sign: {
4391 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4396 AbsArg, ConstantFP::getInfinity(Arg->
getType()),
"isinf");
4402 Value *NegativeOne = ConstantInt::get(
IntTy, -1);
4403 Value *SignResult =
Builder.CreateSelect(IsNeg, NegativeOne, One);
4408 case Builtin::BI__builtin_flt_rounds: {
4413 if (
Result->getType() != ResultType)
4419 case Builtin::BI__builtin_set_flt_rounds: {
4427 case Builtin::BI__builtin_fpclassify: {
4428 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4439 "fpclassify_result");
4443 Value *IsZero =
Builder.CreateFCmpOEQ(
V, Constant::getNullValue(Ty),
4447 Builder.CreateCondBr(IsZero, End, NotZero);
4451 Builder.SetInsertPoint(NotZero);
4455 Builder.CreateCondBr(IsNan, End, NotNan);
4456 Result->addIncoming(NanLiteral, NotZero);
4459 Builder.SetInsertPoint(NotNan);
4462 Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(
V->getType()),
4466 Builder.CreateCondBr(IsInf, End, NotInf);
4467 Result->addIncoming(InfLiteral, NotNan);
4470 Builder.SetInsertPoint(NotInf);
4471 APFloat Smallest = APFloat::getSmallestNormalized(
4474 Builder.CreateFCmpUGE(VAbs, ConstantFP::get(
V->getContext(), Smallest),
4476 Value *NormalResult =
4480 Result->addIncoming(NormalResult, NotInf);
4493 case Builtin::BIalloca:
4494 case Builtin::BI_alloca:
4495 case Builtin::BI__builtin_alloca_uninitialized:
4496 case Builtin::BI__builtin_alloca: {
4500 const Align SuitableAlignmentInBytes =
4504 AllocaInst *AI =
Builder.CreateAlloca(
Builder.getInt8Ty(), Size);
4505 AI->setAlignment(SuitableAlignmentInBytes);
4506 if (BuiltinID != Builtin::BI__builtin_alloca_uninitialized)
4518 case Builtin::BI__builtin_alloca_with_align_uninitialized:
4519 case Builtin::BI__builtin_alloca_with_align: {
4522 auto *AlignmentInBitsCI = cast<ConstantInt>(AlignmentInBitsValue);
4523 unsigned AlignmentInBits = AlignmentInBitsCI->getZExtValue();
4524 const Align AlignmentInBytes =
4526 AllocaInst *AI =
Builder.CreateAlloca(
Builder.getInt8Ty(), Size);
4527 AI->setAlignment(AlignmentInBytes);
4528 if (BuiltinID != Builtin::BI__builtin_alloca_with_align_uninitialized)
4540 case Builtin::BIbzero:
4541 case Builtin::BI__builtin_bzero: {