35#include "llvm/ADT/APFloat.h"
36#include "llvm/ADT/APInt.h"
37#include "llvm/ADT/FloatingPointMode.h"
38#include "llvm/ADT/SmallPtrSet.h"
39#include "llvm/ADT/StringExtras.h"
40#include "llvm/Analysis/ValueTracking.h"
41#include "llvm/IR/DataLayout.h"
42#include "llvm/IR/InlineAsm.h"
43#include "llvm/IR/Intrinsics.h"
44#include "llvm/IR/IntrinsicsAArch64.h"
45#include "llvm/IR/IntrinsicsAMDGPU.h"
46#include "llvm/IR/IntrinsicsARM.h"
47#include "llvm/IR/IntrinsicsBPF.h"
48#include "llvm/IR/IntrinsicsDirectX.h"
49#include "llvm/IR/IntrinsicsHexagon.h"
50#include "llvm/IR/IntrinsicsNVPTX.h"
51#include "llvm/IR/IntrinsicsPowerPC.h"
52#include "llvm/IR/IntrinsicsR600.h"
53#include "llvm/IR/IntrinsicsRISCV.h"
54#include "llvm/IR/IntrinsicsS390.h"
55#include "llvm/IR/IntrinsicsVE.h"
56#include "llvm/IR/IntrinsicsWebAssembly.h"
57#include "llvm/IR/IntrinsicsX86.h"
58#include "llvm/IR/MDBuilder.h"
59#include "llvm/IR/MatrixBuilder.h"
60#include "llvm/IR/MemoryModelRelaxationAnnotations.h"
61#include "llvm/Support/ConvertUTF.h"
62#include "llvm/Support/MathExtras.h"
63#include "llvm/Support/ScopedPrinter.h"
64#include "llvm/TargetParser/AArch64TargetParser.h"
65#include "llvm/TargetParser/RISCVISAInfo.h"
66#include "llvm/TargetParser/X86TargetParser.h"
71using namespace CodeGen;
75 Align AlignmentInBytes) {
77 switch (CGF.
getLangOpts().getTrivialAutoVarInit()) {
78 case LangOptions::TrivialAutoVarInitKind::Uninitialized:
81 case LangOptions::TrivialAutoVarInitKind::Zero:
82 Byte = CGF.
Builder.getInt8(0x00);
84 case LangOptions::TrivialAutoVarInitKind::Pattern: {
86 Byte = llvm::dyn_cast<llvm::ConstantInt>(
94 I->addAnnotationMetadata(
"auto-init");
100 unsigned BuiltinID) {
109 static SmallDenseMap<unsigned, StringRef, 64> F128Builtins{
110 {Builtin::BI__builtin___fprintf_chk,
"__fprintf_chkieee128"},
111 {Builtin::BI__builtin___printf_chk,
"__printf_chkieee128"},
112 {Builtin::BI__builtin___snprintf_chk,
"__snprintf_chkieee128"},
113 {Builtin::BI__builtin___sprintf_chk,
"__sprintf_chkieee128"},
114 {Builtin::BI__builtin___vfprintf_chk,
"__vfprintf_chkieee128"},
115 {Builtin::BI__builtin___vprintf_chk,
"__vprintf_chkieee128"},
116 {Builtin::BI__builtin___vsnprintf_chk,
"__vsnprintf_chkieee128"},
117 {Builtin::BI__builtin___vsprintf_chk,
"__vsprintf_chkieee128"},
118 {Builtin::BI__builtin_fprintf,
"__fprintfieee128"},
119 {Builtin::BI__builtin_printf,
"__printfieee128"},
120 {Builtin::BI__builtin_snprintf,
"__snprintfieee128"},
121 {Builtin::BI__builtin_sprintf,
"__sprintfieee128"},
122 {Builtin::BI__builtin_vfprintf,
"__vfprintfieee128"},
123 {Builtin::BI__builtin_vprintf,
"__vprintfieee128"},
124 {Builtin::BI__builtin_vsnprintf,
"__vsnprintfieee128"},
125 {Builtin::BI__builtin_vsprintf,
"__vsprintfieee128"},
126 {Builtin::BI__builtin_fscanf,
"__fscanfieee128"},
127 {Builtin::BI__builtin_scanf,
"__scanfieee128"},
128 {Builtin::BI__builtin_sscanf,
"__sscanfieee128"},
129 {Builtin::BI__builtin_vfscanf,
"__vfscanfieee128"},
130 {Builtin::BI__builtin_vscanf,
"__vscanfieee128"},
131 {Builtin::BI__builtin_vsscanf,
"__vsscanfieee128"},
132 {Builtin::BI__builtin_nexttowardf128,
"__nexttowardieee128"},
138 static SmallDenseMap<unsigned, StringRef, 4> AIXLongDouble64Builtins{
139 {Builtin::BI__builtin_frexpl,
"frexp"},
140 {Builtin::BI__builtin_ldexpl,
"ldexp"},
141 {Builtin::BI__builtin_modfl,
"modf"},
147 if (FD->
hasAttr<AsmLabelAttr>())
153 &
getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad() &&
154 F128Builtins.contains(BuiltinID))
155 Name = F128Builtins[BuiltinID];
158 &llvm::APFloat::IEEEdouble() &&
159 AIXLongDouble64Builtins.contains(BuiltinID))
160 Name = AIXLongDouble64Builtins[BuiltinID];
165 llvm::FunctionType *Ty =
168 return GetOrCreateLLVMFunction(Name, Ty,
D,
false);
174 QualType T, llvm::IntegerType *IntType) {
177 if (
V->getType()->isPointerTy())
178 return CGF.
Builder.CreatePtrToInt(
V, IntType);
180 assert(
V->getType() == IntType);
188 if (ResultType->isPointerTy())
189 return CGF.
Builder.CreateIntToPtr(
V, ResultType);
191 assert(
V->getType() == ResultType);
202 if (Align % Bytes != 0) {
215 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
225 llvm::IntegerType *IntType = llvm::IntegerType::get(
229 llvm::Type *ValueType = Val->getType();
257 llvm::AtomicRMWInst::BinOp Kind,
266 llvm::AtomicRMWInst::BinOp Kind,
268 Instruction::BinaryOps Op,
269 bool Invert =
false) {
278 llvm::IntegerType *IntType = llvm::IntegerType::get(
282 llvm::Type *ValueType = Val->getType();
286 Kind, DestAddr, Val, llvm::AtomicOrdering::SequentiallyConsistent);
291 llvm::ConstantInt::getAllOnesValue(IntType));
315 llvm::IntegerType *IntType = llvm::IntegerType::get(
319 llvm::Type *ValueType = Cmp->getType();
324 DestAddr, Cmp, New, llvm::AtomicOrdering::SequentiallyConsistent,
325 llvm::AtomicOrdering::SequentiallyConsistent);
328 return CGF.
Builder.CreateZExt(CGF.
Builder.CreateExtractValue(Pair, 1),
351 AtomicOrdering SuccessOrdering = AtomicOrdering::SequentiallyConsistent) {
366 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release ?
367 AtomicOrdering::Monotonic :
375 DestAddr, Comparand, Exchange, SuccessOrdering, FailureOrdering);
376 Result->setVolatile(
true);
394 AtomicOrdering SuccessOrdering) {
395 assert(
E->getNumArgs() == 4);
401 assert(DestPtr->getType()->isPointerTy());
402 assert(!ExchangeHigh->getType()->isPointerTy());
403 assert(!ExchangeLow->getType()->isPointerTy());
406 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release
407 ? AtomicOrdering::Monotonic
412 llvm::Type *Int128Ty = llvm::IntegerType::get(CGF.
getLLVMContext(), 128);
413 Address DestAddr(DestPtr, Int128Ty,
418 ExchangeHigh = CGF.
Builder.CreateZExt(ExchangeHigh, Int128Ty);
419 ExchangeLow = CGF.
Builder.CreateZExt(ExchangeLow, Int128Ty);
421 CGF.
Builder.CreateShl(ExchangeHigh, llvm::ConstantInt::get(Int128Ty, 64));
422 llvm::Value *Exchange = CGF.
Builder.CreateOr(ExchangeHigh, ExchangeLow);
428 SuccessOrdering, FailureOrdering);
434 CXI->setVolatile(
true);
446 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
452 AtomicRMWInst::Add, DestAddr, ConstantInt::get(IntTy, 1), Ordering);
453 return CGF.
Builder.CreateAdd(
Result, ConstantInt::get(IntTy, 1));
458 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
464 AtomicRMWInst::Sub, DestAddr, ConstantInt::get(IntTy, 1), Ordering);
465 return CGF.
Builder.CreateSub(
Result, ConstantInt::get(IntTy, 1));
476 Load->setVolatile(
true);
486 llvm::StoreInst *Store =
488 Store->setVolatile(
true);
497 unsigned ConstrainedIntrinsicID) {
500 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
501 if (CGF.
Builder.getIsFPConstrained()) {
503 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0 });
506 return CGF.
Builder.CreateCall(F, Src0);
514 unsigned ConstrainedIntrinsicID) {
518 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
519 if (CGF.
Builder.getIsFPConstrained()) {
521 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0, Src1 });
524 return CGF.
Builder.CreateCall(F, { Src0, Src1 });
531 llvm::Intrinsic::ID ConstrainedIntrinsicID) {
535 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
536 if (CGF.
Builder.getIsFPConstrained()) {
538 {Src0->getType(), Src1->getType()});
539 return CGF.
Builder.CreateConstrainedFPCall(F, {Src0, Src1});
544 return CGF.
Builder.CreateCall(F, {Src0, Src1});
551 unsigned ConstrainedIntrinsicID) {
556 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
557 if (CGF.
Builder.getIsFPConstrained()) {
559 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0, Src1, Src2 });
562 return CGF.
Builder.CreateCall(F, { Src0, Src1, Src2 });
569 unsigned IntrinsicID,
570 unsigned ConstrainedIntrinsicID,
574 if (CGF.
Builder.getIsFPConstrained())
579 if (CGF.
Builder.getIsFPConstrained())
580 return CGF.
Builder.CreateConstrainedFPCall(F, Args);
582 return CGF.
Builder.CreateCall(F, Args);
590 unsigned IntrinsicID,
591 llvm::StringRef Name =
"") {
592 static_assert(N,
"expect non-empty argument");
594 for (
unsigned I = 0; I < N; ++I)
597 return CGF.
Builder.CreateCall(F, Args, Name);
603 unsigned IntrinsicID) {
608 return CGF.
Builder.CreateCall(F, {Src0, Src1});
614 unsigned IntrinsicID,
615 unsigned ConstrainedIntrinsicID) {
619 if (CGF.
Builder.getIsFPConstrained()) {
620 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
622 {ResultType, Src0->getType()});
623 return CGF.
Builder.CreateConstrainedFPCall(F, {Src0});
627 return CGF.
Builder.CreateCall(F, Src0);
632 llvm::Intrinsic::ID IntrinsicID) {
640 llvm::Value *
Call = CGF.
Builder.CreateCall(F, Src0);
642 llvm::Value *Exp = CGF.
Builder.CreateExtractValue(
Call, 1);
653 Call->setDoesNotAccessMemory();
662 llvm::Type *Ty =
V->getType();
663 int Width = Ty->getPrimitiveSizeInBits();
664 llvm::Type *IntTy = llvm::IntegerType::get(
C, Width);
666 if (Ty->isPPC_FP128Ty()) {
676 Value *ShiftCst = llvm::ConstantInt::get(IntTy, Width);
681 IntTy = llvm::IntegerType::get(
C, Width);
684 Value *Zero = llvm::Constant::getNullValue(IntTy);
685 return CGF.
Builder.CreateICmpSLT(
V, Zero);
689 const CallExpr *
E, llvm::Constant *calleeValue) {
690 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
697 auto IsErrnoIntrinsic = [&]() ->
unsigned {
699 case Builtin::BIexpf:
700 case Builtin::BI__builtin_expf:
701 case Builtin::BI__builtin_expf128:
710 !CGF.
Builder.getIsFPConstrained()) {
715 Instruction *Inst = cast<llvm::Instruction>(
Call.getScalarVal());
732 const llvm::Intrinsic::ID IntrinsicID,
733 llvm::Value *
X, llvm::Value *Y,
734 llvm::Value *&Carry) {
736 assert(
X->getType() == Y->getType() &&
737 "Arguments must be the same type. (Did you forget to make sure both "
738 "arguments have the same integer width?)");
741 llvm::Value *Tmp = CGF.
Builder.CreateCall(Callee, {
X, Y});
742 Carry = CGF.
Builder.CreateExtractValue(Tmp, 1);
743 return CGF.
Builder.CreateExtractValue(Tmp, 0);
750 llvm::ConstantRange CR(APInt(32, low), APInt(32, high));
751 Call->addRangeRetAttr(CR);
752 Call->addRetAttr(llvm::Attribute::AttrKind::NoUndef);
757 struct WidthAndSignedness {
763static WidthAndSignedness
777static struct WidthAndSignedness
779 assert(Types.size() > 0 &&
"Empty list of types.");
783 for (
const auto &
Type : Types) {
792 for (
const auto &
Type : Types) {
794 if (Width < MinWidth) {
803 Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend;
814 return From == To || (From == 0 && To == 1) || (From == 3 && To == 2);
819 return ConstantInt::get(ResType, (
Type & 2) ? 0 : -1,
true);
823CodeGenFunction::evaluateOrEmitBuiltinObjectSize(
const Expr *
E,
unsigned Type,
824 llvm::IntegerType *ResType,
825 llvm::Value *EmittedE,
829 return emitBuiltinObjectSize(
E,
Type, ResType, EmittedE, IsDynamic);
830 return ConstantInt::get(ResType, ObjectSize,
true);
838 uint32_t FieldNo = 0;
844 if ((!FAMDecl || FD == FAMDecl) &&
846 Ctx, FD, FD->getType(), StrictFlexArraysLevel,
874 if (FD->getType()->isCountAttributedType())
886CodeGenFunction::emitFlexibleArrayMemberSize(
const Expr *
E,
unsigned Type,
887 llvm::IntegerType *ResType) {
916 const Expr *Idx =
nullptr;
918 if (
const auto *UO = dyn_cast<UnaryOperator>(
Base);
919 UO && UO->getOpcode() == UO_AddrOf) {
921 if (
const auto *ASE = dyn_cast<ArraySubscriptExpr>(SubExpr)) {
922 Base = ASE->getBase()->IgnoreParenImpCasts();
925 if (
const auto *IL = dyn_cast<IntegerLiteral>(Idx)) {
926 int64_t Val = IL->getValue().getSExtValue();
943 if (
const auto *ME = dyn_cast<MemberExpr>(
Base)) {
945 const ValueDecl *VD = ME->getMemberDecl();
947 FAMDecl = dyn_cast<FieldDecl>(VD);
950 }
else if (
const auto *DRE = dyn_cast<DeclRefExpr>(
Base)) {
952 QualType Ty = DRE->getDecl()->getType();
1011 CountedByInst =
Builder.CreateIntCast(CountedByInst, ResType, IsSigned);
1014 Value *IdxInst =
nullptr;
1022 IdxInst =
Builder.CreateIntCast(IdxInst, ResType, IdxSigned);
1027 Builder.CreateSub(CountedByInst, IdxInst,
"", !IsSigned, IsSigned);
1033 llvm::Constant *ElemSize =
1034 llvm::ConstantInt::get(ResType,
Size.getQuantity(), IsSigned);
1036 Builder.CreateMul(CountedByInst, ElemSize,
"", !IsSigned, IsSigned);
1037 FAMSize =
Builder.CreateIntCast(FAMSize, ResType, IsSigned);
1038 Value *Res = FAMSize;
1040 if (isa<DeclRefExpr>(
Base)) {
1045 llvm::Constant *FAMOffset = ConstantInt::get(ResType, Offset, IsSigned);
1046 Value *OffsetAndFAMSize =
1047 Builder.CreateAdd(FAMOffset, Res,
"", !IsSigned, IsSigned);
1050 llvm::Constant *SizeofStruct =
1056 ?
Builder.CreateBinaryIntrinsic(llvm::Intrinsic::smax,
1057 OffsetAndFAMSize, SizeofStruct)
1058 :
Builder.CreateBinaryIntrinsic(llvm::Intrinsic::umax,
1059 OffsetAndFAMSize, SizeofStruct);
1069 return Builder.CreateSelect(Cmp, Res, ConstantInt::get(ResType, 0, IsSigned));
1082CodeGenFunction::emitBuiltinObjectSize(
const Expr *
E,
unsigned Type,
1083 llvm::IntegerType *ResType,
1084 llvm::Value *EmittedE,
bool IsDynamic) {
1088 auto *Param = dyn_cast<ParmVarDecl>(
D->getDecl());
1089 auto *PS =
D->getDecl()->
getAttr<PassObjectSizeAttr>();
1090 if (Param !=
nullptr && PS !=
nullptr &&
1092 auto Iter = SizeArguments.find(Param);
1093 assert(
Iter != SizeArguments.end());
1096 auto DIter = LocalDeclMap.find(
D);
1097 assert(DIter != LocalDeclMap.end());
1107 if (
Value *
V = emitFlexibleArrayMemberSize(
E,
Type, ResType))
1118 assert(Ptr->
getType()->isPointerTy() &&
1119 "Non-pointer passed to __builtin_object_size?");
1135 enum ActionKind : uint8_t { TestOnly, Complement, Reset,
Set };
1136 enum InterlockingKind : uint8_t {
1145 InterlockingKind Interlocking;
1148 static BitTest decodeBitTestBuiltin(
unsigned BuiltinID);
1153BitTest BitTest::decodeBitTestBuiltin(
unsigned BuiltinID) {
1154 switch (BuiltinID) {
1156 case Builtin::BI_bittest:
1157 return {TestOnly, Unlocked,
false};
1158 case Builtin::BI_bittestandcomplement:
1159 return {Complement, Unlocked,
false};
1160 case Builtin::BI_bittestandreset:
1161 return {Reset, Unlocked,
false};
1162 case Builtin::BI_bittestandset:
1163 return {
Set, Unlocked,
false};
1164 case Builtin::BI_interlockedbittestandreset:
1165 return {Reset, Sequential,
false};
1166 case Builtin::BI_interlockedbittestandset:
1167 return {
Set, Sequential,
false};
1170 case Builtin::BI_bittest64:
1171 return {TestOnly, Unlocked,
true};
1172 case Builtin::BI_bittestandcomplement64:
1173 return {Complement, Unlocked,
true};
1174 case Builtin::BI_bittestandreset64:
1175 return {Reset, Unlocked,
true};
1176 case Builtin::BI_bittestandset64:
1177 return {
Set, Unlocked,
true};
1178 case Builtin::BI_interlockedbittestandreset64:
1179 return {Reset, Sequential,
true};
1180 case Builtin::BI_interlockedbittestandset64:
1181 return {
Set, Sequential,
true};
1184 case Builtin::BI_interlockedbittestandset_acq:
1185 return {
Set, Acquire,
false};
1186 case Builtin::BI_interlockedbittestandset_rel:
1187 return {
Set, Release,
false};
1188 case Builtin::BI_interlockedbittestandset_nf:
1189 return {
Set, NoFence,
false};
1190 case Builtin::BI_interlockedbittestandreset_acq:
1191 return {Reset, Acquire,
false};
1192 case Builtin::BI_interlockedbittestandreset_rel:
1193 return {Reset, Release,
false};
1194 case Builtin::BI_interlockedbittestandreset_nf:
1195 return {Reset, NoFence,
false};
1197 llvm_unreachable(
"expected only bittest intrinsics");
1202 case BitTest::TestOnly:
return '\0';
1203 case BitTest::Complement:
return 'c';
1204 case BitTest::Reset:
return 'r';
1205 case BitTest::Set:
return 's';
1207 llvm_unreachable(
"invalid action");
1215 char SizeSuffix = BT.Is64Bit ?
'q' :
'l';
1219 raw_svector_ostream AsmOS(
Asm);
1220 if (BT.Interlocking != BitTest::Unlocked)
1225 AsmOS << SizeSuffix <<
" $2, ($1)";
1228 std::string Constraints =
"={@ccc},r,r,~{cc},~{memory}";
1230 if (!MachineClobbers.empty()) {
1232 Constraints += MachineClobbers;
1234 llvm::IntegerType *IntType = llvm::IntegerType::get(
1237 llvm::FunctionType *FTy =
1238 llvm::FunctionType::get(CGF.
Int8Ty, {CGF.UnqualPtrTy, IntType},
false);
1240 llvm::InlineAsm *IA =
1241 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
1242 return CGF.
Builder.CreateCall(IA, {BitBase, BitPos});
1245static llvm::AtomicOrdering
1248 case BitTest::Unlocked:
return llvm::AtomicOrdering::NotAtomic;
1249 case BitTest::Sequential:
return llvm::AtomicOrdering::SequentiallyConsistent;
1250 case BitTest::Acquire:
return llvm::AtomicOrdering::Acquire;
1251 case BitTest::Release:
return llvm::AtomicOrdering::Release;
1252 case BitTest::NoFence:
return llvm::AtomicOrdering::Monotonic;
1254 llvm_unreachable(
"invalid interlocking");
1267 BitTest BT = BitTest::decodeBitTestBuiltin(BuiltinID);
1279 BitPos, llvm::ConstantInt::get(BitPos->
getType(), 3),
"bittest.byteidx");
1282 ByteIndex,
"bittest.byteaddr"),
1286 llvm::ConstantInt::get(CGF.
Int8Ty, 0x7));
1289 Value *Mask =
nullptr;
1290 if (BT.Action != BitTest::TestOnly) {
1291 Mask = CGF.
Builder.CreateShl(llvm::ConstantInt::get(CGF.
Int8Ty, 1), PosLow,
1298 Value *OldByte =
nullptr;
1299 if (Ordering != llvm::AtomicOrdering::NotAtomic) {
1302 llvm::AtomicRMWInst::BinOp RMWOp = llvm::AtomicRMWInst::Or;
1303 if (BT.Action == BitTest::Reset) {
1304 Mask = CGF.
Builder.CreateNot(Mask);
1305 RMWOp = llvm::AtomicRMWInst::And;
1311 Value *NewByte =
nullptr;
1312 switch (BT.Action) {
1313 case BitTest::TestOnly:
1316 case BitTest::Complement:
1317 NewByte = CGF.
Builder.CreateXor(OldByte, Mask);
1319 case BitTest::Reset:
1320 NewByte = CGF.
Builder.CreateAnd(OldByte, CGF.
Builder.CreateNot(Mask));
1323 NewByte = CGF.
Builder.CreateOr(OldByte, Mask);
1332 Value *ShiftedByte = CGF.
Builder.CreateLShr(OldByte, PosLow,
"bittest.shr");
1334 ShiftedByte, llvm::ConstantInt::get(CGF.
Int8Ty, 1),
"bittest.res");
1343 raw_svector_ostream AsmOS(
Asm);
1344 llvm::IntegerType *RetType = CGF.
Int32Ty;
1346 switch (BuiltinID) {
1347 case clang::PPC::BI__builtin_ppc_ldarx:
1351 case clang::PPC::BI__builtin_ppc_lwarx:
1355 case clang::PPC::BI__builtin_ppc_lharx:
1359 case clang::PPC::BI__builtin_ppc_lbarx:
1364 llvm_unreachable(
"Expected only PowerPC load reserve intrinsics");
1367 AsmOS <<
"$0, ${1:y}";
1369 std::string Constraints =
"=r,*Z,~{memory}";
1371 if (!MachineClobbers.empty()) {
1373 Constraints += MachineClobbers;
1377 llvm::FunctionType *FTy = llvm::FunctionType::get(RetType, {PtrType},
false);
1379 llvm::InlineAsm *IA =
1380 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
1381 llvm::CallInst *CI = CGF.
Builder.CreateCall(IA, {Addr});
1383 0, Attribute::get(CGF.
getLLVMContext(), Attribute::ElementType, RetType));
1388enum class MSVCSetJmpKind {
1400 llvm::Value *Arg1 =
nullptr;
1401 llvm::Type *Arg1Ty =
nullptr;
1403 bool IsVarArg =
false;
1404 if (SJKind == MSVCSetJmpKind::_setjmp3) {
1407 Arg1 = llvm::ConstantInt::get(CGF.
IntTy, 0);
1410 Name = SJKind == MSVCSetJmpKind::_setjmp ?
"_setjmp" :
"_setjmpex";
1413 Arg1 = CGF.
Builder.CreateCall(
1416 Arg1 = CGF.
Builder.CreateCall(
1418 llvm::ConstantInt::get(CGF.
Int32Ty, 0));
1422 llvm::Type *ArgTypes[2] = {CGF.
Int8PtrTy, Arg1Ty};
1423 llvm::AttributeList ReturnsTwiceAttr = llvm::AttributeList::get(
1425 llvm::Attribute::ReturnsTwice);
1427 llvm::FunctionType::get(CGF.
IntTy, ArgTypes, IsVarArg), Name,
1428 ReturnsTwiceAttr,
true);
1430 llvm::Value *Buf = CGF.
Builder.CreateBitOrPointerCast(
1432 llvm::Value *Args[] = {Buf, Arg1};
1434 CB->setAttributes(ReturnsTwiceAttr);
1482static std::optional<CodeGenFunction::MSVCIntrin>
1485 switch (BuiltinID) {
1487 return std::nullopt;
1488 case clang::ARM::BI_BitScanForward:
1489 case clang::ARM::BI_BitScanForward64:
1490 return MSVCIntrin::_BitScanForward;
1491 case clang::ARM::BI_BitScanReverse:
1492 case clang::ARM::BI_BitScanReverse64:
1493 return MSVCIntrin::_BitScanReverse;
1494 case clang::ARM::BI_InterlockedAnd64:
1495 return MSVCIntrin::_InterlockedAnd;
1496 case clang::ARM::BI_InterlockedExchange64:
1497 return MSVCIntrin::_InterlockedExchange;
1498 case clang::ARM::BI_InterlockedExchangeAdd64:
1499 return MSVCIntrin::_InterlockedExchangeAdd;
1500 case clang::ARM::BI_InterlockedExchangeSub64:
1501 return MSVCIntrin::_InterlockedExchangeSub;
1502 case clang::ARM::BI_InterlockedOr64:
1503 return MSVCIntrin::_InterlockedOr;
1504 case clang::ARM::BI_InterlockedXor64:
1505 return MSVCIntrin::_InterlockedXor;
1506 case clang::ARM::BI_InterlockedDecrement64:
1507 return MSVCIntrin::_InterlockedDecrement;
1508 case clang::ARM::BI_InterlockedIncrement64:
1509 return MSVCIntrin::_InterlockedIncrement;
1510 case clang::ARM::BI_InterlockedExchangeAdd8_acq:
1511 case clang::ARM::BI_InterlockedExchangeAdd16_acq:
1512 case clang::ARM::BI_InterlockedExchangeAdd_acq:
1513 case clang::ARM::BI_InterlockedExchangeAdd64_acq:
1514 return MSVCIntrin::_InterlockedExchangeAdd_acq;
1515 case clang::ARM::BI_InterlockedExchangeAdd8_rel:
1516 case clang::ARM::BI_InterlockedExchangeAdd16_rel:
1517 case clang::ARM::BI_InterlockedExchangeAdd_rel:
1518 case clang::ARM::BI_InterlockedExchangeAdd64_rel:
1519 return MSVCIntrin::_InterlockedExchangeAdd_rel;
1520 case clang::ARM::BI_InterlockedExchangeAdd8_nf:
1521 case clang::ARM::BI_InterlockedExchangeAdd16_nf:
1522 case clang::ARM::BI_InterlockedExchangeAdd_nf:
1523 case clang::ARM::BI_InterlockedExchangeAdd64_nf:
1524 return MSVCIntrin::_InterlockedExchangeAdd_nf;
1525 case clang::ARM::BI_InterlockedExchange8_acq:
1526 case clang::ARM::BI_InterlockedExchange16_acq:
1527 case clang::ARM::BI_InterlockedExchange_acq:
1528 case clang::ARM::BI_InterlockedExchange64_acq:
1529 return MSVCIntrin::_InterlockedExchange_acq;
1530 case clang::ARM::BI_InterlockedExchange8_rel:
1531 case clang::ARM::BI_InterlockedExchange16_rel:
1532 case clang::ARM::BI_InterlockedExchange_rel:
1533 case clang::ARM::BI_InterlockedExchange64_rel:
1534 return MSVCIntrin::_InterlockedExchange_rel;
1535 case clang::ARM::BI_InterlockedExchange8_nf:
1536 case clang::ARM::BI_InterlockedExchange16_nf:
1537 case clang::ARM::BI_InterlockedExchange_nf:
1538 case clang::ARM::BI_InterlockedExchange64_nf:
1539 return MSVCIntrin::_InterlockedExchange_nf;
1540 case clang::ARM::BI_InterlockedCompareExchange8_acq:
1541 case clang::ARM::BI_InterlockedCompareExchange16_acq:
1542 case clang::ARM::BI_InterlockedCompareExchange_acq:
1543 case clang::ARM::BI_InterlockedCompareExchange64_acq:
1544 return MSVCIntrin::_InterlockedCompareExchange_acq;
1545 case clang::ARM::BI_InterlockedCompareExchange8_rel:
1546 case clang::ARM::BI_InterlockedCompareExchange16_rel:
1547 case clang::ARM::BI_InterlockedCompareExchange_rel:
1548 case clang::ARM::BI_InterlockedCompareExchange64_rel:
1549 return MSVCIntrin::_InterlockedCompareExchange_rel;
1550 case clang::ARM::BI_InterlockedCompareExchange8_nf:
1551 case clang::ARM::BI_InterlockedCompareExchange16_nf:
1552 case clang::ARM::BI_InterlockedCompareExchange_nf:
1553 case clang::ARM::BI_InterlockedCompareExchange64_nf:
1554 return MSVCIntrin::_InterlockedCompareExchange_nf;
1555 case clang::ARM::BI_InterlockedOr8_acq:
1556 case clang::ARM::BI_InterlockedOr16_acq:
1557 case clang::ARM::BI_InterlockedOr_acq:
1558 case clang::ARM::BI_InterlockedOr64_acq:
1559 return MSVCIntrin::_InterlockedOr_acq;
1560 case clang::ARM::BI_InterlockedOr8_rel:
1561 case clang::ARM::BI_InterlockedOr16_rel:
1562 case clang::ARM::BI_InterlockedOr_rel:
1563 case clang::ARM::BI_InterlockedOr64_rel:
1564 return MSVCIntrin::_InterlockedOr_rel;
1565 case clang::ARM::BI_InterlockedOr8_nf:
1566 case clang::ARM::BI_InterlockedOr16_nf:
1567 case clang::ARM::BI_InterlockedOr_nf:
1568 case clang::ARM::BI_InterlockedOr64_nf:
1569 return MSVCIntrin::_InterlockedOr_nf;
1570 case clang::ARM::BI_InterlockedXor8_acq:
1571 case clang::ARM::BI_InterlockedXor16_acq:
1572 case clang::ARM::BI_InterlockedXor_acq:
1573 case clang::ARM::BI_InterlockedXor64_acq:
1574 return MSVCIntrin::_InterlockedXor_acq;
1575 case clang::ARM::BI_InterlockedXor8_rel:
1576 case clang::ARM::BI_InterlockedXor16_rel:
1577 case clang::ARM::BI_InterlockedXor_rel:
1578 case clang::ARM::BI_InterlockedXor64_rel:
1579 return MSVCIntrin::_InterlockedXor_rel;
1580 case clang::ARM::BI_InterlockedXor8_nf:
1581 case clang::ARM::BI_InterlockedXor16_nf:
1582 case clang::ARM::BI_InterlockedXor_nf:
1583 case clang::ARM::BI_InterlockedXor64_nf:
1584 return MSVCIntrin::_InterlockedXor_nf;
1585 case clang::ARM::BI_InterlockedAnd8_acq:
1586 case clang::ARM::BI_InterlockedAnd16_acq:
1587 case clang::ARM::BI_InterlockedAnd_acq:
1588 case clang::ARM::BI_InterlockedAnd64_acq:
1589 return MSVCIntrin::_InterlockedAnd_acq;
1590 case clang::ARM::BI_InterlockedAnd8_rel:
1591 case clang::ARM::BI_InterlockedAnd16_rel:
1592 case clang::ARM::BI_InterlockedAnd_rel:
1593 case clang::ARM::BI_InterlockedAnd64_rel:
1594 return MSVCIntrin::_InterlockedAnd_rel;
1595 case clang::ARM::BI_InterlockedAnd8_nf:
1596 case clang::ARM::BI_InterlockedAnd16_nf:
1597 case clang::ARM::BI_InterlockedAnd_nf:
1598 case clang::ARM::BI_InterlockedAnd64_nf:
1599 return MSVCIntrin::_InterlockedAnd_nf;
1600 case clang::ARM::BI_InterlockedIncrement16_acq:
1601 case clang::ARM::BI_InterlockedIncrement_acq:
1602 case clang::ARM::BI_InterlockedIncrement64_acq:
1603 return MSVCIntrin::_InterlockedIncrement_acq;
1604 case clang::ARM::BI_InterlockedIncrement16_rel:
1605 case clang::ARM::BI_InterlockedIncrement_rel:
1606 case clang::ARM::BI_InterlockedIncrement64_rel:
1607 return MSVCIntrin::_InterlockedIncrement_rel;
1608 case clang::ARM::BI_InterlockedIncrement16_nf:
1609 case clang::ARM::BI_InterlockedIncrement_nf:
1610 case clang::ARM::BI_InterlockedIncrement64_nf:
1611 return MSVCIntrin::_InterlockedIncrement_nf;
1612 case clang::ARM::BI_InterlockedDecrement16_acq:
1613 case clang::ARM::BI_InterlockedDecrement_acq:
1614 case clang::ARM::BI_InterlockedDecrement64_acq:
1615 return MSVCIntrin::_InterlockedDecrement_acq;
1616 case clang::ARM::BI_InterlockedDecrement16_rel:
1617 case clang::ARM::BI_InterlockedDecrement_rel:
1618 case clang::ARM::BI_InterlockedDecrement64_rel:
1619 return MSVCIntrin::_InterlockedDecrement_rel;
1620 case clang::ARM::BI_InterlockedDecrement16_nf:
1621 case clang::ARM::BI_InterlockedDecrement_nf:
1622 case clang::ARM::BI_InterlockedDecrement64_nf:
1623 return MSVCIntrin::_InterlockedDecrement_nf;
1625 llvm_unreachable(
"must return from switch");
1628static std::optional<CodeGenFunction::MSVCIntrin>
1631 switch (BuiltinID) {
1633 return std::nullopt;
1634 case clang::AArch64::BI_BitScanForward:
1635 case clang::AArch64::BI_BitScanForward64:
1636 return MSVCIntrin::_BitScanForward;
1637 case clang::AArch64::BI_BitScanReverse:
1638 case clang::AArch64::BI_BitScanReverse64:
1639 return MSVCIntrin::_BitScanReverse;
1640 case clang::AArch64::BI_InterlockedAnd64:
1641 return MSVCIntrin::_InterlockedAnd;
1642 case clang::AArch64::BI_InterlockedExchange64:
1643 return MSVCIntrin::_InterlockedExchange;
1644 case clang::AArch64::BI_InterlockedExchangeAdd64:
1645 return MSVCIntrin::_InterlockedExchangeAdd;
1646 case clang::AArch64::BI_InterlockedExchangeSub64:
1647 return MSVCIntrin::_InterlockedExchangeSub;
1648 case clang::AArch64::BI_InterlockedOr64:
1649 return MSVCIntrin::_InterlockedOr;
1650 case clang::AArch64::BI_InterlockedXor64:
1651 return MSVCIntrin::_InterlockedXor;
1652 case clang::AArch64::BI_InterlockedDecrement64:
1653 return MSVCIntrin::_InterlockedDecrement;
1654 case clang::AArch64::BI_InterlockedIncrement64:
1655 return MSVCIntrin::_InterlockedIncrement;
1656 case clang::AArch64::BI_InterlockedExchangeAdd8_acq:
1657 case clang::AArch64::BI_InterlockedExchangeAdd16_acq:
1658 case clang::AArch64::BI_InterlockedExchangeAdd_acq:
1659 case clang::AArch64::BI_InterlockedExchangeAdd64_acq:
1660 return MSVCIntrin::_InterlockedExchangeAdd_acq;
1661 case clang::AArch64::BI_InterlockedExchangeAdd8_rel:
1662 case clang::AArch64::BI_InterlockedExchangeAdd16_rel:
1663 case clang::AArch64::BI_InterlockedExchangeAdd_rel:
1664 case clang::AArch64::BI_InterlockedExchangeAdd64_rel:
1665 return MSVCIntrin::_InterlockedExchangeAdd_rel;
1666 case clang::AArch64::BI_InterlockedExchangeAdd8_nf:
1667 case clang::AArch64::BI_InterlockedExchangeAdd16_nf:
1668 case clang::AArch64::BI_InterlockedExchangeAdd_nf:
1669 case clang::AArch64::BI_InterlockedExchangeAdd64_nf:
1670 return MSVCIntrin::_InterlockedExchangeAdd_nf;
1671 case clang::AArch64::BI_InterlockedExchange8_acq:
1672 case clang::AArch64::BI_InterlockedExchange16_acq:
1673 case clang::AArch64::BI_InterlockedExchange_acq:
1674 case clang::AArch64::BI_InterlockedExchange64_acq:
1675 return MSVCIntrin::_InterlockedExchange_acq;
1676 case clang::AArch64::BI_InterlockedExchange8_rel:
1677 case clang::AArch64::BI_InterlockedExchange16_rel:
1678 case clang::AArch64::BI_InterlockedExchange_rel:
1679 case clang::AArch64::BI_InterlockedExchange64_rel:
1680 return MSVCIntrin::_InterlockedExchange_rel;
1681 case clang::AArch64::BI_InterlockedExchange8_nf:
1682 case clang::AArch64::BI_InterlockedExchange16_nf:
1683 case clang::AArch64::BI_InterlockedExchange_nf:
1684 case clang::AArch64::BI_InterlockedExchange64_nf:
1685 return MSVCIntrin::_InterlockedExchange_nf;
1686 case clang::AArch64::BI_InterlockedCompareExchange8_acq:
1687 case clang::AArch64::BI_InterlockedCompareExchange16_acq:
1688 case clang::AArch64::BI_InterlockedCompareExchange_acq:
1689 case clang::AArch64::BI_InterlockedCompareExchange64_acq:
1690 return MSVCIntrin::_InterlockedCompareExchange_acq;
1691 case clang::AArch64::BI_InterlockedCompareExchange8_rel:
1692 case clang::AArch64::BI_InterlockedCompareExchange16_rel:
1693 case clang::AArch64::BI_InterlockedCompareExchange_rel:
1694 case clang::AArch64::BI_InterlockedCompareExchange64_rel:
1695 return MSVCIntrin::_InterlockedCompareExchange_rel;
1696 case clang::AArch64::BI_InterlockedCompareExchange8_nf:
1697 case clang::AArch64::BI_InterlockedCompareExchange16_nf:
1698 case clang::AArch64::BI_InterlockedCompareExchange_nf:
1699 case clang::AArch64::BI_InterlockedCompareExchange64_nf:
1700 return MSVCIntrin::_InterlockedCompareExchange_nf;
1701 case clang::AArch64::BI_InterlockedCompareExchange128:
1702 return MSVCIntrin::_InterlockedCompareExchange128;
1703 case clang::AArch64::BI_InterlockedCompareExchange128_acq:
1704 return MSVCIntrin::_InterlockedCompareExchange128_acq;
1705 case clang::AArch64::BI_InterlockedCompareExchange128_nf:
1706 return MSVCIntrin::_InterlockedCompareExchange128_nf;
1707 case clang::AArch64::BI_InterlockedCompareExchange128_rel:
1708 return MSVCIntrin::_InterlockedCompareExchange128_rel;
1709 case clang::AArch64::BI_InterlockedOr8_acq:
1710 case clang::AArch64::BI_InterlockedOr16_acq:
1711 case clang::AArch64::BI_InterlockedOr_acq:
1712 case clang::AArch64::BI_InterlockedOr64_acq:
1713 return MSVCIntrin::_InterlockedOr_acq;
1714 case clang::AArch64::BI_InterlockedOr8_rel:
1715 case clang::AArch64::BI_InterlockedOr16_rel:
1716 case clang::AArch64::BI_InterlockedOr_rel:
1717 case clang::AArch64::BI_InterlockedOr64_rel:
1718 return MSVCIntrin::_InterlockedOr_rel;
1719 case clang::AArch64::BI_InterlockedOr8_nf:
1720 case clang::AArch64::BI_InterlockedOr16_nf:
1721 case clang::AArch64::BI_InterlockedOr_nf:
1722 case clang::AArch64::BI_InterlockedOr64_nf:
1723 return MSVCIntrin::_InterlockedOr_nf;
1724 case clang::AArch64::BI_InterlockedXor8_acq:
1725 case clang::AArch64::BI_InterlockedXor16_acq:
1726 case clang::AArch64::BI_InterlockedXor_acq:
1727 case clang::AArch64::BI_InterlockedXor64_acq:
1728 return MSVCIntrin::_InterlockedXor_acq;
1729 case clang::AArch64::BI_InterlockedXor8_rel:
1730 case clang::AArch64::BI_InterlockedXor16_rel:
1731 case clang::AArch64::BI_InterlockedXor_rel:
1732 case clang::AArch64::BI_InterlockedXor64_rel:
1733 return MSVCIntrin::_InterlockedXor_rel;
1734 case clang::AArch64::BI_InterlockedXor8_nf:
1735 case clang::AArch64::BI_InterlockedXor16_nf:
1736 case clang::AArch64::BI_InterlockedXor_nf:
1737 case clang::AArch64::BI_InterlockedXor64_nf:
1738 return MSVCIntrin::_InterlockedXor_nf;
1739 case clang::AArch64::BI_InterlockedAnd8_acq:
1740 case clang::AArch64::BI_InterlockedAnd16_acq:
1741 case clang::AArch64::BI_InterlockedAnd_acq:
1742 case clang::AArch64::BI_InterlockedAnd64_acq:
1743 return MSVCIntrin::_InterlockedAnd_acq;
1744 case clang::AArch64::BI_InterlockedAnd8_rel:
1745 case clang::AArch64::BI_InterlockedAnd16_rel:
1746 case clang::AArch64::BI_InterlockedAnd_rel:
1747 case clang::AArch64::BI_InterlockedAnd64_rel:
1748 return MSVCIntrin::_InterlockedAnd_rel;
1749 case clang::AArch64::BI_InterlockedAnd8_nf:
1750 case clang::AArch64::BI_InterlockedAnd16_nf:
1751 case clang::AArch64::BI_InterlockedAnd_nf:
1752 case clang::AArch64::BI_InterlockedAnd64_nf:
1753 return MSVCIntrin::_InterlockedAnd_nf;
1754 case clang::AArch64::BI_InterlockedIncrement16_acq:
1755 case clang::AArch64::BI_InterlockedIncrement_acq:
1756 case clang::AArch64::BI_InterlockedIncrement64_acq:
1757 return MSVCIntrin::_InterlockedIncrement_acq;
1758 case clang::AArch64::BI_InterlockedIncrement16_rel:
1759 case clang::AArch64::BI_InterlockedIncrement_rel:
1760 case clang::AArch64::BI_InterlockedIncrement64_rel:
1761 return MSVCIntrin::_InterlockedIncrement_rel;
1762 case clang::AArch64::BI_InterlockedIncrement16_nf:
1763 case clang::AArch64::BI_InterlockedIncrement_nf:
1764 case clang::AArch64::BI_InterlockedIncrement64_nf:
1765 return MSVCIntrin::_InterlockedIncrement_nf;
1766 case clang::AArch64::BI_InterlockedDecrement16_acq:
1767 case clang::AArch64::BI_InterlockedDecrement_acq:
1768 case clang::AArch64::BI_InterlockedDecrement64_acq:
1769 return MSVCIntrin::_InterlockedDecrement_acq;
1770 case clang::AArch64::BI_InterlockedDecrement16_rel:
1771 case clang::AArch64::BI_InterlockedDecrement_rel:
1772 case clang::AArch64::BI_InterlockedDecrement64_rel:
1773 return MSVCIntrin::_InterlockedDecrement_rel;
1774 case clang::AArch64::BI_InterlockedDecrement16_nf:
1775 case clang::AArch64::BI_InterlockedDecrement_nf:
1776 case clang::AArch64::BI_InterlockedDecrement64_nf:
1777 return MSVCIntrin::_InterlockedDecrement_nf;
1779 llvm_unreachable(
"must return from switch");
1782static std::optional<CodeGenFunction::MSVCIntrin>
1785 switch (BuiltinID) {
1787 return std::nullopt;
1788 case clang::X86::BI_BitScanForward:
1789 case clang::X86::BI_BitScanForward64:
1790 return MSVCIntrin::_BitScanForward;
1791 case clang::X86::BI_BitScanReverse:
1792 case clang::X86::BI_BitScanReverse64:
1793 return MSVCIntrin::_BitScanReverse;
1794 case clang::X86::BI_InterlockedAnd64:
1795 return MSVCIntrin::_InterlockedAnd;
1796 case clang::X86::BI_InterlockedCompareExchange128:
1797 return MSVCIntrin::_InterlockedCompareExchange128;
1798 case clang::X86::BI_InterlockedExchange64:
1799 return MSVCIntrin::_InterlockedExchange;
1800 case clang::X86::BI_InterlockedExchangeAdd64:
1801 return MSVCIntrin::_InterlockedExchangeAdd;
1802 case clang::X86::BI_InterlockedExchangeSub64:
1803 return MSVCIntrin::_InterlockedExchangeSub;
1804 case clang::X86::BI_InterlockedOr64:
1805 return MSVCIntrin::_InterlockedOr;
1806 case clang::X86::BI_InterlockedXor64:
1807 return MSVCIntrin::_InterlockedXor;
1808 case clang::X86::BI_InterlockedDecrement64:
1809 return MSVCIntrin::_InterlockedDecrement;
1810 case clang::X86::BI_InterlockedIncrement64:
1811 return MSVCIntrin::_InterlockedIncrement;
1813 llvm_unreachable(
"must return from switch");
1819 switch (BuiltinID) {
1820 case MSVCIntrin::_BitScanForward:
1821 case MSVCIntrin::_BitScanReverse: {
1825 llvm::Type *ArgType = ArgValue->
getType();
1826 llvm::Type *IndexType = IndexAddress.getElementType();
1829 Value *ArgZero = llvm::Constant::getNullValue(ArgType);
1830 Value *ResZero = llvm::Constant::getNullValue(ResultType);
1831 Value *ResOne = llvm::ConstantInt::get(ResultType, 1);
1836 PHINode *
Result =
Builder.CreatePHI(ResultType, 2,
"bitscan_result");
1839 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, ArgZero);
1841 Builder.CreateCondBr(IsZero, End, NotZero);
1844 Builder.SetInsertPoint(NotZero);
1846 if (BuiltinID == MSVCIntrin::_BitScanForward) {
1849 ZeroCount =
Builder.CreateIntCast(ZeroCount, IndexType,
false);
1852 unsigned ArgWidth = cast<llvm::IntegerType>(ArgType)->getBitWidth();
1853 Value *ArgTypeLastIndex = llvm::ConstantInt::get(IndexType, ArgWidth - 1);
1857 ZeroCount =
Builder.CreateIntCast(ZeroCount, IndexType,
false);
1858 Value *Index =
Builder.CreateNSWSub(ArgTypeLastIndex, ZeroCount);
1862 Result->addIncoming(ResOne, NotZero);
1867 case MSVCIntrin::_InterlockedAnd:
1869 case MSVCIntrin::_InterlockedExchange:
1871 case MSVCIntrin::_InterlockedExchangeAdd:
1873 case MSVCIntrin::_InterlockedExchangeSub:
1875 case MSVCIntrin::_InterlockedOr:
1877 case MSVCIntrin::_InterlockedXor:
1879 case MSVCIntrin::_InterlockedExchangeAdd_acq:
1881 AtomicOrdering::Acquire);
1882 case MSVCIntrin::_InterlockedExchangeAdd_rel:
1884 AtomicOrdering::Release);
1885 case MSVCIntrin::_InterlockedExchangeAdd_nf:
1887 AtomicOrdering::Monotonic);
1888 case MSVCIntrin::_InterlockedExchange_acq:
1890 AtomicOrdering::Acquire);
1891 case MSVCIntrin::_InterlockedExchange_rel:
1893 AtomicOrdering::Release);
1894 case MSVCIntrin::_InterlockedExchange_nf:
1896 AtomicOrdering::Monotonic);
1897 case MSVCIntrin::_InterlockedCompareExchange_acq:
1899 case MSVCIntrin::_InterlockedCompareExchange_rel:
1901 case MSVCIntrin::_InterlockedCompareExchange_nf:
1903 case MSVCIntrin::_InterlockedCompareExchange128:
1905 *
this,
E, AtomicOrdering::SequentiallyConsistent);
1906 case MSVCIntrin::_InterlockedCompareExchange128_acq:
1908 case MSVCIntrin::_InterlockedCompareExchange128_rel:
1910 case MSVCIntrin::_InterlockedCompareExchange128_nf:
1912 case MSVCIntrin::_InterlockedOr_acq:
1914 AtomicOrdering::Acquire);
1915 case MSVCIntrin::_InterlockedOr_rel:
1917 AtomicOrdering::Release);
1918 case MSVCIntrin::_InterlockedOr_nf:
1920 AtomicOrdering::Monotonic);
1921 case MSVCIntrin::_InterlockedXor_acq:
1923 AtomicOrdering::Acquire);
1924 case MSVCIntrin::_InterlockedXor_rel:
1926 AtomicOrdering::Release);
1927 case MSVCIntrin::_InterlockedXor_nf:
1929 AtomicOrdering::Monotonic);
1930 case MSVCIntrin::_InterlockedAnd_acq:
1932 AtomicOrdering::Acquire);
1933 case MSVCIntrin::_InterlockedAnd_rel:
1935 AtomicOrdering::Release);
1936 case MSVCIntrin::_InterlockedAnd_nf:
1938 AtomicOrdering::Monotonic);
1939 case MSVCIntrin::_InterlockedIncrement_acq:
1941 case MSVCIntrin::_InterlockedIncrement_rel:
1943 case MSVCIntrin::_InterlockedIncrement_nf:
1945 case MSVCIntrin::_InterlockedDecrement_acq:
1947 case MSVCIntrin::_InterlockedDecrement_rel:
1949 case MSVCIntrin::_InterlockedDecrement_nf:
1952 case MSVCIntrin::_InterlockedDecrement:
1954 case MSVCIntrin::_InterlockedIncrement:
1957 case MSVCIntrin::__fastfail: {
1962 StringRef
Asm, Constraints;
1967 case llvm::Triple::x86:
1968 case llvm::Triple::x86_64:
1970 Constraints =
"{cx}";
1972 case llvm::Triple::thumb:
1974 Constraints =
"{r0}";
1976 case llvm::Triple::aarch64:
1977 Asm =
"brk #0xF003";
1978 Constraints =
"{w0}";
1980 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy, {
Int32Ty},
false);
1981 llvm::InlineAsm *IA =
1982 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
1983 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
1985 llvm::Attribute::NoReturn);
1987 CI->setAttributes(NoReturnAttr);
1991 llvm_unreachable(
"Incorrect MSVC intrinsic!");
1997 CallObjCArcUse(llvm::Value *
object) : object(object) {}
1998 llvm::Value *object;
2007 BuiltinCheckKind Kind) {
2009 &&
"Unsupported builtin check kind");
2015 SanitizerScope SanScope(
this);
2017 ArgValue, llvm::Constant::getNullValue(ArgValue->
getType()));
2018 EmitCheck(std::make_pair(Cond, SanitizerKind::Builtin),
2019 SanitizerHandler::InvalidBuiltin,
2021 llvm::ConstantInt::get(
Builder.getInt8Ty(), Kind)},
2027 return CGF.
Builder.CreateBinaryIntrinsic(
2028 Intrinsic::abs, ArgValue,
2029 ConstantInt::get(CGF.
Builder.getInt1Ty(), HasNSW));
2033 bool SanitizeOverflow) {
2037 if (
const auto *VCI = dyn_cast<llvm::ConstantInt>(ArgValue)) {
2038 if (!VCI->isMinSignedValue())
2039 return EmitAbs(CGF, ArgValue,
true);
2042 CodeGenFunction::SanitizerScope SanScope(&CGF);
2044 Constant *Zero = Constant::getNullValue(ArgValue->
getType());
2045 Value *ResultAndOverflow = CGF.
Builder.CreateBinaryIntrinsic(
2046 Intrinsic::ssub_with_overflow, Zero, ArgValue);
2049 CGF.
Builder.CreateExtractValue(ResultAndOverflow, 1));
2052 if (SanitizeOverflow) {
2053 CGF.
EmitCheck({{NotOverflow, SanitizerKind::SignedIntegerOverflow}},
2054 SanitizerHandler::NegateOverflow,
2059 CGF.
EmitTrapCheck(NotOverflow, SanitizerHandler::SubOverflow);
2061 Value *CmpResult = CGF.
Builder.CreateICmpSLT(ArgValue, Zero,
"abscond");
2062 return CGF.
Builder.CreateSelect(CmpResult,
Result, ArgValue,
"abs");
2067 QualType UnsignedTy =
C.getIntTypeForBitwidth(Size * 8,
false);
2068 return C.getCanonicalType(UnsignedTy);
2078 raw_svector_ostream OS(Name);
2079 OS <<
"__os_log_helper";
2083 for (
const auto &Item : Layout.
Items)
2084 OS <<
"_" <<
int(Item.getSizeByte()) <<
"_"
2085 <<
int(Item.getDescriptorByte());
2088 if (llvm::Function *F =
CGM.
getModule().getFunction(Name))
2098 for (
unsigned int I = 0,
E = Layout.
Items.size(); I <
E; ++I) {
2099 char Size = Layout.
Items[I].getSizeByte();
2106 &Ctx.
Idents.
get(std::string(
"arg") + llvm::to_string(I)), ArgTy,
2108 ArgTys.emplace_back(ArgTy);
2119 llvm::Function *
Fn = llvm::Function::Create(
2120 FuncTy, llvm::GlobalValue::LinkOnceODRLinkage, Name, &
CGM.
getModule());
2121 Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);
2124 Fn->setDoesNotThrow();
2128 Fn->addFnAttr(llvm::Attribute::NoInline);
2146 for (
const auto &Item : Layout.
Items) {
2148 Builder.getInt8(Item.getDescriptorByte()),
2151 Builder.getInt8(Item.getSizeByte()),
2155 if (!
Size.getQuantity())
2172 assert(
E.getNumArgs() >= 2 &&
2173 "__builtin_os_log_format takes at least 2 arguments");
2184 for (
const auto &Item : Layout.
Items) {
2185 int Size = Item.getSizeByte();
2189 llvm::Value *ArgVal;
2193 for (
unsigned I = 0,
E = Item.getMaskType().size(); I <
E; ++I)
2194 Val |= ((
uint64_t)Item.getMaskType()[I]) << I * 8;
2195 ArgVal = llvm::Constant::getIntegerValue(
Int64Ty, llvm::APInt(64, Val));
2196 }
else if (
const Expr *TheExpr = Item.getExpr()) {
2202 auto LifetimeExtendObject = [&](
const Expr *
E) {
2210 if (isa<CallExpr>(
E) || isa<ObjCMessageExpr>(
E))
2215 if (TheExpr->getType()->isObjCRetainableType() &&
2216 getLangOpts().ObjCAutoRefCount && LifetimeExtendObject(TheExpr)) {
2218 "Only scalar can be a ObjC retainable type");
2219 if (!isa<Constant>(ArgVal)) {
2233 pushCleanupAfterFullExpr<CallObjCArcUse>(Cleanup, ArgVal);
2237 ArgVal =
Builder.getInt32(Item.getConstValue().getQuantity());
2240 unsigned ArgValSize =
2244 ArgVal =
Builder.CreateBitOrPointerCast(ArgVal,
IntTy);
2260 unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info,
2261 WidthAndSignedness ResultInfo) {
2262 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2263 Op1Info.Width == Op2Info.Width && Op2Info.Width == ResultInfo.Width &&
2264 !Op1Info.Signed && !Op2Info.Signed && ResultInfo.Signed;
2269 const clang::Expr *Op2, WidthAndSignedness Op2Info,
2271 WidthAndSignedness ResultInfo) {
2273 Builtin::BI__builtin_mul_overflow, Op1Info, Op2Info, ResultInfo) &&
2274 "Cannot specialize this multiply");
2279 llvm::Value *HasOverflow;
2281 CGF, llvm::Intrinsic::umul_with_overflow, V1, V2, HasOverflow);
2286 auto IntMax = llvm::APInt::getSignedMaxValue(ResultInfo.Width);
2287 llvm::Value *IntMaxValue = llvm::ConstantInt::get(
Result->getType(), IntMax);
2289 llvm::Value *IntMaxOverflow = CGF.
Builder.CreateICmpUGT(
Result, IntMaxValue);
2290 HasOverflow = CGF.
Builder.CreateOr(HasOverflow, IntMaxOverflow);
2302 WidthAndSignedness Op1Info,
2303 WidthAndSignedness Op2Info,
2304 WidthAndSignedness ResultInfo) {
2305 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2306 std::max(Op1Info.Width, Op2Info.Width) >= ResultInfo.Width &&
2307 Op1Info.Signed != Op2Info.Signed;
2314 WidthAndSignedness Op1Info,
const clang::Expr *Op2,
2315 WidthAndSignedness Op2Info,
2317 WidthAndSignedness ResultInfo) {
2319 Op2Info, ResultInfo) &&
2320 "Not a mixed-sign multipliction we can specialize");
2323 const clang::Expr *SignedOp = Op1Info.Signed ? Op1 : Op2;
2324 const clang::Expr *UnsignedOp = Op1Info.Signed ? Op2 : Op1;
2327 unsigned SignedOpWidth = Op1Info.Signed ? Op1Info.Width : Op2Info.Width;
2328 unsigned UnsignedOpWidth = Op1Info.Signed ? Op2Info.Width : Op1Info.Width;
2331 if (SignedOpWidth < UnsignedOpWidth)
2333 if (UnsignedOpWidth < SignedOpWidth)
2336 llvm::Type *OpTy =
Signed->getType();
2337 llvm::Value *Zero = llvm::Constant::getNullValue(OpTy);
2340 unsigned OpWidth = std::max(Op1Info.Width, Op2Info.Width);
2343 llvm::Value *IsNegative = CGF.
Builder.CreateICmpSLT(
Signed, Zero);
2344 llvm::Value *AbsOfNegative = CGF.
Builder.CreateSub(Zero,
Signed);
2345 llvm::Value *AbsSigned =
2346 CGF.
Builder.CreateSelect(IsNegative, AbsOfNegative,
Signed);
2349 llvm::Value *UnsignedOverflow;
2350 llvm::Value *UnsignedResult =
2354 llvm::Value *Overflow, *
Result;
2355 if (ResultInfo.Signed) {
2359 llvm::APInt::getSignedMaxValue(ResultInfo.Width).zext(OpWidth);
2360 llvm::Value *MaxResult =
2361 CGF.
Builder.CreateAdd(llvm::ConstantInt::get(OpTy, IntMax),
2362 CGF.
Builder.CreateZExt(IsNegative, OpTy));
2363 llvm::Value *SignedOverflow =
2364 CGF.
Builder.CreateICmpUGT(UnsignedResult, MaxResult);
2365 Overflow = CGF.
Builder.CreateOr(UnsignedOverflow, SignedOverflow);
2368 llvm::Value *NegativeResult = CGF.
Builder.CreateNeg(UnsignedResult);
2369 llvm::Value *SignedResult =
2370 CGF.
Builder.CreateSelect(IsNegative, NegativeResult, UnsignedResult);
2374 llvm::Value *Underflow = CGF.
Builder.CreateAnd(
2375 IsNegative, CGF.
Builder.CreateIsNotNull(UnsignedResult));
2376 Overflow = CGF.
Builder.CreateOr(UnsignedOverflow, Underflow);
2377 if (ResultInfo.Width < OpWidth) {
2379 llvm::APInt::getMaxValue(ResultInfo.Width).zext(OpWidth);
2380 llvm::Value *TruncOverflow = CGF.
Builder.CreateICmpUGT(
2381 UnsignedResult, llvm::ConstantInt::get(OpTy, IntMax));
2382 Overflow = CGF.
Builder.CreateOr(Overflow, TruncOverflow);
2387 IsNegative, CGF.
Builder.CreateNeg(UnsignedResult), UnsignedResult);
2391 assert(Overflow &&
Result &&
"Missing overflow or result");
2402 llvm::SmallPtrSetImpl<const Decl *> &Seen) {
2411 if (!Seen.insert(
Record).second)
2414 assert(
Record->hasDefinition() &&
2415 "Incomplete types should already be diagnosed");
2417 if (
Record->isDynamicClass())
2442 llvm::Type *Ty = Src->getType();
2443 ShiftAmt =
Builder.CreateIntCast(ShiftAmt, Ty,
false);
2446 unsigned IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2453 switch (BuiltinID) {
2454#define MUTATE_LDBL(func) \
2455 case Builtin::BI__builtin_##func##l: \
2456 return Builtin::BI__builtin_##func##f128;
2525 if (CGF.
Builder.getIsFPConstrained() &&
2526 CGF.
Builder.getDefaultConstrainedExcept() != fp::ebIgnore) {
2538 auto UBF = CGF->
CGM.
getModule().getOrInsertFunction(Name, FnTy);
2541 for (
auto &&FormalTy : FnTy->params())
2542 Args.push_back(llvm::PoisonValue::get(FormalTy));
2555 !
Result.hasSideEffects()) {
2559 if (
Result.Val.isFloat())
2568 if (
getTarget().getTriple().isPPC64() &&
2569 &
getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad())
2576 const unsigned BuiltinIDIfNoAsmLabel =
2577 FD->
hasAttr<AsmLabelAttr>() ? 0 : BuiltinID;
2579 std::optional<bool> ErrnoOverriden;
2583 if (
E->hasStoredFPFeatures()) {
2585 if (OP.hasMathErrnoOverride())
2586 ErrnoOverriden = OP.getMathErrnoOverride();
2595 bool ErrnoOverridenToFalseWithOpt =
2596 ErrnoOverriden.has_value() && !ErrnoOverriden.value() && !OptNone &&
2614 switch (BuiltinID) {
2615 case Builtin::BI__builtin_fma:
2616 case Builtin::BI__builtin_fmaf:
2617 case Builtin::BI__builtin_fmal:
2618 case Builtin::BI__builtin_fmaf16:
2619 case Builtin::BIfma:
2620 case Builtin::BIfmaf:
2621 case Builtin::BIfmal: {
2623 if (Trip.isGNUEnvironment() || Trip.isOSMSVCRT())
2631 bool ConstWithoutErrnoAndExceptions =
2633 bool ConstWithoutExceptions =
2651 bool ConstWithoutErrnoOrExceptions =
2652 ConstWithoutErrnoAndExceptions || ConstWithoutExceptions;
2653 bool GenerateIntrinsics =
2654 (ConstAlways && !OptNone) ||
2656 !(ErrnoOverriden.has_value() && ErrnoOverriden.value()) && !OptNone);
2657 if (!GenerateIntrinsics) {
2658 GenerateIntrinsics =
2659 ConstWithoutErrnoOrExceptions && !ConstWithoutErrnoAndExceptions;
2660 if (!GenerateIntrinsics)
2661 GenerateIntrinsics =
2662 ConstWithoutErrnoOrExceptions &&
2664 !(ErrnoOverriden.has_value() && ErrnoOverriden.value()) && !OptNone);
2665 if (!GenerateIntrinsics)
2666 GenerateIntrinsics =
2667 ConstWithoutErrnoOrExceptions && ErrnoOverridenToFalseWithOpt;
2669 if (GenerateIntrinsics) {
2670 switch (BuiltinIDIfNoAsmLabel) {
2671 case Builtin::BIacos:
2672 case Builtin::BIacosf:
2673 case Builtin::BIacosl:
2674 case Builtin::BI__builtin_acos:
2675 case Builtin::BI__builtin_acosf:
2676 case Builtin::BI__builtin_acosf16:
2677 case Builtin::BI__builtin_acosl:
2678 case Builtin::BI__builtin_acosf128:
2680 *
this,
E, Intrinsic::acos, Intrinsic::experimental_constrained_acos));
2682 case Builtin::BIasin:
2683 case Builtin::BIasinf:
2684 case Builtin::BIasinl:
2685 case Builtin::BI__builtin_asin:
2686 case Builtin::BI__builtin_asinf:
2687 case Builtin::BI__builtin_asinf16:
2688 case Builtin::BI__builtin_asinl:
2689 case Builtin::BI__builtin_asinf128:
2691 *
this,
E, Intrinsic::asin, Intrinsic::experimental_constrained_asin));
2693 case Builtin::BIatan:
2694 case Builtin::BIatanf:
2695 case Builtin::BIatanl:
2696 case Builtin::BI__builtin_atan:
2697 case Builtin::BI__builtin_atanf:
2698 case Builtin::BI__builtin_atanf16:
2699 case Builtin::BI__builtin_atanl:
2700 case Builtin::BI__builtin_atanf128:
2702 *
this,
E, Intrinsic::atan, Intrinsic::experimental_constrained_atan));
2704 case Builtin::BIceil:
2705 case Builtin::BIceilf:
2706 case Builtin::BIceill:
2707 case Builtin::BI__builtin_ceil:
2708 case Builtin::BI__builtin_ceilf:
2709 case Builtin::BI__builtin_ceilf16:
2710 case Builtin::BI__builtin_ceill:
2711 case Builtin::BI__builtin_ceilf128:
2714 Intrinsic::experimental_constrained_ceil));
2716 case Builtin::BIcopysign:
2717 case Builtin::BIcopysignf:
2718 case Builtin::BIcopysignl:
2719 case Builtin::BI__builtin_copysign:
2720 case Builtin::BI__builtin_copysignf:
2721 case Builtin::BI__builtin_copysignf16:
2722 case Builtin::BI__builtin_copysignl:
2723 case Builtin::BI__builtin_copysignf128:
2725 emitBuiltinWithOneOverloadedType<2>(*
this,
E, Intrinsic::copysign));
2727 case Builtin::BIcos:
2728 case Builtin::BIcosf:
2729 case Builtin::BIcosl:
2730 case Builtin::BI__builtin_cos:
2731 case Builtin::BI__builtin_cosf:
2732 case Builtin::BI__builtin_cosf16:
2733 case Builtin::BI__builtin_cosl:
2734 case Builtin::BI__builtin_cosf128:
2737 Intrinsic::experimental_constrained_cos));
2739 case Builtin::BIcosh:
2740 case Builtin::BIcoshf:
2741 case Builtin::BIcoshl:
2742 case Builtin::BI__builtin_cosh:
2743 case Builtin::BI__builtin_coshf:
2744 case Builtin::BI__builtin_coshf16:
2745 case Builtin::BI__builtin_coshl:
2746 case Builtin::BI__builtin_coshf128:
2748 *
this,
E, Intrinsic::cosh, Intrinsic::experimental_constrained_cosh));
2750 case Builtin::BIexp:
2751 case Builtin::BIexpf:
2752 case Builtin::BIexpl:
2753 case Builtin::BI__builtin_exp:
2754 case Builtin::BI__builtin_expf:
2755 case Builtin::BI__builtin_expf16:
2756 case Builtin::BI__builtin_expl:
2757 case Builtin::BI__builtin_expf128:
2760 Intrinsic::experimental_constrained_exp));
2762 case Builtin::BIexp2:
2763 case Builtin::BIexp2f:
2764 case Builtin::BIexp2l:
2765 case Builtin::BI__builtin_exp2:
2766 case Builtin::BI__builtin_exp2f:
2767 case Builtin::BI__builtin_exp2f16:
2768 case Builtin::BI__builtin_exp2l:
2769 case Builtin::BI__builtin_exp2f128:
2772 Intrinsic::experimental_constrained_exp2));
2773 case Builtin::BI__builtin_exp10:
2774 case Builtin::BI__builtin_exp10f:
2775 case Builtin::BI__builtin_exp10f16:
2776 case Builtin::BI__builtin_exp10l:
2777 case Builtin::BI__builtin_exp10f128: {
2779 if (
Builder.getIsFPConstrained())
2782 emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::exp10));
2784 case Builtin::BIfabs:
2785 case Builtin::BIfabsf:
2786 case Builtin::BIfabsl:
2787 case Builtin::BI__builtin_fabs:
2788 case Builtin::BI__builtin_fabsf:
2789 case Builtin::BI__builtin_fabsf16:
2790 case Builtin::BI__builtin_fabsl:
2791 case Builtin::BI__builtin_fabsf128:
2793 emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::fabs));
2795 case Builtin::BIfloor:
2796 case Builtin::BIfloorf:
2797 case Builtin::BIfloorl:
2798 case Builtin::BI__builtin_floor:
2799 case Builtin::BI__builtin_floorf:
2800 case Builtin::BI__builtin_floorf16:
2801 case Builtin::BI__builtin_floorl:
2802 case Builtin::BI__builtin_floorf128:
2805 Intrinsic::experimental_constrained_floor));
2807 case Builtin::BIfma:
2808 case Builtin::BIfmaf:
2809 case Builtin::BIfmal:
2810 case Builtin::BI__builtin_fma:
2811 case Builtin::BI__builtin_fmaf:
2812 case Builtin::BI__builtin_fmaf16:
2813 case Builtin::BI__builtin_fmal:
2814 case Builtin::BI__builtin_fmaf128:
2817 Intrinsic::experimental_constrained_fma));
2819 case Builtin::BIfmax:
2820 case Builtin::BIfmaxf:
2821 case Builtin::BIfmaxl:
2822 case Builtin::BI__builtin_fmax:
2823 case Builtin::BI__builtin_fmaxf:
2824 case Builtin::BI__builtin_fmaxf16:
2825 case Builtin::BI__builtin_fmaxl:
2826 case Builtin::BI__builtin_fmaxf128:
2829 Intrinsic::experimental_constrained_maxnum));
2831 case Builtin::BIfmin:
2832 case Builtin::BIfminf:
2833 case Builtin::BIfminl:
2834 case Builtin::BI__builtin_fmin:
2835 case Builtin::BI__builtin_fminf:
2836 case Builtin::BI__builtin_fminf16:
2837 case Builtin::BI__builtin_fminl:
2838 case Builtin::BI__builtin_fminf128:
2841 Intrinsic::experimental_constrained_minnum));
2845 case Builtin::BIfmod:
2846 case Builtin::BIfmodf:
2847 case Builtin::BIfmodl:
2848 case Builtin::BI__builtin_fmod:
2849 case Builtin::BI__builtin_fmodf:
2850 case Builtin::BI__builtin_fmodf16:
2851 case Builtin::BI__builtin_fmodl:
2852 case Builtin::BI__builtin_fmodf128: {
2853 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
2859 case Builtin::BIlog:
2860 case Builtin::BIlogf:
2861 case Builtin::BIlogl:
2862 case Builtin::BI__builtin_log:
2863 case Builtin::BI__builtin_logf:
2864 case Builtin::BI__builtin_logf16:
2865 case Builtin::BI__builtin_logl:
2866 case Builtin::BI__builtin_logf128:
2869 Intrinsic::experimental_constrained_log));
2871 case Builtin::BIlog10:
2872 case Builtin::BIlog10f:
2873 case Builtin::BIlog10l:
2874 case Builtin::BI__builtin_log10:
2875 case Builtin::BI__builtin_log10f:
2876 case Builtin::BI__builtin_log10f16:
2877 case Builtin::BI__builtin_log10l:
2878 case Builtin::BI__builtin_log10f128:
2881 Intrinsic::experimental_constrained_log10));
2883 case Builtin::BIlog2:
2884 case Builtin::BIlog2f:
2885 case Builtin::BIlog2l:
2886 case Builtin::BI__builtin_log2:
2887 case Builtin::BI__builtin_log2f:
2888 case Builtin::BI__builtin_log2f16:
2889 case Builtin::BI__builtin_log2l:
2890 case Builtin::BI__builtin_log2f128:
2893 Intrinsic::experimental_constrained_log2));
2895 case Builtin::BInearbyint:
2896 case Builtin::BInearbyintf:
2897 case Builtin::BInearbyintl:
2898 case Builtin::BI__builtin_nearbyint:
2899 case Builtin::BI__builtin_nearbyintf:
2900 case Builtin::BI__builtin_nearbyintl:
2901 case Builtin::BI__builtin_nearbyintf128:
2903 Intrinsic::nearbyint,
2904 Intrinsic::experimental_constrained_nearbyint));
2906 case Builtin::BIpow:
2907 case Builtin::BIpowf:
2908 case Builtin::BIpowl:
2909 case Builtin::BI__builtin_pow:
2910 case Builtin::BI__builtin_powf:
2911 case Builtin::BI__builtin_powf16:
2912 case Builtin::BI__builtin_powl:
2913 case Builtin::BI__builtin_powf128:
2916 Intrinsic::experimental_constrained_pow));
2918 case Builtin::BIrint:
2919 case Builtin::BIrintf:
2920 case Builtin::BIrintl:
2921 case Builtin::BI__builtin_rint:
2922 case Builtin::BI__builtin_rintf:
2923 case Builtin::BI__builtin_rintf16:
2924 case Builtin::BI__builtin_rintl:
2925 case Builtin::BI__builtin_rintf128:
2928 Intrinsic::experimental_constrained_rint));
2930 case Builtin::BIround:
2931 case Builtin::BIroundf:
2932 case Builtin::BIroundl:
2933 case Builtin::BI__builtin_round:
2934 case Builtin::BI__builtin_roundf:
2935 case Builtin::BI__builtin_roundf16:
2936 case Builtin::BI__builtin_roundl:
2937 case Builtin::BI__builtin_roundf128:
2940 Intrinsic::experimental_constrained_round));
2942 case Builtin::BIroundeven:
2943 case Builtin::BIroundevenf:
2944 case Builtin::BIroundevenl:
2945 case Builtin::BI__builtin_roundeven:
2946 case Builtin::BI__builtin_roundevenf:
2947 case Builtin::BI__builtin_roundevenf16:
2948 case Builtin::BI__builtin_roundevenl:
2949 case Builtin::BI__builtin_roundevenf128:
2951 Intrinsic::roundeven,
2952 Intrinsic::experimental_constrained_roundeven));
2954 case Builtin::BIsin:
2955 case Builtin::BIsinf:
2956 case Builtin::BIsinl:
2957 case Builtin::BI__builtin_sin:
2958 case Builtin::BI__builtin_sinf:
2959 case Builtin::BI__builtin_sinf16:
2960 case Builtin::BI__builtin_sinl:
2961 case Builtin::BI__builtin_sinf128:
2964 Intrinsic::experimental_constrained_sin));
2966 case Builtin::BIsinh:
2967 case Builtin::BIsinhf:
2968 case Builtin::BIsinhl:
2969 case Builtin::BI__builtin_sinh:
2970 case Builtin::BI__builtin_sinhf:
2971 case Builtin::BI__builtin_sinhf16:
2972 case Builtin::BI__builtin_sinhl:
2973 case Builtin::BI__builtin_sinhf128:
2975 *
this,
E, Intrinsic::sinh, Intrinsic::experimental_constrained_sinh));
2977 case Builtin::BIsqrt:
2978 case Builtin::BIsqrtf:
2979 case Builtin::BIsqrtl:
2980 case Builtin::BI__builtin_sqrt:
2981 case Builtin::BI__builtin_sqrtf:
2982 case Builtin::BI__builtin_sqrtf16:
2983 case Builtin::BI__builtin_sqrtl:
2984 case Builtin::BI__builtin_sqrtf128:
2985 case Builtin::BI__builtin_elementwise_sqrt: {
2987 *
this,
E, Intrinsic::sqrt, Intrinsic::experimental_constrained_sqrt);
2992 case Builtin::BItan:
2993 case Builtin::BItanf:
2994 case Builtin::BItanl:
2995 case Builtin::BI__builtin_tan:
2996 case Builtin::BI__builtin_tanf:
2997 case Builtin::BI__builtin_tanf16:
2998 case Builtin::BI__builtin_tanl:
2999 case Builtin::BI__builtin_tanf128:
3001 *
this,
E, Intrinsic::tan, Intrinsic::experimental_constrained_tan));
3003 case Builtin::BItanh:
3004 case Builtin::BItanhf:
3005 case Builtin::BItanhl:
3006 case Builtin::BI__builtin_tanh:
3007 case Builtin::BI__builtin_tanhf:
3008 case Builtin::BI__builtin_tanhf16:
3009 case Builtin::BI__builtin_tanhl:
3010 case Builtin::BI__builtin_tanhf128:
3012 *
this,
E, Intrinsic::tanh, Intrinsic::experimental_constrained_tanh));
3014 case Builtin::BItrunc:
3015 case Builtin::BItruncf:
3016 case Builtin::BItruncl:
3017 case Builtin::BI__builtin_trunc:
3018 case Builtin::BI__builtin_truncf:
3019 case Builtin::BI__builtin_truncf16:
3020 case Builtin::BI__builtin_truncl:
3021 case Builtin::BI__builtin_truncf128:
3024 Intrinsic::experimental_constrained_trunc));
3026 case Builtin::BIlround:
3027 case Builtin::BIlroundf:
3028 case Builtin::BIlroundl:
3029 case Builtin::BI__builtin_lround:
3030 case Builtin::BI__builtin_lroundf:
3031 case Builtin::BI__builtin_lroundl:
3032 case Builtin::BI__builtin_lroundf128:
3034 *
this,
E, Intrinsic::lround,
3035 Intrinsic::experimental_constrained_lround));
3037 case Builtin::BIllround:
3038 case Builtin::BIllroundf:
3039 case Builtin::BIllroundl:
3040 case Builtin::BI__builtin_llround:
3041 case Builtin::BI__builtin_llroundf:
3042 case Builtin::BI__builtin_llroundl:
3043 case Builtin::BI__builtin_llroundf128:
3045 *
this,
E, Intrinsic::llround,
3046 Intrinsic::experimental_constrained_llround));
3048 case Builtin::BIlrint:
3049 case Builtin::BIlrintf:
3050 case Builtin::BIlrintl:
3051 case Builtin::BI__builtin_lrint:
3052 case Builtin::BI__builtin_lrintf:
3053 case Builtin::BI__builtin_lrintl:
3054 case Builtin::BI__builtin_lrintf128:
3056 *
this,
E, Intrinsic::lrint,
3057 Intrinsic::experimental_constrained_lrint));
3059 case Builtin::BIllrint:
3060 case Builtin::BIllrintf:
3061 case Builtin::BIllrintl:
3062 case Builtin::BI__builtin_llrint:
3063 case Builtin::BI__builtin_llrintf:
3064 case Builtin::BI__builtin_llrintl:
3065 case Builtin::BI__builtin_llrintf128:
3067 *
this,
E, Intrinsic::llrint,
3068 Intrinsic::experimental_constrained_llrint));
3069 case Builtin::BI__builtin_ldexp:
3070 case Builtin::BI__builtin_ldexpf:
3071 case Builtin::BI__builtin_ldexpl:
3072 case Builtin::BI__builtin_ldexpf16:
3073 case Builtin::BI__builtin_ldexpf128: {
3075 *
this,
E, Intrinsic::ldexp,
3076 Intrinsic::experimental_constrained_ldexp));
3086 Value *Val = A.emitRawPointer(*
this);
3092 SkippedChecks.
set(SanitizerKind::All);
3093 SkippedChecks.
clear(SanitizerKind::Alignment);
3096 if (
auto *CE = dyn_cast<ImplicitCastExpr>(Arg))
3097 if (CE->getCastKind() == CK_BitCast)
3098 Arg = CE->getSubExpr();
3104 switch (BuiltinIDIfNoAsmLabel) {
3106 case Builtin::BI__builtin___CFStringMakeConstantString:
3107 case Builtin::BI__builtin___NSStringMakeConstantString:
3109 case Builtin::BI__builtin_stdarg_start:
3110 case Builtin::BI__builtin_va_start:
3111 case Builtin::BI__va_start:
3112 case Builtin::BI__builtin_va_end:
3116 BuiltinID != Builtin::BI__builtin_va_end);
3118 case Builtin::BI__builtin_va_copy: {
3125 case Builtin::BIabs:
3126 case Builtin::BIlabs:
3127 case Builtin::BIllabs:
3128 case Builtin::BI__builtin_abs:
3129 case Builtin::BI__builtin_labs:
3130 case Builtin::BI__builtin_llabs: {
3131 bool SanitizeOverflow =
SanOpts.
has(SanitizerKind::SignedIntegerOverflow);
3134 switch (
getLangOpts().getSignedOverflowBehavior()) {
3139 if (!SanitizeOverflow) {
3151 case Builtin::BI__builtin_complex: {
3156 case Builtin::BI__builtin_conj:
3157 case Builtin::BI__builtin_conjf:
3158 case Builtin::BI__builtin_conjl:
3159 case Builtin::BIconj:
3160 case Builtin::BIconjf:
3161 case Builtin::BIconjl: {
3163 Value *Real = ComplexVal.first;
3164 Value *Imag = ComplexVal.second;
3165 Imag =
Builder.CreateFNeg(Imag,
"neg");
3168 case Builtin::BI__builtin_creal:
3169 case Builtin::BI__builtin_crealf:
3170 case Builtin::BI__builtin_creall:
3171 case Builtin::BIcreal:
3172 case Builtin::BIcrealf:
3173 case Builtin::BIcreall: {
3178 case Builtin::BI__builtin_preserve_access_index: {
3199 case Builtin::BI__builtin_cimag:
3200 case Builtin::BI__builtin_cimagf:
3201 case Builtin::BI__builtin_cimagl:
3202 case Builtin::BIcimag:
3203 case Builtin::BIcimagf:
3204 case Builtin::BIcimagl: {
3209 case Builtin::BI__builtin_clrsb:
3210 case Builtin::BI__builtin_clrsbl:
3211 case Builtin::BI__builtin_clrsbll: {
3215 llvm::Type *ArgType = ArgValue->
getType();
3219 Value *
Zero = llvm::Constant::getNullValue(ArgType);
3220 Value *IsNeg =
Builder.CreateICmpSLT(ArgValue, Zero,
"isneg");
3222 Value *Tmp =
Builder.CreateSelect(IsNeg, Inverse, ArgValue);
3229 case Builtin::BI__builtin_ctzs:
3230 case Builtin::BI__builtin_ctz:
3231 case Builtin::BI__builtin_ctzl:
3232 case Builtin::BI__builtin_ctzll:
3233 case Builtin::BI__builtin_ctzg: {
3234 bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_ctzg &&
3235 E->getNumArgs() > 1;
3241 llvm::Type *ArgType = ArgValue->
getType();
3248 if (
Result->getType() != ResultType)
3254 Value *
Zero = Constant::getNullValue(ArgType);
3255 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, Zero,
"iszero");
3257 Value *ResultOrFallback =
3258 Builder.CreateSelect(IsZero, FallbackValue,
Result,
"ctzg");
3261 case Builtin::BI__builtin_clzs:
3262 case Builtin::BI__builtin_clz:
3263 case Builtin::BI__builtin_clzl:
3264 case Builtin::BI__builtin_clzll:
3265 case Builtin::BI__builtin_clzg: {
3266 bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_clzg &&
3267 E->getNumArgs() > 1;
3273 llvm::Type *ArgType = ArgValue->
getType();
3280 if (
Result->getType() != ResultType)
3286 Value *
Zero = Constant::getNullValue(ArgType);
3287 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, Zero,
"iszero");
3289 Value *ResultOrFallback =
3290 Builder.CreateSelect(IsZero, FallbackValue,
Result,
"clzg");
3293 case Builtin::BI__builtin_ffs:
3294 case Builtin::BI__builtin_ffsl:
3295 case Builtin::BI__builtin_ffsll: {
3299 llvm::Type *ArgType = ArgValue->
getType();
3304 Builder.CreateAdd(
Builder.CreateCall(F, {ArgValue, Builder.getTrue()}),
3305 llvm::ConstantInt::get(ArgType, 1));
3306 Value *
Zero = llvm::Constant::getNullValue(ArgType);
3307 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, Zero,
"iszero");
3309 if (
Result->getType() != ResultType)
3314 case Builtin::BI__builtin_parity:
3315 case Builtin::BI__builtin_parityl:
3316 case Builtin::BI__builtin_parityll: {
3320 llvm::Type *ArgType = ArgValue->
getType();
3326 if (
Result->getType() != ResultType)
3331 case Builtin::BI__lzcnt16:
3332 case Builtin::BI__lzcnt:
3333 case Builtin::BI__lzcnt64: {
3336 llvm::Type *ArgType = ArgValue->
getType();
3341 if (
Result->getType() != ResultType)
3346 case Builtin::BI__popcnt16:
3347 case Builtin::BI__popcnt:
3348 case Builtin::BI__popcnt64:
3349 case Builtin::BI__builtin_popcount:
3350 case Builtin::BI__builtin_popcountl:
3351 case Builtin::BI__builtin_popcountll:
3352 case Builtin::BI__builtin_popcountg: {
3355 llvm::Type *ArgType = ArgValue->
getType();
3360 if (
Result->getType() != ResultType)
3365 case Builtin::BI__builtin_unpredictable: {
3371 case Builtin::BI__builtin_expect: {
3373 llvm::Type *ArgType = ArgValue->
getType();
3384 Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue},
"expval");
3387 case Builtin::BI__builtin_expect_with_probability: {
3389 llvm::Type *ArgType = ArgValue->
getType();
3392 llvm::APFloat Probability(0.0);
3393 const Expr *ProbArg =
E->getArg(2);
3395 assert(EvalSucceed &&
"probability should be able to evaluate as float");
3397 bool LoseInfo =
false;
3398 Probability.convert(llvm::APFloat::IEEEdouble(),
3399 llvm::RoundingMode::Dynamic, &LoseInfo);
3401 Constant *Confidence = ConstantFP::get(Ty, Probability);
3411 FnExpect, {ArgValue, ExpectedValue, Confidence},
"expval");
3414 case Builtin::BI__builtin_assume_aligned: {
3415 const Expr *Ptr =
E->getArg(0);
3417 Value *OffsetValue =
3421 ConstantInt *AlignmentCI = cast<ConstantInt>(AlignmentValue);
3422 if (AlignmentCI->getValue().ugt(llvm::Value::MaximumAlignment))
3423 AlignmentCI = ConstantInt::get(AlignmentCI->getIntegerType(),
3424 llvm::Value::MaximumAlignment);
3428 AlignmentCI, OffsetValue);
3431 case Builtin::BI__assume:
3432 case Builtin::BI__builtin_assume: {
3438 Builder.CreateCall(FnAssume, ArgValue);
3441 case Builtin::BI__builtin_assume_separate_storage: {
3442 const Expr *Arg0 =
E->getArg(0);
3443 const Expr *Arg1 =
E->getArg(1);
3448 Value *Values[] = {Value0, Value1};
3449 OperandBundleDefT<Value *> OBD(
"separate_storage", Values);
3453 case Builtin::BI__builtin_allow_runtime_check: {
3457 llvm::Value *Allow =
Builder.CreateCall(
3459 llvm::MetadataAsValue::get(Ctx, llvm::MDString::get(Ctx, Kind)));
3462 case Builtin::BI__arithmetic_fence: {
3465 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
3466 llvm::FastMathFlags FMF =
Builder.getFastMathFlags();
3467 bool isArithmeticFenceEnabled =
3468 FMF.allowReassoc() &&
3472 if (isArithmeticFenceEnabled) {
3475 Value *Real =
Builder.CreateArithmeticFence(ComplexVal.first,
3477 Value *Imag =
Builder.CreateArithmeticFence(ComplexVal.second,
3482 Value *Real = ComplexVal.first;
3483 Value *Imag = ComplexVal.second;
3487 if (isArithmeticFenceEnabled)
3492 case Builtin::BI__builtin_bswap16:
3493 case Builtin::BI__builtin_bswap32:
3494 case Builtin::BI__builtin_bswap64:
3495 case Builtin::BI_byteswap_ushort:
3496 case Builtin::BI_byteswap_ulong:
3497 case Builtin::BI_byteswap_uint64: {
3499 emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::bswap));
3501 case Builtin::BI__builtin_bitreverse8:
3502 case Builtin::BI__builtin_bitreverse16:
3503 case Builtin::BI__builtin_bitreverse32:
3504 case Builtin::BI__builtin_bitreverse64: {
3506 emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::bitreverse));
3508 case Builtin::BI__builtin_rotateleft8:
3509 case Builtin::BI__builtin_rotateleft16:
3510 case Builtin::BI__builtin_rotateleft32:
3511 case Builtin::BI__builtin_rotateleft64:
3512 case Builtin::BI_rotl8:
3513 case Builtin::BI_rotl16:
3514 case Builtin::BI_rotl:
3515 case Builtin::BI_lrotl:
3516 case Builtin::BI_rotl64:
3519 case Builtin::BI__builtin_rotateright8:
3520 case Builtin::BI__builtin_rotateright16:
3521 case Builtin::BI__builtin_rotateright32:
3522 case Builtin::BI__builtin_rotateright64:
3523 case Builtin::BI_rotr8:
3524 case Builtin::BI_rotr16:
3525 case Builtin::BI_rotr:
3526 case Builtin::BI_lrotr:
3527 case Builtin::BI_rotr64:
3530 case Builtin::BI__builtin_constant_p: {
3533 const Expr *Arg =
E->getArg(0);
3541 return RValue::get(ConstantInt::get(ResultType, 0));
3546 return RValue::get(ConstantInt::get(ResultType, 0));
3558 if (
Result->getType() != ResultType)
3562 case Builtin::BI__builtin_dynamic_object_size:
3563 case Builtin::BI__builtin_object_size: {
3570 bool IsDynamic = BuiltinID == Builtin::BI__builtin_dynamic_object_size;
3572 nullptr, IsDynamic));
3574 case Builtin::BI__builtin_prefetch: {
3578 llvm::ConstantInt::get(
Int32Ty, 0);
3580 llvm::ConstantInt::get(
Int32Ty, 3);
3586 case Builtin::BI__builtin_readcyclecounter: {
3590 case Builtin::BI__builtin_readsteadycounter: {
3594 case Builtin::BI__builtin___clear_cache: {
3600 case Builtin::BI__builtin_trap:
3603 case Builtin::BI__builtin_verbose_trap: {
3604 llvm::DILocation *TrapLocation =
Builder.getCurrentDebugLocation();
3615 case Builtin::BI__debugbreak:
3618 case Builtin::BI__builtin_unreachable: {
3627 case Builtin::BI__builtin_powi:
3628 case Builtin::BI__builtin_powif:
3629 case Builtin::BI__builtin_powil: {
3633 if (
Builder.getIsFPConstrained()) {
3636 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
3643 { Src0->getType(), Src1->getType() });
3646 case Builtin::BI__builtin_frexpl: {
3650 if (&
getTarget().getLongDoubleFormat() == &llvm::APFloat::PPCDoubleDouble())
3654 case Builtin::BI__builtin_frexp:
3655 case Builtin::BI__builtin_frexpf:
3656 case Builtin::BI__builtin_frexpf128:
3657 case Builtin::BI__builtin_frexpf16:
3659 case Builtin::BI__builtin_isgreater:
3660 case Builtin::BI__builtin_isgreaterequal:
3661 case Builtin::BI__builtin_isless:
3662 case Builtin::BI__builtin_islessequal:
3663 case Builtin::BI__builtin_islessgreater:
3664 case Builtin::BI__builtin_isunordered: {
3667 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
3671 switch (BuiltinID) {
3672 default: llvm_unreachable(
"Unknown ordered comparison");
3673 case Builtin::BI__builtin_isgreater:
3674 LHS =
Builder.CreateFCmpOGT(LHS, RHS,
"cmp");
3676 case Builtin::BI__builtin_isgreaterequal:
3677 LHS =
Builder.CreateFCmpOGE(LHS, RHS,
"cmp");
3679 case Builtin::BI__builtin_isless:
3680 LHS =
Builder.CreateFCmpOLT(LHS, RHS,
"cmp");
3682 case Builtin::BI__builtin_islessequal:
3683 LHS =
Builder.CreateFCmpOLE(LHS, RHS,
"cmp");
3685 case Builtin::BI__builtin_islessgreater:
3686 LHS =
Builder.CreateFCmpONE(LHS, RHS,
"cmp");
3688 case Builtin::BI__builtin_isunordered:
3689 LHS =
Builder.CreateFCmpUNO(LHS, RHS,
"cmp");
3696 case Builtin::BI__builtin_isnan: {
3697 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
3706 case Builtin::BI__builtin_issignaling: {
3707 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
3714 case Builtin::BI__builtin_isinf: {
3715 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
3724 case Builtin::BIfinite:
3725 case Builtin::BI__finite:
3726 case Builtin::BIfinitef:
3727 case Builtin::BI__finitef:
3728 case Builtin::BIfinitel:
3729 case Builtin::BI__finitel:
3730 case Builtin::BI__builtin_isfinite: {
3731 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
3740 case Builtin::BI__builtin_isnormal: {
3741 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
3748 case Builtin::BI__builtin_issubnormal: {
3749 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
3752 Builder.CreateZExt(
Builder.createIsFPClass(
V, FPClassTest::fcSubnormal),
3756 case Builtin::BI__builtin_iszero: {
3757 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
3764 case Builtin::BI__builtin_isfpclass: {
3769 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
3775 case Builtin::BI__builtin_nondeterministic_value: {
3784 case Builtin::BI__builtin_elementwise_abs: {
3789 QT = VecTy->getElementType();
3793 Builder.getFalse(),
nullptr,
"elt.abs");
3795 Result = emitBuiltinWithOneOverloadedType<1>(
3796 *
this,
E, llvm::Intrinsic::fabs,
"elt.abs");
3800 case Builtin::BI__builtin_elementwise_acos:
3801 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3802 *
this,
E, llvm::Intrinsic::acos,
"elt.acos"));
3803 case Builtin::BI__builtin_elementwise_asin:
3804 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3805 *
this,
E, llvm::Intrinsic::asin,
"elt.asin"));
3806 case Builtin::BI__builtin_elementwise_atan:
3807 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3808 *
this,
E, llvm::Intrinsic::atan,
"elt.atan"));
3809 case Builtin::BI__builtin_elementwise_ceil:
3810 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3811 *
this,
E, llvm::Intrinsic::ceil,
"elt.ceil"));
3812 case Builtin::BI__builtin_elementwise_exp:
3813 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3814 *
this,
E, llvm::Intrinsic::exp,
"elt.exp"));
3815 case Builtin::BI__builtin_elementwise_exp2:
3816 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3817 *
this,
E, llvm::Intrinsic::exp2,
"elt.exp2"));
3818 case Builtin::BI__builtin_elementwise_log:
3819 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3820 *
this,
E, llvm::Intrinsic::log,
"elt.log"));
3821 case Builtin::BI__builtin_elementwise_log2:
3822 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3823 *
this,
E, llvm::Intrinsic::log2,
"elt.log2"));
3824 case Builtin::BI__builtin_elementwise_log10:
3825 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3826 *
this,
E, llvm::Intrinsic::log10,
"elt.log10"));
3827 case Builtin::BI__builtin_elementwise_pow: {
3829 emitBuiltinWithOneOverloadedType<2>(*
this,
E, llvm::Intrinsic::pow));
3831 case Builtin::BI__builtin_elementwise_bitreverse:
3832 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3833 *
this,
E, llvm::Intrinsic::bitreverse,
"elt.bitreverse"));
3834 case Builtin::BI__builtin_elementwise_cos:
3835 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3836 *
this,
E, llvm::Intrinsic::cos,
"elt.cos"));
3837 case Builtin::BI__builtin_elementwise_cosh:
3838 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3839 *
this,
E, llvm::Intrinsic::cosh,
"elt.cosh"));
3840 case Builtin::BI__builtin_elementwise_floor:
3841 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3842 *
this,
E, llvm::Intrinsic::floor,
"elt.floor"));
3843 case Builtin::BI__builtin_elementwise_roundeven:
3844 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3845 *
this,
E, llvm::Intrinsic::roundeven,
"elt.roundeven"));
3846 case Builtin::BI__builtin_elementwise_round:
3847 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3848 *
this,
E, llvm::Intrinsic::round,
"elt.round"));
3849 case Builtin::BI__builtin_elementwise_rint:
3850 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3851 *
this,
E, llvm::Intrinsic::rint,
"elt.rint"));
3852 case Builtin::BI__builtin_elementwise_nearbyint:
3853 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3854 *
this,
E, llvm::Intrinsic::nearbyint,
"elt.nearbyint"));
3855 case Builtin::BI__builtin_elementwise_sin:
3856 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3857 *
this,
E, llvm::Intrinsic::sin,
"elt.sin"));
3858 case Builtin::BI__builtin_elementwise_sinh:
3859 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3860 *
this,
E, llvm::Intrinsic::sinh,
"elt.sinh"));
3861 case Builtin::BI__builtin_elementwise_tan:
3862 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3863 *
this,
E, llvm::Intrinsic::tan,
"elt.tan"));
3864 case Builtin::BI__builtin_elementwise_tanh:
3865 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3866 *
this,
E, llvm::Intrinsic::tanh,
"elt.tanh"));
3867 case Builtin::BI__builtin_elementwise_trunc:
3868 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3869 *
this,
E, llvm::Intrinsic::trunc,
"elt.trunc"));
3870 case Builtin::BI__builtin_elementwise_canonicalize:
3871 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3872 *
this,
E, llvm::Intrinsic::canonicalize,
"elt.canonicalize"));
3873 case Builtin::BI__builtin_elementwise_copysign:
3874 return RValue::get(emitBuiltinWithOneOverloadedType<2>(
3875 *
this,
E, llvm::Intrinsic::copysign));
3876 case Builtin::BI__builtin_elementwise_fma:
3878 emitBuiltinWithOneOverloadedType<3>(*
this,
E, llvm::Intrinsic::fma));
3879 case Builtin::BI__builtin_elementwise_add_sat:
3880 case Builtin::BI__builtin_elementwise_sub_sat: {
3884 assert(Op0->
getType()->isIntOrIntVectorTy() &&
"integer type expected");
3887 Ty = VecTy->getElementType();
3890 if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_add_sat)
3891 Opc = IsSigned ? llvm::Intrinsic::sadd_sat : llvm::Intrinsic::uadd_sat;
3893 Opc = IsSigned ? llvm::Intrinsic::ssub_sat : llvm::Intrinsic::usub_sat;
3894 Result =
Builder.CreateBinaryIntrinsic(Opc, Op0, Op1,
nullptr,
"elt.sat");
3898 case Builtin::BI__builtin_elementwise_max: {
3902 if (Op0->
getType()->isIntOrIntVectorTy()) {
3905 Ty = VecTy->getElementType();
3907 ? llvm::Intrinsic::smax
3908 : llvm::Intrinsic::umax,
3909 Op0, Op1,
nullptr,
"elt.max");
3914 case Builtin::BI__builtin_elementwise_min: {
3918 if (Op0->
getType()->isIntOrIntVectorTy()) {
3921 Ty = VecTy->getElementType();
3923 ? llvm::Intrinsic::smin
3924 : llvm::Intrinsic::umin,
3925 Op0, Op1,
nullptr,
"elt.min");
3931 case Builtin::BI__builtin_reduce_max: {
3932 auto GetIntrinsicID = [
this](
QualType QT) {
3934 QT = VecTy->getElementType();
3939 return llvm::Intrinsic::vector_reduce_smax;
3941 return llvm::Intrinsic::vector_reduce_umax;
3943 return llvm::Intrinsic::vector_reduce_fmax;
3945 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3946 *
this,
E, GetIntrinsicID(
E->getArg(0)->
getType()),
"rdx.min"));
3949 case Builtin::BI__builtin_reduce_min: {
3950 auto GetIntrinsicID = [
this](
QualType QT) {
3952 QT = VecTy->getElementType();
3957 return llvm::Intrinsic::vector_reduce_smin;
3959 return llvm::Intrinsic::vector_reduce_umin;
3961 return llvm::Intrinsic::vector_reduce_fmin;
3964 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3965 *
this,
E, GetIntrinsicID(
E->getArg(0)->
getType()),
"rdx.min"));
3968 case Builtin::BI__builtin_reduce_add:
3969 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3970 *
this,
E, llvm::Intrinsic::vector_reduce_add,
"rdx.add"));
3971 case Builtin::BI__builtin_reduce_mul:
3972 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3973 *
this,
E, llvm::Intrinsic::vector_reduce_mul,
"rdx.mul"));
3974 case Builtin::BI__builtin_reduce_xor:
3975 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3976 *
this,
E, llvm::Intrinsic::vector_reduce_xor,
"rdx.xor"));
3977 case Builtin::BI__builtin_reduce_or:
3978 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3979 *
this,
E, llvm::Intrinsic::vector_reduce_or,
"rdx.or"));
3980 case Builtin::BI__builtin_reduce_and:
3981 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
3982 *
this,
E, llvm::Intrinsic::vector_reduce_and,
"rdx.and"));
3984 case Builtin::BI__builtin_matrix_transpose: {
3988 Value *
Result = MB.CreateMatrixTranspose(MatValue, MatrixTy->getNumRows(),
3989 MatrixTy->getNumColumns());
3993 case Builtin::BI__builtin_matrix_column_major_load: {
3999 assert(PtrTy &&
"arg0 must be of pointer type");
4009 ResultTy->getNumRows(), ResultTy->getNumColumns(),
"matrix");
4013 case Builtin::BI__builtin_matrix_column_major_store: {
4021 assert(PtrTy &&
"arg1 must be of pointer type");
4030 MatrixTy->getNumRows(), MatrixTy->getNumColumns());
4034 case Builtin::BI__builtin_isinf_sign: {
4036 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4041 AbsArg, ConstantFP::getInfinity(Arg->
getType()),
"isinf");
4047 Value *NegativeOne = ConstantInt::get(
IntTy, -1);
4048 Value *SignResult =
Builder.CreateSelect(IsNeg, NegativeOne, One);
4053 case Builtin::BI__builtin_flt_rounds: {
4058 if (
Result->getType() != ResultType)
4064 case Builtin::BI__builtin_set_flt_rounds: {
4072 case Builtin::BI__builtin_fpclassify: {
4073 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4084 "fpclassify_result");
4088 Value *IsZero =
Builder.CreateFCmpOEQ(
V, Constant::getNullValue(Ty),
4092 Builder.CreateCondBr(IsZero, End, NotZero);
4096 Builder.SetInsertPoint(NotZero);
4100 Builder.CreateCondBr(IsNan, End, NotNan);
4101 Result->addIncoming(NanLiteral, NotZero);
4104 Builder.SetInsertPoint(NotNan);
4107 Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(
V->getType()),
4111 Builder.CreateCondBr(IsInf, End, NotInf);
4112 Result->addIncoming(InfLiteral, NotNan);
4115 Builder.SetInsertPoint(NotInf);
4116 APFloat Smallest = APFloat::getSmallestNormalized(
4119 Builder.CreateFCmpUGE(VAbs, ConstantFP::get(
V->getContext(), Smallest),
4121 Value *NormalResult =
4125 Result->addIncoming(NormalResult, NotInf);
4138 case Builtin::BIalloca:
4139 case Builtin::BI_alloca:
4140 case Builtin::BI__builtin_alloca_uninitialized:
4141 case Builtin::BI__builtin_alloca: {
4145 const Align SuitableAlignmentInBytes =
4149 AllocaInst *AI =
Builder.CreateAlloca(
Builder.getInt8Ty(), Size);
4150 AI->setAlignment(SuitableAlignmentInBytes);
4151 if (BuiltinID != Builtin::BI__builtin_alloca_uninitialized)
4163 case Builtin::BI__builtin_alloca_with_align_uninitialized:
4164 case Builtin::BI__builtin_alloca_with_align: {
4167 auto *AlignmentInBitsCI = cast<ConstantInt>(AlignmentInBitsValue);
4168 unsigned AlignmentInBits = AlignmentInBitsCI->getZExtValue();
4169 const Align AlignmentInBytes =
4171 AllocaInst *AI =
Builder.CreateAlloca(
Builder.getInt8Ty(), Size);
4172 AI->setAlignment(AlignmentInBytes);
4173 if (BuiltinID != Builtin::BI__builtin_alloca_with_align_uninitialized)
4185 case Builtin::BIbzero:
4186 case Builtin::BI__builtin_bzero: {
4195 case Builtin::BIbcopy:
4196 case Builtin::BI__builtin_bcopy: {
4210 case Builtin::BImemcpy:
4211 case Builtin::BI__builtin_memcpy:
4212 case Builtin::BImempcpy:
4213 case Builtin::BI__builtin_mempcpy: {
4217 EmitArgCheck(
TCK_Store, Dest,
E->getArg(0), 0);
4218 EmitArgCheck(
TCK_Load, Src,
E->getArg(1), 1);
4220 if (BuiltinID == Builtin::BImempcpy ||
4221 BuiltinID == Builtin::BI__builtin_mempcpy)
4228 case Builtin::BI__builtin_memcpy_inline: {
4233 EmitArgCheck(
TCK_Store, Dest,
E->getArg(0), 0);
4234 EmitArgCheck(
TCK_Load, Src,
E->getArg(1), 1);
4239 case Builtin::BI__builtin_char_memchr:
4240 BuiltinID = Builtin::BI__builtin_memchr;
4243 case Builtin::BI__builtin___memcpy_chk: {
4250 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
4251 if (
Size.ugt(DstSize))
4255 Value *SizeVal = llvm::ConstantInt::get(
Builder.getContext(), Size);
4260 case Builtin::BI__builtin_objc_memmove_collectable: {
4265 DestAddr, SrcAddr, SizeVal);
4269 case Builtin::BI__builtin___memmove_chk: {
4276 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
4277 if (
Size.ugt(DstSize))
4281 Value *SizeVal = llvm::ConstantInt::get(
Builder.getContext(), Size);
4286 case Builtin::BImemmove:
4287 case Builtin::BI__builtin_memmove: {
4291 EmitArgCheck(
TCK_Store, Dest,
E->getArg(0), 0);
4292 EmitArgCheck(
TCK_Load, Src,
E->getArg(1), 1);
4296 case Builtin::BImemset:
4297 case Builtin::BI__builtin_memset: {
4307 case Builtin::BI__builtin_memset_inline: {
4319 case Builtin::BI__builtin___memset_chk: {
4326 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
4327 if (
Size.ugt(DstSize))
4332 Value *SizeVal = llvm::ConstantInt::get(
Builder.getContext(), Size);
4336 case Builtin::BI__builtin_wmemchr: {
4339 if (!
getTarget().getTriple().isOSMSVCRT())
4347 BasicBlock *Entry =
Builder.GetInsertBlock();
4352 Builder.CreateCondBr(SizeEq0, Exit, CmpEq);
4356 StrPhi->addIncoming(Str, Entry);
4358 SizePhi->addIncoming(Size, Entry);
4362 Value *FoundChr =
Builder.CreateConstInBoundsGEP1_32(WCharTy, StrPhi, 0);
4364 Builder.CreateCondBr(StrEqChr, Exit, Next);
4367 Value *NextStr =
Builder.CreateConstInBoundsGEP1_32(WCharTy, StrPhi, 1);
4369 Value *NextSizeEq0 =
4370 Builder.CreateICmpEQ(NextSize, ConstantInt::get(
SizeTy, 0));
4371 Builder.CreateCondBr(NextSizeEq0, Exit, CmpEq);
4372 StrPhi->addIncoming(NextStr, Next);
4373 SizePhi->addIncoming(NextSize, Next);
4377 Ret->addIncoming(llvm::Constant::getNullValue(Str->
getType()), Entry);
4378 Ret->addIncoming(llvm::Constant::getNullValue(Str->
getType()), Next);
4379 Ret->addIncoming(FoundChr, CmpEq);
4382 case Builtin::BI__builtin_wmemcmp: {
4385 if (!
getTarget().getTriple().isOSMSVCRT())
4394 BasicBlock *Entry =
Builder.GetInsertBlock();
4400 Builder.CreateCondBr(SizeEq0, Exit, CmpGT);
4404 DstPhi->addIncoming(Dst, Entry);
4406 SrcPhi->addIncoming(Src, Entry);
4408 SizePhi->addIncoming(Size, Entry);
4414 Builder.CreateCondBr(DstGtSrc, Exit, CmpLT);
4418 Builder.CreateCondBr(DstLtSrc, Exit, Next);
4421 Value *NextDst =
Builder.CreateConstInBoundsGEP1_32(WCharTy, DstPhi, 1);
4422 Value *NextSrc =
Builder.CreateConstInBoundsGEP1_32(WCharTy, SrcPhi, 1);
4424 Value *NextSizeEq0 =
4425 Builder.CreateICmpEQ(NextSize, ConstantInt::get(
SizeTy, 0));
4426 Builder.CreateCondBr(NextSizeEq0, Exit, CmpGT);
4427 DstPhi->addIncoming(NextDst, Next);
4428 SrcPhi->addIncoming(NextSrc, Next);
4429 SizePhi->addIncoming(NextSize, Next);
4433 Ret->addIncoming(ConstantInt::get(
IntTy, 0), Entry);
4434 Ret->addIncoming(ConstantInt::get(
IntTy, 1), CmpGT);
4435 Ret->addIncoming(ConstantInt::get(
IntTy, -1), CmpLT);
4436 Ret->addIncoming(ConstantInt::get(
IntTy, 0), Next);
4439 case Builtin::BI__builtin_dwarf_cfa: {
4452 llvm::ConstantInt::get(
Int32Ty, Offset)));
4454 case Builtin::BI__builtin_return_address: {
4460 case Builtin::BI_ReturnAddress: {
4464 case Builtin::BI__builtin_frame_address: {
4470 case Builtin::BI__builtin_extract_return_addr: {
4475 case Builtin::BI__builtin_frob_return_addr: {
4480 case Builtin::BI__builtin_dwarf_sp_column: {
4481 llvm::IntegerType *Ty
4490 case Builtin::BI__builtin_init_dwarf_reg_size_table: {
4496 case Builtin::BI__builtin_eh_return: {
4500 llvm::IntegerType *
IntTy = cast<llvm::IntegerType>(
Int->getType());
4501 assert((
IntTy->getBitWidth() == 32 ||
IntTy->getBitWidth() == 64) &&
4502 "LLVM's __builtin_eh_return only supports 32- and 64-bit variants");
4505 : Intrinsic::eh_return_i64);
4514 case Builtin::BI__builtin_unwind_init: {
4519 case Builtin::BI__builtin_extend_pointer: {
4544 case Builtin::BI__builtin_setjmp: {
4551 ConstantInt::get(
Int32Ty, 0));
4565 case Builtin::BI__builtin_longjmp: {
4579 case Builtin::BI__builtin_launder: {
4580 const Expr *Arg =
E->getArg(0);
4588 case Builtin::BI__sync_fetch_and_add:
4589 case Builtin::BI__sync_fetch_and_sub:
4590 case Builtin::BI__sync_fetch_and_or:
4591 case Builtin::BI__sync_fetch_and_and:
4592 case Builtin::BI__sync_fetch_and_xor:
4593 case Builtin::BI__sync_fetch_and_nand:
4594 case Builtin::BI__sync_add_and_fetch:
4595 case Builtin::BI__sync_sub_and_fetch:
4596 case Builtin::BI__sync_and_and_fetch:
4597 case Builtin::BI__sync_or_and_fetch:
4598 case Builtin::BI__sync_xor_and_fetch:
4599 case Builtin::BI__sync_nand_and_fetch:
4600 case Builtin::BI__sync_val_compare_and_swap:
4601 case Builtin::BI__sync_bool_compare_and_swap:
4602 case Builtin::BI__sync_lock_test_and_set:
4603 case Builtin::BI__sync_lock_release:
4604 case Builtin::BI__sync_swap:
4605 llvm_unreachable(
"Shouldn't make it through sema");
4606 case Builtin::BI__sync_fetch_and_add_1:
4607 case Builtin::BI__sync_fetch_and_add_2:
4608 case Builtin::BI__sync_fetch_and_add_4:
4609 case Builtin::BI__sync_fetch_and_add_8:
4610 case Builtin::BI__sync_fetch_and_add_16:
4612 case Builtin::BI__sync_fetch_and_sub_1:
4613 case Builtin::BI__sync_fetch_and_sub_2:
4614 case Builtin::BI__sync_fetch_and_sub_4:
4615 case Builtin::BI__sync_fetch_and_sub_8:
4616 case Builtin::BI__sync_fetch_and_sub_16:
4618 case Builtin::BI__sync_fetch_and_or_1:
4619 case Builtin::BI__sync_fetch_and_or_2:
4620 case Builtin::BI__sync_fetch_and_or_4:
4621 case Builtin::BI__sync_fetch_and_or_8:
4622 case Builtin::BI__sync_fetch_and_or_16:
4624 case Builtin::BI__sync_fetch_and_and_1:
4625 case Builtin::BI__sync_fetch_and_and_2:
4626 case Builtin::BI__sync_fetch_and_and_4:
4627 case Builtin::BI__sync_fetch_and_and_8:
4628 case Builtin::BI__sync_fetch_and_and_16:
4630 case Builtin::BI__sync_fetch_and_xor_1:
4631 case Builtin::BI__sync_fetch_and_xor_2:
4632 case Builtin::BI__sync_fetch_and_xor_4:
4633 case Builtin::BI__sync_fetch_and_xor_8:
4634 case Builtin::BI__sync_fetch_and_xor_16:
4636 case Builtin::BI__sync_fetch_and_nand_1:
4637 case Builtin::BI__sync_fetch_and_nand_2:
4638 case Builtin::BI__sync_fetch_and_nand_4:
4639 case Builtin::BI__sync_fetch_and_nand_8:
4640 case Builtin::BI__sync_fetch_and_nand_16:
4644 case Builtin::BI__sync_fetch_and_min:
4646 case Builtin::BI__sync_fetch_and_max:
4648 case Builtin::BI__sync_fetch_and_umin:
4650 case Builtin::BI__sync_fetch_and_umax:
4653 case Builtin::BI__sync_add_and_fetch_1:
4654 case Builtin::BI__sync_add_and_fetch_2:
4655 case Builtin::BI__sync_add_and_fetch_4:
4656 case Builtin::BI__sync_add_and_fetch_8:
4657 case Builtin::BI__sync_add_and_fetch_16:
4659 llvm::Instruction::Add);
4660 case Builtin::BI__sync_sub_and_fetch_1:
4661 case Builtin::BI__sync_sub_and_fetch_2:
4662 case Builtin::BI__sync_sub_and_fetch_4:
4663 case Builtin::BI__sync_sub_and_fetch_8:
4664 case Builtin::BI__sync_sub_and_fetch_16:
4666 llvm::Instruction::Sub);
4667 case Builtin::BI__sync_and_and_fetch_1:
4668 case Builtin::BI__sync_and_and_fetch_2:
4669 case Builtin::BI__sync_and_and_fetch_4:
4670 case Builtin::BI__sync_and_and_fetch_8:
4671 case Builtin::BI__sync_and_and_fetch_16:
4673 llvm::Instruction::And);
4674 case Builtin::BI__sync_or_and_fetch_1:
4675 case Builtin::BI__sync_or_and_fetch_2:
4676 case Builtin::BI__sync_or_and_fetch_4:
4677 case Builtin::BI__sync_or_and_fetch_8:
4678 case Builtin::BI__sync_or_and_fetch_16:
4680 llvm::Instruction::Or);
4681 case Builtin::BI__sync_xor_and_fetch_1:
4682 case Builtin::BI__sync_xor_and_fetch_2:
4683 case Builtin::BI__sync_xor_and_fetch_4:
4684 case Builtin::BI__sync_xor_and_fetch_8:
4685 case Builtin::BI__sync_xor_and_fetch_16:
4687 llvm::Instruction::Xor);
4688 case Builtin::BI__sync_nand_and_fetch_1:
4689 case Builtin::BI__sync_nand_and_fetch_2:
4690 case Builtin::BI__sync_nand_and_fetch_4:
4691 case Builtin::BI__sync_nand_and_fetch_8:
4692 case Builtin::BI__sync_nand_and_fetch_16:
4694 llvm::Instruction::And,
true);
4696 case Builtin::BI__sync_val_compare_and_swap_1:
4697 case Builtin::BI__sync_val_compare_and_swap_2:
4698 case Builtin::BI__sync_val_compare_and_swap_4:
4699 case Builtin::BI__sync_val_compare_and_swap_8:
4700 case Builtin::BI__sync_val_compare_and_swap_16:
4703 case Builtin::BI__sync_bool_compare_and_swap_1:
4704 case Builtin::BI__sync_bool_compare_and_swap_2:
4705 case Builtin::BI__sync_bool_compare_and_swap_4:
4706 case Builtin::BI__sync_bool_compare_and_swap_8:
4707 case Builtin::BI__sync_bool_compare_and_swap_16:
4710 case Builtin::BI__sync_swap_1:
4711 case Builtin::BI__sync_swap_2:
4712 case Builtin::BI__sync_swap_4:
4713 case Builtin::BI__sync_swap_8:
4714 case Builtin::BI__sync_swap_16:
4717 case Builtin::BI__sync_lock_test_and_set_1:
4718 case Builtin::BI__sync_lock_test_and_set_2:
4719 case Builtin::BI__sync_lock_test_and_set_4:
4720 case Builtin::BI__sync_lock_test_and_set_8:
4721 case Builtin::BI__sync_lock_test_and_set_16:
4724 case Builtin::BI__sync_lock_release_1:
4725 case Builtin::BI__sync_lock_release_2:
4726 case Builtin::BI__sync_lock_release_4:
4727 case Builtin::BI__sync_lock_release_8:
4728 case Builtin::BI__sync_lock_release_16: {
4734 llvm::StoreInst *
Store =
4736 Store->setAtomic(llvm::AtomicOrdering::Release);
4740 case Builtin::BI__sync_synchronize: {
4748 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent);
4752 case Builtin::BI__builtin_nontemporal_load:
4754 case Builtin::BI__builtin_nontemporal_store:
4756 case Builtin::BI__c11_atomic_is_lock_free:
4757 case Builtin::BI__atomic_is_lock_free: {
4761 const char *LibCallName =
"__atomic_is_lock_free";
4765 if (BuiltinID == Builtin::BI__atomic_is_lock_free)
4779 case Builtin::BI__atomic_test_and_set: {
4791 if (isa<llvm::ConstantInt>(Order)) {
4792 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
4793 AtomicRMWInst *
Result =
nullptr;
4798 llvm::AtomicOrdering::Monotonic);
4803 llvm::AtomicOrdering::Acquire);
4807 llvm::AtomicOrdering::Release);
4812 llvm::AtomicOrdering::AcquireRelease);
4816 llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
4817 llvm::AtomicOrdering::SequentiallyConsistent);
4820 Result->setVolatile(Volatile);
4826 llvm::BasicBlock *BBs[5] = {
4833 llvm::AtomicOrdering Orders[5] = {
4834 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Acquire,
4835 llvm::AtomicOrdering::Release, llvm::AtomicOrdering::AcquireRelease,
4836 llvm::AtomicOrdering::SequentiallyConsistent};
4838 Order =
Builder.CreateIntCast(Order,
Builder.getInt32Ty(),
false);
4839 llvm::SwitchInst *SI =
Builder.CreateSwitch(Order, BBs[0]);
4841 Builder.SetInsertPoint(ContBB);
4844 for (
unsigned i = 0; i < 5; ++i) {
4845 Builder.SetInsertPoint(BBs[i]);
4847 Ptr, NewVal, Orders[i]);
4848 RMW->setVolatile(Volatile);
4849 Result->addIncoming(RMW, BBs[i]);
4853 SI->addCase(
Builder.getInt32(0), BBs[0]);
4854 SI->addCase(
Builder.getInt32(1), BBs[1]);
4855 SI->addCase(
Builder.getInt32(2), BBs[1]);
4856 SI->addCase(
Builder.getInt32(3), BBs[2]);
4857 SI->addCase(
Builder.getInt32(4), BBs[3]);
4858 SI->addCase(
Builder.getInt32(5), BBs[4]);
4860 Builder.SetInsertPoint(ContBB);
4864 case Builtin::BI__atomic_clear: {
4873 if (isa<llvm::ConstantInt>(Order)) {
4874 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
4879 Store->setOrdering(llvm::AtomicOrdering::Monotonic);
4882 Store->setOrdering(llvm::AtomicOrdering::Release);
4885 Store->setOrdering(llvm::AtomicOrdering::SequentiallyConsistent);
4893 llvm::BasicBlock *BBs[3] = {
4898 llvm::AtomicOrdering Orders[3] = {
4899 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Release,
4900 llvm::AtomicOrdering::SequentiallyConsistent};
4902 Order =
Builder.CreateIntCast(Order,
Builder.getInt32Ty(),
false);
4903 llvm::SwitchInst *SI =
Builder.CreateSwitch(Order, BBs[0]);
4905 for (
unsigned i = 0; i < 3; ++i) {
4906 Builder.SetInsertPoint(BBs[i]);
4908 Store->setOrdering(Orders[i]);
4912 SI->addCase(
Builder.getInt32(0), BBs[0]);
4913 SI->addCase(
Builder.getInt32(3), BBs[1]);
4914 SI->addCase(
Builder.getInt32(5), BBs[2]);
4916 Builder.SetInsertPoint(ContBB);
4920 case Builtin::BI__atomic_thread_fence:
4921 case Builtin::BI__atomic_signal_fence:
4922 case Builtin::BI__c11_atomic_thread_fence:
4923 case Builtin::BI__c11_atomic_signal_fence: {
4924 llvm::SyncScope::ID SSID;
4925 if (BuiltinID == Builtin::BI__atomic_signal_fence ||
4926 BuiltinID == Builtin::BI__c11_atomic_signal_fence)
4927 SSID = llvm::SyncScope::SingleThread;
4929 SSID = llvm::SyncScope::System;
4931 if (isa<llvm::ConstantInt>(Order)) {
4932 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
4939 Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
4942 Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
4945 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
4948 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
4954 llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB;
4961 Order =
Builder.CreateIntCast(Order,
Builder.getInt32Ty(),
false);
4962 llvm::SwitchInst *SI =
Builder.CreateSwitch(Order, ContBB);
4964 Builder.SetInsertPoint(AcquireBB);
4965 Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
4967 SI->addCase(
Builder.getInt32(1), AcquireBB);
4968 SI->addCase(
Builder.getInt32(2), AcquireBB);
4970 Builder.SetInsertPoint(ReleaseBB);
4971 Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
4973 SI->addCase(
Builder.getInt32(3), ReleaseBB);
4975 Builder.SetInsertPoint(AcqRelBB);
4976 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
4978 SI->addCase(
Builder.getInt32(4), AcqRelBB);
4980 Builder.SetInsertPoint(SeqCstBB);
4981 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
4983 SI->addCase(
Builder.getInt32(5), SeqCstBB);
4985 Builder.SetInsertPoint(ContBB);
4989 case Builtin::BI__builtin_signbit:
4990 case Builtin::BI__builtin_signbitf:
4991 case Builtin::BI__builtin_signbitl: {
4996 case Builtin::BI__warn_memset_zero_len:
4998 case Builtin::BI__annotation: {
5001 for (
const Expr *Arg :
E->arguments()) {
5003 assert(Str->getCharByteWidth() == 2);
5004 StringRef WideBytes = Str->getBytes();
5005 std::string StrUtf8;
5006 if (!convertUTF16ToUTF8String(
5007 ArrayRef(WideBytes.data(), WideBytes.size()), StrUtf8)) {
5011 Strings.push_back(llvm::MDString::get(
getLLVMContext(), StrUtf8));
5021 case Builtin::BI__builtin_annotation: {
5030 StringRef Str = cast<StringLiteral>(AnnotationStrExpr)->getString();
5034 case Builtin::BI__builtin_addcb:
5035 case Builtin::BI__builtin_addcs:
5036 case Builtin::BI__builtin_addc:
5037 case Builtin::BI__builtin_addcl:
5038 case Builtin::BI__builtin_addcll:
5039 case Builtin::BI__builtin_subcb:
5040 case Builtin::BI__builtin_subcs:
5041 case Builtin::BI__builtin_subc:
5042 case Builtin::BI__builtin_subcl:
5043 case Builtin::BI__builtin_subcll: {
5069 llvm::Intrinsic::ID IntrinsicId;
5070 switch (BuiltinID) {
5071 default: llvm_unreachable(
"Unknown multiprecision builtin id.");
5072 case Builtin::BI__builtin_addcb:
5073 case Builtin::BI__builtin_addcs:
5074 case Builtin::BI__builtin_addc:
5075 case Builtin::BI__builtin_addcl:
5076 case Builtin::BI__builtin_addcll:
5077 IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
5079 case Builtin::BI__builtin_subcb:
5080 case Builtin::BI__builtin_subcs:
5081 case Builtin::BI__builtin_subc:
5082 case Builtin::BI__builtin_subcl:
5083 case Builtin::BI__builtin_subcll:
5084 IntrinsicId = llvm::Intrinsic::usub_with_overflow;
5089 llvm::Value *Carry1;
5092 llvm::Value *Carry2;
5094 Sum1, Carryin, Carry2);
5095 llvm::Value *CarryOut =
Builder.CreateZExt(
Builder.CreateOr(Carry1, Carry2),
5101 case Builtin::BI__builtin_add_overflow:
5102 case Builtin::BI__builtin_sub_overflow:
5103 case Builtin::BI__builtin_mul_overflow: {
5111 WidthAndSignedness LeftInfo =
5113 WidthAndSignedness RightInfo =
5115 WidthAndSignedness ResultInfo =
5122 RightInfo, ResultArg, ResultQTy,
5128 *
this, LeftArg, LeftInfo, RightArg, RightInfo, ResultArg, ResultQTy,
5131 WidthAndSignedness EncompassingInfo =
5134 llvm::Type *EncompassingLLVMTy =
5139 llvm::Intrinsic::ID IntrinsicId;
5140 switch (BuiltinID) {
5142 llvm_unreachable(
"Unknown overflow builtin id.");
5143 case Builtin::BI__builtin_add_overflow:
5144 IntrinsicId = EncompassingInfo.Signed
5145 ? llvm::Intrinsic::sadd_with_overflow
5146 : llvm::Intrinsic::uadd_with_overflow;
5148 case Builtin::BI__builtin_sub_overflow:
5149 IntrinsicId = EncompassingInfo.Signed
5150 ? llvm::Intrinsic::ssub_with_overflow
5151 : llvm::Intrinsic::usub_with_overflow;
5153 case Builtin::BI__builtin_mul_overflow:
5154 IntrinsicId = EncompassingInfo.Signed
5155 ? llvm::Intrinsic::smul_with_overflow
5156 : llvm::Intrinsic::umul_with_overflow;
5165 Left =
Builder.CreateIntCast(Left, EncompassingLLVMTy, LeftInfo.Signed);
5166 Right =
Builder.CreateIntCast(Right, EncompassingLLVMTy, RightInfo.Signed);
5169 llvm::Value *Overflow, *
Result;
5172 if (EncompassingInfo.Width > ResultInfo.Width) {
5175 llvm::Value *ResultTrunc =
Builder.CreateTrunc(
Result, ResultLLVMTy);
5179 llvm::Value *ResultTruncExt =
Builder.CreateIntCast(
5180 ResultTrunc, EncompassingLLVMTy, ResultInfo.Signed);
5181 llvm::Value *TruncationOverflow =
5184 Overflow =
Builder.CreateOr(Overflow, TruncationOverflow);
5196 case Builtin::BI__builtin_uadd_overflow:
5197 case Builtin::BI__builtin_uaddl_overflow:
5198 case Builtin::BI__builtin_uaddll_overflow:
5199 case Builtin::BI__builtin_usub_overflow:
5200 case Builtin::BI__builtin_usubl_overflow:
5201 case Builtin::BI__builtin_usubll_overflow:
5202 case Builtin::BI__builtin_umul_overflow:
5203 case Builtin::BI__builtin_umull_overflow:
5204 case Builtin::BI__builtin_umulll_overflow:
5205 case Builtin::BI__builtin_sadd_overflow:
5206 case Builtin::BI__builtin_saddl_overflow:
5207 case Builtin::BI__builtin_saddll_overflow:
5208 case Builtin::BI__builtin_ssub_overflow:
5209 case Builtin::BI__builtin_ssubl_overflow:
5210 case Builtin::BI__builtin_ssubll_overflow:
5211 case Builtin::BI__builtin_smul_overflow:
5212 case Builtin::BI__builtin_smull_overflow:
5213 case Builtin::BI__builtin_smulll_overflow: {
5223 llvm::Intrinsic::ID IntrinsicId;
5224 switch (BuiltinID) {
5225 default: llvm_unreachable(
"Unknown overflow builtin id.");
5226 case Builtin::BI__builtin_uadd_overflow:
5227 case Builtin::BI__builtin_uaddl_overflow:
5228 case Builtin::BI__builtin_uaddll_overflow:
5229 IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
5231 case Builtin::BI__builtin_usub_overflow:
5232 case Builtin::BI__builtin_usubl_overflow:
5233 case Builtin::BI__builtin_usubll_overflow:
5234 IntrinsicId = llvm::Intrinsic::usub_with_overflow;
5236 case Builtin::BI__builtin_umul_overflow:
5237 case Builtin::BI__builtin_umull_overflow:
5238 case Builtin::BI__builtin_umulll_overflow:
5239 IntrinsicId = llvm::Intrinsic::umul_with_overflow;
5241 case Builtin::BI__builtin_sadd_overflow:
5242 case Builtin::BI__builtin_saddl_overflow:
5243 case Builtin::BI__builtin_saddll_overflow:
5244 IntrinsicId = llvm::Intrinsic::sadd_with_overflow;
5246 case Builtin::BI__builtin_ssub_overflow:
5247 case Builtin::BI__builtin_ssubl_overflow:
5248 case Builtin::BI__builtin_ssubll_overflow:
5249 IntrinsicId = llvm::Intrinsic::ssub_with_overflow;
5251 case Builtin::BI__builtin_smul_overflow:
5252 case Builtin::BI__builtin_smull_overflow:
5253 case Builtin::BI__builtin_smulll_overflow:
5254 IntrinsicId = llvm::Intrinsic::smul_with_overflow;
5265 case Builtin::BIaddressof:
5266 case Builtin::BI__addressof:
5267 case Builtin::BI__builtin_addressof:
5269 case Builtin::BI__builtin_function_start:
5272 case Builtin::BI__builtin_operator_new:
5275 case Builtin::BI__builtin_operator_delete:
5280 case Builtin::BI__builtin_is_aligned:
5282 case Builtin::BI__builtin_align_up:
5284 case Builtin::BI__builtin_align_down:
5287 case Builtin::BI__noop:
5290 case Builtin::BI__builtin_call_with_static_chain: {
5292 const Expr *Chain =
E->getArg(1);
5297 case Builtin::BI_InterlockedExchange8:
5298 case Builtin::BI_InterlockedExchange16:
5299 case Builtin::BI_InterlockedExchange:
5300 case Builtin::BI_InterlockedExchangePointer:
5303 case Builtin::BI_InterlockedCompareExchangePointer:
5304 case Builtin::BI_InterlockedCompareExchangePointer_nf: {
5306 llvm::IntegerType *IntType = IntegerType::get(
5312 RTy = Exchange->getType();
5313 Exchange =
Builder.CreatePtrToInt(Exchange, IntType);
5315 llvm::Value *Comparand =
5319 BuiltinID == Builtin::BI_InterlockedCompareExchangePointer_nf ?
5320 AtomicOrdering::Monotonic : AtomicOrdering::SequentiallyConsistent;
5323 Ordering, Ordering);
5324 Result->setVolatile(
true);
5330 case Builtin::BI_InterlockedCompareExchange8:
5331 case Builtin::BI_InterlockedCompareExchange16:
5332 case Builtin::BI_InterlockedCompareExchange:
5333 case Builtin::BI_InterlockedCompareExchange64:
5335 case Builtin::BI_InterlockedIncrement16:
5336 case Builtin::BI_InterlockedIncrement:
5339 case Builtin::BI_InterlockedDecrement16:
5340 case Builtin::BI_InterlockedDecrement:
5343 case Builtin::BI_InterlockedAnd8:
5344 case Builtin::BI_InterlockedAnd16:
5345 case Builtin::BI_InterlockedAnd:
5347 case Builtin::BI_InterlockedExchangeAdd8:
5348 case Builtin::BI_InterlockedExchangeAdd16:
5349 case Builtin::BI_InterlockedExchangeAdd:
5352 case Builtin::BI_InterlockedExchangeSub8:
5353 case Builtin::BI_InterlockedExchangeSub16:
5354 case Builtin::BI_InterlockedExchangeSub:
5357 case Builtin::BI_InterlockedOr8:
5358 case Builtin::BI_InterlockedOr16:
5359 case Builtin::BI_InterlockedOr:
5361 case Builtin::BI_InterlockedXor8:
5362 case Builtin::BI_InterlockedXor16:
5363 case Builtin::BI_InterlockedXor:
5366 case Builtin::BI_bittest64:
5367 case Builtin::BI_bittest:
5368 case Builtin::BI_bittestandcomplement64:
5369 case Builtin::BI_bittestandcomplement:
5370 case Builtin::BI_bittestandreset64:
5371 case Builtin::BI_bittestandreset:
5372 case Builtin::BI_bittestandset64:
5373 case Builtin::BI_bittestandset:
5374 case Builtin::BI_interlockedbittestandreset:
5375 case Builtin::BI_interlockedbittestandreset64:
5376 case Builtin::BI_interlockedbittestandset64:
5377 case Builtin::BI_interlockedbittestandset:
5378 case Builtin::BI_interlockedbittestandset_acq:
5379 case Builtin::BI_interlockedbittestandset_rel:
5380 case Builtin::BI_interlockedbittestandset_nf:
5381 case Builtin::BI_interlockedbittestandreset_acq:
5382 case Builtin::BI_interlockedbittestandreset_rel:
5383 case Builtin::BI_interlockedbittestandreset_nf:
5388 case Builtin::BI__iso_volatile_load8:
5389 case Builtin::BI__iso_volatile_load16:
5390 case Builtin::BI__iso_volatile_load32:
5391 case Builtin::BI__iso_volatile_load64:
5393 case Builtin::BI__iso_volatile_store8:
5394 case Builtin::BI__iso_volatile_store16:
5395 case Builtin::BI__iso_volatile_store32:
5396 case Builtin::BI__iso_volatile_store64:
5399 case Builtin::BI__builtin_ptrauth_sign_constant:
5402 case Builtin::BI__builtin_ptrauth_auth:
5403 case Builtin::BI__builtin_ptrauth_auth_and_resign:
5404 case Builtin::BI__builtin_ptrauth_blend_discriminator:
5405 case Builtin::BI__builtin_ptrauth_sign_generic_data:
5406 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
5407 case Builtin::BI__builtin_ptrauth_strip: {
5410 for (
auto argExpr :
E->arguments())
5414 llvm::Type *OrigValueType = Args[0]->getType();
5415 if (OrigValueType->isPointerTy())
5418 switch (BuiltinID) {
5419 case Builtin::BI__builtin_ptrauth_auth_and_resign:
5420 if (Args[4]->getType()->isPointerTy())
5424 case Builtin::BI__builtin_ptrauth_auth:
5425 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
5426 if (Args[2]->getType()->isPointerTy())
5430 case Builtin::BI__builtin_ptrauth_sign_generic_data:
5431 if (Args[1]->getType()->isPointerTy())
5435 case Builtin::BI__builtin_ptrauth_blend_discriminator:
5436 case Builtin::BI__builtin_ptrauth_strip:
5441 auto IntrinsicID = [&]() ->
unsigned {
5442 switch (BuiltinID) {
5443 case Builtin::BI__builtin_ptrauth_auth:
5444 return llvm::Intrinsic::ptrauth_auth;
5445 case Builtin::BI__builtin_ptrauth_auth_and_resign:
5446 return llvm::Intrinsic::ptrauth_resign;
5447 case Builtin::BI__builtin_ptrauth_blend_discriminator:
5448 return llvm::Intrinsic::ptrauth_blend;
5449 case Builtin::BI__builtin_ptrauth_sign_generic_data:
5450 return llvm::Intrinsic::ptrauth_sign_generic;
5451 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
5452 return llvm::Intrinsic::ptrauth_sign;
5453 case Builtin::BI__builtin_ptrauth_strip:
5454 return llvm::Intrinsic::ptrauth_strip;
5456 llvm_unreachable(
"bad ptrauth intrinsic");
5461 if (BuiltinID != Builtin::BI__builtin_ptrauth_sign_generic_data &&
5462 BuiltinID != Builtin::BI__builtin_ptrauth_blend_discriminator &&
5463 OrigValueType->isPointerTy()) {
5469 case Builtin::BI__exception_code:
5470 case Builtin::BI_exception_code:
5472 case Builtin::BI__exception_info:
5473 case Builtin::BI_exception_info:
5475 case Builtin::BI__abnormal_termination:
5476 case Builtin::BI_abnormal_termination:
5478 case Builtin::BI_setjmpex:
5479 if (
getTarget().getTriple().isOSMSVCRT() &&
E->getNumArgs() == 1 &&
5483 case Builtin::BI_setjmp:
5484 if (
getTarget().getTriple().isOSMSVCRT() &&
E->getNumArgs() == 1 &&
5486 if (
getTarget().getTriple().getArch() == llvm::Triple::x86)
5488 else if (
getTarget().getTriple().getArch() == llvm::Triple::aarch64)
5495 case Builtin::BImove:
5496 case Builtin::BImove_if_noexcept:
5497 case Builtin::BIforward:
5498 case Builtin::BIforward_like:
5499 case Builtin::BIas_const:
5501 case Builtin::BI__GetExceptionInfo: {
5502 if (llvm::GlobalVariable *GV =
5508 case Builtin::BI__fastfail:
5511 case Builtin::BI__builtin_coro_id:
5513 case Builtin::BI__builtin_coro_promise:
5515 case Builtin::BI__builtin_coro_resume:
5518 case Builtin::BI__builtin_coro_frame:
5520 case Builtin::BI__builtin_coro_noop:
5522 case Builtin::BI__builtin_coro_free:
5524 case Builtin::BI__builtin_coro_destroy:
5527 case Builtin::BI__builtin_coro_done:
5529 case Builtin::BI__builtin_coro_alloc:
5531 case Builtin::BI__builtin_coro_begin:
5533 case Builtin::BI__builtin_coro_end:
5535 case Builtin::BI__builtin_coro_suspend:
5537 case Builtin::BI__builtin_coro_size:
5539 case Builtin::BI__builtin_coro_align:
5543 case Builtin::BIread_pipe:
5544 case Builtin::BIwrite_pipe: {
5548 Value *PacketSize = OpenCLRT.getPipeElemSize(
E->getArg(0));
5549 Value *PacketAlign = OpenCLRT.getPipeElemAlign(
E->getArg(0));
5552 unsigned GenericAS =
5554 llvm::Type *I8PTy = llvm::PointerType::get(
getLLVMContext(), GenericAS);
5557 if (2U ==
E->getNumArgs()) {
5558 const char *Name = (BuiltinID == Builtin::BIread_pipe) ?
"__read_pipe_2"
5563 llvm::FunctionType *FTy = llvm::FunctionType::get(
5568 {Arg0, BCast, PacketSize, PacketAlign}));
5570 assert(4 ==
E->getNumArgs() &&
5571 "Illegal number of parameters to pipe function");
5572 const char *Name = (BuiltinID == Builtin::BIread_pipe) ?
"__read_pipe_4"
5579 llvm::FunctionType *FTy = llvm::FunctionType::get(
5588 {Arg0, Arg1, Arg2, BCast, PacketSize, PacketAlign}));
5593 case Builtin::BIreserve_read_pipe:
5594 case Builtin::BIreserve_write_pipe:
5595 case Builtin::BIwork_group_reserve_read_pipe:
5596 case Builtin::BIwork_group_reserve_write_pipe:
5597 case Builtin::BIsub_group_reserve_read_pipe:
5598 case Builtin::BIsub_group_reserve_write_pipe: {
5601 if (BuiltinID == Builtin::BIreserve_read_pipe)
5602 Name =
"__reserve_read_pipe";
5603 else if (BuiltinID == Builtin::BIreserve_write_pipe)
5604 Name =
"__reserve_write_pipe";
5605 else if (BuiltinID == Builtin::BIwork_group_reserve_read_pipe)
5606 Name =
"__work_group_reserve_read_pipe";
5607 else if (BuiltinID == Builtin::BIwork_group_reserve_write_pipe)
5608 Name =
"__work_group_reserve_write_pipe";
5609 else if (BuiltinID == Builtin::BIsub_group_reserve_read_pipe)
5610 Name =
"__sub_group_reserve_read_pipe";
5612 Name =
"__sub_group_reserve_write_pipe";
5618 Value *PacketSize = OpenCLRT.getPipeElemSize(
E->getArg(0));
5619 Value *PacketAlign = OpenCLRT.getPipeElemAlign(
E->getArg(0));
5623 llvm::FunctionType *FTy = llvm::FunctionType::get(
5630 {Arg0, Arg1, PacketSize, PacketAlign}));
5634 case Builtin::BIcommit_read_pipe:
5635 case Builtin::BIcommit_write_pipe:
5636 case Builtin::BIwork_group_commit_read_pipe:
5637 case Builtin::BIwork_group_commit_write_pipe:
5638 case Builtin::BIsub_group_commit_read_pipe:
5639 case Builtin::BIsub_group_commit_write_pipe: {
5641 if (BuiltinID == Builtin::BIcommit_read_pipe)
5642 Name =
"__commit_read_pipe";
5643 else if (BuiltinID == Builtin::BIcommit_write_pipe)
5644 Name =
"__commit_write_pipe";
5645 else if (BuiltinID == Builtin::BIwork_group_commit_read_pipe)
5646 Name =
"__work_group_commit_read_pipe";
5647 else if (BuiltinID == Builtin::BIwork_group_commit_write_pipe)
5648 Name =
"__work_group_commit_write_pipe";
5649 else if (BuiltinID == Builtin::BIsub_group_commit_read_pipe)
5650 Name =
"__sub_group_commit_read_pipe";
5652 Name =
"__sub_group_commit_write_pipe";
5657 Value *PacketSize = OpenCLRT.getPipeElemSize(
E->getArg(0));
5658 Value *PacketAlign = OpenCLRT.getPipeElemAlign(
E->getArg(0));
5662 llvm::FunctionType *FTy =
5667 {Arg0, Arg1, PacketSize, PacketAlign}));
5670 case Builtin::BIget_pipe_num_packets:
5671 case Builtin::BIget_pipe_max_packets: {
5672 const char *BaseName;
5674 if (BuiltinID == Builtin::BIget_pipe_num_packets)
5675 BaseName =
"__get_pipe_num_packets";
5677 BaseName =
"__get_pipe_max_packets";
5678 std::string Name = std::string(BaseName) +
5679 std::string(PipeTy->isReadOnly() ?
"_ro" :
"_wo");
5684 Value *PacketSize = OpenCLRT.getPipeElemSize(
E->getArg(0));
5685 Value *PacketAlign = OpenCLRT.getPipeElemAlign(
E->getArg(0));
5687 llvm::FunctionType *FTy = llvm::FunctionType::get(
5691 {Arg0, PacketSize, PacketAlign}));
5695 case Builtin::BIto_global:
5696 case Builtin::BIto_local:
5697 case Builtin::BIto_private: {
5699 auto NewArgT = llvm::PointerType::get(
5702 auto NewRetT = llvm::PointerType::get(
5706 auto FTy = llvm::FunctionType::get(NewRetT, {NewArgT},
false);
5707 llvm::Value *NewArg;
5708 if (Arg0->
getType()->getPointerAddressSpace() !=
5709 NewArgT->getPointerAddressSpace())
5712 NewArg =
Builder.CreateBitOrPointerCast(Arg0, NewArgT);
5713 auto NewName = std::string(
"__") +
E->getDirectCallee()->getName().str();
5728 case Builtin::BIenqueue_kernel: {
5730 unsigned NumArgs =
E->getNumArgs();
5733 llvm::Type *GenericVoidPtrTy =
Builder.getPtrTy(
5745 Name =
"__enqueue_kernel_basic";
5746 llvm::Type *ArgTys[] = {QueueTy,
Int32Ty, RangeTy, GenericVoidPtrTy,
5748 llvm::FunctionType *FTy = llvm::FunctionType::get(
5754 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5755 llvm::Value *
Block =
5756 Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5758 AttrBuilder B(
Builder.getContext());
5760 llvm::AttributeList ByValAttrSet =
5761 llvm::AttributeList::get(
CGM.
getModule().getContext(), 3U, B);
5765 {Queue, Flags, Range, Kernel, Block});
5766 RTCall->setAttributes(ByValAttrSet);
5769 assert(NumArgs >= 5 &&
"Invalid enqueue_kernel signature");
5773 auto CreateArrayForSizeVar = [=](
unsigned First)
5774 -> std::tuple<llvm::Value *, llvm::Value *, llvm::Value *> {
5775 llvm::APInt ArraySize(32, NumArgs -
First);
5777 getContext().getSizeType(), ArraySize,
nullptr,
5781 llvm::Value *TmpPtr = Tmp.getPointer();
5784 llvm::Value *ElemPtr;
5787 auto *
Zero = llvm::ConstantInt::get(
IntTy, 0);
5788 for (
unsigned I =
First; I < NumArgs; ++I) {
5789 auto *Index = llvm::ConstantInt::get(
IntTy, I -
First);
5799 return std::tie(ElemPtr, TmpSize, TmpPtr);
5805 Name =
"__enqueue_kernel_varargs";
5809 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5810 auto *
Block =
Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5811 llvm::Value *ElemPtr, *TmpSize, *TmpPtr;
5812 std::tie(ElemPtr, TmpSize, TmpPtr) = CreateArrayForSizeVar(4);
5816 llvm::Value *
const Args[] = {Queue, Flags,
5820 llvm::Type *
const ArgTys[] = {
5821 QueueTy,
IntTy, RangeTy, GenericVoidPtrTy,
5822 GenericVoidPtrTy,
IntTy, ElemPtr->getType()};
5824 llvm::FunctionType *FTy = llvm::FunctionType::get(
Int32Ty, ArgTys,
false);
5833 llvm::PointerType *PtrTy = llvm::PointerType::get(
5837 llvm::Value *NumEvents =
5843 llvm::Value *EventWaitList =
nullptr;
5846 EventWaitList = llvm::ConstantPointerNull::get(PtrTy);
5853 EventWaitList =
Builder.CreatePointerCast(EventWaitList, PtrTy);
5855 llvm::Value *EventRet =
nullptr;
5858 EventRet = llvm::ConstantPointerNull::get(PtrTy);
5867 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5868 llvm::Value *
Block =
5869 Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5871 std::vector<llvm::Type *> ArgTys = {
5873 PtrTy, PtrTy, GenericVoidPtrTy, GenericVoidPtrTy};
5875 std::vector<llvm::Value *> Args = {Queue, Flags,
Range,
5876 NumEvents, EventWaitList, EventRet,
5881 Name =
"__enqueue_kernel_basic_events";
5882 llvm::FunctionType *FTy = llvm::FunctionType::get(
5890 Args.push_back(ConstantInt::get(
Int32Ty, NumArgs - 7));
5892 Name =
"__enqueue_kernel_events_varargs";
5894 llvm::Value *ElemPtr, *TmpSize, *TmpPtr;
5895 std::tie(ElemPtr, TmpSize, TmpPtr) = CreateArrayForSizeVar(7);
5896 Args.push_back(ElemPtr);
5897 ArgTys.push_back(ElemPtr->getType());
5899 llvm::FunctionType *FTy = llvm::FunctionType::get(
5908 llvm_unreachable(
"Unexpected enqueue_kernel signature");
5912 case Builtin::BIget_kernel_work_group_size: {
5913 llvm::Type *GenericVoidPtrTy =
Builder.getPtrTy(
5918 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5919 Value *Arg =
Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5922 llvm::FunctionType::get(
IntTy, {GenericVoidPtrTy, GenericVoidPtrTy},
5924 "__get_kernel_work_group_size_impl"),
5927 case Builtin::BIget_kernel_preferred_work_group_size_multiple: {
5928 llvm::Type *GenericVoidPtrTy =
Builder.getPtrTy(
5933 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5934 Value *Arg =
Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5937 llvm::FunctionType::get(
IntTy, {GenericVoidPtrTy, GenericVoidPtrTy},
5939 "__get_kernel_preferred_work_group_size_multiple_impl"),
5942 case Builtin::BIget_kernel_max_sub_group_size_for_ndrange:
5943 case Builtin::BIget_kernel_sub_group_count_for_ndrange: {
5944 llvm::Type *GenericVoidPtrTy =
Builder.getPtrTy(
5951 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5954 BuiltinID == Builtin::BIget_kernel_max_sub_group_size_for_ndrange
5955 ?
"__get_kernel_max_sub_group_size_for_ndrange_impl"
5956 :
"__get_kernel_sub_group_count_for_ndrange_impl";
5959 llvm::FunctionType::get(
5960 IntTy, {NDRange->getType(), GenericVoidPtrTy, GenericVoidPtrTy},
5963 {NDRange, Kernel, Block}));
5965 case Builtin::BI__builtin_store_half:
5966 case Builtin::BI__builtin_store_halff: {
5973 case Builtin::BI__builtin_load_half: {
5978 case Builtin::BI__builtin_load_halff: {
5983 case Builtin::BI__builtin_printf:
5984 case Builtin::BIprintf:
5985 if (
getTarget().getTriple().isNVPTX() ||
5988 getTarget().getTriple().getVendor() == Triple::VendorType::AMD)) {
5993 if ((
getTarget().getTriple().isAMDGCN() ||
6000 case Builtin::BI__builtin_canonicalize:
6001 case Builtin::BI__builtin_canonicalizef:
6002 case Builtin::BI__builtin_canonicalizef16:
6003 case Builtin::BI__builtin_canonicalizel:
6005 emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::canonicalize));
6007 case Builtin::BI__builtin_thread_pointer: {
6008 if (!
getContext().getTargetInfo().isTLSSupported())
6013 case Builtin::BI__builtin_os_log_format:
6016 case Builtin::BI__xray_customevent: {
6029 auto FTy = F->getFunctionType();
6030 auto Arg0 =
E->getArg(0);
6032 auto Arg0Ty = Arg0->
getType();
6033 auto PTy0 = FTy->getParamType(0);
6034 if (PTy0 != Arg0Val->getType()) {
6035 if (Arg0Ty->isArrayType())
6038 Arg0Val =
Builder.CreatePointerCast(Arg0Val, PTy0);
6041 auto PTy1 = FTy->getParamType(1);
6043 Arg1 =
Builder.CreateTruncOrBitCast(Arg1, PTy1);
6047 case Builtin::BI__xray_typedevent: {
6063 auto FTy = F->getFunctionType();
6065 auto PTy0 = FTy->getParamType(0);
6067 Arg0 =
Builder.CreateTruncOrBitCast(Arg0, PTy0);
6068 auto Arg1 =
E->getArg(1);
6070 auto Arg1Ty = Arg1->
getType();
6071 auto PTy1 = FTy->getParamType(1);
6072 if (PTy1 != Arg1Val->getType()) {
6073 if (Arg1Ty->isArrayType())
6076 Arg1Val =
Builder.CreatePointerCast(Arg1Val, PTy1);
6079 auto PTy2 = FTy->getParamType(2);
6081 Arg2 =
Builder.CreateTruncOrBitCast(Arg2, PTy2);
6085 case Builtin::BI__builtin_ms_va_start:
6086 case Builtin::BI__builtin_ms_va_end:
6089 BuiltinID == Builtin::BI__builtin_ms_va_start));
6091 case Builtin::BI__builtin_ms_va_copy: {
6108 case Builtin::BI__builtin_get_device_side_mangled_name: {
6136 LargestVectorWidth = std::max(LargestVectorWidth, VectorWidth);
6140 Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic;
6142 llvm::Triple::getArchTypePrefix(
getTarget().getTriple().getArch());
6143 if (!Prefix.empty()) {
6144 IntrinsicID = Intrinsic::getIntrinsicForClangBuiltin(Prefix.data(), Name);
6145 if (IntrinsicID == Intrinsic::not_intrinsic && Prefix ==
"spv" &&
6146 getTarget().getTriple().getOS() == llvm::Triple::OSType::AMDHSA)
6147 IntrinsicID = Intrinsic::getIntrinsicForClangBuiltin(
"amdgcn", Name);
6151 if (IntrinsicID == Intrinsic::not_intrinsic)
6152 IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(Prefix.data(), Name);
6155 if (IntrinsicID != Intrinsic::not_intrinsic) {
6160 unsigned ICEArguments = 0;
6166 llvm::FunctionType *FTy = F->getFunctionType();
6168 for (
unsigned i = 0, e =
E->getNumArgs(); i != e; ++i) {
6172 llvm::Type *PTy = FTy->getParamType(i);
6173 if (PTy != ArgValue->
getType()) {
6175 if (
auto *PtrTy = dyn_cast<llvm::PointerType>(PTy)) {
6176 if (PtrTy->getAddressSpace() !=
6177 ArgValue->
getType()->getPointerAddressSpace()) {
6180 PtrTy->getAddressSpace()));
6186 if (PTy->isX86_AMXTy())
6187 ArgValue =
Builder.CreateIntrinsic(Intrinsic::x86_cast_vector_to_tile,
6188 {ArgValue->
getType()}, {ArgValue});
6190 ArgValue =
Builder.CreateBitCast(ArgValue, PTy);
6193 Args.push_back(ArgValue);
6199 llvm::Type *RetTy =
VoidTy;
6203 if (RetTy !=
V->getType()) {
6205 if (
auto *PtrTy = dyn_cast<llvm::PointerType>(RetTy)) {
6206 if (PtrTy->getAddressSpace() !=
V->getType()->getPointerAddressSpace()) {
6209 PtrTy->getAddressSpace()));
6215 if (
V->getType()->isX86_AMXTy())
6216 V =
Builder.CreateIntrinsic(Intrinsic::x86_cast_tile_to_vector, {RetTy},
6222 if (RetTy->isVoidTy())
6242 if (
V->getType()->isVoidTy())
6249 llvm_unreachable(
"No current target builtin returns complex");
6251 llvm_unreachable(
"Bad evaluation kind in EmitBuiltinExpr");
6270 llvm::Triple::ArchType Arch) {
6282 case llvm::Triple::arm:
6283 case llvm::Triple::armeb:
6284 case llvm::Triple::thumb:
6285 case llvm::Triple::thumbeb:
6287 case llvm::Triple::aarch64:
6288 case llvm::Triple::aarch64_32:
6289 case llvm::Triple::aarch64_be:
6291 case llvm::Triple::bpfeb:
6292 case llvm::Triple::bpfel:
6294 case llvm::Triple::x86:
6295 case llvm::Triple::x86_64:
6297 case llvm::Triple::ppc:
6298 case llvm::Triple::ppcle:
6299 case llvm::Triple::ppc64:
6300 case llvm::Triple::ppc64le:
6302 case llvm::Triple::r600:
6303 case llvm::Triple::amdgcn:
6305 case llvm::Triple::systemz:
6307 case llvm::Triple::nvptx:
6308 case llvm::Triple::nvptx64:
6310 case llvm::Triple::wasm32:
6311 case llvm::Triple::wasm64:
6313 case llvm::Triple::hexagon:
6315 case llvm::Triple::riscv32:
6316 case llvm::Triple::riscv64:
6318 case llvm::Triple::spirv64:
6331 assert(
getContext().getAuxTargetInfo() &&
"Missing aux target info");
6343 bool HasLegalHalfType =
true,
6345 bool AllowBFloatArgsAndRet =
true) {
6346 int IsQuad = TypeFlags.
isQuad();
6350 return llvm::FixedVectorType::get(CGF->
Int8Ty, V1Ty ? 1 : (8 << IsQuad));
6353 return llvm::FixedVectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
6355 if (AllowBFloatArgsAndRet)
6356 return llvm::FixedVectorType::get(CGF->
BFloatTy, V1Ty ? 1 : (4 << IsQuad));
6358 return llvm::FixedVectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
6360 if (HasLegalHalfType)
6361 return llvm::FixedVectorType::get(CGF->
HalfTy, V1Ty ? 1 : (4 << IsQuad));
6363 return llvm::FixedVectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
6365 return llvm::FixedVectorType::get(CGF->
Int32Ty, V1Ty ? 1 : (2 << IsQuad));
6368 return llvm::FixedVectorType::get(CGF->
Int64Ty, V1Ty ? 1 : (1 << IsQuad));
6373 return llvm::FixedVectorType::get(CGF->
Int8Ty, 16);
6375 return llvm::FixedVectorType::get(CGF->
FloatTy, V1Ty ? 1 : (2 << IsQuad));
6377 return llvm::FixedVectorType::get(CGF->
DoubleTy, V1Ty ? 1 : (1 << IsQuad));
6379 llvm_unreachable(
"Unknown vector element type!");
6384 int IsQuad = IntTypeFlags.
isQuad();
6387 return llvm::FixedVectorType::get(CGF->
HalfTy, (4 << IsQuad));
6389 return llvm::FixedVectorType::get(CGF->
FloatTy, (2 << IsQuad));
6391 return llvm::FixedVectorType::get(CGF->
DoubleTy, (1 << IsQuad));
6393 llvm_unreachable(
"Type can't be converted to floating-point!");
6398 const ElementCount &Count) {
6399 Value *SV = llvm::ConstantVector::getSplat(Count,
C);
6400 return Builder.CreateShuffleVector(
V,
V, SV,
"lane");
6404 ElementCount EC = cast<llvm::VectorType>(
V->getType())->getElementCount();
6410 unsigned shift,
bool rightshift) {
6412 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
6413 ai != ae; ++ai, ++j) {
6414 if (F->isConstrainedFPIntrinsic())
6415 if (ai->getType()->isMetadataTy())
6417 if (shift > 0 && shift == j)
6420 Ops[j] =
Builder.CreateBitCast(Ops[j], ai->getType(), name);
6423 if (F->isConstrainedFPIntrinsic())
6424 return Builder.CreateConstrainedFPCall(F, Ops, name);
6426 return Builder.CreateCall(F, Ops, name);
6431 int SV = cast<ConstantInt>(
V)->getSExtValue();
6432 return ConstantInt::get(Ty, neg ? -SV : SV);
6437 llvm::Type *Ty,
bool usgn,
6439 llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
6441 int ShiftAmt = cast<ConstantInt>(Shift)->getSExtValue();
6442 int EltSize = VTy->getScalarSizeInBits();
6444 Vec =
Builder.CreateBitCast(Vec, Ty);
6448 if (ShiftAmt == EltSize) {
6451 return llvm::ConstantAggregateZero::get(VTy);
6456 Shift = ConstantInt::get(VTy->getElementType(), ShiftAmt);
6462 return Builder.CreateLShr(Vec, Shift, name);
6464 return Builder.CreateAShr(Vec, Shift, name);
6490struct ARMVectorIntrinsicInfo {
6491 const char *NameHint;
6493 unsigned LLVMIntrinsic;
6494 unsigned AltLLVMIntrinsic;
6497 bool operator<(
unsigned RHSBuiltinID)
const {
6498 return BuiltinID < RHSBuiltinID;
6500 bool operator<(
const ARMVectorIntrinsicInfo &TE)
const {
6501 return BuiltinID < TE.BuiltinID;
6506#define NEONMAP0(NameBase) \
6507 { #NameBase, NEON::BI__builtin_neon_ ## NameBase, 0, 0, 0 }
6509#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
6510 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
6511 Intrinsic::LLVMIntrinsic, 0, TypeModifier }
6513#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier) \
6514 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
6515 Intrinsic::LLVMIntrinsic, Intrinsic::AltLLVMIntrinsic, \
6519 NEONMAP1(__a32_vcvt_bf16_f32, arm_neon_vcvtfp2bf, 0),
6526 NEONMAP1(vabs_v, arm_neon_vabs, 0),
6527 NEONMAP1(vabsq_v, arm_neon_vabs, 0),
6531 NEONMAP1(vaesdq_u8, arm_neon_aesd, 0),
6532 NEONMAP1(vaeseq_u8, arm_neon_aese, 0),
6533 NEONMAP1(vaesimcq_u8, arm_neon_aesimc, 0),
6534 NEONMAP1(vaesmcq_u8, arm_neon_aesmc, 0),
6535 NEONMAP1(vbfdot_f32, arm_neon_bfdot, 0),
6536 NEONMAP1(vbfdotq_f32, arm_neon_bfdot, 0),
6537 NEONMAP1(vbfmlalbq_f32, arm_neon_bfmlalb, 0),
6538 NEONMAP1(vbfmlaltq_f32, arm_neon_bfmlalt, 0),
6539 NEONMAP1(vbfmmlaq_f32, arm_neon_bfmmla, 0),
6552 NEONMAP1(vcage_v, arm_neon_vacge, 0),
6553 NEONMAP1(vcageq_v, arm_neon_vacge, 0),
6554 NEONMAP1(vcagt_v, arm_neon_vacgt, 0),
6555 NEONMAP1(vcagtq_v, arm_neon_vacgt, 0),
6556 NEONMAP1(vcale_v, arm_neon_vacge, 0),
6557 NEONMAP1(vcaleq_v, arm_neon_vacge, 0),
6558 NEONMAP1(vcalt_v, arm_neon_vacgt, 0),
6559 NEONMAP1(vcaltq_v, arm_neon_vacgt, 0),
6576 NEONMAP1(vcvt_f16_f32, arm_neon_vcvtfp2hf, 0),
6579 NEONMAP1(vcvt_f32_f16, arm_neon_vcvthf2fp, 0),
6581 NEONMAP1(vcvt_n_f16_s16, arm_neon_vcvtfxs2fp, 0),
6582 NEONMAP1(vcvt_n_f16_u16, arm_neon_vcvtfxu2fp, 0),
6583 NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
6584 NEONMAP1(vcvt_n_s16_f16, arm_neon_vcvtfp2fxs, 0),
6585 NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0),
6586 NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0),
6587 NEONMAP1(vcvt_n_u16_f16, arm_neon_vcvtfp2fxu, 0),
6588 NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0),
6589 NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0),
6596 NEONMAP1(vcvta_s16_f16, arm_neon_vcvtas, 0),
6597 NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0),
6598 NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0),
6599 NEONMAP1(vcvta_u16_f16, arm_neon_vcvtau, 0),
6600 NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0),
6601 NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0),
6602 NEONMAP1(vcvtaq_s16_f16, arm_neon_vcvtas, 0),
6603 NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0),
6604 NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0),
6605 NEONMAP1(vcvtaq_u16_f16, arm_neon_vcvtau, 0),
6606 NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0),
6607 NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0),
6608 NEONMAP1(vcvth_bf16_f32, arm_neon_vcvtbfp2bf, 0),
6609 NEONMAP1(vcvtm_s16_f16, arm_neon_vcvtms, 0),
6610 NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0),
6611 NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0),
6612 NEONMAP1(vcvtm_u16_f16, arm_neon_vcvtmu, 0),
6613 NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0),
6614 NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0),
6615 NEONMAP1(vcvtmq_s16_f16, arm_neon_vcvtms, 0),
6616 NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0),
6617 NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0),
6618 NEONMAP1(vcvtmq_u16_f16, arm_neon_vcvtmu, 0),
6619 NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0),
6620 NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0),
6621 NEONMAP1(vcvtn_s16_f16, arm_neon_vcvtns, 0),
6622 NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0),
6623 NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0),
6624 NEONMAP1(vcvtn_u16_f16, arm_neon_vcvtnu, 0),
6625 NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0),
6626 NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0),
6627 NEONMAP1(vcvtnq_s16_f16, arm_neon_vcvtns, 0),
6628 NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0),
6629 NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0),
6630 NEONMAP1(vcvtnq_u16_f16, arm_neon_vcvtnu, 0),
6631 NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0),
6632 NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0),
6633 NEONMAP1(vcvtp_s16_f16, arm_neon_vcvtps, 0),
6634 NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0),
6635 NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0),
6636 NEONMAP1(vcvtp_u16_f16, arm_neon_vcvtpu, 0),
6637 NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0),
6638 NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0),
6639 NEONMAP1(vcvtpq_s16_f16, arm_neon_vcvtps, 0),
6640 NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0),
6641 NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0),
6642 NEONMAP1(vcvtpq_u16_f16, arm_neon_vcvtpu, 0),
6643 NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0),
6644 NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0),
6648 NEONMAP1(vcvtq_n_f16_s16, arm_neon_vcvtfxs2fp, 0),
6649 NEONMAP1(vcvtq_n_f16_u16, arm_neon_vcvtfxu2fp, 0),
6650 NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
6651 NEONMAP1(vcvtq_n_s16_f16, arm_neon_vcvtfp2fxs, 0),
6652 NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0),
6653 NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0),
6654 NEONMAP1(vcvtq_n_u16_f16, arm_neon_vcvtfp2fxu, 0),
6655 NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0),
6656 NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0),
6663 NEONMAP1(vdot_s32, arm_neon_sdot, 0),
6664 NEONMAP1(vdot_u32, arm_neon_udot, 0),
6665 NEONMAP1(vdotq_s32, arm_neon_sdot, 0),
6666 NEONMAP1(vdotq_u32, arm_neon_udot, 0),
6676 NEONMAP1(vld1_v, arm_neon_vld1, 0),
6677 NEONMAP1(vld1_x2_v, arm_neon_vld1x2, 0),
6678 NEONMAP1(vld1_x3_v, arm_neon_vld1x3, 0),
6679 NEONMAP1(vld1_x4_v, arm_neon_vld1x4, 0),
6681 NEONMAP1(vld1q_v, arm_neon_vld1, 0),
6682 NEONMAP1(vld1q_x2_v, arm_neon_vld1x2, 0),
6683 NEONMAP1(vld1q_x3_v, arm_neon_vld1x3, 0),
6684 NEONMAP1(vld1q_x4_v, arm_neon_vld1x4, 0),
6685 NEONMAP1(vld2_dup_v, arm_neon_vld2dup, 0),
6686 NEONMAP1(vld2_lane_v, arm_neon_vld2lane, 0),
6687 NEONMAP1(vld2_v, arm_neon_vld2, 0),
6688 NEONMAP1(vld2q_dup_v, arm_neon_vld2dup, 0),
6689 NEONMAP1(vld2q_lane_v, arm_neon_vld2lane, 0),
6690 NEONMAP1(vld2q_v, arm_neon_vld2, 0),
6691 NEONMAP1(vld3_dup_v, arm_neon_vld3dup, 0),
6692 NEONMAP1(vld3_lane_v, arm_neon_vld3lane, 0),
6693 NEONMAP1(vld3_v, arm_neon_vld3, 0),
6694 NEONMAP1(vld3q_dup_v, arm_neon_vld3dup, 0),
6695 NEONMAP1(vld3q_lane_v, arm_neon_vld3lane, 0),
6696 NEONMAP1(vld3q_v, arm_neon_vld3, 0),
6697 NEONMAP1(vld4_dup_v, arm_neon_vld4dup, 0),
6698 NEONMAP1(vld4_lane_v, arm_neon_vld4lane, 0),
6699 NEONMAP1(vld4_v, arm_neon_vld4, 0),
6700 NEONMAP1(vld4q_dup_v, arm_neon_vld4dup, 0),
6701 NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0),
6702 NEONMAP1(vld4q_v, arm_neon_vld4, 0),
6711 NEONMAP1(vmmlaq_s32, arm_neon_smmla, 0),
6712 NEONMAP1(vmmlaq_u32, arm_neon_ummla, 0),
6730 NEONMAP2(vqdmlal_v, arm_neon_vqdmull, sadd_sat, 0),
6731 NEONMAP2(vqdmlsl_v, arm_neon_vqdmull, ssub_sat, 0),
6755 NEONMAP1(vqshlu_n_v, arm_neon_vqshiftsu, 0),
6756 NEONMAP1(vqshluq_n_v, arm_neon_vqshiftsu, 0),
6760 NEONMAP2(vrecpe_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
6761 NEONMAP2(vrecpeq_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
6784 NEONMAP2(vrsqrte_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
6785 NEONMAP2(vrsqrteq_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
6789 NEONMAP1(vsha1su0q_u32, arm_neon_sha1su0, 0),
6790 NEONMAP1(vsha1su1q_u32, arm_neon_sha1su1, 0),
6791 NEONMAP1(vsha256h2q_u32, arm_neon_sha256h2, 0),
6792 NEONMAP1(vsha256hq_u32, arm_neon_sha256h, 0),
6793 NEONMAP1(vsha256su0q_u32, arm_neon_sha256su0, 0),
6794 NEONMAP1(vsha256su1q_u32, arm_neon_sha256su1, 0),
6803 NEONMAP1(vst1_v, arm_neon_vst1, 0),
6804 NEONMAP1(vst1_x2_v, arm_neon_vst1x2, 0),
6805 NEONMAP1(vst1_x3_v, arm_neon_vst1x3, 0),
6806 NEONMAP1(vst1_x4_v, arm_neon_vst1x4, 0),
6807 NEONMAP1(vst1q_v, arm_neon_vst1, 0),
6808 NEONMAP1(vst1q_x2_v, arm_neon_vst1x2, 0),
6809 NEONMAP1(vst1q_x3_v, arm_neon_vst1x3, 0),
6810 NEONMAP1(vst1q_x4_v, arm_neon_vst1x4, 0),
6811 NEONMAP1(vst2_lane_v, arm_neon_vst2lane, 0),
6812 NEONMAP1(vst2_v, arm_neon_vst2, 0),
6813 NEONMAP1(vst2q_lane_v, arm_neon_vst2lane, 0),
6814 NEONMAP1(vst2q_v, arm_neon_vst2, 0),
6815 NEONMAP1(vst3_lane_v, arm_neon_vst3lane, 0),
6816 NEONMAP1(vst3_v, arm_neon_vst3, 0),
6817 NEONMAP1(vst3q_lane_v, arm_neon_vst3lane, 0),
6818 NEONMAP1(vst3q_v, arm_neon_vst3, 0),
6819 NEONMAP1(vst4_lane_v, arm_neon_vst4lane, 0),
6820 NEONMAP1(vst4_v, arm_neon_vst4, 0),
6821 NEONMAP1(vst4q_lane_v, arm_neon_vst4lane, 0),
6822 NEONMAP1(vst4q_v, arm_neon_vst4, 0),
6828 NEONMAP1(vusdot_s32, arm_neon_usdot, 0),
6829 NEONMAP1(vusdotq_s32, arm_neon_usdot, 0),
6830 NEONMAP1(vusmmlaq_s32, arm_neon_usmmla, 0),
6838 NEONMAP1(__a64_vcvtq_low_bf16_f32, aarch64_neon_bfcvtn, 0),
6843 NEONMAP1(vabs_v, aarch64_neon_abs, 0),
6844 NEONMAP1(vabsq_v, aarch64_neon_abs, 0),
6849 NEONMAP1(vaesdq_u8, aarch64_crypto_aesd, 0),
6850 NEONMAP1(vaeseq_u8, aarch64_crypto_aese, 0),
6851 NEONMAP1(vaesimcq_u8, aarch64_crypto_aesimc, 0),
6852 NEONMAP1(vaesmcq_u8, aarch64_crypto_aesmc, 0),
6861 NEONMAP1(vbfdot_f32, aarch64_neon_bfdot, 0),
6862 NEONMAP1(vbfdotq_f32, aarch64_neon_bfdot, 0),
6863 NEONMAP1(vbfmlalbq_f32, aarch64_neon_bfmlalb, 0),
6864 NEONMAP1(vbfmlaltq_f32, aarch64_neon_bfmlalt, 0),
6865 NEONMAP1(vbfmmlaq_f32, aarch64_neon_bfmmla, 0),
6876 NEONMAP1(vcage_v, aarch64_neon_facge, 0),
6877 NEONMAP1(vcageq_v, aarch64_neon_facge, 0),
6878 NEONMAP1(vcagt_v, aarch64_neon_facgt, 0),
6879 NEONMAP1(vcagtq_v, aarch64_neon_facgt, 0),
6880 NEONMAP1(vcale_v, aarch64_neon_facge, 0),
6881 NEONMAP1(vcaleq_v, aarch64_neon_facge, 0),
6882 NEONMAP1(vcalt_v, aarch64_neon_facgt, 0),
6883 NEONMAP1(vcaltq_v, aarch64_neon_facgt, 0),
6920 NEONMAP1(vcvt_f16_f32, aarch64_neon_vcvtfp2hf, 0),
6923 NEONMAP1(vcvt_f32_f16, aarch64_neon_vcvthf2fp, 0),
6925 NEONMAP1(vcvt_n_f16_s16, aarch64_neon_vcvtfxs2fp, 0),
6926 NEONMAP1(vcvt_n_f16_u16, aarch64_neon_vcvtfxu2fp, 0),
6927 NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
6928 NEONMAP2(vcvt_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
6929 NEONMAP1(vcvt_n_s16_f16, aarch64_neon_vcvtfp2fxs, 0),
6930 NEONMAP1(vcvt_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
6931 NEONMAP1(vcvt_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
6932 NEONMAP1(vcvt_n_u16_f16, aarch64_neon_vcvtfp2fxu, 0),
6933 NEONMAP1(vcvt_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
6934 NEONMAP1(vcvt_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
6938 NEONMAP1(vcvtq_high_bf16_f32, aarch64_neon_bfcvtn2, 0),
6939 NEONMAP1(vcvtq_n_f16_s16, aarch64_neon_vcvtfxs2fp, 0),
6940 NEONMAP1(vcvtq_n_f16_u16, aarch64_neon_vcvtfxu2fp, 0),
6941 NEONMAP2(vcvtq_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
6942 NEONMAP2(vcvtq_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
6943 NEONMAP1(vcvtq_n_s16_f16, aarch64_neon_vcvtfp2fxs, 0),
6944 NEONMAP1(vcvtq_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
6945 NEONMAP1(vcvtq_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
6946 NEONMAP1(vcvtq_n_u16_f16, aarch64_neon_vcvtfp2fxu, 0),
6947 NEONMAP1(vcvtq_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
6948 NEONMAP1(vcvtq_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
6950 NEONMAP1(vdot_s32, aarch64_neon_sdot, 0),
6951 NEONMAP1(vdot_u32, aarch64_neon_udot, 0),
6952 NEONMAP1(vdotq_s32, aarch64_neon_sdot, 0),
6953 NEONMAP1(vdotq_u32, aarch64_neon_udot, 0),
6966 NEONMAP1(vfmlal_high_f16, aarch64_neon_fmlal2, 0),
6967 NEONMAP1(vfmlal_low_f16, aarch64_neon_fmlal, 0),
6968 NEONMAP1(vfmlalq_high_f16, aarch64_neon_fmlal2, 0),
6969 NEONMAP1(vfmlalq_low_f16, aarch64_neon_fmlal, 0),
6970 NEONMAP1(vfmlsl_high_f16, aarch64_neon_fmlsl2, 0),
6971 NEONMAP1(vfmlsl_low_f16, aarch64_neon_fmlsl, 0),
6972 NEONMAP1(vfmlslq_high_f16, aarch64_neon_fmlsl2, 0),
6973 NEONMAP1(vfmlslq_low_f16, aarch64_neon_fmlsl, 0),
6978 NEONMAP1(vld1_x2_v, aarch64_neon_ld1x2, 0),
6979 NEONMAP1(vld1_x3_v, aarch64_neon_ld1x3, 0),
6980 NEONMAP1(vld1_x4_v, aarch64_neon_ld1x4, 0),
6981 NEONMAP1(vld1q_x2_v, aarch64_neon_ld1x2, 0),
6982 NEONMAP1(vld1q_x3_v, aarch64_neon_ld1x3, 0),
6983 NEONMAP1(vld1q_x4_v, aarch64_neon_ld1x4, 0),
6984 NEONMAP1(vmmlaq_s32, aarch64_neon_smmla, 0),
6985 NEONMAP1(vmmlaq_u32, aarch64_neon_ummla, 0),
6998 NEONMAP2(vqdmlal_v, aarch64_neon_sqdmull, aarch64_neon_sqadd, 0),
6999 NEONMAP2(vqdmlsl_v, aarch64_neon_sqdmull, aarch64_neon_sqsub, 0),
7000 NEONMAP1(vqdmulh_lane_v, aarch64_neon_sqdmulh_lane, 0),
7001 NEONMAP1(vqdmulh_laneq_v, aarch64_neon_sqdmulh_laneq, 0),
7003 NEONMAP1(vqdmulhq_lane_v, aarch64_neon_sqdmulh_lane, 0),
7004 NEONMAP1(vqdmulhq_laneq_v, aarch64_neon_sqdmulh_laneq, 0),
7019 NEONMAP1(vqrdmulh_lane_v, aarch64_neon_sqrdmulh_lane, 0),
7020 NEONMAP1(vqrdmulh_laneq_v, aarch64_neon_sqrdmulh_laneq, 0),
7022 NEONMAP1(vqrdmulhq_lane_v, aarch64_neon_sqrdmulh_lane, 0),
7023 NEONMAP1(vqrdmulhq_laneq_v, aarch64_neon_sqrdmulh_laneq, 0),
7031 NEONMAP1(vqshlu_n_v, aarch64_neon_sqshlu, 0),
7032 NEONMAP1(vqshluq_n_v, aarch64_neon_sqshlu, 0),
7036 NEONMAP1(vrax1q_u64, aarch64_crypto_rax1, 0),
7037 NEONMAP2(vrecpe_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
7038 NEONMAP2(vrecpeq_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
7065 NEONMAP2(vrsqrte_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
7066 NEONMAP2(vrsqrteq_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
7070 NEONMAP1(vsha1su0q_u32, aarch64_crypto_sha1su0, 0),
7071 NEONMAP1(vsha1su1q_u32, aarch64_crypto_sha1su1, 0),
7072 NEONMAP1(vsha256h2q_u32, aarch64_crypto_sha256h2, 0),
7073 NEONMAP1(vsha256hq_u32, aarch64_crypto_sha256h, 0),
7074 NEONMAP1(vsha256su0q_u32, aarch64_crypto_sha256su0, 0),
7075 NEONMAP1(vsha256su1q_u32, aarch64_crypto_sha256su1, 0),
7076 NEONMAP1(vsha512h2q_u64, aarch64_crypto_sha512h2, 0),
7077 NEONMAP1(vsha512hq_u64, aarch64_crypto_sha512h, 0),
7078 NEONMAP1(vsha512su0q_u64, aarch64_crypto_sha512su0, 0),
7079 NEONMAP1(vsha512su1q_u64, aarch64_crypto_sha512su1, 0),
7088 NEONMAP1(vsm3partw1q_u32, aarch64_crypto_sm3partw1, 0),
7089 NEONMAP1(vsm3partw2q_u32, aarch64_crypto_sm3partw2, 0),
7090 NEONMAP1(vsm3ss1q_u32, aarch64_crypto_sm3ss1, 0),
7091 NEONMAP1(vsm3tt1aq_u32, aarch64_crypto_sm3tt1a, 0),
7092 NEONMAP1(vsm3tt1bq_u32, aarch64_crypto_sm3tt1b, 0),
7093 NEONMAP1(vsm3tt2aq_u32, aarch64_crypto_sm3tt2a, 0),
7094 NEONMAP1(vsm3tt2bq_u32, aarch64_crypto_sm3tt2b, 0),
7095 NEONMAP1(vsm4ekeyq_u32, aarch64_crypto_sm4ekey, 0),
7096 NEONMAP1(vsm4eq_u32, aarch64_crypto_sm4e, 0),
7097 NEONMAP1(vst1_x2_v, aarch64_neon_st1x2, 0),
7098 NEONMAP1(vst1_x3_v, aarch64_neon_st1x3, 0),
7099 NEONMAP1(vst1_x4_v, aarch64_neon_st1x4, 0),
7100 NEONMAP1(vst1q_x2_v, aarch64_neon_st1x2, 0),
7101 NEONMAP1(vst1q_x3_v, aarch64_neon_st1x3, 0),
7102 NEONMAP1(vst1q_x4_v, aarch64_neon_st1x4, 0),
7106 NEONMAP1(vusdot_s32, aarch64_neon_usdot, 0),
7107 NEONMAP1(vusdotq_s32, aarch64_neon_usdot, 0),
7108 NEONMAP1(vusmmlaq_s32, aarch64_neon_usmmla, 0),
7109 NEONMAP1(vxarq_u64, aarch64_crypto_xar, 0),
7147 NEONMAP1(vcvth_bf16_f32, aarch64_neon_bfcvt, 0),
7166 NEONMAP1(vcvtxd_f32_f64, aarch64_sisd_fcvtxn, 0),
7187 NEONMAP1(vmull_p64, aarch64_neon_pmull64, 0),
7215 NEONMAP1(vqdmulls_s32, aarch64_neon_sqdmulls_scalar, 0),
7296 NEONMAP1(vsha1cq_u32, aarch64_crypto_sha1c, 0),
7297 NEONMAP1(vsha1h_u32, aarch64_crypto_sha1h, 0),
7298 NEONMAP1(vsha1mq_u32, aarch64_crypto_sha1m, 0),
7299 NEONMAP1(vsha1pq_u32, aarch64_crypto_sha1p, 0),
7353 { NEON::BI__builtin_neon_splat_lane_bf16, NEON::BI__builtin_neon_splat_lane_v, },
7354 { NEON::BI__builtin_neon_splat_laneq_bf16, NEON::BI__builtin_neon_splat_laneq_v, },
7355 { NEON::BI__builtin_neon_splatq_lane_bf16, NEON::BI__builtin_neon_splatq_lane_v, },
7356 { NEON::BI__builtin_neon_splatq_laneq_bf16, NEON::BI__builtin_neon_splatq_laneq_v, },
7357 { NEON::BI__builtin_neon_vabd_f16, NEON::BI__builtin_neon_vabd_v, },
7358 { NEON::BI__builtin_neon_vabdq_f16, NEON::BI__builtin_neon_vabdq_v, },
7359 { NEON::BI__builtin_neon_vabs_f16, NEON::BI__builtin_neon_vabs_v, },
7360 { NEON::BI__builtin_neon_vabsq_f16, NEON::BI__builtin_neon_vabsq_v, },
7361 { NEON::BI__builtin_neon_vcage_f16, NEON::BI__builtin_neon_vcage_v, },
7362 { NEON::BI__builtin_neon_vcageq_f16, NEON::BI__builtin_neon_vcageq_v, },
7363 { NEON::BI__builtin_neon_vcagt_f16, NEON::BI__builtin_neon_vcagt_v, },
7364 { NEON::BI__builtin_neon_vcagtq_f16, NEON::BI__builtin_neon_vcagtq_v, },
7365 { NEON::BI__builtin_neon_vcale_f16, NEON::BI__builtin_neon_vcale_v, },
7366 { NEON::BI__builtin_neon_vcaleq_f16, NEON::BI__builtin_neon_vcaleq_v, },
7367 { NEON::BI__builtin_neon_vcalt_f16, NEON::BI__builtin_neon_vcalt_v, },
7368 { NEON::BI__builtin_neon_vcaltq_f16, NEON::BI__builtin_neon_vcaltq_v, },
7369 { NEON::BI__builtin_neon_vceqz_f16, NEON::BI__builtin_neon_vceqz_v, },
7370 { NEON::BI__builtin_neon_vceqzq_f16, NEON::BI__builtin_neon_vceqzq_v, },
7371 { NEON::BI__builtin_neon_vcgez_f16, NEON::BI__builtin_neon_vcgez_v, },
7372 { NEON::BI__builtin_neon_vcgezq_f16, NEON::BI__builtin_neon_vcgezq_v, },
7373 { NEON::BI__builtin_neon_vcgtz_f16, NEON::BI__builtin_neon_vcgtz_v, },
7374 { NEON::BI__builtin_neon_vcgtzq_f16, NEON::BI__builtin_neon_vcgtzq_v, },
7375 { NEON::BI__builtin_neon_vclez_f16, NEON::BI__builtin_neon_vclez_v, },
7376 { NEON::BI__builtin_neon_vclezq_f16, NEON::BI__builtin_neon_vclezq_v, },
7377 { NEON::BI__builtin_neon_vcltz_f16, NEON::BI__builtin_neon_vcltz_v, },
7378 { NEON::BI__builtin_neon_vcltzq_f16, NEON::BI__builtin_neon_vcltzq_v, },
7379 { NEON::BI__builtin_neon_vfma_f16, NEON::BI__builtin_neon_vfma_v, },
7380 { NEON::BI__builtin_neon_vfma_lane_f16, NEON::BI__builtin_neon_vfma_lane_v, },
7381 { NEON::BI__builtin_neon_vfma_laneq_f16, NEON::BI__builtin_neon_vfma_laneq_v, },
7382 { NEON::BI__builtin_neon_vfmaq_f16, NEON::BI__builtin_neon_vfmaq_v, },
7383 { NEON::BI__builtin_neon_vfmaq_lane_f16, NEON::BI__builtin_neon_vfmaq_lane_v, },
7384 { NEON::BI__builtin_neon_vfmaq_laneq_f16, NEON::BI__builtin_neon_vfmaq_laneq_v, },
7385 { NEON::BI__builtin_neon_vld1_bf16_x2, NEON::BI__builtin_neon_vld1_x2_v },
7386 { NEON::BI__builtin_neon_vld1_bf16_x3, NEON::BI__builtin_neon_vld1_x3_v },
7387 { NEON::BI__builtin_neon_vld1_bf16_x4, NEON::BI__builtin_neon_vld1_x4_v },
7388 { NEON::BI__builtin_neon_vld1_bf16, NEON::BI__builtin_neon_vld1_v },
7389 { NEON::BI__builtin_neon_vld1_dup_bf16, NEON::BI__builtin_neon_vld1_dup_v },
7390 { NEON::BI__builtin_neon_vld1_lane_bf16, NEON::BI__builtin_neon_vld1_lane_v },
7391 { NEON::BI__builtin_neon_vld1q_bf16_x2, NEON::BI__builtin_neon_vld1q_x2_v },
7392 { NEON::BI__builtin_neon_vld1q_bf16_x3, NEON::BI__builtin_neon_vld1q_x3_v },
7393 { NEON::BI__builtin_neon_vld1q_bf16_x4, NEON::BI__builtin_neon_vld1q_x4_v },
7394 { NEON::BI__builtin_neon_vld1q_bf16, NEON::BI__builtin_neon_vld1q_v },
7395 { NEON::BI__builtin_neon_vld1q_dup_bf16, NEON::BI__builtin_neon_vld1q_dup_v },
7396 { NEON::BI__builtin_neon_vld1q_lane_bf16, NEON::BI__builtin_neon_vld1q_lane_v },
7397 { NEON::BI__builtin_neon_vld2_bf16, NEON::BI__builtin_neon_vld2_v },
7398 { NEON::BI__builtin_neon_vld2_dup_bf16, NEON::BI__builtin_neon_vld2_dup_v },
7399 { NEON::BI__builtin_neon_vld2_lane_bf16, NEON::BI__builtin_neon_vld2_lane_v },
7400 { NEON::BI__builtin_neon_vld2q_bf16, NEON::BI__builtin_neon_vld2q_v },
7401 { NEON::BI__builtin_neon_vld2q_dup_bf16, NEON::BI__builtin_neon_vld2q_dup_v },
7402 { NEON::BI__builtin_neon_vld2q_lane_bf16, NEON::BI__builtin_neon_vld2q_lane_v },
7403 { NEON::BI__builtin_neon_vld3_bf16, NEON::BI__builtin_neon_vld3_v },
7404 { NEON::BI__builtin_neon_vld3_dup_bf16, NEON::BI__builtin_neon_vld3_dup_v },
7405 { NEON::BI__builtin_neon_vld3_lane_bf16, NEON::BI__builtin_neon_vld3_lane_v },
7406 { NEON::BI__builtin_neon_vld3q_bf16, NEON::BI__builtin_neon_vld3q_v },
7407 { NEON::BI__builtin_neon_vld3q_dup_bf16, NEON::BI__builtin_neon_vld3q_dup_v },
7408 { NEON::BI__builtin_neon_vld3q_lane_bf16, NEON::BI__builtin_neon_vld3q_lane_v },
7409 { NEON::BI__builtin_neon_vld4_bf16, NEON::BI__builtin_neon_vld4_v },
7410 { NEON::BI__builtin_neon_vld4_dup_bf16, NEON::BI__builtin_neon_vld4_dup_v },
7411 { NEON::BI__builtin_neon_vld4_lane_bf16, NEON::BI__builtin_neon_vld4_lane_v },
7412 { NEON::BI__builtin_neon_vld4q_bf16, NEON::BI__builtin_neon_vld4q_v },
7413 { NEON::BI__builtin_neon_vld4q_dup_bf16, NEON::BI__builtin_neon_vld4q_dup_v },
7414 { NEON::BI__builtin_neon_vld4q_lane_bf16, NEON::BI__builtin_neon_vld4q_lane_v },
7415 { NEON::BI__builtin_neon_vmax_f16, NEON::BI__builtin_neon_vmax_v, },
7416 { NEON::BI__builtin_neon_vmaxnm_f16, NEON::BI__builtin_neon_vmaxnm_v, },
7417 { NEON::BI__builtin_neon_vmaxnmq_f16, NEON::BI__builtin_neon_vmaxnmq_v, },
7418 { NEON::BI__builtin_neon_vmaxq_f16, NEON::BI__builtin_neon_vmaxq_v, },
7419 { NEON::BI__builtin_neon_vmin_f16, NEON::BI__builtin_neon_vmin_v, },
7420 { NEON::BI__builtin_neon_vminnm_f16, NEON::BI__builtin_neon_vminnm_v, },
7421 { NEON::BI__builtin_neon_vminnmq_f16, NEON::BI__builtin_neon_vminnmq_v, },
7422 { NEON::BI__builtin_neon_vminq_f16, NEON::BI__builtin_neon_vminq_v, },
7423 { NEON::BI__builtin_neon_vmulx_f16, NEON::BI__builtin_neon_vmulx_v, },
7424 { NEON::BI__builtin_neon_vmulxq_f16, NEON::BI__builtin_neon_vmulxq_v, },
7425 { NEON::BI__builtin_neon_vpadd_f16, NEON::BI__builtin_neon_vpadd_v, },
7426 { NEON::BI__builtin_neon_vpaddq_f16, NEON::BI__builtin_neon_vpaddq_v, },
7427 { NEON::BI__builtin_neon_vpmax_f16, NEON::BI__builtin_neon_vpmax_v, },
7428 { NEON::BI__builtin_neon_vpmaxnm_f16, NEON::BI__builtin_neon_vpmaxnm_v, },
7429 { NEON::BI__builtin_neon_vpmaxnmq_f16, NEON::BI__builtin_neon_vpmaxnmq_v, },
7430 { NEON::BI__builtin_neon_vpmaxq_f16, NEON::BI__builtin_neon_vpmaxq_v, },
7431 { NEON::BI__builtin_neon_vpmin_f16, NEON::BI__builtin_neon_vpmin_v, },
7432 { NEON::BI__builtin_neon_vpminnm_f16, NEON::BI__builtin_neon_vpminnm_v, },
7433 { NEON::BI__builtin_neon_vpminnmq_f16, NEON::BI__builtin_neon_vpminnmq_v, },
7434 { NEON::BI__builtin_neon_vpminq_f16, NEON::BI__builtin_neon_vpminq_v, },
7435 { NEON::BI__builtin_neon_vrecpe_f16, NEON::BI__builtin_neon_vrecpe_v, },
7436 { NEON::BI__builtin_neon_vrecpeq_f16, NEON::BI__builtin_neon_vrecpeq_v, },
7437 { NEON::BI__builtin_neon_vrecps_f16, NEON::BI__builtin_neon_vrecps_v, },
7438 { NEON::BI__builtin_neon_vrecpsq_f16, NEON::BI__builtin_neon_vrecpsq_v, },
7439 { NEON::BI__builtin_neon_vrnd_f16, NEON::BI__builtin_neon_vrnd_v, },
7440 { NEON::BI__builtin_neon_vrnda_f16, NEON::BI__builtin_neon_vrnda_v, },
7441 { NEON::BI__builtin_neon_vrndaq_f16, NEON::BI__builtin_neon_vrndaq_v, },
7442 { NEON::BI__builtin_neon_vrndi_f16, NEON::BI__builtin_neon_vrndi_v, },
7443 { NEON::BI__builtin_neon_vrndiq_f16, NEON::BI__builtin_neon_vrndiq_v, },
7444 { NEON::BI__builtin_neon_vrndm_f16, NEON::BI__builtin_neon_vrndm_v, },
7445 { NEON::BI__builtin_neon_vrndmq_f16, NEON::BI__builtin_neon_vrndmq_v, },
7446 { NEON::BI__builtin_neon_vrndn_f16, NEON::BI__builtin_neon_vrndn_v, },
7447 { NEON::BI__builtin_neon_vrndnq_f16, NEON::BI__builtin_neon_vrndnq_v, },
7448 { NEON::BI__builtin_neon_vrndp_f16, NEON::BI__builtin_neon_vrndp_v, },
7449 { NEON::BI__builtin_neon_vrndpq_f16, NEON::BI__builtin_neon_vrndpq_v, },
7450 { NEON::BI__builtin_neon_vrndq_f16, NEON::BI__builtin_neon_vrndq_v, },
7451 { NEON::BI__builtin_neon_vrndx_f16, NEON::BI__builtin_neon_vrndx_v, },
7452 { NEON::BI__builtin_neon_vrndxq_f16, NEON::BI__builtin_neon_vrndxq_v, },
7453 { NEON::BI__builtin_neon_vrsqrte_f16, NEON::BI__builtin_neon_vrsqrte_v, },
7454 { NEON::BI__builtin_neon_vrsqrteq_f16, NEON::BI__builtin_neon_vrsqrteq_v, },
7455 { NEON::BI__builtin_neon_vrsqrts_f16, NEON::BI__builtin_neon_vrsqrts_v, },
7456 { NEON::BI__builtin_neon_vrsqrtsq_f16, NEON::BI__builtin_neon_vrsqrtsq_v, },
7457 { NEON::BI__builtin_neon_vsqrt_f16, NEON::BI__builtin_neon_vsqrt_v, },
7458 { NEON::BI__builtin_neon_vsqrtq_f16, NEON::BI__builtin_neon_vsqrtq_v, },
7459 { NEON::BI__builtin_neon_vst1_bf16_x2, NEON::BI__builtin_neon_vst1_x2_v },
7460 { NEON::BI__builtin_neon_vst1_bf16_x3, NEON::BI__builtin_neon_vst1_x3_v },
7461 { NEON::BI__builtin_neon_vst1_bf16_x4, NEON::BI__builtin_neon_vst1_x4_v },
7462 { NEON::BI__builtin_neon_vst1_bf16, NEON::BI__builtin_neon_vst1_v },
7463 { NEON::BI__builtin_neon_vst1_lane_bf16, NEON::BI__builtin_neon_vst1_lane_v },
7464 { NEON::BI__builtin_neon_vst1q_bf16_x2, NEON::BI__builtin_neon_vst1q_x2_v },
7465 { NEON::BI__builtin_neon_vst1q_bf16_x3, NEON::BI__builtin_neon_vst1q_x3_v },
7466 { NEON::BI__builtin_neon_vst1q_bf16_x4, NEON::BI__builtin_neon_vst1q_x4_v },
7467 { NEON::BI__builtin_neon_vst1q_bf16, NEON::BI__builtin_neon_vst1q_v },
7468 { NEON::BI__builtin_neon_vst1q_lane_bf16, NEON::BI__builtin_neon_vst1q_lane_v },
7469 { NEON::BI__builtin_neon_vst2_bf16, NEON::BI__builtin_neon_vst2_v },
7470 { NEON::BI__builtin_neon_vst2_lane_bf16, NEON::BI__builtin_neon_vst2_lane_v },
7471 { NEON::BI__builtin_neon_vst2q_bf16, NEON::BI__builtin_neon_vst2q_v },
7472 { NEON::BI__builtin_neon_vst2q_lane_bf16, NEON::BI__builtin_neon_vst2q_lane_v },
7473 { NEON::BI__builtin_neon_vst3_bf16, NEON::BI__builtin_neon_vst3_v },
7474 { NEON::BI__builtin_neon_vst3_lane_bf16, NEON::BI__builtin_neon_vst3_lane_v },
7475 { NEON::BI__builtin_neon_vst3q_bf16, NEON::BI__builtin_neon_vst3q_v },
7476 { NEON::BI__builtin_neon_vst3q_lane_bf16, NEON::BI__builtin_neon_vst3q_lane_v },
7477 { NEON::BI__builtin_neon_vst4_bf16, NEON::BI__builtin_neon_vst4_v },
7478 { NEON::BI__builtin_neon_vst4_lane_bf16, NEON::BI__builtin_neon_vst4_lane_v },
7479 { NEON::BI__builtin_neon_vst4q_bf16, NEON::BI__builtin_neon_vst4q_v },
7480 { NEON::BI__builtin_neon_vst4q_lane_bf16, NEON::BI__builtin_neon_vst4q_lane_v },
7484 { NEON::BI__builtin_neon_vldap1_lane_u64, NEON::BI__builtin_neon_vldap1_lane_s64 },
7485 { NEON::BI__builtin_neon_vldap1_lane_f64, NEON::BI__builtin_neon_vldap1_lane_s64 },
7486 { NEON::BI__builtin_neon_vldap1_lane_p64, NEON::BI__builtin_neon_vldap1_lane_s64 },
7487 { NEON::BI__builtin_neon_vldap1q_lane_u64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
7488 { NEON::BI__builtin_neon_vldap1q_lane_f64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
7489 { NEON::BI__builtin_neon_vldap1q_lane_p64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
7490 { NEON::BI__builtin_neon_vstl1_lane_u64, NEON::BI__builtin_neon_vstl1_lane_s64 },
7491 { NEON::BI__builtin_neon_vstl1_lane_f64, NEON::BI__builtin_neon_vstl1_lane_s64 },
7492 { NEON::BI__builtin_neon_vstl1_lane_p64, NEON::BI__builtin_neon_vstl1_lane_s64 },
7493 { NEON::BI__builtin_neon_vstl1q_lane_u64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
7494 { NEON::BI__builtin_neon_vstl1q_lane_f64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
7495 { NEON::BI__builtin_neon_vstl1q_lane_p64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
7502#define SVEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
7504 #NameBase, SVE::BI__builtin_sve_##NameBase, Intrinsic::LLVMIntrinsic, 0, \
7508#define SVEMAP2(NameBase, TypeModifier) \
7509 { #NameBase, SVE::BI__builtin_sve_##NameBase, 0, 0, TypeModifier }
7511#define GET_SVE_LLVM_INTRINSIC_MAP
7512#include "clang/Basic/arm_sve_builtin_cg.inc"
7513#include "clang/Basic/BuiltinsAArch64NeonSVEBridge_cg.def"
7514#undef GET_SVE_LLVM_INTRINSIC_MAP
7520#define SMEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
7522 #NameBase, SME::BI__builtin_sme_##NameBase, Intrinsic::LLVMIntrinsic, 0, \
7526#define SMEMAP2(NameBase, TypeModifier) \
7527 { #NameBase, SME::BI__builtin_sme_##NameBase, 0, 0, TypeModifier }
7529#define GET_SME_LLVM_INTRINSIC_MAP
7530#include "clang/Basic/arm_sme_builtin_cg.inc"
7531#undef GET_SME_LLVM_INTRINSIC_MAP
7544static const ARMVectorIntrinsicInfo *
7546 unsigned BuiltinID,
bool &MapProvenSorted) {
7549 if (!MapProvenSorted) {
7550 assert(llvm::is_sorted(IntrinsicMap));
7551 MapProvenSorted =
true;
7555 const ARMVectorIntrinsicInfo *Builtin =
7556 llvm::lower_bound(IntrinsicMap, BuiltinID);
7558 if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID)
7566 llvm::Type *ArgType,
7579 Ty = llvm::FixedVectorType::get(
7580 Ty, VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1);
7587 int Elts = VectorSize ? VectorSize / ArgType->getPrimitiveSizeInBits() : 1;
7588 ArgType = llvm::FixedVectorType::get(ArgType, Elts);
7592 Tys.push_back(ArgType);
7595 Tys.push_back(ArgType);
7606 unsigned BuiltinID = SISDInfo.BuiltinID;
7607 unsigned int Int = SISDInfo.LLVMIntrinsic;
7608 unsigned Modifier = SISDInfo.TypeModifier;
7609 const char *
s = SISDInfo.NameHint;
7611 switch (BuiltinID) {
7612 case NEON::BI__builtin_neon_vcled_s64:
7613 case NEON::BI__builtin_neon_vcled_u64:
7614 case NEON::BI__builtin_neon_vcles_f32:
7615 case NEON::BI__builtin_neon_vcled_f64:
7616 case NEON::BI__builtin_neon_vcltd_s64:
7617 case NEON::BI__builtin_neon_vcltd_u64:
7618 case NEON::BI__builtin_neon_vclts_f32:
7619 case NEON::BI__builtin_neon_vcltd_f64:
7620 case NEON::BI__builtin_neon_vcales_f32:
7621 case NEON::BI__builtin_neon_vcaled_f64:
7622 case NEON::BI__builtin_neon_vcalts_f32:
7623 case NEON::BI__builtin_neon_vcaltd_f64:
7627 std::swap(Ops[0], Ops[1]);
7631 assert(Int &&
"Generic code assumes a valid intrinsic");
7634 const Expr *Arg =
E->getArg(0);
7639 ConstantInt *C0 = ConstantInt::get(CGF.
SizeTy, 0);
7640 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
7641 ai != ae; ++ai, ++j) {
7642 llvm::Type *ArgTy = ai->getType();
7643 if (Ops[j]->getType()->getPrimitiveSizeInBits() ==
7644 ArgTy->getPrimitiveSizeInBits())
7647 assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy());
7650 Ops[j] = CGF.
Builder.CreateTruncOrBitCast(
7651 Ops[j], cast<llvm::VectorType>(ArgTy)->getElementType());
7653 CGF.
Builder.CreateInsertElement(PoisonValue::get(ArgTy), Ops[j], C0);
7658 if (ResultType->getPrimitiveSizeInBits().getFixedValue() <
7659 Result->getType()->getPrimitiveSizeInBits().getFixedValue())
7666 unsigned BuiltinID,
unsigned LLVMIntrinsic,
unsigned AltLLVMIntrinsic,
7667 const char *NameHint,
unsigned Modifier,
const CallExpr *
E,
7669 llvm::Triple::ArchType Arch) {
7671 const Expr *Arg =
E->getArg(
E->getNumArgs() - 1);
7672 std::optional<llvm::APSInt> NeonTypeConst =
7679 bool Usgn =
Type.isUnsigned();
7680 bool Quad =
Type.isQuad();
7682 const bool AllowBFloatArgsAndRet =
7685 llvm::FixedVectorType *VTy =
7686 GetNeonType(
this,
Type, HasLegalHalfType,
false, AllowBFloatArgsAndRet);
7687 llvm::Type *Ty = VTy;
7691 auto getAlignmentValue32 = [&](
Address addr) ->
Value* {
7692 return Builder.getInt32(addr.getAlignment().getQuantity());
7695 unsigned Int = LLVMIntrinsic;
7697 Int = AltLLVMIntrinsic;
7699 switch (BuiltinID) {
7701 case NEON::BI__builtin_neon_splat_lane_v:
7702 case NEON::BI__builtin_neon_splat_laneq_v:
7703 case NEON::BI__builtin_neon_splatq_lane_v:
7704 case NEON::BI__builtin_neon_splatq_laneq_v: {
7705 auto NumElements = VTy->getElementCount();
7706 if (BuiltinID == NEON::BI__builtin_neon_splatq_lane_v)
7707 NumElements = NumElements * 2;
7708 if (BuiltinID == NEON::BI__builtin_neon_splat_laneq_v)
7709 NumElements = NumElements.divideCoefficientBy(2);
7711 Ops[0] =
Builder.CreateBitCast(Ops[0], VTy);
7712 return EmitNeonSplat(Ops[0], cast<ConstantInt>(Ops[1]), NumElements);
7714 case NEON::BI__builtin_neon_vpadd_v:
7715 case NEON::BI__builtin_neon_vpaddq_v:
7717 if (VTy->getElementType()->isFloatingPointTy() &&
7718 Int == Intrinsic::aarch64_neon_addp)
7719 Int = Intrinsic::aarch64_neon_faddp;
7721 case NEON::BI__builtin_neon_vabs_v:
7722 case NEON::BI__builtin_neon_vabsq_v:
7723 if (VTy->getElementType()->isFloatingPointTy())
7726 case NEON::BI__builtin_neon_vadd_v:
7727 case NEON::BI__builtin_neon_vaddq_v: {
7728 llvm::Type *VTy = llvm::FixedVectorType::get(
Int8Ty, Quad ? 16 : 8);
7729 Ops[0] =
Builder.CreateBitCast(Ops[0], VTy);
7730 Ops[1] =
Builder.CreateBitCast(Ops[1], VTy);
7731 Ops[0] =
Builder.CreateXor(Ops[0], Ops[1]);
7732 return Builder.CreateBitCast(Ops[0], Ty);
7734 case NEON::BI__builtin_neon_vaddhn_v: {
7735 llvm::FixedVectorType *SrcTy =
7736 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
7739 Ops[0] =
Builder.CreateBitCast(Ops[0], SrcTy);
7740 Ops[1] =
Builder.CreateBitCast(Ops[1], SrcTy);
7741 Ops[0] =
Builder.CreateAdd(Ops[0], Ops[1],
"vaddhn");
7744 Constant *ShiftAmt =
7745 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
7746 Ops[0] =
Builder.CreateLShr(Ops[0], ShiftAmt,
"vaddhn");
7749 return Builder.CreateTrunc(Ops[0], VTy,
"vaddhn");
7751 case NEON::BI__builtin_neon_vcale_v:
7752 case NEON::BI__builtin_neon_vcaleq_v:
7753 case NEON::BI__builtin_neon_vcalt_v:
7754 case NEON::BI__builtin_neon_vcaltq_v:
7755 std::swap(Ops[0], Ops[1]);
7757 case NEON::BI__builtin_neon_vcage_v:
7758 case NEON::BI__builtin_neon_vcageq_v:
7759 case NEON::BI__builtin_neon_vcagt_v:
7760 case NEON::BI__builtin_neon_vcagtq_v: {
7762 switch (VTy->getScalarSizeInBits()) {
7763 default: llvm_unreachable(
"unexpected type");
7774 auto *VecFlt = llvm::FixedVectorType::get(Ty, VTy->getNumElements());
7775 llvm::Type *Tys[] = { VTy, VecFlt };
7779 case NEON::BI__builtin_neon_vceqz_v:
7780 case NEON::BI__builtin_neon_vceqzq_v:
7782 ICmpInst::ICMP_EQ,
"vceqz");
7783 case NEON::BI__builtin_neon_vcgez_v:
7784 case NEON::BI__builtin_neon_vcgezq_v:
7786 ICmpInst::ICMP_SGE,
"vcgez");
7787 case NEON::BI__builtin_neon_vclez_v:
7788 case NEON::BI__builtin_neon_vclezq_v:
7790 ICmpInst::ICMP_SLE,
"vclez");
7791 case NEON::BI__builtin_neon_vcgtz_v:
7792 case NEON::BI__builtin_neon_vcgtzq_v:
7794 ICmpInst::ICMP_SGT,
"vcgtz");
7795 case NEON::BI__builtin_neon_vcltz_v:
7796 case NEON::BI__builtin_neon_vcltzq_v:
7798 ICmpInst::ICMP_SLT,
"vcltz");
7799 case NEON::BI__builtin_neon_vclz_v:
7800 case NEON::BI__builtin_neon_vclzq_v:
7805 case NEON::BI__builtin_neon_vcvt_f32_v:
7806 case NEON::BI__builtin_neon_vcvtq_f32_v:
7807 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
7810 return Usgn ?
Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
7811 :
Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
7812 case NEON::BI__builtin_neon_vcvt_f16_s16:
7813 case NEON::BI__builtin_neon_vcvt_f16_u16:
7814 case NEON::BI__builtin_neon_vcvtq_f16_s16:
7815 case NEON::BI__builtin_neon_vcvtq_f16_u16:
7816 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
7819 return Usgn ?
Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
7820 :
Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
7821 case NEON::BI__builtin_neon_vcvt_n_f16_s16:
7822 case NEON::BI__builtin_neon_vcvt_n_f16_u16:
7823 case NEON::BI__builtin_neon_vcvtq_n_f16_s16:
7824 case NEON::BI__builtin_neon_vcvtq_n_f16_u16: {
7829 case NEON::BI__builtin_neon_vcvt_n_f32_v:
7830 case NEON::BI__builtin_neon_vcvt_n_f64_v:
7831 case NEON::BI__builtin_neon_vcvtq_n_f32_v:
7832 case NEON::BI__builtin_neon_vcvtq_n_f64_v: {
7834 Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
7838 case NEON::BI__builtin_neon_vcvt_n_s16_f16:
7839 case NEON::BI__builtin_neon_vcvt_n_s32_v:
7840 case NEON::BI__builtin_neon_vcvt_n_u16_f16:
7841 case NEON::BI__builtin_neon_vcvt_n_u32_v:
7842 case NEON::BI__builtin_neon_vcvt_n_s64_v:
7843 case NEON::BI__builtin_neon_vcvt_n_u64_v:
7844 case NEON::BI__builtin_neon_vcvtq_n_s16_f16:
7845 case NEON::BI__builtin_neon_vcvtq_n_s32_v:
7846 case NEON::BI__builtin_neon_vcvtq_n_u16_f16:
7847 case NEON::BI__builtin_neon_vcvtq_n_u32_v:
7848 case NEON::BI__builtin_neon_vcvtq_n_s64_v:
7849 case NEON::BI__builtin_neon_vcvtq_n_u64_v: {
7854 case NEON::BI__builtin_neon_vcvt_s32_v:
7855 case NEON::BI__builtin_neon_vcvt_u32_v:
7856 case NEON::BI__builtin_neon_vcvt_s64_v:
7857 case NEON::BI__builtin_neon_vcvt_u64_v:
7858 case NEON::BI__builtin_neon_vcvt_s16_f16:
7859 case NEON::BI__builtin_neon_vcvt_u16_f16:
7860 case NEON::BI__builtin_neon_vcvtq_s32_v:
7861 case NEON::BI__builtin_neon_vcvtq_u32_v:
7862 case NEON::BI__builtin_neon_vcvtq_s64_v:
7863 case NEON::BI__builtin_neon_vcvtq_u64_v:
7864 case NEON::BI__builtin_neon_vcvtq_s16_f16:
7865 case NEON::BI__builtin_neon_vcvtq_u16_f16: {
7867 return Usgn ?
Builder.CreateFPToUI(Ops[0], Ty,
"vcvt")
7868 :
Builder.CreateFPToSI(Ops[0], Ty,
"vcvt");
7870 case NEON::BI__builtin_neon_vcvta_s16_f16:
7871 case NEON::BI__builtin_neon_vcvta_s32_v:
7872 case NEON::BI__builtin_neon_vcvta_s64_v:
7873 case NEON::BI__builtin_neon_vcvta_u16_f16:
7874 case NEON::BI__builtin_neon_vcvta_u32_v:
7875 case NEON::BI__builtin_neon_vcvta_u64_v:
7876 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
7877 case NEON::BI__builtin_neon_vcvtaq_s32_v:
7878 case NEON::BI__builtin_neon_vcvtaq_s64_v:
7879 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
7880 case NEON::BI__builtin_neon_vcvtaq_u32_v:
7881 case NEON::BI__builtin_neon_vcvtaq_u64_v:
7882 case NEON::BI__builtin_neon_vcvtn_s16_f16:
7883 case NEON::BI__builtin_neon_vcvtn_s32_v:
7884 case NEON::BI__builtin_neon_vcvtn_s64_v:
7885 case NEON::BI__builtin_neon_vcvtn_u16_f16:
7886 case NEON::BI__builtin_neon_vcvtn_u32_v:
7887 case NEON::BI__builtin_neon_vcvtn_u64_v:
7888 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
7889 case NEON::BI__builtin_neon_vcvtnq_s32_v:
7890 case NEON::BI__builtin_neon_vcvtnq_s64_v:
7891 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
7892 case NEON::BI__builtin_neon_vcvtnq_u32_v:
7893 case NEON::BI__builtin_neon_vcvtnq_u64_v:
7894 case NEON::BI__builtin_neon_vcvtp_s16_f16:
7895 case NEON::BI__builtin_neon_vcvtp_s32_v:
7896 case NEON::BI__builtin_neon_vcvtp_s64_v:
7897 case NEON::BI__builtin_neon_vcvtp_u16_f16:
7898 case NEON::BI__builtin_neon_vcvtp_u32_v:
7899 case NEON::BI__builtin_neon_vcvtp_u64_v:
7900 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
7901 case NEON::BI__builtin_neon_vcvtpq_s32_v:
7902 case NEON::BI__builtin_neon_vcvtpq_s64_v:
7903 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
7904 case NEON::BI__builtin_neon_vcvtpq_u32_v:
7905 case NEON::BI__builtin_neon_vcvtpq_u64_v:
7906 case NEON::BI__builtin_neon_vcvtm_s16_f16:
7907 case NEON::BI__builtin_neon_vcvtm_s32_v:
7908 case NEON::BI__builtin_neon_vcvtm_s64_v:
7909 case NEON::BI__builtin_neon_vcvtm_u16_f16:
7910 case NEON::BI__builtin_neon_vcvtm_u32_v:
7911 case NEON::BI__builtin_neon_vcvtm_u64_v:
7912 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
7913 case NEON::BI__builtin_neon_vcvtmq_s32_v:
7914 case NEON::BI__builtin_neon_vcvtmq_s64_v:
7915 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
7916 case NEON::BI__builtin_neon_vcvtmq_u32_v:
7917 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
7921 case NEON::BI__builtin_neon_vcvtx_f32_v: {
7922 llvm::Type *Tys[2] = { VTy->getTruncatedElementVectorType(VTy), Ty};
7926 case NEON::BI__builtin_neon_vext_v:
7927 case NEON::BI__builtin_neon_vextq_v: {
7928 int CV = cast<ConstantInt>(Ops[2])->getSExtValue();
7930 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
7931 Indices.push_back(i+CV);
7933 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
7934 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
7935 return Builder.CreateShuffleVector(Ops[0], Ops[1], Indices,
"vext");
7937 case NEON::BI__builtin_neon_vfma_v:
7938 case NEON::BI__builtin_neon_vfmaq_v: {
7939 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
7940 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
7941 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
7945 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
7946 {Ops[1], Ops[2], Ops[0]});
7948 case NEON::BI__builtin_neon_vld1_v:
7949 case NEON::BI__builtin_neon_vld1q_v: {
7951 Ops.push_back(getAlignmentValue32(PtrOp0));
7954 case NEON::BI__builtin_neon_vld1_x2_v:
7955 case NEON::BI__builtin_neon_vld1q_x2_v:
7956 case NEON::BI__builtin_neon_vld1_x3_v:
7957 case NEON::BI__builtin_neon_vld1q_x3_v:
7958 case NEON::BI__builtin_neon_vld1_x4_v:
7959 case NEON::BI__builtin_neon_vld1q_x4_v: {
7962 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld1xN");
7965 case NEON::BI__builtin_neon_vld2_v:
7966 case NEON::BI__builtin_neon_vld2q_v:
7967 case NEON::BI__builtin_neon_vld3_v:
7968 case NEON::BI__builtin_neon_vld3q_v:
7969 case NEON::BI__builtin_neon_vld4_v:
7970 case NEON::BI__builtin_neon_vld4q_v:
7971 case NEON::BI__builtin_neon_vld2_dup_v:
7972 case NEON::BI__builtin_neon_vld2q_dup_v:
7973 case NEON::BI__builtin_neon_vld3_dup_v:
7974 case NEON::BI__builtin_neon_vld3q_dup_v:
7975 case NEON::BI__builtin_neon_vld4_dup_v:
7976 case NEON::BI__builtin_neon_vld4q_dup_v: {
7979 Value *Align = getAlignmentValue32(PtrOp1);
7980 Ops[1] =
Builder.CreateCall(F, {Ops[1], Align}, NameHint);
7983 case NEON::BI__builtin_neon_vld1_dup_v:
7984 case NEON::BI__builtin_neon_vld1q_dup_v: {
7985 Value *
V = PoisonValue::get(Ty);
7988 llvm::Constant *CI = ConstantInt::get(
SizeTy, 0);
7989 Ops[0] =
Builder.CreateInsertElement(
V, Ld, CI);
7992 case NEON::BI__builtin_neon_vld2_lane_v:
7993 case NEON::BI__builtin_neon_vld2q_lane_v:
7994 case NEON::BI__builtin_neon_vld3_lane_v:
7995 case NEON::BI__builtin_neon_vld3q_lane_v:
7996 case NEON::BI__builtin_neon_vld4_lane_v:
7997 case NEON::BI__builtin_neon_vld4q_lane_v: {
8000 for (
unsigned I = 2; I < Ops.size() - 1; ++I)
8001 Ops[I] =
Builder.CreateBitCast(Ops[I], Ty);
8002 Ops.push_back(getAlignmentValue32(PtrOp1));
8006 case NEON::BI__builtin_neon_vmovl_v: {
8007 llvm::FixedVectorType *DTy =
8008 llvm::FixedVectorType::getTruncatedElementVectorType(VTy);
8009 Ops[0] =
Builder.CreateBitCast(Ops[0], DTy);
8011 return Builder.CreateZExt(Ops[0], Ty,
"vmovl");
8012 return Builder.CreateSExt(Ops[0], Ty,
"vmovl");
8014 case NEON::BI__builtin_neon_vmovn_v: {
8015 llvm::FixedVectorType *QTy =
8016 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
8017 Ops[0] =
Builder.CreateBitCast(Ops[0], QTy);
8018 return Builder.CreateTrunc(Ops[0], Ty,
"vmovn");
8020 case NEON::BI__builtin_neon_vmull_v:
8026 Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls;
8029 case NEON::BI__builtin_neon_vpadal_v:
8030 case NEON::BI__builtin_neon_vpadalq_v: {
8032 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
8036 llvm::FixedVectorType::get(EltTy, VTy->getNumElements() * 2);
8037 llvm::Type *Tys[2] = { Ty, NarrowTy };
8040 case NEON::BI__builtin_neon_vpaddl_v:
8041 case NEON::BI__builtin_neon_vpaddlq_v: {
8043 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
8044 llvm::Type *EltTy = llvm::IntegerType::get(
getLLVMContext(), EltBits / 2);
8046 llvm::FixedVectorType::get(EltTy, VTy->getNumElements() * 2);
8047 llvm::Type *Tys[2] = { Ty, NarrowTy };
8050 case NEON::BI__builtin_neon_vqdmlal_v:
8051 case NEON::BI__builtin_neon_vqdmlsl_v: {
8058 case NEON::BI__builtin_neon_vqdmulhq_lane_v:
8059 case NEON::BI__builtin_neon_vqdmulh_lane_v:
8060 case NEON::BI__builtin_neon_vqrdmulhq_lane_v:
8061 case NEON::BI__builtin_neon_vqrdmulh_lane_v: {
8062 auto *RTy = cast<llvm::FixedVectorType>(Ty);
8063 if (BuiltinID == NEON::BI__builtin_neon_vqdmulhq_lane_v ||
8064 BuiltinID == NEON::BI__builtin_neon_vqrdmulhq_lane_v)
8065 RTy = llvm::FixedVectorType::get(RTy->getElementType(),
8066 RTy->getNumElements() * 2);
8067 llvm::Type *Tys[2] = {
8072 case NEON::BI__builtin_neon_vqdmulhq_laneq_v:
8073 case NEON::BI__builtin_neon_vqdmulh_laneq_v:
8074 case NEON::BI__builtin_neon_vqrdmulhq_laneq_v:
8075 case NEON::BI__builtin_neon_vqrdmulh_laneq_v: {
8076 llvm::Type *Tys[2] = {
8081 case NEON::BI__builtin_neon_vqshl_n_v:
8082 case NEON::BI__builtin_neon_vqshlq_n_v:
8085 case NEON::BI__builtin_neon_vqshlu_n_v:
8086 case NEON::BI__builtin_neon_vqshluq_n_v:
8089 case NEON::BI__builtin_neon_vrecpe_v:
8090 case NEON::BI__builtin_neon_vrecpeq_v:
8091 case NEON::BI__builtin_neon_vrsqrte_v:
8092 case NEON::BI__builtin_neon_vrsqrteq_v:
8093 Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic;
8095 case NEON::BI__builtin_neon_vrndi_v:
8096 case NEON::BI__builtin_neon_vrndiq_v:
8098 ? Intrinsic::experimental_constrained_nearbyint
8099 : Intrinsic::nearbyint;
8101 case NEON::BI__builtin_neon_vrshr_n_v:
8102 case NEON::BI__builtin_neon_vrshrq_n_v:
8105 case NEON::BI__builtin_neon_vsha512hq_u64:
8106 case NEON::BI__builtin_neon_vsha512h2q_u64:
8107 case NEON::BI__builtin_neon_vsha512su0q_u64:
8108 case NEON::BI__builtin_neon_vsha512su1q_u64: {
8112 case NEON::BI__builtin_neon_vshl_n_v:
8113 case NEON::BI__builtin_neon_vshlq_n_v:
8115 return Builder.CreateShl(
Builder.CreateBitCast(Ops[0],Ty), Ops[1],
8117 case NEON::BI__builtin_neon_vshll_n_v: {
8118 llvm::FixedVectorType *SrcTy =
8119 llvm::FixedVectorType::getTruncatedElementVectorType(VTy);
8120 Ops[0] =
Builder.CreateBitCast(Ops[0], SrcTy);
8122 Ops[0] =
Builder.CreateZExt(Ops[0], VTy);
8124 Ops[0] =
Builder.CreateSExt(Ops[0], VTy);
8126 return Builder.CreateShl(Ops[0], Ops[1],
"vshll_n");
8128 case NEON::BI__builtin_neon_vshrn_n_v: {
8129 llvm::FixedVectorType *SrcTy =
8130 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
8131 Ops[0] =
Builder.CreateBitCast(Ops[0], SrcTy);
8134 Ops[0] =
Builder.CreateLShr(Ops[0], Ops[1]);
8136 Ops[0] =
Builder.CreateAShr(Ops[0], Ops[1]);
8137 return Builder.CreateTrunc(Ops[0], Ty,
"vshrn_n");
8139 case NEON::BI__builtin_neon_vshr_n_v:
8140 case NEON::BI__builtin_neon_vshrq_n_v:
8142 case NEON::BI__builtin_neon_vst1_v:
8143 case NEON::BI__builtin_neon_vst1q_v:
8144 case NEON::BI__builtin_neon_vst2_v:
8145 case NEON::BI__builtin_neon_vst2q_v:
8146 case NEON::BI__builtin_neon_vst3_v:
8147 case NEON::BI__builtin_neon_vst3q_v:
8148 case NEON::BI__builtin_neon_vst4_v:
8149 case NEON::BI__builtin_neon_vst4q_v:
8150 case NEON::BI__builtin_neon_vst2_lane_v:
8151 case NEON::BI__builtin_neon_vst2q_lane_v:
8152 case NEON::BI__builtin_neon_vst3_lane_v:
8153 case NEON::BI__builtin_neon_vst3q_lane_v:
8154 case NEON::BI__builtin_neon_vst4_lane_v:
8155 case NEON::BI__builtin_neon_vst4q_lane_v: {
8157 Ops.push_back(getAlignmentValue32(PtrOp0));
8160 case NEON::BI__builtin_neon_vsm3partw1q_u32:
8161 case NEON::BI__builtin_neon_vsm3partw2q_u32:
8162 case NEON::BI__builtin_neon_vsm3ss1q_u32:
8163 case NEON::BI__builtin_neon_vsm4ekeyq_u32:
8164 case NEON::BI__builtin_neon_vsm4eq_u32: {
8168 case NEON::BI__builtin_neon_vsm3tt1aq_u32:
8169 case NEON::BI__builtin_neon_vsm3tt1bq_u32:
8170 case NEON::BI__builtin_neon_vsm3tt2aq_u32:
8171 case NEON::BI__builtin_neon_vsm3tt2bq_u32: {
8176 case NEON::BI__builtin_neon_vst1_x2_v:
8177 case NEON::BI__builtin_neon_vst1q_x2_v:
8178 case NEON::BI__builtin_neon_vst1_x3_v:
8179 case NEON::BI__builtin_neon_vst1q_x3_v:
8180 case NEON::BI__builtin_neon_vst1_x4_v:
8181 case NEON::BI__builtin_neon_vst1q_x4_v: {
8184 if (Arch == llvm::Triple::aarch64 || Arch == llvm::Triple::aarch64_be ||
8185 Arch == llvm::Triple::aarch64_32) {
8187 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
8193 case NEON::BI__builtin_neon_vsubhn_v: {
8194 llvm::FixedVectorType *SrcTy =
8195 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
8198 Ops[0] =
Builder.CreateBitCast(Ops[0], SrcTy);
8199 Ops[1] =
Builder.CreateBitCast(Ops[1], SrcTy);
8200 Ops[0] =
Builder.CreateSub(Ops[0], Ops[1],
"vsubhn");
8203 Constant *ShiftAmt =
8204 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
8205 Ops[0] =
Builder.CreateLShr(Ops[0], ShiftAmt,
"vsubhn");
8208 return Builder.CreateTrunc(Ops[0], VTy,
"vsubhn");
8210 case NEON::BI__builtin_neon_vtrn_v:
8211 case NEON::BI__builtin_neon_vtrnq_v: {
8212 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
8213 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
8214 Value *SV =
nullptr;
8216 for (
unsigned vi = 0; vi != 2; ++vi) {
8218 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
8219 Indices.push_back(i+vi);
8220 Indices.push_back(i+e+vi);
8222 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8223 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vtrn");
8228 case NEON::BI__builtin_neon_vtst_v:
8229 case NEON::BI__builtin_neon_vtstq_v: {
8230 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
8231 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
8232 Ops[0] =
Builder.CreateAnd(Ops[0], Ops[1]);
8233 Ops[0] =
Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
8234 ConstantAggregateZero::get(Ty));
8235 return Builder.CreateSExt(Ops[0], Ty,
"vtst");
8237 case NEON::BI__builtin_neon_vuzp_v:
8238 case NEON::BI__builtin_neon_vuzpq_v: {
8239 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
8240 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
8241 Value *SV =
nullptr;
8243 for (
unsigned vi = 0; vi != 2; ++vi) {
8245 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
8246 Indices.push_back(2*i+vi);
8248 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8249 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vuzp");
8254 case NEON::BI__builtin_neon_vxarq_u64: {
8259 case NEON::BI__builtin_neon_vzip_v:
8260 case NEON::BI__builtin_neon_vzipq_v: {
8261 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
8262 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
8263 Value *SV =
nullptr;
8265 for (
unsigned vi = 0; vi != 2; ++vi) {
8267 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
8268 Indices.push_back((i + vi*e) >> 1);
8269 Indices.push_back(((i + vi*e) >> 1)+e);
8271 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8272 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vzip");
8277 case NEON::BI__builtin_neon_vdot_s32:
8278 case NEON::BI__builtin_neon_vdot_u32:
8279 case NEON::BI__builtin_neon_vdotq_s32:
8280 case NEON::BI__builtin_neon_vdotq_u32: {
8282 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8283 llvm::Type *Tys[2] = { Ty, InputTy };
8286 case NEON::BI__builtin_neon_vfmlal_low_f16:
8287 case NEON::BI__builtin_neon_vfmlalq_low_f16: {
8289 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8290 llvm::Type *Tys[2] = { Ty, InputTy };
8293 case NEON::BI__builtin_neon_vfmlsl_low_f16:
8294 case NEON::BI__builtin_neon_vfmlslq_low_f16: {
8296 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8297 llvm::Type *Tys[2] = { Ty, InputTy };
8300 case NEON::BI__builtin_neon_vfmlal_high_f16:
8301 case NEON::BI__builtin_neon_vfmlalq_high_f16: {
8303 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8304 llvm::Type *Tys[2] = { Ty, InputTy };
8307 case NEON::BI__builtin_neon_vfmlsl_high_f16:
8308 case NEON::BI__builtin_neon_vfmlslq_high_f16: {
8310 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8311 llvm::Type *Tys[2] = { Ty, InputTy };
8314 case NEON::BI__builtin_neon_vmmlaq_s32:
8315 case NEON::BI__builtin_neon_vmmlaq_u32: {
8317 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8318 llvm::Type *Tys[2] = { Ty, InputTy };
8321 case NEON::BI__builtin_neon_vusmmlaq_s32: {
8323 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8324 llvm::Type *Tys[2] = { Ty, InputTy };
8327 case NEON::BI__builtin_neon_vusdot_s32:
8328 case NEON::BI__builtin_neon_vusdotq_s32: {
8330 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8331 llvm::Type *Tys[2] = { Ty, InputTy };
8334 case NEON::BI__builtin_neon_vbfdot_f32:
8335 case NEON::BI__builtin_neon_vbfdotq_f32: {
8336 llvm::Type *InputTy =
8337 llvm::FixedVectorType::get(
BFloatTy, Ty->getPrimitiveSizeInBits() / 16);
8338 llvm::Type *Tys[2] = { Ty, InputTy };
8341 case NEON::BI__builtin_neon___a32_vcvt_bf16_f32: {
8342 llvm::Type *Tys[1] = { Ty };
8349 assert(Int &&
"Expected valid intrinsic number");
8362 Value *Op, llvm::Type *Ty,
const CmpInst::Predicate Fp,
8363 const CmpInst::Predicate Ip,
const Twine &Name) {
8364 llvm::Type *OTy = Op->
getType();
8370 if (BitCastInst *BI = dyn_cast<BitCastInst>(Op))
8371 OTy = BI->getOperand(0)->getType();
8373 Op =
Builder.CreateBitCast(Op, OTy);
8374 if (OTy->getScalarType()->isFloatingPointTy()) {
8375 if (Fp == CmpInst::FCMP_OEQ)
8376 Op =
Builder.CreateFCmp(Fp, Op, Constant::getNullValue(OTy));
8378 Op =
Builder.CreateFCmpS(Fp, Op, Constant::getNullValue(OTy));
8380 Op =
Builder.CreateICmp(Ip, Op, Constant::getNullValue(OTy));
8382 return Builder.CreateSExt(Op, Ty, Name);
8387 llvm::Type *ResTy,
unsigned IntID,
8391 TblOps.push_back(ExtOp);
8395 auto *TblTy = cast<llvm::FixedVectorType>(Ops[0]->getType());
8396 for (
unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) {
8397 Indices.push_back(2*i);
8398 Indices.push_back(2*i+1);
8401 int PairPos = 0, End = Ops.size() - 1;
8402 while (PairPos < End) {
8403 TblOps.push_back(CGF.
Builder.CreateShuffleVector(Ops[PairPos],
8404 Ops[PairPos+1], Indices,
8411 if (PairPos == End) {
8412 Value *ZeroTbl = ConstantAggregateZero::get(TblTy);
8413 TblOps.push_back(CGF.
Builder.CreateShuffleVector(Ops[PairPos],
8414 ZeroTbl, Indices, Name));
8418 TblOps.push_back(IndexOp);
8424Value *CodeGenFunction::GetValueForARMHint(
unsigned BuiltinID) {
8426 switch (BuiltinID) {
8429 case clang::ARM::BI__builtin_arm_nop:
8432 case clang::ARM::BI__builtin_arm_yield:
8433 case clang::ARM::BI__yield:
8436 case clang::ARM::BI__builtin_arm_wfe:
8437 case clang::ARM::BI__wfe:
8440 case clang::ARM::BI__builtin_arm_wfi:
8441 case clang::ARM::BI__wfi:
8444 case clang::ARM::BI__builtin_arm_sev:
8445 case clang::ARM::BI__sev:
8448 case clang::ARM::BI__builtin_arm_sevl:
8449 case clang::ARM::BI__sevl:
8467 llvm::Type *RegisterType,
8468 llvm::Type *ValueType,
bool isExecHi) {
8473 llvm::Value *
Call = Builder.CreateCall(F, {Builder.getInt1(
true)});
8476 Value *Rt2 = Builder.CreateLShr(
Call, 32);
8477 Rt2 = Builder.CreateTrunc(Rt2, CGF.
Int32Ty);
8489 llvm::Type *RegisterType,
8490 llvm::Type *ValueType,
8492 StringRef SysReg =
"") {
8494 assert((RegisterType->isIntegerTy(32) || RegisterType->isIntegerTy(64) ||
8495 RegisterType->isIntegerTy(128)) &&
8496 "Unsupported size for register.");
8502 if (SysReg.empty()) {
8504 SysReg = cast<clang::StringLiteral>(SysRegStrExpr)->getString();
8507 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) };
8508 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
8509 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
8511 llvm::Type *Types[] = { RegisterType };
8513 bool MixedTypes = RegisterType->isIntegerTy(64) && ValueType->isIntegerTy(32);
8514 assert(!(RegisterType->isIntegerTy(32) && ValueType->isIntegerTy(64))
8515 &&
"Can't fit 64-bit value in 32-bit register");
8517 if (AccessKind !=
Write) {
8520 AccessKind ==
VolatileRead ? llvm::Intrinsic::read_volatile_register
8521 : llvm::Intrinsic::read_register,
8523 llvm::Value *
Call = Builder.CreateCall(F, Metadata);
8527 return Builder.CreateTrunc(
Call, ValueType);
8529 if (ValueType->isPointerTy())
8531 return Builder.CreateIntToPtr(
Call, ValueType);
8536 llvm::Function *F = CGM.
getIntrinsic(llvm::Intrinsic::write_register, Types);
8540 ArgValue = Builder.CreateZExt(ArgValue, RegisterType);
8541 return Builder.CreateCall(F, { Metadata, ArgValue });
8544 if (ValueType->isPointerTy()) {
8546 ArgValue = Builder.CreatePtrToInt(ArgValue, RegisterType);
8547 return Builder.CreateCall(F, { Metadata, ArgValue });
8550 return Builder.CreateCall(F, { Metadata, ArgValue });
8556 switch (BuiltinID) {
8558 case NEON::BI__builtin_neon_vget_lane_i8:
8559 case NEON::BI__builtin_neon_vget_lane_i16:
8560 case NEON::BI__builtin_neon_vget_lane_bf16:
8561 case NEON::BI__builtin_neon_vget_lane_i32:
8562 case NEON::BI__builtin_neon_vget_lane_i64:
8563 case NEON::BI__builtin_neon_vget_lane_f32:
8564 case NEON::BI__builtin_neon_vgetq_lane_i8:
8565 case NEON::BI__builtin_neon_vgetq_lane_i16:
8566 case NEON::BI__builtin_neon_vgetq_lane_bf16:
8567 case NEON::BI__builtin_neon_vgetq_lane_i32:
8568 case NEON::BI__builtin_neon_vgetq_lane_i64:
8569 case NEON::BI__builtin_neon_vgetq_lane_f32:
8570 case NEON::BI__builtin_neon_vduph_lane_bf16:
8571 case NEON::BI__builtin_neon_vduph_laneq_bf16:
8572 case NEON::BI__builtin_neon_vset_lane_i8:
8573 case NEON::BI__builtin_neon_vset_lane_i16:
8574 case NEON::BI__builtin_neon_vset_lane_bf16:
8575 case NEON::BI__builtin_neon_vset_lane_i32:
8576 case NEON::BI__builtin_neon_vset_lane_i64:
8577 case NEON::BI__builtin_neon_vset_lane_f32:
8578 case NEON::BI__builtin_neon_vsetq_lane_i8:
8579 case NEON::BI__builtin_neon_vsetq_lane_i16:
8580 case NEON::BI__builtin_neon_vsetq_lane_bf16:
8581 case NEON::BI__builtin_neon_vsetq_lane_i32:
8582 case NEON::BI__builtin_neon_vsetq_lane_i64:
8583 case NEON::BI__builtin_neon_vsetq_lane_f32:
8584 case NEON::BI__builtin_neon_vsha1h_u32:
8585 case NEON::BI__builtin_neon_vsha1cq_u32:
8586 case NEON::BI__builtin_neon_vsha1pq_u32:
8587 case NEON::BI__builtin_neon_vsha1mq_u32:
8588 case NEON::BI__builtin_neon_vcvth_bf16_f32:
8589 case clang::ARM::BI_MoveToCoprocessor:
8590 case clang::ARM::BI_MoveToCoprocessor2:
8599 llvm::Triple::ArchType Arch) {
8600 if (
auto Hint = GetValueForARMHint(BuiltinID))
8603 if (BuiltinID == clang::ARM::BI__emit) {
8605 llvm::FunctionType *FTy =
8606 llvm::FunctionType::get(
VoidTy,
false);
8610 llvm_unreachable(
"Sema will ensure that the parameter is constant");
8613 uint64_t ZExtValue =
Value.zextOrTrunc(IsThumb ? 16 : 32).getZExtValue();
8615 llvm::InlineAsm *Emit =
8616 IsThumb ? InlineAsm::get(FTy,
".inst.n 0x" + utohexstr(ZExtValue),
"",
8618 : InlineAsm::get(FTy,
".inst 0x" + utohexstr(ZExtValue),
"",
8621 return Builder.CreateCall(Emit);
8624 if (BuiltinID == clang::ARM::BI__builtin_arm_dbg) {
8629 if (BuiltinID == clang::ARM::BI__builtin_arm_prefetch) {
8641 if (BuiltinID == clang::ARM::BI__builtin_arm_rbit) {
8644 CGM.
getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
8647 if (BuiltinID == clang::ARM::BI__builtin_arm_clz ||
8648 BuiltinID == clang::ARM::BI__builtin_arm_clz64) {
8652 if (BuiltinID == clang::ARM::BI__builtin_arm_clz64)
8658 if (BuiltinID == clang::ARM::BI__builtin_arm_cls) {
8662 if (BuiltinID == clang::ARM::BI__builtin_arm_cls64) {
8668 if (BuiltinID == clang::ARM::BI__clear_cache) {
8669 assert(
E->getNumArgs() == 2 &&
"__clear_cache takes 2 arguments");
8672 for (
unsigned i = 0; i < 2; i++)
8675 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
8676 StringRef Name = FD->
getName();
8680 if (BuiltinID == clang::ARM::BI__builtin_arm_mcrr ||
8681 BuiltinID == clang::ARM::BI__builtin_arm_mcrr2) {
8684 switch (BuiltinID) {
8685 default: llvm_unreachable(
"unexpected builtin");
8686 case clang::ARM::BI__builtin_arm_mcrr:
8689 case clang::ARM::BI__builtin_arm_mcrr2:
8711 return Builder.CreateCall(F, {Coproc, Opc1, Rt, Rt2, CRm});
8714 if (BuiltinID == clang::ARM::BI__builtin_arm_mrrc ||
8715 BuiltinID == clang::ARM::BI__builtin_arm_mrrc2) {
8718 switch (BuiltinID) {
8719 default: llvm_unreachable(
"unexpected builtin");
8720 case clang::ARM::BI__builtin_arm_mrrc:
8723 case clang::ARM::BI__builtin_arm_mrrc2:
8731 Value *RtAndRt2 =
Builder.CreateCall(F, {Coproc, Opc1, CRm});
8741 Value *ShiftCast = llvm::ConstantInt::get(
Int64Ty, 32);
8742 RtAndRt2 =
Builder.CreateShl(Rt, ShiftCast,
"shl",
true);
8743 RtAndRt2 =
Builder.CreateOr(RtAndRt2, Rt1);
8748 if (BuiltinID == clang::ARM::BI__builtin_arm_ldrexd ||
8749 ((BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
8750 BuiltinID == clang::ARM::BI__builtin_arm_ldaex) &&
8752 BuiltinID == clang::ARM::BI__ldrexd) {
8755 switch (BuiltinID) {
8756 default: llvm_unreachable(
"unexpected builtin");
8757 case clang::ARM::BI__builtin_arm_ldaex:
8760 case clang::ARM::BI__builtin_arm_ldrexd:
8761 case clang::ARM::BI__builtin_arm_ldrex:
8762 case clang::ARM::BI__ldrexd:
8776 Val =
Builder.CreateShl(Val0, ShiftCst,
"shl",
true );
8777 Val =
Builder.CreateOr(Val, Val1);
8781 if (BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
8782 BuiltinID == clang::ARM::BI__builtin_arm_ldaex) {
8791 BuiltinID == clang::ARM::BI__builtin_arm_ldaex ? Intrinsic::arm_ldaex
8792 : Intrinsic::arm_ldrex,
8794 CallInst *Val =
Builder.CreateCall(F, LoadAddr,
"ldrex");
8798 if (RealResTy->isPointerTy())
8799 return Builder.CreateIntToPtr(Val, RealResTy);
8801 llvm::Type *IntResTy = llvm::IntegerType::get(
8803 return Builder.CreateBitCast(
Builder.CreateTruncOrBitCast(Val, IntResTy),
8808 if (BuiltinID == clang::ARM::BI__builtin_arm_strexd ||
8809 ((BuiltinID == clang::ARM::BI__builtin_arm_stlex ||
8810 BuiltinID == clang::ARM::BI__builtin_arm_strex) &&
8813 BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlexd
8814 : Intrinsic::arm_strexd);
8827 return Builder.CreateCall(F, {Arg0, Arg1, StPtr},
"strexd");
8830 if (BuiltinID == clang::ARM::BI__builtin_arm_strex ||
8831 BuiltinID == clang::ARM::BI__builtin_arm_stlex) {
8836 llvm::Type *StoreTy =
8839 if (StoreVal->
getType()->isPointerTy())
8842 llvm::Type *
IntTy = llvm::IntegerType::get(
8850 BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlex
8851 : Intrinsic::arm_strex,
8854 CallInst *CI =
Builder.CreateCall(F, {StoreVal, StoreAddr},
"strex");
8856 1, Attribute::get(
getLLVMContext(), Attribute::ElementType, StoreTy));
8860 if (BuiltinID == clang::ARM::BI__builtin_arm_clrex) {
8866 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
8867 switch (BuiltinID) {
8868 case clang::ARM::BI__builtin_arm_crc32b:
8869 CRCIntrinsicID = Intrinsic::arm_crc32b;
break;
8870 case clang::ARM::BI__builtin_arm_crc32cb:
8871 CRCIntrinsicID = Intrinsic::arm_crc32cb;
break;
8872 case clang::ARM::BI__builtin_arm_crc32h:
8873 CRCIntrinsicID = Intrinsic::arm_crc32h;
break;
8874 case clang::ARM::BI__builtin_arm_crc32ch:
8875 CRCIntrinsicID = Intrinsic::arm_crc32ch;
break;
8876 case clang::ARM::BI__builtin_arm_crc32w:
8877 case clang::ARM::BI__builtin_arm_crc32d:
8878 CRCIntrinsicID = Intrinsic::arm_crc32w;
break;
8879 case clang::ARM::BI__builtin_arm_crc32cw:
8880 case clang::ARM::BI__builtin_arm_crc32cd:
8881 CRCIntrinsicID = Intrinsic::arm_crc32cw;
break;
8884 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
8890 if (BuiltinID == clang::ARM::BI__builtin_arm_crc32d ||
8891 BuiltinID == clang::ARM::BI__builtin_arm_crc32cd) {
8899 return Builder.CreateCall(F, {Res, Arg1b});
8904 return Builder.CreateCall(F, {Arg0, Arg1});
8908 if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
8909 BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
8910 BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
8911 BuiltinID == clang::ARM::BI__builtin_arm_wsr ||
8912 BuiltinID == clang::ARM::BI__builtin_arm_wsr64 ||
8913 BuiltinID == clang::ARM::BI__builtin_arm_wsrp) {
8916 if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
8917 BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
8918 BuiltinID == clang::ARM::BI__builtin_arm_rsrp)
8921 bool IsPointerBuiltin = BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
8922 BuiltinID == clang::ARM::BI__builtin_arm_wsrp;
8924 bool Is64Bit = BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
8925 BuiltinID == clang::ARM::BI__builtin_arm_wsr64;
8927 llvm::Type *ValueType;
8928 llvm::Type *RegisterType;
8929 if (IsPointerBuiltin) {
8932 }
else if (Is64Bit) {
8933 ValueType = RegisterType =
Int64Ty;
8935 ValueType = RegisterType =
Int32Ty;
8942 if (BuiltinID == ARM::BI__builtin_sponentry) {
8961 return P.first == BuiltinID;
8964 BuiltinID = It->second;
8968 unsigned ICEArguments = 0;
8973 auto getAlignmentValue32 = [&](
Address addr) ->
Value* {
8974 return Builder.getInt32(addr.getAlignment().getQuantity());
8981 unsigned NumArgs =
E->getNumArgs() - (HasExtraArg ? 1 : 0);
8982 for (
unsigned i = 0, e = NumArgs; i != e; i++) {
8984 switch (BuiltinID) {
8985 case NEON::BI__builtin_neon_vld1_v:
8986 case NEON::BI__builtin_neon_vld1q_v:
8987 case NEON::BI__builtin_neon_vld1q_lane_v:
8988 case NEON::BI__builtin_neon_vld1_lane_v:
8989 case NEON::BI__builtin_neon_vld1_dup_v:
8990 case NEON::BI__builtin_neon_vld1q_dup_v:
8991 case NEON::BI__builtin_neon_vst1_v:
8992 case NEON::BI__builtin_neon_vst1q_v:
8993 case NEON::BI__builtin_neon_vst1q_lane_v:
8994 case NEON::BI__builtin_neon_vst1_lane_v:
8995 case NEON::BI__builtin_neon_vst2_v:
8996 case NEON::BI__builtin_neon_vst2q_v:
8997 case NEON::BI__builtin_neon_vst2_lane_v:
8998 case NEON::BI__builtin_neon_vst2q_lane_v:
8999 case NEON::BI__builtin_neon_vst3_v:
9000 case NEON::BI__builtin_neon_vst3q_v:
9001 case NEON::BI__builtin_neon_vst3_lane_v:
9002 case NEON::BI__builtin_neon_vst3q_lane_v:
9003 case NEON::BI__builtin_neon_vst4_v:
9004 case NEON::BI__builtin_neon_vst4q_v:
9005 case NEON::BI__builtin_neon_vst4_lane_v:
9006 case NEON::BI__builtin_neon_vst4q_lane_v:
9015 switch (BuiltinID) {
9016 case NEON::BI__builtin_neon_vld2_v:
9017 case NEON::BI__builtin_neon_vld2q_v:
9018 case NEON::BI__builtin_neon_vld3_v:
9019 case NEON::BI__builtin_neon_vld3q_v:
9020 case NEON::BI__builtin_neon_vld4_v:
9021 case NEON::BI__builtin_neon_vld4q_v:
9022 case NEON::BI__builtin_neon_vld2_lane_v:
9023 case NEON::BI__builtin_neon_vld2q_lane_v:
9024 case NEON::BI__builtin_neon_vld3_lane_v:
9025 case NEON::BI__builtin_neon_vld3q_lane_v:
9026 case NEON::BI__builtin_neon_vld4_lane_v:
9027 case NEON::BI__builtin_neon_vld4q_lane_v:
9028 case NEON::BI__builtin_neon_vld2_dup_v:
9029 case NEON::BI__builtin_neon_vld2q_dup_v:
9030 case NEON::BI__builtin_neon_vld3_dup_v:
9031 case NEON::BI__builtin_neon_vld3q_dup_v:
9032 case NEON::BI__builtin_neon_vld4_dup_v:
9033 case NEON::BI__builtin_neon_vld4q_dup_v:
9045 switch (BuiltinID) {
9048 case NEON::BI__builtin_neon_vget_lane_i8:
9049 case NEON::BI__builtin_neon_vget_lane_i16:
9050 case NEON::BI__builtin_neon_vget_lane_i32:
9051 case NEON::BI__builtin_neon_vget_lane_i64:
9052 case NEON::BI__builtin_neon_vget_lane_bf16:
9053 case NEON::BI__builtin_neon_vget_lane_f32:
9054 case NEON::BI__builtin_neon_vgetq_lane_i8:
9055 case NEON::BI__builtin_neon_vgetq_lane_i16:
9056 case NEON::BI__builtin_neon_vgetq_lane_i32:
9057 case NEON::BI__builtin_neon_vgetq_lane_i64:
9058 case NEON::BI__builtin_neon_vgetq_lane_bf16:
9059 case NEON::BI__builtin_neon_vgetq_lane_f32:
9060 case NEON::BI__builtin_neon_vduph_lane_bf16:
9061 case NEON::BI__builtin_neon_vduph_laneq_bf16:
9062 return Builder.CreateExtractElement(Ops[0], Ops[1],
"vget_lane");
9064 case NEON::BI__builtin_neon_vrndns_f32: {
9066 llvm::Type *Tys[] = {Arg->
getType()};
9068 return Builder.CreateCall(F, {Arg},
"vrndn"); }
9070 case NEON::BI__builtin_neon_vset_lane_i8:
9071 case NEON::BI__builtin_neon_vset_lane_i16:
9072 case NEON::BI__builtin_neon_vset_lane_i32:
9073 case NEON::BI__builtin_neon_vset_lane_i64:
9074 case NEON::BI__builtin_neon_vset_lane_bf16:
9075 case NEON::BI__builtin_neon_vset_lane_f32:
9076 case NEON::BI__builtin_neon_vsetq_lane_i8:
9077 case NEON::BI__builtin_neon_vsetq_lane_i16:
9078 case NEON::BI__builtin_neon_vsetq_lane_i32:
9079 case NEON::BI__builtin_neon_vsetq_lane_i64:
9080 case NEON::BI__builtin_neon_vsetq_lane_bf16:
9081 case NEON::BI__builtin_neon_vsetq_lane_f32:
9082 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
9084 case NEON::BI__builtin_neon_vsha1h_u32:
9087 case NEON::BI__builtin_neon_vsha1cq_u32:
9090 case NEON::BI__builtin_neon_vsha1pq_u32:
9093 case NEON::BI__builtin_neon_vsha1mq_u32:
9097 case NEON::BI__builtin_neon_vcvth_bf16_f32: {
9104 case clang::ARM::BI_MoveToCoprocessor:
9105 case clang::ARM::BI_MoveToCoprocessor2: {
9107 ? Intrinsic::arm_mcr
9108 : Intrinsic::arm_mcr2);
9109 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0],
9110 Ops[3], Ops[4], Ops[5]});
9115 assert(HasExtraArg);
9116 const Expr *Arg =
E->getArg(
E->getNumArgs()-1);
9117 std::optional<llvm::APSInt>
Result =
9122 if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f ||
9123 BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_d) {
9126 if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f)
9132 bool usgn =
Result->getZExtValue() == 1;
9133 unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr;
9137 return Builder.CreateCall(F, Ops,
"vcvtr");
9142 bool usgn =
Type.isUnsigned();
9143 bool rightShift =
false;
9145 llvm::FixedVectorType *VTy =
9148 llvm::Type *Ty = VTy;
9159 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
9160 Builtin->NameHint, Builtin->TypeModifier,
E, Ops, PtrOp0, PtrOp1, Arch);
9163 switch (BuiltinID) {
9164 default:
return nullptr;
9165 case NEON::BI__builtin_neon_vld1q_lane_v:
9168 if (VTy->getElementType()->isIntegerTy(64)) {
9170 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
9171 int Lane = cast<ConstantInt>(Ops[2])->getZExtValue();
9172 Value *SV = llvm::ConstantVector::get(ConstantInt::get(
Int32Ty, 1-Lane));
9173 Ops[1] =
Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
9175 Ty = llvm::FixedVectorType::get(VTy->getElementType(), 1);
9178 Value *Align = getAlignmentValue32(PtrOp0);
9181 int Indices[] = {1 - Lane, Lane};
9182 return Builder.CreateShuffleVector(Ops[1], Ld, Indices,
"vld1q_lane");
9185 case NEON::BI__builtin_neon_vld1_lane_v: {
9186 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
9189 return Builder.CreateInsertElement(Ops[1], Ld, Ops[2],
"vld1_lane");
9191 case NEON::BI__builtin_neon_vqrshrn_n_v:
9193 usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns;
9196 case NEON::BI__builtin_neon_vqrshrun_n_v:
9198 Ops,
"vqrshrun_n", 1,
true);
9199 case NEON::BI__builtin_neon_vqshrn_n_v:
9200 Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns;
9203 case NEON::BI__builtin_neon_vqshrun_n_v:
9205 Ops,
"vqshrun_n", 1,
true);
9206 case NEON::BI__builtin_neon_vrecpe_v:
9207 case NEON::BI__builtin_neon_vrecpeq_v:
9210 case NEON::BI__builtin_neon_vrshrn_n_v:
9212 Ops,
"vrshrn_n", 1,
true);
9213 case NEON::BI__builtin_neon_vrsra_n_v:
9214 case NEON::BI__builtin_neon_vrsraq_n_v:
9215 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
9216 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
9218 Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
9220 return Builder.CreateAdd(Ops[0], Ops[1],
"vrsra_n");
9221 case NEON::BI__builtin_neon_vsri_n_v:
9222 case NEON::BI__builtin_neon_vsriq_n_v:
9225 case NEON::BI__builtin_neon_vsli_n_v:
9226 case NEON::BI__builtin_neon_vsliq_n_v:
9230 case NEON::BI__builtin_neon_vsra_n_v:
9231 case NEON::BI__builtin_neon_vsraq_n_v:
9232 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
9234 return Builder.CreateAdd(Ops[0], Ops[1]);
9235 case NEON::BI__builtin_neon_vst1q_lane_v:
9238 if (VTy->getElementType()->isIntegerTy(64)) {
9239 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
9240 Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2]));
9241 Ops[1] =
Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
9242 Ops[2] = getAlignmentValue32(PtrOp0);
9243 llvm::Type *Tys[] = {
Int8PtrTy, Ops[1]->getType()};
9248 case NEON::BI__builtin_neon_vst1_lane_v: {
9249 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
9250 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2]);
9254 case NEON::BI__builtin_neon_vtbl1_v:
9257 case NEON::BI__builtin_neon_vtbl2_v:
9260 case NEON::BI__builtin_neon_vtbl3_v:
9263 case NEON::BI__builtin_neon_vtbl4_v:
9266 case NEON::BI__builtin_neon_vtbx1_v:
9269 case NEON::BI__builtin_neon_vtbx2_v:
9272 case NEON::BI__builtin_neon_vtbx3_v:
9275 case NEON::BI__builtin_neon_vtbx4_v:
9281template<
typename Integer>
9290 return Unsigned ? Builder.CreateZExt(
V,
T) : Builder.CreateSExt(
V,
T);
9300 unsigned LaneBits = cast<llvm::VectorType>(
V->getType())
9302 ->getPrimitiveSizeInBits();
9303 if (Shift == LaneBits) {
9308 return llvm::Constant::getNullValue(
V->getType());
9312 return Unsigned ? Builder.CreateLShr(
V, Shift) : Builder.CreateAShr(
V, Shift);
9319 unsigned Elements = 128 /
V->getType()->getPrimitiveSizeInBits();
9320 return Builder.CreateVectorSplat(Elements,
V);
9326 llvm::Type *DestType) {
9339 V->getType()->getScalarSizeInBits() != DestType->getScalarSizeInBits()) {
9340 return Builder.CreateCall(
9342 {DestType, V->getType()}),
9345 return Builder.CreateBitCast(
V, DestType);
9353 unsigned InputElements =
9354 cast<llvm::FixedVectorType>(
V->getType())->getNumElements();
9355 for (
unsigned i = 0; i < InputElements; i += 2)
9356 Indices.push_back(i + Odd);
9357 return Builder.CreateShuffleVector(
V, Indices);
9363 assert(V0->getType() == V1->getType() &&
"Can't zip different vector types");
9365 unsigned InputElements =
9366 cast<llvm::FixedVectorType>(V0->getType())->getNumElements();
9367 for (
unsigned i = 0; i < InputElements; i++) {
9368 Indices.push_back(i);
9369 Indices.push_back(i + InputElements);
9371 return Builder.CreateShuffleVector(V0, V1, Indices);
9374template<
unsigned HighBit,
unsigned OtherBits>
9378 llvm::Type *
T = cast<llvm::VectorType>(VT)->getElementType();
9379 unsigned LaneBits =
T->getPrimitiveSizeInBits();
9380 uint32_t
Value = HighBit << (LaneBits - 1);
9382 Value |= (1UL << (LaneBits - 1)) - 1;
9383 llvm::Value *Lane = llvm::ConstantInt::get(
T,
Value);
9389 unsigned ReverseWidth) {
9393 unsigned LaneSize =
V->getType()->getScalarSizeInBits();
9394 unsigned Elements = 128 / LaneSize;
9395 unsigned Mask = ReverseWidth / LaneSize - 1;
9396 for (
unsigned i = 0; i < Elements; i++)
9397 Indices.push_back(i ^ Mask);
9398 return Builder.CreateShuffleVector(
V, Indices);
9404 llvm::Triple::ArchType Arch) {
9405 enum class CustomCodeGen { VLD24, VST24 } CustomCodeGenType;
9406 Intrinsic::ID IRIntr;
9407 unsigned NumVectors;
9410 switch (BuiltinID) {
9411 #include "clang/Basic/arm_mve_builtin_cg.inc"
9422 switch (CustomCodeGenType) {
9424 case CustomCodeGen::VLD24: {
9430 assert(MvecLType->isStructTy() &&
9431 "Return type for vld[24]q should be a struct");
9432 assert(MvecLType->getStructNumElements() == 1 &&
9433 "Return-type struct for vld[24]q should have one element");
9434 auto MvecLTypeInner = MvecLType->getStructElementType(0);
9435 assert(MvecLTypeInner->isArrayTy() &&
9436 "Return-type struct for vld[24]q should contain an array");
9437 assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
9438 "Array member of return-type struct vld[24]q has wrong length");
9439 auto VecLType = MvecLTypeInner->getArrayElementType();
9441 Tys.push_back(VecLType);
9443 auto Addr =
E->getArg(0);
9449 Value *MvecOut = PoisonValue::get(MvecLType);
9450 for (
unsigned i = 0; i < NumVectors; ++i) {
9451 Value *Vec =
Builder.CreateExtractValue(LoadResult, i);
9452 MvecOut =
Builder.CreateInsertValue(MvecOut, Vec, {0, i});
9461 case CustomCodeGen::VST24: {
9465 auto Addr =
E->getArg(0);
9469 auto MvecCType =
E->getArg(1)->
getType();
9471 assert(MvecLType->isStructTy() &&
"Data type for vst2q should be a struct");
9472 assert(MvecLType->getStructNumElements() == 1 &&
9473 "Data-type struct for vst2q should have one element");
9474 auto MvecLTypeInner = MvecLType->getStructElementType(0);
9475 assert(MvecLTypeInner->isArrayTy() &&
9476 "Data-type struct for vst2q should contain an array");
9477 assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
9478 "Array member of return-type struct vld[24]q has wrong length");
9479 auto VecLType = MvecLTypeInner->getArrayElementType();
9481 Tys.push_back(VecLType);
9486 for (
unsigned i = 0; i < NumVectors; i++)
9487 Ops.push_back(
Builder.CreateExtractValue(Mvec, {0, i}));
9490 Value *ToReturn =
nullptr;
9491 for (
unsigned i = 0; i < NumVectors; i++) {
9492 Ops.push_back(llvm::ConstantInt::get(
Int32Ty, i));
9493 ToReturn =
Builder.CreateCall(F, Ops);
9499 llvm_unreachable(
"unknown custom codegen type.");
9505 llvm::Triple::ArchType Arch) {
9506 switch (BuiltinID) {
9509#include "clang/Basic/arm_cde_builtin_cg.inc"
9516 llvm::Triple::ArchType Arch) {
9517 unsigned int Int = 0;
9518 const char *
s =
nullptr;
9520 switch (BuiltinID) {
9523 case NEON::BI__builtin_neon_vtbl1_v:
9524 case NEON::BI__builtin_neon_vqtbl1_v:
9525 case NEON::BI__builtin_neon_vqtbl1q_v:
9526 case NEON::BI__builtin_neon_vtbl2_v:
9527 case NEON::BI__builtin_neon_vqtbl2_v:
9528 case NEON::BI__builtin_neon_vqtbl2q_v:
9529 case NEON::BI__builtin_neon_vtbl3_v:
9530 case NEON::BI__builtin_neon_vqtbl3_v:
9531 case NEON::BI__builtin_neon_vqtbl3q_v:
9532 case NEON::BI__builtin_neon_vtbl4_v:
9533 case NEON::BI__builtin_neon_vqtbl4_v:
9534 case NEON::BI__builtin_neon_vqtbl4q_v:
9536 case NEON::BI__builtin_neon_vtbx1_v:
9537 case NEON::BI__builtin_neon_vqtbx1_v:
9538 case NEON::BI__builtin_neon_vqtbx1q_v:
9539 case NEON::BI__builtin_neon_vtbx2_v:
9540 case NEON::BI__builtin_neon_vqtbx2_v:
9541 case NEON::BI__builtin_neon_vqtbx2q_v:
9542 case NEON::BI__builtin_neon_vtbx3_v:
9543 case NEON::BI__builtin_neon_vqtbx3_v:
9544 case NEON::BI__builtin_neon_vqtbx3q_v:
9545 case NEON::BI__builtin_neon_vtbx4_v:
9546 case NEON::BI__builtin_neon_vqtbx4_v:
9547 case NEON::BI__builtin_neon_vqtbx4q_v:
9551 assert(
E->getNumArgs() >= 3);
9554 const Expr *Arg =
E->getArg(
E->getNumArgs() - 1);
9555 std::optional<llvm::APSInt>
Result =
9570 switch (BuiltinID) {
9571 case NEON::BI__builtin_neon_vtbl1_v: {
9573 Ty, Intrinsic::aarch64_neon_tbl1,
"vtbl1");
9575 case NEON::BI__builtin_neon_vtbl2_v: {
9577 Ty, Intrinsic::aarch64_neon_tbl1,
"vtbl1");
9579 case NEON::BI__builtin_neon_vtbl3_v: {
9581 Ty, Intrinsic::aarch64_neon_tbl2,
"vtbl2");
9583 case NEON::BI__builtin_neon_vtbl4_v: {
9585 Ty, Intrinsic::aarch64_neon_tbl2,
"vtbl2");
9587 case NEON::BI__builtin_neon_vtbx1_v: {
9590 Intrinsic::aarch64_neon_tbl1,
"vtbl1");
9592 llvm::Constant *EightV = ConstantInt::get(Ty, 8);
9593 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV);
9594 CmpRes = Builder.CreateSExt(CmpRes, Ty);
9596 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
9597 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
9598 return Builder.CreateOr(EltsFromInput, EltsFromTbl,
"vtbx");
9600 case NEON::BI__builtin_neon_vtbx2_v: {
9602 Ty, Intrinsic::aarch64_neon_tbx1,
"vtbx1");
9604 case NEON::BI__builtin_neon_vtbx3_v: {
9607 Intrinsic::aarch64_neon_tbl2,
"vtbl2");
9609 llvm::Constant *TwentyFourV = ConstantInt::get(Ty, 24);
9610 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4],
9612 CmpRes = Builder.CreateSExt(CmpRes, Ty);
9614 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
9615 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
9616 return Builder.CreateOr(EltsFromInput, EltsFromTbl,
"vtbx");
9618 case NEON::BI__builtin_neon_vtbx4_v: {
9620 Ty, Intrinsic::aarch64_neon_tbx2,
"vtbx2");
9622 case NEON::BI__builtin_neon_vqtbl1_v:
9623 case NEON::BI__builtin_neon_vqtbl1q_v:
9624 Int = Intrinsic::aarch64_neon_tbl1;
s =
"vtbl1";
break;
9625 case NEON::BI__builtin_neon_vqtbl2_v:
9626 case NEON::BI__builtin_neon_vqtbl2q_v: {
9627 Int = Intrinsic::aarch64_neon_tbl2;
s =
"vtbl2";
break;
9628 case NEON::BI__builtin_neon_vqtbl3_v:
9629 case NEON::BI__builtin_neon_vqtbl3q_v:
9630 Int = Intrinsic::aarch64_neon_tbl3;
s =
"vtbl3";
break;
9631 case NEON::BI__builtin_neon_vqtbl4_v:
9632 case NEON::BI__builtin_neon_vqtbl4q_v:
9633 Int = Intrinsic::aarch64_neon_tbl4;
s =
"vtbl4";
break;
9634 case NEON::BI__builtin_neon_vqtbx1_v:
9635 case NEON::BI__builtin_neon_vqtbx1q_v:
9636 Int = Intrinsic::aarch64_neon_tbx1;
s =
"vtbx1";
break;
9637 case NEON::BI__builtin_neon_vqtbx2_v:
9638 case NEON::BI__builtin_neon_vqtbx2q_v:
9639 Int = Intrinsic::aarch64_neon_tbx2;
s =
"vtbx2";
break;
9640 case NEON::BI__builtin_neon_vqtbx3_v:
9641 case NEON::BI__builtin_neon_vqtbx3q_v:
9642 Int = Intrinsic::aarch64_neon_tbx3;
s =
"vtbx3";
break;
9643 case NEON::BI__builtin_neon_vqtbx4_v:
9644 case NEON::BI__builtin_neon_vqtbx4q_v:
9645 Int = Intrinsic::aarch64_neon_tbx4;
s =
"vtbx4";
break;
9657 auto *VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
9659 Value *
V = PoisonValue::get(VTy);
9660 llvm::Constant *CI = ConstantInt::get(
SizeTy, 0);
9661 Op =
Builder.CreateInsertElement(
V, Op, CI);
9670 case SVETypeFlags::MemEltTyDefault:
9672 case SVETypeFlags::MemEltTyInt8:
9674 case SVETypeFlags::MemEltTyInt16:
9676 case SVETypeFlags::MemEltTyInt32:
9678 case SVETypeFlags::MemEltTyInt64:
9681 llvm_unreachable(
"Unknown MemEltType");
9687 llvm_unreachable(
"Invalid SVETypeFlag!");
9689 case SVETypeFlags::EltTyInt8:
9691 case SVETypeFlags::EltTyInt16:
9693 case SVETypeFlags::EltTyInt32:
9695 case SVETypeFlags::EltTyInt64:
9697 case SVETypeFlags::EltTyInt128:
9700 case SVETypeFlags::EltTyFloat16:
9702 case SVETypeFlags::EltTyFloat32:
9704 case SVETypeFlags::EltTyFloat64:
9707 case SVETypeFlags::EltTyBFloat16:
9710 case SVETypeFlags::EltTyBool8:
9711 case SVETypeFlags::EltTyBool16:
9712 case SVETypeFlags::EltTyBool32:
9713 case SVETypeFlags::EltTyBool64:
9720llvm::ScalableVectorType *
9723 default: llvm_unreachable(
"Unhandled SVETypeFlag!");
9725 case SVETypeFlags::EltTyInt8:
9726 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 16);
9727 case SVETypeFlags::EltTyInt16:
9728 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 8);
9729 case SVETypeFlags::EltTyInt32:
9730 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 4);
9731 case SVETypeFlags::EltTyInt64:
9732 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 2);
9734 case SVETypeFlags::EltTyBFloat16:
9735 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 8);
9736 case SVETypeFlags::EltTyFloat16:
9737 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 8);
9738 case SVETypeFlags::EltTyFloat32:
9739 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 4);
9740 case SVETypeFlags::EltTyFloat64:
9741 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 2);
9743 case SVETypeFlags::EltTyBool8:
9744 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 16);
9745 case SVETypeFlags::EltTyBool16:
9746 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 8);
9747 case SVETypeFlags::EltTyBool32:
9748 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 4);
9749 case SVETypeFlags::EltTyBool64:
9750 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 2);
9755llvm::ScalableVectorType *
9759 llvm_unreachable(
"Invalid SVETypeFlag!");
9761 case SVETypeFlags::EltTyInt8:
9762 return llvm::ScalableVectorType::get(
Builder.getInt8Ty(), 16);
9763 case SVETypeFlags::EltTyInt16:
9764 return llvm::ScalableVectorType::get(
Builder.getInt16Ty(), 8);
9765 case SVETypeFlags::EltTyInt32:
9766 return llvm::ScalableVectorType::get(
Builder.getInt32Ty(), 4);
9767 case SVETypeFlags::EltTyInt64:
9768 return llvm::ScalableVectorType::get(
Builder.getInt64Ty(), 2);
9770 case SVETypeFlags::EltTyFloat16:
9771 return llvm::ScalableVectorType::get(
Builder.getHalfTy(), 8);
9772 case SVETypeFlags::EltTyBFloat16:
9773 return llvm::ScalableVectorType::get(
Builder.getBFloatTy(), 8);
9774 case SVETypeFlags::EltTyFloat32:
9775 return llvm::ScalableVectorType::get(
Builder.getFloatTy(), 4);
9776 case SVETypeFlags::EltTyFloat64:
9777 return llvm::ScalableVectorType::get(
Builder.getDoubleTy(), 2);
9779 case SVETypeFlags::EltTyBool8:
9780 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 16);
9781 case SVETypeFlags::EltTyBool16:
9782 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 8);
9783 case SVETypeFlags::EltTyBool32:
9784 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 4);
9785 case SVETypeFlags::EltTyBool64:
9786 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 2);
9801 return llvm::ScalableVectorType::get(EltTy, NumElts);
9807 llvm::ScalableVectorType *VTy) {
9809 if (isa<TargetExtType>(Pred->
getType()) &&
9810 cast<TargetExtType>(Pred->
getType())->getName() ==
"aarch64.svcount")
9813 auto *RTy = llvm::VectorType::get(IntegerType::get(
getLLVMContext(), 1), VTy);
9818 llvm::Type *IntrinsicTy;
9819 switch (VTy->getMinNumElements()) {
9821 llvm_unreachable(
"unsupported element count!");
9826 IntID = Intrinsic::aarch64_sve_convert_from_svbool;
9830 IntID = Intrinsic::aarch64_sve_convert_to_svbool;
9831 IntrinsicTy = Pred->
getType();
9837 assert(
C->getType() == RTy &&
"Unexpected return type!");
9845 auto *OverloadedTy =
9849 if (Ops[1]->getType()->isVectorTy())
9869 Ops[0], cast<llvm::ScalableVectorType>(F->getArg(0)->getType()));
9874 if (Ops.size() == 2) {
9875 assert(Ops[1]->getType()->isVectorTy() &&
"Scalar base requires an offset");
9876 Ops.push_back(ConstantInt::get(
Int64Ty, 0));
9881 if (!TypeFlags.
isByteIndexed() && Ops[1]->getType()->isVectorTy()) {
9882 unsigned BytesPerElt =
9883 OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
9884 Ops[2] =
Builder.CreateShl(Ops[2], Log2_32(BytesPerElt));
9899 auto *OverloadedTy =
9904 Ops.insert(Ops.begin(), Ops.pop_back_val());
9907 if (Ops[2]->getType()->isVectorTy())
9922 if (Ops.size() == 3) {
9923 assert(Ops[1]->getType()->isVectorTy() &&
"Scalar base requires an offset");
9924 Ops.push_back(ConstantInt::get(
Int64Ty, 0));
9929 Ops[0] =
Builder.CreateTrunc(Ops[0], OverloadedTy);
9939 Ops[1], cast<llvm::ScalableVectorType>(F->getArg(1)->getType()));
9943 if (!TypeFlags.
isByteIndexed() && Ops[2]->getType()->isVectorTy()) {
9944 unsigned BytesPerElt =
9945 OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
9946 Ops[3] =
Builder.CreateShl(Ops[3], Log2_32(BytesPerElt));
9949 return Builder.CreateCall(F, Ops);
9957 auto *OverloadedTy = dyn_cast<llvm::ScalableVectorType>(Ops[1]->getType());
9959 OverloadedTy = cast<llvm::ScalableVectorType>(Ops[2]->getType());
9965 if (Ops[1]->getType()->isVectorTy()) {
9966 if (Ops.size() == 3) {
9968 Ops.push_back(ConstantInt::get(
Int64Ty, 0));
9971 std::swap(Ops[2], Ops[3]);
9975 unsigned BytesPerElt = MemEltTy->getPrimitiveSizeInBits() / 8;
9976 if (BytesPerElt > 1)
9977 Ops[2] =
Builder.CreateShl(Ops[2], Log2_32(BytesPerElt));
9982 return Builder.CreateCall(F, Ops);
9988 llvm::ScalableVectorType *VTy =
getSVEType(TypeFlags);
9992 case Intrinsic::aarch64_sve_ld2_sret:
9993 case Intrinsic::aarch64_sve_ld1_pn_x2:
9994 case Intrinsic::aarch64_sve_ldnt1_pn_x2:
9995 case Intrinsic::aarch64_sve_ld2q_sret:
9998 case Intrinsic::aarch64_sve_ld3_sret:
9999 case Intrinsic::aarch64_sve_ld3q_sret:
10002 case Intrinsic::aarch64_sve_ld4_sret:
10003 case Intrinsic::aarch64_sve_ld1_pn_x4:
10004 case Intrinsic::aarch64_sve_ldnt1_pn_x4:
10005 case Intrinsic::aarch64_sve_ld4q_sret:
10009 llvm_unreachable(
"unknown intrinsic!");
10011 auto RetTy = llvm::VectorType::get(VTy->getElementType(),
10012 VTy->getElementCount() * N);
10015 Value *BasePtr = Ops[1];
10018 if (Ops.size() > 2)
10023 unsigned MinElts = VTy->getMinNumElements();
10024 Value *
Ret = llvm::PoisonValue::get(RetTy);
10025 for (
unsigned I = 0; I < N; I++) {
10028 Ret =
Builder.CreateInsertVector(RetTy, Ret, SRet, Idx);
10036 llvm::ScalableVectorType *VTy =
getSVEType(TypeFlags);
10040 case Intrinsic::aarch64_sve_st2:
10041 case Intrinsic::aarch64_sve_st1_pn_x2:
10042 case Intrinsic::aarch64_sve_stnt1_pn_x2:
10043 case Intrinsic::aarch64_sve_st2q:
10046 case Intrinsic::aarch64_sve_st3:
10047 case Intrinsic::aarch64_sve_st3q:
10050 case Intrinsic::aarch64_sve_st4:
10051 case Intrinsic::aarch64_sve_st1_pn_x4:
10052 case Intrinsic::aarch64_sve_stnt1_pn_x4:
10053 case Intrinsic::aarch64_sve_st4q:
10057 llvm_unreachable(
"unknown intrinsic!");
10061 Value *BasePtr = Ops[1];
10064 if (Ops.size() > (2 + N))
10070 for (
unsigned I = Ops.size() - N; I < Ops.size(); ++I)
10071 Operands.push_back(Ops[I]);
10072 Operands.append({Predicate, BasePtr});
10075 return Builder.CreateCall(F, Operands);
10083 unsigned BuiltinID) {
10095 llvm::ScalableVectorType *Ty =
getSVEType(TypeFlags);
10101 llvm::Type *OverloadedTy =
getSVEType(TypeFlags);
10108 unsigned BuiltinID) {
10111 auto *MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
10114 Value *BasePtr = Ops[1];
10117 if (Ops.size() > 3)
10120 Value *PrfOp = Ops.back();
10123 return Builder.CreateCall(F, {Predicate, BasePtr, PrfOp});
10127 llvm::Type *ReturnTy,
10129 unsigned IntrinsicID,
10130 bool IsZExtReturn) {
10137 auto VectorTy = cast<llvm::ScalableVectorType>(ReturnTy);
10138 llvm::ScalableVectorType *MemoryTy =
nullptr;
10139 llvm::ScalableVectorType *PredTy =
nullptr;
10140 bool IsQuadLoad =
false;
10141 switch (IntrinsicID) {
10142 case Intrinsic::aarch64_sve_ld1uwq:
10143 case Intrinsic::aarch64_sve_ld1udq:
10144 MemoryTy = llvm::ScalableVectorType::get(MemEltTy, 1);
10145 PredTy = llvm::ScalableVectorType::get(
10150 MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
10156 Value *BasePtr = Ops[1];
10159 if (Ops.size() > 2)
10164 cast<llvm::Instruction>(
Builder.CreateCall(F, {Predicate, BasePtr}));
10171 return IsZExtReturn ?
Builder.CreateZExt(Load, VectorTy)
10172 :
Builder.CreateSExt(Load, VectorTy);
10177 unsigned IntrinsicID) {
10184 auto VectorTy = cast<llvm::ScalableVectorType>(Ops.back()->getType());
10185 auto MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
10187 auto PredTy = MemoryTy;
10188 auto AddrMemoryTy = MemoryTy;
10189 bool IsQuadStore =
false;
10191 switch (IntrinsicID) {
10192 case Intrinsic::aarch64_sve_st1wq:
10193 case Intrinsic::aarch64_sve_st1dq:
10194 AddrMemoryTy = llvm::ScalableVectorType::get(MemEltTy, 1);
10196 llvm::ScalableVectorType::get(IntegerType::get(
getLLVMContext(), 1), 1);
10197 IsQuadStore =
true;
10203 Value *BasePtr = Ops[1];
10206 if (Ops.size() == 4)
10211 IsQuadStore ? Ops.back() :
Builder.CreateTrunc(Ops.back(), MemoryTy);
10216 cast<llvm::Instruction>(
Builder.CreateCall(F, {Val, Predicate, BasePtr}));
10229 NewOps.push_back(Ops[2]);
10231 llvm::Value *BasePtr = Ops[3];
10235 if (Ops.size() == 5) {
10238 llvm::Value *StreamingVectorLengthCall =
10239 Builder.CreateCall(StreamingVectorLength);
10240 llvm::Value *Mulvl =
10241 Builder.CreateMul(StreamingVectorLengthCall, Ops[4],
"mulvl");
10245 NewOps.push_back(BasePtr);
10246 NewOps.push_back(Ops[0]);
10247 NewOps.push_back(Ops[1]);
10249 return Builder.CreateCall(F, NewOps);
10261 return Builder.CreateCall(F, Ops);
10268 if (Ops.size() == 0)
10269 Ops.push_back(llvm::ConstantInt::get(
Int32Ty, 255));
10271 return Builder.CreateCall(F, Ops);
10277 if (Ops.size() == 2)
10278 Ops.push_back(
Builder.getInt32(0));
10282 return Builder.CreateCall(F, Ops);
10288 return Builder.CreateVectorSplat(
10289 cast<llvm::VectorType>(Ty)->getElementCount(), Scalar);
10303 return Builder.CreateBitCast(Val, Ty);
10308 auto *SplatZero = Constant::getNullValue(Ty);
10309 Ops.insert(Ops.begin(), SplatZero);
10314 auto *SplatUndef = UndefValue::get(Ty);
10315 Ops.insert(Ops.begin(), SplatUndef);
10320 llvm::Type *ResultType,
10325 llvm::Type *DefaultType =
getSVEType(TypeFlags);
10328 return {DefaultType, Ops[1]->getType()};
10334 return {Ops[0]->getType(), Ops.back()->getType()};
10336 if (TypeFlags.
isReductionQV() && !ResultType->isScalableTy() &&
10337 ResultType->isVectorTy())
10338 return {ResultType, Ops[1]->getType()};
10341 return {DefaultType};
10348 "Expects TypleFlags.isTupleSet() or TypeFlags.isTupleGet()");
10350 unsigned I = cast<ConstantInt>(Ops[1])->getSExtValue();
10351 auto *SingleVecTy = dyn_cast<llvm::ScalableVectorType>(
10352 TypeFlags.
isTupleSet() ? Ops[2]->getType() : Ty);
10358 I * SingleVecTy->getMinNumElements());
10361 return Builder.CreateInsertVector(Ty, Ops[0], Ops[2], Idx);
10362 return Builder.CreateExtractVector(Ty, Ops[0], Idx);
10368 assert(TypeFlags.
isTupleCreate() &&
"Expects TypleFlag isTupleCreate");
10370 auto *SrcTy = dyn_cast<llvm::ScalableVectorType>(Ops[0]->getType());
10375 unsigned MinElts = SrcTy->getMinNumElements();
10376 Value *
Call = llvm::PoisonValue::get(Ty);
10377 for (
unsigned I = 0; I < Ops.size(); I++) {
10388 auto *StructTy = dyn_cast<StructType>(
Call->getType());
10392 auto *VTy = dyn_cast<ScalableVectorType>(StructTy->getTypeAtIndex(0
U));
10395 unsigned N = StructTy->getNumElements();
10398 bool IsPredTy = VTy->getElementType()->isIntegerTy(1);
10399 unsigned MinElts = IsPredTy ? 16 : VTy->getMinNumElements();
10401 ScalableVectorType *WideVTy =
10402 ScalableVectorType::get(VTy->getElementType(), MinElts * N);
10403 Value *
Ret = llvm::PoisonValue::get(WideVTy);
10404 for (
unsigned I = 0; I < N; ++I) {
10406 assert(SRet->
getType() == VTy &&
"Unexpected type for result value");
10411 SRet, ScalableVectorType::get(
Builder.getInt1Ty(), 16));
10413 Ret =
Builder.CreateInsertVector(WideVTy, Ret, SRet, Idx);
10424 unsigned ICEArguments = 0;
10433 for (
unsigned i = 0, e =
E->getNumArgs(); i != e; i++) {
10434 bool IsICE = ICEArguments & (1 << i);
10440 std::optional<llvm::APSInt>
Result =
10442 assert(
Result &&
"Expected argument to be a constant");
10452 if (IsTupleGetOrSet || !isa<ScalableVectorType>(Arg->getType())) {
10453 Ops.push_back(Arg);
10457 auto *VTy = cast<ScalableVectorType>(Arg->getType());
10458 unsigned MinElts = VTy->getMinNumElements();
10459 bool IsPred = VTy->getElementType()->isIntegerTy(1);
10460 unsigned N = (MinElts * VTy->getScalarSizeInBits()) / (IsPred ? 16 : 128);
10463 Ops.push_back(Arg);
10467 for (
unsigned I = 0; I < N; ++I) {
10470 ScalableVectorType::get(VTy->getElementType(), MinElts / N);
10471 Ops.push_back(
Builder.CreateExtractVector(NewVTy, Arg, Idx));
10479 if (BuiltinID >= SVE::BI__builtin_sve_reinterpret_s8_s8 &&
10480 BuiltinID <= SVE::BI__builtin_sve_reinterpret_f64_f64_x4) {
10495 else if (TypeFlags.
isStore())
10513 else if (TypeFlags.
isUndef())
10514 return UndefValue::get(Ty);
10515 else if (Builtin->LLVMIntrinsic != 0) {
10516 if (TypeFlags.
getMergeType() == SVETypeFlags::MergeZeroExp)
10519 if (TypeFlags.
getMergeType() == SVETypeFlags::MergeAnyExp)
10525 Ops.push_back(
Builder.getInt32( 31));
10527 Ops.insert(&Ops[1],
Builder.getInt32( 31));
10530 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
10531 if (
auto PredTy = dyn_cast<llvm::VectorType>(Ops[i]->getType()))
10532 if (PredTy->getElementType()->isIntegerTy(1))
10542 std::swap(Ops[1], Ops[2]);
10544 std::swap(Ops[1], Ops[2]);
10547 std::swap(Ops[1], Ops[2]);
10550 std::swap(Ops[1], Ops[3]);
10553 if (TypeFlags.
getMergeType() == SVETypeFlags::MergeZero) {
10554 llvm::Type *OpndTy = Ops[1]->getType();
10555 auto *SplatZero = Constant::getNullValue(OpndTy);
10556 Ops[1] =
Builder.CreateSelect(Ops[0], Ops[1], SplatZero);
10564 if (
auto PredTy = dyn_cast<llvm::VectorType>(
Call->getType()))
10565 if (PredTy->getScalarType()->isIntegerTy(1))
10571 switch (BuiltinID) {
10575 case SVE::BI__builtin_sve_svreinterpret_b: {
10579 CGM.
getIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool, SVCountTy);
10580 return Builder.CreateCall(CastFromSVCountF, Ops[0]);
10582 case SVE::BI__builtin_sve_svreinterpret_c: {
10586 CGM.
getIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool, SVCountTy);
10587 return Builder.CreateCall(CastToSVCountF, Ops[0]);
10590 case SVE::BI__builtin_sve_svpsel_lane_b8:
10591 case SVE::BI__builtin_sve_svpsel_lane_b16:
10592 case SVE::BI__builtin_sve_svpsel_lane_b32:
10593 case SVE::BI__builtin_sve_svpsel_lane_b64:
10594 case SVE::BI__builtin_sve_svpsel_lane_c8:
10595 case SVE::BI__builtin_sve_svpsel_lane_c16:
10596 case SVE::BI__builtin_sve_svpsel_lane_c32:
10597 case SVE::BI__builtin_sve_svpsel_lane_c64: {
10598 bool IsSVCount = isa<TargetExtType>(Ops[0]->getType());
10599 assert(((!IsSVCount || cast<TargetExtType>(Ops[0]->getType())->
getName() ==
10600 "aarch64.svcount")) &&
10601 "Unexpected TargetExtType");
10605 CGM.
getIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool, SVCountTy);
10607 CGM.
getIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool, SVCountTy);
10611 llvm::Value *Ops0 =
10612 IsSVCount ?
Builder.CreateCall(CastFromSVCountF, Ops[0]) : Ops[0];
10614 llvm::Value *PSel =
Builder.CreateCall(F, {Ops0, Ops1, Ops[2]});
10615 return IsSVCount ?
Builder.CreateCall(CastToSVCountF, PSel) : PSel;
10617 case SVE::BI__builtin_sve_svmov_b_z: {
10620 llvm::Type* OverloadedTy =
getSVEType(TypeFlags);
10622 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[1]});
10625 case SVE::BI__builtin_sve_svnot_b_z: {
10628 llvm::Type* OverloadedTy =
getSVEType(TypeFlags);
10630 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[0]});
10633 case SVE::BI__builtin_sve_svmovlb_u16:
10634 case SVE::BI__builtin_sve_svmovlb_u32:
10635 case SVE::BI__builtin_sve_svmovlb_u64:
10636 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllb);
10638 case SVE::BI__builtin_sve_svmovlb_s16:
10639 case SVE::BI__builtin_sve_svmovlb_s32:
10640 case SVE::BI__builtin_sve_svmovlb_s64:
10641 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllb);
10643 case SVE::BI__builtin_sve_svmovlt_u16:
10644 case SVE::BI__builtin_sve_svmovlt_u32:
10645 case SVE::BI__builtin_sve_svmovlt_u64:
10646 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllt);
10648 case SVE::BI__builtin_sve_svmovlt_s16:
10649 case SVE::BI__builtin_sve_svmovlt_s32:
10650 case SVE::BI__builtin_sve_svmovlt_s64:
10651 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllt);
10653 case SVE::BI__builtin_sve_svpmullt_u16:
10654 case SVE::BI__builtin_sve_svpmullt_u64:
10655 case SVE::BI__builtin_sve_svpmullt_n_u16:
10656 case SVE::BI__builtin_sve_svpmullt_n_u64:
10657 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullt_pair);
10659 case SVE::BI__builtin_sve_svpmullb_u16:
10660 case SVE::BI__builtin_sve_svpmullb_u64:
10661 case SVE::BI__builtin_sve_svpmullb_n_u16:
10662 case SVE::BI__builtin_sve_svpmullb_n_u64:
10663 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullb_pair);
10665 case SVE::BI__builtin_sve_svdup_n_b8:
10666 case SVE::BI__builtin_sve_svdup_n_b16:
10667 case SVE::BI__builtin_sve_svdup_n_b32:
10668 case SVE::BI__builtin_sve_svdup_n_b64: {
10670 Builder.CreateICmpNE(Ops[0], Constant::getNullValue(Ops[0]->getType()));
10671 llvm::ScalableVectorType *OverloadedTy =
getSVEType(TypeFlags);
10676 case SVE::BI__builtin_sve_svdupq_n_b8:
10677 case SVE::BI__builtin_sve_svdupq_n_b16:
10678 case SVE::BI__builtin_sve_svdupq_n_b32:
10679 case SVE::BI__builtin_sve_svdupq_n_b64:
10680 case SVE::BI__builtin_sve_svdupq_n_u8:
10681 case SVE::BI__builtin_sve_svdupq_n_s8:
10682 case SVE::BI__builtin_sve_svdupq_n_u64:
10683 case SVE::BI__builtin_sve_svdupq_n_f64:
10684 case SVE::BI__builtin_sve_svdupq_n_s64:
10685 case SVE::BI__builtin_sve_svdupq_n_u16:
10686 case SVE::BI__builtin_sve_svdupq_n_f16:
10687 case SVE::BI__builtin_sve_svdupq_n_bf16:
10688 case SVE::BI__builtin_sve_svdupq_n_s16:
10689 case SVE::BI__builtin_sve_svdupq_n_u32:
10690 case SVE::BI__builtin_sve_svdupq_n_f32:
10691 case SVE::BI__builtin_sve_svdupq_n_s32: {
10694 unsigned NumOpnds = Ops.size();
10697 cast<llvm::VectorType>(Ty)->getElementType()->isIntegerTy(1);
10702 llvm::Type *EltTy = Ops[0]->getType();
10707 for (
unsigned I = 0; I < NumOpnds; ++I)
10708 VecOps.push_back(
Builder.CreateZExt(Ops[I], EltTy));
10713 OverloadedTy, PoisonValue::get(OverloadedTy), Vec,
Builder.getInt64(0));
10728 : Intrinsic::aarch64_sve_cmpne_wide,
10735 case SVE::BI__builtin_sve_svpfalse_b:
10736 return ConstantInt::getFalse(Ty);
10738 case SVE::BI__builtin_sve_svpfalse_c: {
10739 auto SVBoolTy = ScalableVectorType::get(
Builder.getInt1Ty(), 16);
10742 return Builder.CreateCall(CastToSVCountF, ConstantInt::getFalse(SVBoolTy));
10745 case SVE::BI__builtin_sve_svlen_bf16:
10746 case SVE::BI__builtin_sve_svlen_f16:
10747 case SVE::BI__builtin_sve_svlen_f32:
10748 case SVE::BI__builtin_sve_svlen_f64:
10749 case SVE::BI__builtin_sve_svlen_s8:
10750 case SVE::BI__builtin_sve_svlen_s16:
10751 case SVE::BI__builtin_sve_svlen_s32:
10752 case SVE::BI__builtin_sve_svlen_s64:
10753 case SVE::BI__builtin_sve_svlen_u8:
10754 case SVE::BI__builtin_sve_svlen_u16:
10755 case SVE::BI__builtin_sve_svlen_u32:
10756 case SVE::BI__builtin_sve_svlen_u64: {
10758 auto VTy = cast<llvm::VectorType>(
getSVEType(TF));
10760 llvm::ConstantInt::get(Ty, VTy->getElementCount().getKnownMinValue());
10766 case SVE::BI__builtin_sve_svtbl2_u8:
10767 case SVE::BI__builtin_sve_svtbl2_s8:
10768 case SVE::BI__builtin_sve_svtbl2_u16:
10769 case SVE::BI__builtin_sve_svtbl2_s16:
10770 case SVE::BI__builtin_sve_svtbl2_u32:
10771 case SVE::BI__builtin_sve_svtbl2_s32:
10772 case SVE::BI__builtin_sve_svtbl2_u64:
10773 case SVE::BI__builtin_sve_svtbl2_s64:
10774 case SVE::BI__builtin_sve_svtbl2_f16:
10775 case SVE::BI__builtin_sve_svtbl2_bf16:
10776 case SVE::BI__builtin_sve_svtbl2_f32:
10777 case SVE::BI__builtin_sve_svtbl2_f64: {
10779 auto VTy = cast<llvm::ScalableVectorType>(
getSVEType(TF));
10781 return Builder.CreateCall(F, Ops);
10784 case SVE::BI__builtin_sve_svset_neonq_s8:
10785 case SVE::BI__builtin_sve_svset_neonq_s16:
10786 case SVE::BI__builtin_sve_svset_neonq_s32:
10787 case SVE::BI__builtin_sve_svset_neonq_s64:
10788 case SVE::BI__builtin_sve_svset_neonq_u8:
10789 case SVE::BI__builtin_sve_svset_neonq_u16:
10790 case SVE::BI__builtin_sve_svset_neonq_u32:
10791 case SVE::BI__builtin_sve_svset_neonq_u64:
10792 case SVE::BI__builtin_sve_svset_neonq_f16:
10793 case SVE::BI__builtin_sve_svset_neonq_f32:
10794 case SVE::BI__builtin_sve_svset_neonq_f64:
10795 case SVE::BI__builtin_sve_svset_neonq_bf16: {
10796 return Builder.CreateInsertVector(Ty, Ops[0], Ops[1],
Builder.getInt64(0));
10799 case SVE::BI__builtin_sve_svget_neonq_s8:
10800 case SVE::BI__builtin_sve_svget_neonq_s16:
10801 case SVE::BI__builtin_sve_svget_neonq_s32:
10802 case SVE::BI__builtin_sve_svget_neonq_s64:
10803 case SVE::BI__builtin_sve_svget_neonq_u8:
10804 case SVE::BI__builtin_sve_svget_neonq_u16:
10805 case SVE::BI__builtin_sve_svget_neonq_u32:
10806 case SVE::BI__builtin_sve_svget_neonq_u64:
10807 case SVE::BI__builtin_sve_svget_neonq_f16:
10808 case SVE::BI__builtin_sve_svget_neonq_f32:
10809 case SVE::BI__builtin_sve_svget_neonq_f64:
10810 case SVE::BI__builtin_sve_svget_neonq_bf16: {
10811 return Builder.CreateExtractVector(Ty, Ops[0],
Builder.getInt64(0));
10814 case SVE::BI__builtin_sve_svdup_neonq_s8:
10815 case SVE::BI__builtin_sve_svdup_neonq_s16:
10816 case SVE::BI__builtin_sve_svdup_neonq_s32:
10817 case SVE::BI__builtin_sve_svdup_neonq_s64:
10818 case SVE::BI__builtin_sve_svdup_neonq_u8:
10819 case SVE::BI__builtin_sve_svdup_neonq_u16:
10820 case SVE::BI__builtin_sve_svdup_neonq_u32:
10821 case SVE::BI__builtin_sve_svdup_neonq_u64:
10822 case SVE::BI__builtin_sve_svdup_neonq_f16:
10823 case SVE::BI__builtin_sve_svdup_neonq_f32:
10824 case SVE::BI__builtin_sve_svdup_neonq_f64:
10825 case SVE::BI__builtin_sve_svdup_neonq_bf16: {
10828 return Builder.CreateIntrinsic(Intrinsic::aarch64_sve_dupq_lane, {Ty},
10840 switch (BuiltinID) {
10843 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x1:
10846 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x2:
10847 case SME::BI__builtin_sme_svsudot_za32_s8_vg1x2:
10850 case SME::BI__builtin_sme_svsudot_za32_s8_vg1x4:
10851 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x4:
10857 for (
unsigned I = 0; I < MultiVec; ++I)
10858 std::swap(Ops[I + 1], Ops[I + 1 + MultiVec]);
10871 return EmitSMELd1St1(TypeFlags, Ops, Builtin->LLVMIntrinsic);
10874 else if (BuiltinID == SME::BI__builtin_sme_svzero_mask_za ||
10875 BuiltinID == SME::BI__builtin_sme_svzero_za)
10876 return EmitSMEZero(TypeFlags, Ops, Builtin->LLVMIntrinsic);
10877 else if (BuiltinID == SME::BI__builtin_sme_svldr_vnum_za ||
10878 BuiltinID == SME::BI__builtin_sme_svstr_vnum_za ||
10879 BuiltinID == SME::BI__builtin_sme_svldr_za ||
10880 BuiltinID == SME::BI__builtin_sme_svstr_za)
10881 return EmitSMELdrStr(TypeFlags, Ops, Builtin->LLVMIntrinsic);
10887 if (Builtin->LLVMIntrinsic == 0)
10891 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
10892 if (
auto PredTy = dyn_cast<llvm::VectorType>(Ops[i]->getType()))
10893 if (PredTy->getElementType()->isIntegerTy(1))
10907 llvm::Triple::ArchType Arch) {
10916 if (BuiltinID == Builtin::BI__builtin_cpu_supports)
10917 return EmitAArch64CpuSupports(
E);
10919 unsigned HintID =
static_cast<unsigned>(-1);
10920 switch (BuiltinID) {
10922 case clang::AArch64::BI__builtin_arm_nop:
10925 case clang::AArch64::BI__builtin_arm_yield:
10926 case clang::AArch64::BI__yield:
10929 case clang::AArch64::BI__builtin_arm_wfe:
10930 case clang::AArch64::BI__wfe:
10933 case clang::AArch64::BI__builtin_arm_wfi:
10934 case clang::AArch64::BI__wfi:
10937 case clang::AArch64::BI__builtin_arm_sev:
10938 case clang::AArch64::BI__sev:
10941 case clang::AArch64::BI__builtin_arm_sevl:
10942 case clang::AArch64::BI__sevl:
10947 if (HintID !=
static_cast<unsigned>(-1)) {
10949 return Builder.CreateCall(F, llvm::ConstantInt::get(
Int32Ty, HintID));
10952 if (BuiltinID == clang::AArch64::BI__builtin_arm_trap) {
10958 if (BuiltinID == clang::AArch64::BI__builtin_arm_get_sme_state) {
10963 "__arm_sme_state"));
10965 "aarch64_pstate_sm_compatible");
10966 CI->setAttributes(Attrs);
10967 CI->setCallingConv(
10968 llvm::CallingConv::
10969 AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2);
10976 if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit) {
10978 "rbit of unusual size!");
10981 CGM.
getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
10983 if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit64) {
10985 "rbit of unusual size!");
10988 CGM.
getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
10991 if (BuiltinID == clang::AArch64::BI__builtin_arm_clz ||
10992 BuiltinID == clang::AArch64::BI__builtin_arm_clz64) {
10996 if (BuiltinID == clang::AArch64::BI__builtin_arm_clz64)
11001 if (BuiltinID == clang::AArch64::BI__builtin_arm_cls) {
11006 if (BuiltinID == clang::AArch64::BI__builtin_arm_cls64) {
11012 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint32zf ||
11013 BuiltinID == clang::AArch64::BI__builtin_arm_rint32z) {
11015 llvm::Type *Ty = Arg->getType();
11020 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint64zf ||
11021 BuiltinID == clang::AArch64::BI__builtin_arm_rint64z) {
11023 llvm::Type *Ty = Arg->getType();
11028 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint32xf ||
11029 BuiltinID == clang::AArch64::BI__builtin_arm_rint32x) {
11031 llvm::Type *Ty = Arg->getType();
11036 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint64xf ||
11037 BuiltinID == clang::AArch64::BI__builtin_arm_rint64x) {
11039 llvm::Type *Ty = Arg->getType();
11044 if (BuiltinID == clang::AArch64::BI__builtin_arm_jcvt) {
11046 "__jcvt of unusual size!");
11052 if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b ||
11053 BuiltinID == clang::AArch64::BI__builtin_arm_st64b ||
11054 BuiltinID == clang::AArch64::BI__builtin_arm_st64bv ||
11055 BuiltinID == clang::AArch64::BI__builtin_arm_st64bv0) {
11059 if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b) {
11063 llvm::Value *Val =
Builder.CreateCall(F, MemAddr);
11064 llvm::Value *ToRet;
11065 for (
size_t i = 0; i < 8; i++) {
11066 llvm::Value *ValOffsetPtr =
11077 Args.push_back(MemAddr);
11078 for (
size_t i = 0; i < 8; i++) {
11079 llvm::Value *ValOffsetPtr =
11086 auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_st64b
11087 ? Intrinsic::aarch64_st64b
11088 : BuiltinID == clang::AArch64::BI__builtin_arm_st64bv
11089 ? Intrinsic::aarch64_st64bv
11090 : Intrinsic::aarch64_st64bv0);
11092 return Builder.CreateCall(F, Args);
11096 if (BuiltinID == clang::AArch64::BI__builtin_arm_rndr ||
11097 BuiltinID == clang::AArch64::BI__builtin_arm_rndrrs) {
11099 auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_rndr
11100 ? Intrinsic::aarch64_rndr
11101 : Intrinsic::aarch64_rndrrs);
11103 llvm::Value *Val =
Builder.CreateCall(F);
11104 Value *RandomValue =
Builder.CreateExtractValue(Val, 0);
11113 if (BuiltinID == clang::AArch64::BI__clear_cache) {
11114 assert(
E->getNumArgs() == 2 &&
"__clear_cache takes 2 arguments");
11117 for (
unsigned i = 0; i < 2; i++)
11120 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
11121 StringRef Name = FD->
getName();
11125 if ((BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
11126 BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) &&
11130 ? Intrinsic::aarch64_ldaxp
11131 : Intrinsic::aarch64_ldxp);
11138 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
11139 Val0 =
Builder.CreateZExt(Val0, Int128Ty);
11140 Val1 =
Builder.CreateZExt(Val1, Int128Ty);
11142 Value *ShiftCst = llvm::ConstantInt::get(Int128Ty, 64);
11143 Val =
Builder.CreateShl(Val0, ShiftCst,
"shl",
true );
11144 Val =
Builder.CreateOr(Val, Val1);
11146 }
else if (BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
11147 BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) {
11152 llvm::Type *
IntTy =
11157 ? Intrinsic::aarch64_ldaxr
11158 : Intrinsic::aarch64_ldxr,
11160 CallInst *Val =
Builder.CreateCall(F, LoadAddr,
"ldxr");
11164 if (RealResTy->isPointerTy())
11165 return Builder.CreateIntToPtr(Val, RealResTy);
11167 llvm::Type *IntResTy = llvm::IntegerType::get(
11169 return Builder.CreateBitCast(
Builder.CreateTruncOrBitCast(Val, IntResTy),
11173 if ((BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
11174 BuiltinID == clang::AArch64::BI__builtin_arm_stlex) &&
11178 ? Intrinsic::aarch64_stlxp
11179 : Intrinsic::aarch64_stxp);
11191 return Builder.CreateCall(F, {Arg0, Arg1, StPtr},
"stxp");
11194 if (BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
11195 BuiltinID == clang::AArch64::BI__builtin_arm_stlex) {
11200 llvm::Type *StoreTy =
11203 if (StoreVal->
getType()->isPointerTy())
11206 llvm::Type *
IntTy = llvm::IntegerType::get(
11215 ? Intrinsic::aarch64_stlxr
11216 : Intrinsic::aarch64_stxr,
11218 CallInst *CI =
Builder.CreateCall(F, {StoreVal, StoreAddr},
"stxr");
11220 1, Attribute::get(
getLLVMContext(), Attribute::ElementType, StoreTy));
11224 if (BuiltinID == clang::AArch64::BI__getReg) {
11227 llvm_unreachable(
"Sema will ensure that the parameter is constant");
11233 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, Reg)};
11234 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
11235 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
11237 llvm::Function *F =
11239 return Builder.CreateCall(F, Metadata);
11242 if (BuiltinID == clang::AArch64::BI__break) {
11245 llvm_unreachable(
"Sema will ensure that the parameter is constant");
11247 llvm::Function *F =
CGM.
getIntrinsic(llvm::Intrinsic::aarch64_break);
11251 if (BuiltinID == clang::AArch64::BI__builtin_arm_clrex) {
11253 return Builder.CreateCall(F);
11256 if (BuiltinID == clang::AArch64::BI_ReadWriteBarrier)
11257 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
11258 llvm::SyncScope::SingleThread);
11261 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
11262 switch (BuiltinID) {
11263 case clang::AArch64::BI__builtin_arm_crc32b:
11264 CRCIntrinsicID = Intrinsic::aarch64_crc32b;
break;
11265 case clang::AArch64::BI__builtin_arm_crc32cb:
11266 CRCIntrinsicID = Intrinsic::aarch64_crc32cb;
break;
11267 case clang::AArch64::BI__builtin_arm_crc32h:
11268 CRCIntrinsicID = Intrinsic::aarch64_crc32h;
break;
11269 case clang::AArch64::BI__builtin_arm_crc32ch:
11270 CRCIntrinsicID = Intrinsic::aarch64_crc32ch;
break;
11271 case clang::AArch64::BI__builtin_arm_crc32w:
11272 CRCIntrinsicID = Intrinsic::aarch64_crc32w;
break;
11273 case clang::AArch64::BI__builtin_arm_crc32cw:
11274 CRCIntrinsicID = Intrinsic::aarch64_crc32cw;
break;
11275 case clang::AArch64::BI__builtin_arm_crc32d:
11276 CRCIntrinsicID = Intrinsic::aarch64_crc32x;
break;
11277 case clang::AArch64::BI__builtin_arm_crc32cd:
11278 CRCIntrinsicID = Intrinsic::aarch64_crc32cx;
break;
11281 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
11286 llvm::Type *DataTy = F->getFunctionType()->getParamType(1);
11287 Arg1 =
Builder.CreateZExtOrBitCast(Arg1, DataTy);
11289 return Builder.CreateCall(F, {Arg0, Arg1});
11293 if (BuiltinID == AArch64::BI__builtin_arm_mops_memset_tag) {
11301 CGM.
getIntrinsic(Intrinsic::aarch64_mops_memset_tag), {Dst, Val, Size});
11305 Intrinsic::ID MTEIntrinsicID = Intrinsic::not_intrinsic;
11306 switch (BuiltinID) {
11307 case clang::AArch64::BI__builtin_arm_irg:
11308 MTEIntrinsicID = Intrinsic::aarch64_irg;
break;
11309 case clang::AArch64::BI__builtin_arm_addg:
11310 MTEIntrinsicID = Intrinsic::aarch64_addg;
break;
11311 case clang::AArch64::BI__builtin_arm_gmi:
11312 MTEIntrinsicID = Intrinsic::aarch64_gmi;
break;
11313 case clang::AArch64::BI__builtin_arm_ldg:
11314 MTEIntrinsicID = Intrinsic::aarch64_ldg;
break;
11315 case clang::AArch64::BI__builtin_arm_stg:
11316 MTEIntrinsicID = Intrinsic::aarch64_stg;
break;
11317 case clang::AArch64::BI__builtin_arm_subp:
11318 MTEIntrinsicID = Intrinsic::aarch64_subp;
break;
11321 if (MTEIntrinsicID != Intrinsic::not_intrinsic) {
11324 if (MTEIntrinsicID == Intrinsic::aarch64_irg) {
11332 return Builder.CreatePointerCast(RV,
T);
11334 if (MTEIntrinsicID == Intrinsic::aarch64_addg) {
11342 return Builder.CreatePointerCast(RV,
T);
11344 if (MTEIntrinsicID == Intrinsic::aarch64_gmi) {
11356 if (MTEIntrinsicID == Intrinsic::aarch64_ldg) {
11361 return Builder.CreatePointerCast(RV,
T);
11366 if (MTEIntrinsicID == Intrinsic::aarch64_stg) {
11372 if (MTEIntrinsicID == Intrinsic::aarch64_subp) {
11382 if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
11383 BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
11384 BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
11385 BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
11386 BuiltinID == clang::AArch64::BI__builtin_arm_wsr ||
11387 BuiltinID == clang::AArch64::BI__builtin_arm_wsr64 ||
11388 BuiltinID == clang::AArch64::BI__builtin_arm_wsr128 ||
11389 BuiltinID == clang::AArch64::BI__builtin_arm_wsrp) {
11392 if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
11393 BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
11394 BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
11395 BuiltinID == clang::AArch64::BI__builtin_arm_rsrp)
11398 bool IsPointerBuiltin = BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
11399 BuiltinID == clang::AArch64::BI__builtin_arm_wsrp;
11401 bool Is32Bit = BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
11402 BuiltinID == clang::AArch64::BI__builtin_arm_wsr;
11404 bool Is128Bit = BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
11405 BuiltinID == clang::AArch64::BI__builtin_arm_wsr128;
11407 llvm::Type *ValueType;
11408 llvm::Type *RegisterType =
Int64Ty;
11411 }
else if (Is128Bit) {
11412 llvm::Type *Int128Ty =
11414 ValueType = Int128Ty;
11415 RegisterType = Int128Ty;
11416 }
else if (IsPointerBuiltin) {
11426 if (BuiltinID == clang::AArch64::BI_ReadStatusReg ||
11427 BuiltinID == clang::AArch64::BI_WriteStatusReg) {
11433 std::string SysRegStr;
11434 llvm::raw_string_ostream(SysRegStr) <<
11435 ((1 << 1) | ((SysReg >> 14) & 1)) <<
":" <<
11436 ((SysReg >> 11) & 7) <<
":" <<
11437 ((SysReg >> 7) & 15) <<
":" <<
11438 ((SysReg >> 3) & 15) <<
":" <<
11441 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysRegStr) };
11442 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
11443 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
11445 llvm::Type *RegisterType =
Int64Ty;
11446 llvm::Type *Types[] = { RegisterType };
11448 if (BuiltinID == clang::AArch64::BI_ReadStatusReg) {
11449 llvm::Function *F =
CGM.
getIntrinsic(llvm::Intrinsic::read_register, Types);
11451 return Builder.CreateCall(F, Metadata);
11454 llvm::Function *F =
CGM.
getIntrinsic(llvm::Intrinsic::write_register, Types);
11457 return Builder.CreateCall(F, { Metadata, ArgValue });
11460 if (BuiltinID == clang::AArch64::BI_AddressOfReturnAddress) {
11461 llvm::Function *F =
11463 return Builder.CreateCall(F);
11466 if (BuiltinID == clang::AArch64::BI__builtin_sponentry) {
11468 return Builder.CreateCall(F);
11471 if (BuiltinID == clang::AArch64::BI__mulh ||
11472 BuiltinID == clang::AArch64::BI__umulh) {
11474 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
11476 bool IsSigned = BuiltinID == clang::AArch64::BI__mulh;
11482 Value *MulResult, *HigherBits;
11484 MulResult =
Builder.CreateNSWMul(LHS, RHS);
11485 HigherBits =
Builder.CreateAShr(MulResult, 64);
11487 MulResult =
Builder.CreateNUWMul(LHS, RHS);
11488 HigherBits =
Builder.CreateLShr(MulResult, 64);
11490 HigherBits =
Builder.CreateIntCast(HigherBits, ResType, IsSigned);
11495 if (BuiltinID == AArch64::BI__writex18byte ||
11496 BuiltinID == AArch64::BI__writex18word ||
11497 BuiltinID == AArch64::BI__writex18dword ||
11498 BuiltinID == AArch64::BI__writex18qword) {
11501 llvm::Metadata *Ops[] = {llvm::MDString::get(Context,
"x18")};
11502 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
11503 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
11504 llvm::Function *F =
11506 llvm::Value *X18 =
Builder.CreateCall(F, Metadata);
11517 if (BuiltinID == AArch64::BI__readx18byte ||
11518 BuiltinID == AArch64::BI__readx18word ||
11519 BuiltinID == AArch64::BI__readx18dword ||
11520 BuiltinID == AArch64::BI__readx18qword) {
11525 llvm::Metadata *Ops[] = {llvm::MDString::get(Context,
"x18")};
11526 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
11527 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
11528 llvm::Function *F =
11530 llvm::Value *X18 =
Builder.CreateCall(F, Metadata);
11540 if (BuiltinID == AArch64::BI_CopyDoubleFromInt64 ||
11541 BuiltinID == AArch64::BI_CopyFloatFromInt32 ||
11542 BuiltinID == AArch64::BI_CopyInt32FromFloat ||
11543 BuiltinID == AArch64::BI_CopyInt64FromDouble) {
11546 return Builder.CreateBitCast(Arg, RetTy);
11549 if (BuiltinID == AArch64::BI_CountLeadingOnes ||
11550 BuiltinID == AArch64::BI_CountLeadingOnes64 ||
11551 BuiltinID == AArch64::BI_CountLeadingZeros ||
11552 BuiltinID == AArch64::BI_CountLeadingZeros64) {
11554 llvm::Type *ArgType = Arg->
getType();
11556 if (BuiltinID == AArch64::BI_CountLeadingOnes ||
11557 BuiltinID == AArch64::BI_CountLeadingOnes64)
11558 Arg =
Builder.CreateXor(Arg, Constant::getAllOnesValue(ArgType));
11563 if (BuiltinID == AArch64::BI_CountLeadingOnes64 ||
11564 BuiltinID == AArch64::BI_CountLeadingZeros64)
11569 if (BuiltinID == AArch64::BI_CountLeadingSigns ||
11570 BuiltinID == AArch64::BI_CountLeadingSigns64) {
11573 Function *F = (BuiltinID == AArch64::BI_CountLeadingSigns)
11578 if (BuiltinID == AArch64::BI_CountLeadingSigns64)
11583 if (BuiltinID == AArch64::BI_CountOneBits ||
11584 BuiltinID == AArch64::BI_CountOneBits64) {
11586 llvm::Type *ArgType = ArgValue->
getType();
11590 if (BuiltinID == AArch64::BI_CountOneBits64)
11595 if (BuiltinID == AArch64::BI__prefetch) {
11604 if (BuiltinID == AArch64::BI__hlt) {
11610 return ConstantInt::get(
Builder.getInt32Ty(), 0);
11615 if (std::optional<MSVCIntrin> MsvcIntId =
11621 return P.first == BuiltinID;
11624 BuiltinID = It->second;
11628 unsigned ICEArguments = 0;
11635 for (
unsigned i = 0, e =
E->getNumArgs() - 1; i != e; i++) {
11637 switch (BuiltinID) {
11638 case NEON::BI__builtin_neon_vld1_v:
11639 case NEON::BI__builtin_neon_vld1q_v:
11640 case NEON::BI__builtin_neon_vld1_dup_v:
11641 case NEON::BI__builtin_neon_vld1q_dup_v:
11642 case NEON::BI__builtin_neon_vld1_lane_v:
11643 case NEON::BI__builtin_neon_vld1q_lane_v:
11644 case NEON::BI__builtin_neon_vst1_v:
11645 case NEON::BI__builtin_neon_vst1q_v:
11646 case NEON::BI__builtin_neon_vst1_lane_v:
11647 case NEON::BI__builtin_neon_vst1q_lane_v:
11648 case NEON::BI__builtin_neon_vldap1_lane_s64:
11649 case NEON::BI__builtin_neon_vldap1q_lane_s64:
11650 case NEON::BI__builtin_neon_vstl1_lane_s64:
11651 case NEON::BI__builtin_neon_vstl1q_lane_s64:
11669 assert(
Result &&
"SISD intrinsic should have been handled");
11673 const Expr *Arg =
E->getArg(
E->getNumArgs()-1);
11675 if (std::optional<llvm::APSInt>
Result =
11680 bool usgn =
Type.isUnsigned();
11681 bool quad =
Type.isQuad();
11684 switch (BuiltinID) {
11686 case NEON::BI__builtin_neon_vabsh_f16:
11689 case NEON::BI__builtin_neon_vaddq_p128: {
11692 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
11693 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
11694 Ops[0] =
Builder.CreateXor(Ops[0], Ops[1]);
11695 llvm::Type *Int128Ty = llvm::Type::getIntNTy(
getLLVMContext(), 128);
11696 return Builder.CreateBitCast(Ops[0], Int128Ty);
11698 case NEON::BI__builtin_neon_vldrq_p128: {
11699 llvm::Type *Int128Ty = llvm::Type::getIntNTy(
getLLVMContext(), 128);
11704 case NEON::BI__builtin_neon_vstrq_p128: {
11705 Value *Ptr = Ops[0];
11708 case NEON::BI__builtin_neon_vcvts_f32_u32:
11709 case NEON::BI__builtin_neon_vcvtd_f64_u64:
11712 case NEON::BI__builtin_neon_vcvts_f32_s32:
11713 case NEON::BI__builtin_neon_vcvtd_f64_s64: {
11715 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
11718 Ops[0] =
Builder.CreateBitCast(Ops[0], InTy);
11720 return Builder.CreateUIToFP(Ops[0], FTy);
11721 return Builder.CreateSIToFP(Ops[0], FTy);
11723 case NEON::BI__builtin_neon_vcvth_f16_u16:
11724 case NEON::BI__builtin_neon_vcvth_f16_u32:
11725 case NEON::BI__builtin_neon_vcvth_f16_u64:
11728 case NEON::BI__builtin_neon_vcvth_f16_s16:
11729 case NEON::BI__builtin_neon_vcvth_f16_s32:
11730 case NEON::BI__builtin_neon_vcvth_f16_s64: {
11732 llvm::Type *FTy =
HalfTy;
11734 if (Ops[0]->getType()->getPrimitiveSizeInBits() == 64)
11736 else if (Ops[0]->getType()->getPrimitiveSizeInBits() == 32)
11740 Ops[0] =
Builder.CreateBitCast(Ops[0], InTy);
11742 return Builder.CreateUIToFP(Ops[0], FTy);
11743 return Builder.CreateSIToFP(Ops[0], FTy);
11745 case NEON::BI__builtin_neon_vcvtah_u16_f16:
11746 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
11747 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
11748 case NEON::BI__builtin_neon_vcvtph_u16_f16:
11749 case NEON::BI__builtin_neon_vcvth_u16_f16:
11750 case NEON::BI__builtin_neon_vcvtah_s16_f16:
11751 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
11752 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
11753 case NEON::BI__builtin_neon_vcvtph_s16_f16:
11754 case NEON::BI__builtin_neon_vcvth_s16_f16: {
11757 llvm::Type* FTy =
HalfTy;
11758 llvm::Type *Tys[2] = {InTy, FTy};
11760 switch (BuiltinID) {
11761 default: llvm_unreachable(
"missing builtin ID in switch!");
11762 case NEON::BI__builtin_neon_vcvtah_u16_f16:
11763 Int = Intrinsic::aarch64_neon_fcvtau;
break;
11764 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
11765 Int = Intrinsic::aarch64_neon_fcvtmu;
break;
11766 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
11767 Int = Intrinsic::aarch64_neon_fcvtnu;
break;
11768 case NEON::BI__builtin_neon_vcvtph_u16_f16:
11769 Int = Intrinsic::aarch64_neon_fcvtpu;
break;
11770 case NEON::BI__builtin_neon_vcvth_u16_f16:
11771 Int = Intrinsic::aarch64_neon_fcvtzu;
break;
11772 case NEON::BI__builtin_neon_vcvtah_s16_f16:
11773 Int = Intrinsic::aarch64_neon_fcvtas;
break;
11774 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
11775 Int = Intrinsic::aarch64_neon_fcvtms;
break;
11776 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
11777 Int = Intrinsic::aarch64_neon_fcvtns;
break;
11778 case NEON::BI__builtin_neon_vcvtph_s16_f16:
11779 Int = Intrinsic::aarch64_neon_fcvtps;
break;
11780 case NEON::BI__builtin_neon_vcvth_s16_f16:
11781 Int = Intrinsic::aarch64_neon_fcvtzs;
break;
11786 case NEON::BI__builtin_neon_vcaleh_f16:
11787 case NEON::BI__builtin_neon_vcalth_f16:
11788 case NEON::BI__builtin_neon_vcageh_f16:
11789 case NEON::BI__builtin_neon_vcagth_f16: {
11792 llvm::Type* FTy =
HalfTy;
11793 llvm::Type *Tys[2] = {InTy, FTy};
11795 switch (BuiltinID) {
11796 default: llvm_unreachable(
"missing builtin ID in switch!");
11797 case NEON::BI__builtin_neon_vcageh_f16:
11798 Int = Intrinsic::aarch64_neon_facge;
break;
11799 case NEON::BI__builtin_neon_vcagth_f16:
11800 Int = Intrinsic::aarch64_neon_facgt;
break;
11801 case NEON::BI__builtin_neon_vcaleh_f16:
11802 Int = Intrinsic::aarch64_neon_facge; std::swap(Ops[0], Ops[1]);
break;
11803 case NEON::BI__builtin_neon_vcalth_f16:
11804 Int = Intrinsic::aarch64_neon_facgt; std::swap(Ops[0], Ops[1]);
break;
11809 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
11810 case NEON::BI__builtin_neon_vcvth_n_u16_f16: {
11813 llvm::Type* FTy =
HalfTy;
11814 llvm::Type *Tys[2] = {InTy, FTy};
11816 switch (BuiltinID) {
11817 default: llvm_unreachable(
"missing builtin ID in switch!");
11818 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
11819 Int = Intrinsic::aarch64_neon_vcvtfp2fxs;
break;
11820 case NEON::BI__builtin_neon_vcvth_n_u16_f16:
11821 Int = Intrinsic::aarch64_neon_vcvtfp2fxu;
break;
11826 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
11827 case NEON::BI__builtin_neon_vcvth_n_f16_u16: {
11829 llvm::Type* FTy =
HalfTy;
11831 llvm::Type *Tys[2] = {FTy, InTy};
11833 switch (BuiltinID) {
11834 default: llvm_unreachable(
"missing builtin ID in switch!");
11835 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
11836 Int = Intrinsic::aarch64_neon_vcvtfxs2fp;
11837 Ops[0] =
Builder.CreateSExt(Ops[0], InTy,
"sext");
11839 case NEON::BI__builtin_neon_vcvth_n_f16_u16:
11840 Int = Intrinsic::aarch64_neon_vcvtfxu2fp;
11841 Ops[0] =
Builder.CreateZExt(Ops[0], InTy);
11846 case NEON::BI__builtin_neon_vpaddd_s64: {
11847 auto *Ty = llvm::FixedVectorType::get(
Int64Ty, 2);
11850 Vec =
Builder.CreateBitCast(Vec, Ty,
"v2i64");
11851 llvm::Value *Idx0 = llvm::ConstantInt::get(
SizeTy, 0);
11852 llvm::Value *Idx1 = llvm::ConstantInt::get(
SizeTy, 1);
11853 Value *Op0 =
Builder.CreateExtractElement(Vec, Idx0,
"lane0");
11854 Value *Op1 =
Builder.CreateExtractElement(Vec, Idx1,
"lane1");
11856 return Builder.CreateAdd(Op0, Op1,
"vpaddd");
11858 case NEON::BI__builtin_neon_vpaddd_f64: {
11859 auto *Ty = llvm::FixedVectorType::get(
DoubleTy, 2);
11862 Vec =
Builder.CreateBitCast(Vec, Ty,
"v2f64");
11863 llvm::Value *Idx0 = llvm::ConstantInt::get(
SizeTy, 0);
11864 llvm::Value *Idx1 = llvm::ConstantInt::get(
SizeTy, 1);
11865 Value *Op0 =
Builder.CreateExtractElement(Vec, Idx0,
"lane0");
11866 Value *Op1 =
Builder.CreateExtractElement(Vec, Idx1,
"lane1");
11868 return Builder.CreateFAdd(Op0, Op1,
"vpaddd");
11870 case NEON::BI__builtin_neon_vpadds_f32: {
11871 auto *Ty = llvm::FixedVectorType::get(
FloatTy, 2);
11874 Vec =
Builder.CreateBitCast(Vec, Ty,
"v2f32");
11875 llvm::Value *Idx0 = llvm::ConstantInt::get(
SizeTy, 0);
11876 llvm::Value *Idx1 = llvm::ConstantInt::get(
SizeTy, 1);
11877 Value *Op0 =
Builder.CreateExtractElement(Vec, Idx0,
"lane0");
11878 Value *Op1 =
Builder.CreateExtractElement(Vec, Idx1,
"lane1");
11880 return Builder.CreateFAdd(Op0, Op1,
"vpaddd");
11882 case NEON::BI__builtin_neon_vceqzd_s64:
11883 case NEON::BI__builtin_neon_vceqzd_f64:
11884 case NEON::BI__builtin_neon_vceqzs_f32:
11885 case NEON::BI__builtin_neon_vceqzh_f16:
11889 ICmpInst::FCMP_OEQ, ICmpInst::ICMP_EQ,
"vceqz");
11890 case NEON::BI__builtin_neon_vcgezd_s64:
11891 case NEON::BI__builtin_neon_vcgezd_f64:
11892 case NEON::BI__builtin_neon_vcgezs_f32:
11893 case NEON::BI__builtin_neon_vcgezh_f16:
11897 ICmpInst::FCMP_OGE, ICmpInst::ICMP_SGE,
"vcgez");
11898 case NEON::BI__builtin_neon_vclezd_s64:
11899 case NEON::BI__builtin_neon_vclezd_f64:
11900 case NEON::BI__builtin_neon_vclezs_f32:
11901 case NEON::BI__builtin_neon_vclezh_f16:
11905 ICmpInst::FCMP_OLE, ICmpInst::ICMP_SLE,
"vclez");
11906 case NEON::BI__builtin_neon_vcgtzd_s64:
11907 case NEON::BI__builtin_neon_vcgtzd_f64:
11908 case NEON::BI__builtin_neon_vcgtzs_f32:
11909 case NEON::BI__builtin_neon_vcgtzh_f16:
11913 ICmpInst::FCMP_OGT, ICmpInst::ICMP_SGT,
"vcgtz");
11914 case NEON::BI__builtin_neon_vcltzd_s64:
11915 case NEON::BI__builtin_neon_vcltzd_f64:
11916 case NEON::BI__builtin_neon_vcltzs_f32:
11917 case NEON::BI__builtin_neon_vcltzh_f16:
11921 ICmpInst::FCMP_OLT, ICmpInst::ICMP_SLT,
"vcltz");
11923 case NEON::BI__builtin_neon_vceqzd_u64: {
11927 Builder.CreateICmpEQ(Ops[0], llvm::Constant::getNullValue(
Int64Ty));
11930 case NEON::BI__builtin_neon_vceqd_f64:
11931 case NEON::BI__builtin_neon_vcled_f64:
11932 case NEON::BI__builtin_neon_vcltd_f64:
11933 case NEON::BI__builtin_neon_vcged_f64:
11934 case NEON::BI__builtin_neon_vcgtd_f64: {
11935 llvm::CmpInst::Predicate
P;
11936 switch (BuiltinID) {
11937 default: llvm_unreachable(
"missing builtin ID in switch!");
11938 case NEON::BI__builtin_neon_vceqd_f64:
P = llvm::FCmpInst::FCMP_OEQ;
break;
11939 case NEON::BI__builtin_neon_vcled_f64:
P = llvm::FCmpInst::FCMP_OLE;
break;
11940 case NEON::BI__builtin_neon_vcltd_f64:
P = llvm::FCmpInst::FCMP_OLT;
break;
11941 case NEON::BI__builtin_neon_vcged_f64:
P = llvm::FCmpInst::FCMP_OGE;
break;
11942 case NEON::BI__builtin_neon_vcgtd_f64:
P = llvm::FCmpInst::FCMP_OGT;
break;
11947 if (
P == llvm::FCmpInst::FCMP_OEQ)
11948 Ops[0] =
Builder.CreateFCmp(
P, Ops[0], Ops[1]);
11950 Ops[0] =
Builder.CreateFCmpS(
P, Ops[0], Ops[1]);
11953 case NEON::BI__builtin_neon_vceqs_f32:
11954 case NEON::BI__builtin_neon_vcles_f32:
11955 case NEON::BI__builtin_neon_vclts_f32:
11956 case NEON::BI__builtin_neon_vcges_f32:
11957 case NEON::BI__builtin_neon_vcgts_f32: {
11958 llvm::CmpInst::Predicate
P;
11959 switch (BuiltinID) {
11960 default: llvm_unreachable(
"missing builtin ID in switch!");
11961 case NEON::BI__builtin_neon_vceqs_f32:
P = llvm::FCmpInst::FCMP_OEQ;
break;
11962 case NEON::BI__builtin_neon_vcles_f32:
P = llvm::FCmpInst::FCMP_OLE;
break;
11963 case NEON::BI__builtin_neon_vclts_f32:
P = llvm::FCmpInst::FCMP_OLT;
break;
11964 case NEON::BI__builtin_neon_vcges_f32:
P = llvm::FCmpInst::FCMP_OGE;
break;
11965 case NEON::BI__builtin_neon_vcgts_f32:
P = llvm::FCmpInst::FCMP_OGT;
break;
11970 if (
P == llvm::FCmpInst::FCMP_OEQ)
11971 Ops[0] =
Builder.CreateFCmp(
P, Ops[0], Ops[1]);
11973 Ops[0] =
Builder.CreateFCmpS(
P, Ops[0], Ops[1]);
11976 case NEON::BI__builtin_neon_vceqh_f16:
11977 case NEON::BI__builtin_neon_vcleh_f16:
11978 case NEON::BI__builtin_neon_vclth_f16:
11979 case NEON::BI__builtin_neon_vcgeh_f16:
11980 case NEON::BI__builtin_neon_vcgth_f16: {
11981 llvm::CmpInst::Predicate
P;
11982 switch (BuiltinID) {
11983 default: llvm_unreachable(
"missing builtin ID in switch!");
11984 case NEON::BI__builtin_neon_vceqh_f16:
P = llvm::FCmpInst::FCMP_OEQ;
break;
11985 case NEON::BI__builtin_neon_vcleh_f16:
P = llvm::FCmpInst::FCMP_OLE;
break;
11986 case NEON::BI__builtin_neon_vclth_f16:
P = llvm::FCmpInst::FCMP_OLT;
break;
11987 case NEON::BI__builtin_neon_vcgeh_f16:
P = llvm::FCmpInst::FCMP_OGE;
break;
11988 case NEON::BI__builtin_neon_vcgth_f16:
P = llvm::FCmpInst::FCMP_OGT;
break;
11993 if (
P == llvm::FCmpInst::FCMP_OEQ)
11994 Ops[0] =
Builder.CreateFCmp(
P, Ops[0], Ops[1]);
11996 Ops[0] =
Builder.CreateFCmpS(
P, Ops[0], Ops[1]);
11999 case NEON::BI__builtin_neon_vceqd_s64:
12000 case NEON::BI__builtin_neon_vceqd_u64:
12001 case NEON::BI__builtin_neon_vcgtd_s64:
12002 case NEON::BI__builtin_neon_vcgtd_u64:
12003 case NEON::BI__builtin_neon_vcltd_s64:
12004 case NEON::BI__builtin_neon_vcltd_u64:
12005 case NEON::BI__builtin_neon_vcged_u64:
12006 case NEON::BI__builtin_neon_vcged_s64:
12007 case NEON::BI__builtin_neon_vcled_u64:
12008 case NEON::BI__builtin_neon_vcled_s64: {
12009 llvm::CmpInst::Predicate
P;
12010 switch (BuiltinID) {
12011 default: llvm_unreachable(
"missing builtin ID in switch!");
12012 case NEON::BI__builtin_neon_vceqd_s64:
12013 case NEON::BI__builtin_neon_vceqd_u64:
P = llvm::ICmpInst::ICMP_EQ;
break;
12014 case NEON::BI__builtin_neon_vcgtd_s64:
P = llvm::ICmpInst::ICMP_SGT;
break;
12015 case NEON::BI__builtin_neon_vcgtd_u64:
P = llvm::ICmpInst::ICMP_UGT;
break;
12016 case NEON::BI__builtin_neon_vcltd_s64:
P = llvm::ICmpInst::ICMP_SLT;
break;
12017 case NEON::BI__builtin_neon_vcltd_u64:
P = llvm::ICmpInst::ICMP_ULT;
break;
12018 case NEON::BI__builtin_neon_vcged_u64:
P = llvm::ICmpInst::ICMP_UGE;
break;
12019 case NEON::BI__builtin_neon_vcged_s64:
P = llvm::ICmpInst::ICMP_SGE;
break;
12020 case NEON::BI__builtin_neon_vcled_u64:
P = llvm::ICmpInst::ICMP_ULE;
break;
12021 case NEON::BI__builtin_neon_vcled_s64:
P = llvm::ICmpInst::ICMP_SLE;
break;
12026 Ops[0] =
Builder.CreateICmp(
P, Ops[0], Ops[1]);
12029 case NEON::BI__builtin_neon_vtstd_s64:
12030 case NEON::BI__builtin_neon_vtstd_u64: {
12034 Ops[0] =
Builder.CreateAnd(Ops[0], Ops[1]);
12035 Ops[0] =
Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
12036 llvm::Constant::getNullValue(
Int64Ty));
12039 case NEON::BI__builtin_neon_vset_lane_i8:
12040 case NEON::BI__builtin_neon_vset_lane_i16:
12041 case NEON::BI__builtin_neon_vset_lane_i32:
12042 case NEON::BI__builtin_neon_vset_lane_i64:
12043 case NEON::BI__builtin_neon_vset_lane_bf16:
12044 case NEON::BI__builtin_neon_vset_lane_f32:
12045 case NEON::BI__builtin_neon_vsetq_lane_i8:
12046 case NEON::BI__builtin_neon_vsetq_lane_i16:
12047 case NEON::BI__builtin_neon_vsetq_lane_i32:
12048 case NEON::BI__builtin_neon_vsetq_lane_i64:
12049 case NEON::BI__builtin_neon_vsetq_lane_bf16:
12050 case NEON::BI__builtin_neon_vsetq_lane_f32:
12052 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
12053 case NEON::BI__builtin_neon_vset_lane_f64:
12056 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(
DoubleTy, 1));
12058 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
12059 case NEON::BI__builtin_neon_vsetq_lane_f64:
12062 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(
DoubleTy, 2));
12064 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
12066 case NEON::BI__builtin_neon_vget_lane_i8:
12067 case NEON::BI__builtin_neon_vdupb_lane_i8:
12069 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int8Ty, 8));
12072 case NEON::BI__builtin_neon_vgetq_lane_i8:
12073 case NEON::BI__builtin_neon_vdupb_laneq_i8:
12075 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int8Ty, 16));
12078 case NEON::BI__builtin_neon_vget_lane_i16:
12079 case NEON::BI__builtin_neon_vduph_lane_i16:
12081 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int16Ty, 4));
12084 case NEON::BI__builtin_neon_vgetq_lane_i16:
12085 case NEON::BI__builtin_neon_vduph_laneq_i16:
12087 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int16Ty, 8));
12090 case NEON::BI__builtin_neon_vget_lane_i32:
12091 case NEON::BI__builtin_neon_vdups_lane_i32:
12093 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int32Ty, 2));
12096 case NEON::BI__builtin_neon_vdups_lane_f32:
12098 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
FloatTy, 2));
12101 case NEON::BI__builtin_neon_vgetq_lane_i32:
12102 case NEON::BI__builtin_neon_vdups_laneq_i32:
12104 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int32Ty, 4));
12107 case NEON::BI__builtin_neon_vget_lane_i64:
12108 case NEON::BI__builtin_neon_vdupd_lane_i64:
12110 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int64Ty, 1));
12113 case NEON::BI__builtin_neon_vdupd_lane_f64:
12115 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
DoubleTy, 1));
12118 case NEON::BI__builtin_neon_vgetq_lane_i64:
12119 case NEON::BI__builtin_neon_vdupd_laneq_i64:
12121 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int64Ty, 2));
12124 case NEON::BI__builtin_neon_vget_lane_f32:
12126 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
FloatTy, 2));
12129 case NEON::BI__builtin_neon_vget_lane_f64:
12131 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
DoubleTy, 1));
12134 case NEON::BI__builtin_neon_vgetq_lane_f32:
12135 case NEON::BI__builtin_neon_vdups_laneq_f32:
12137 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
FloatTy, 4));
12140 case NEON::BI__builtin_neon_vgetq_lane_f64:
12141 case NEON::BI__builtin_neon_vdupd_laneq_f64:
12143 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
DoubleTy, 2));
12146 case NEON::BI__builtin_neon_vaddh_f16:
12148 return Builder.CreateFAdd(Ops[0], Ops[1],
"vaddh");
12149 case NEON::BI__builtin_neon_vsubh_f16:
12151 return Builder.CreateFSub(Ops[0], Ops[1],
"vsubh");
12152 case NEON::BI__builtin_neon_vmulh_f16:
12154 return Builder.CreateFMul(Ops[0], Ops[1],
"vmulh");
12155 case NEON::BI__builtin_neon_vdivh_f16:
12157 return Builder.CreateFDiv(Ops[0], Ops[1],
"vdivh");
12158 case NEON::BI__builtin_neon_vfmah_f16:
12161 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma,
HalfTy,
12163 case NEON::BI__builtin_neon_vfmsh_f16: {
12168 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma,
HalfTy,
12171 case NEON::BI__builtin_neon_vaddd_s64:
12172 case NEON::BI__builtin_neon_vaddd_u64:
12174 case NEON::BI__builtin_neon_vsubd_s64:
12175 case NEON::BI__builtin_neon_vsubd_u64:
12177 case NEON::BI__builtin_neon_vqdmlalh_s16:
12178 case NEON::BI__builtin_neon_vqdmlslh_s16: {
12182 auto *VTy = llvm::FixedVectorType::get(
Int32Ty, 4);
12184 ProductOps,
"vqdmlXl");
12185 Constant *CI = ConstantInt::get(
SizeTy, 0);
12186 Ops[1] =
Builder.CreateExtractElement(Ops[1], CI,
"lane0");
12188 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlalh_s16
12189 ? Intrinsic::aarch64_neon_sqadd
12190 : Intrinsic::aarch64_neon_sqsub;
12193 case NEON::BI__builtin_neon_vqshlud_n_s64: {
12199 case NEON::BI__builtin_neon_vqshld_n_u64:
12200 case NEON::BI__builtin_neon_vqshld_n_s64: {
12201 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vqshld_n_u64
12202 ? Intrinsic::aarch64_neon_uqshl
12203 : Intrinsic::aarch64_neon_sqshl;
12208 case NEON::BI__builtin_neon_vrshrd_n_u64:
12209 case NEON::BI__builtin_neon_vrshrd_n_s64: {
12210 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrshrd_n_u64
12211 ? Intrinsic::aarch64_neon_urshl
12212 : Intrinsic::aarch64_neon_srshl;
12214 int SV = cast<ConstantInt>(Ops[1])->getSExtValue();
12215 Ops[1] = ConstantInt::get(
Int64Ty, -SV);
12218 case NEON::BI__builtin_neon_vrsrad_n_u64:
12219 case NEON::BI__builtin_neon_vrsrad_n_s64: {
12220 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrsrad_n_u64
12221 ? Intrinsic::aarch64_neon_urshl
12222 : Intrinsic::aarch64_neon_srshl;
12226 {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)});
12229 case NEON::BI__builtin_neon_vshld_n_s64:
12230 case NEON::BI__builtin_neon_vshld_n_u64: {
12231 llvm::ConstantInt *Amt = cast<ConstantInt>(
EmitScalarExpr(
E->getArg(1)));
12233 Ops[0], ConstantInt::get(
Int64Ty, Amt->getZExtValue()),
"shld_n");
12235 case NEON::BI__builtin_neon_vshrd_n_s64: {
12236 llvm::ConstantInt *Amt = cast<ConstantInt>(
EmitScalarExpr(
E->getArg(1)));
12238 Ops[0], ConstantInt::get(
Int64Ty, std::min(
static_cast<uint64_t>(63),
12239 Amt->getZExtValue())),
12242 case NEON::BI__builtin_neon_vshrd_n_u64: {
12243 llvm::ConstantInt *Amt = cast<ConstantInt>(
EmitScalarExpr(
E->getArg(1)));
12244 uint64_t ShiftAmt = Amt->getZExtValue();
12246 if (ShiftAmt == 64)
12247 return ConstantInt::get(
Int64Ty, 0);
12248 return Builder.CreateLShr(Ops[0], ConstantInt::get(
Int64Ty, ShiftAmt),
12251 case NEON::BI__builtin_neon_vsrad_n_s64: {
12252 llvm::ConstantInt *Amt = cast<ConstantInt>(
EmitScalarExpr(
E->getArg(2)));
12254 Ops[1], ConstantInt::get(
Int64Ty, std::min(
static_cast<uint64_t>(63),
12255 Amt->getZExtValue())),
12257 return Builder.CreateAdd(Ops[0], Ops[1]);
12259 case NEON::BI__builtin_neon_vsrad_n_u64: {
12260 llvm::ConstantInt *Amt = cast<ConstantInt>(
EmitScalarExpr(
E->getArg(2)));
12261 uint64_t ShiftAmt = Amt->getZExtValue();
12264 if (ShiftAmt == 64)
12266 Ops[1] =
Builder.CreateLShr(Ops[1], ConstantInt::get(
Int64Ty, ShiftAmt),
12268 return Builder.CreateAdd(Ops[0], Ops[1]);
12270 case NEON::BI__builtin_neon_vqdmlalh_lane_s16:
12271 case NEON::BI__builtin_neon_vqdmlalh_laneq_s16:
12272 case NEON::BI__builtin_neon_vqdmlslh_lane_s16:
12273 case NEON::BI__builtin_neon_vqdmlslh_laneq_s16: {
12279 auto *VTy = llvm::FixedVectorType::get(
Int32Ty, 4);
12281 ProductOps,
"vqdmlXl");
12282 Constant *CI = ConstantInt::get(
SizeTy, 0);
12283 Ops[1] =
Builder.CreateExtractElement(Ops[1], CI,
"lane0");
12286 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlalh_lane_s16 ||
12287 BuiltinID == NEON::BI__builtin_neon_vqdmlalh_laneq_s16)
12288 ? Intrinsic::aarch64_neon_sqadd
12289 : Intrinsic::aarch64_neon_sqsub;
12292 case NEON::BI__builtin_neon_vqdmlals_s32:
12293 case NEON::BI__builtin_neon_vqdmlsls_s32: {
12295 ProductOps.push_back(Ops[1]);
12299 ProductOps,
"vqdmlXl");
12301 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlals_s32
12302 ? Intrinsic::aarch64_neon_sqadd
12303 : Intrinsic::aarch64_neon_sqsub;
12306 case NEON::BI__builtin_neon_vqdmlals_lane_s32:
12307 case NEON::BI__builtin_neon_vqdmlals_laneq_s32:
12308 case NEON::BI__builtin_neon_vqdmlsls_lane_s32:
12309 case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: {
12313 ProductOps.push_back(Ops[1]);
12314 ProductOps.push_back(Ops[2]);
12317 ProductOps,
"vqdmlXl");
12320 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlals_lane_s32 ||
12321 BuiltinID == NEON::BI__builtin_neon_vqdmlals_laneq_s32)
12322 ? Intrinsic::aarch64_neon_sqadd
12323 : Intrinsic::aarch64_neon_sqsub;
12326 case NEON::BI__builtin_neon_vget_lane_bf16:
12327 case NEON::BI__builtin_neon_vduph_lane_bf16:
12328 case NEON::BI__builtin_neon_vduph_lane_f16: {
12332 case NEON::BI__builtin_neon_vgetq_lane_bf16:
12333 case NEON::BI__builtin_neon_vduph_laneq_bf16:
12334 case NEON::BI__builtin_neon_vduph_laneq_f16: {
12339 case clang::AArch64::BI_InterlockedAdd:
12340 case clang::AArch64::BI_InterlockedAdd64: {
12343 AtomicRMWInst *RMWI =
12345 llvm::AtomicOrdering::SequentiallyConsistent);
12346 return Builder.CreateAdd(RMWI, Val);
12351 llvm::Type *Ty = VTy;
12362 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
12363 Builtin->NameHint, Builtin->TypeModifier,
E, Ops,
12370 switch (BuiltinID) {
12371 default:
return nullptr;
12372 case NEON::BI__builtin_neon_vbsl_v:
12373 case NEON::BI__builtin_neon_vbslq_v: {
12374 llvm::Type *BitTy = llvm::VectorType::getInteger(VTy);
12375 Ops[0] =
Builder.CreateBitCast(Ops[0], BitTy,
"vbsl");
12376 Ops[1] =
Builder.CreateBitCast(Ops[1], BitTy,
"vbsl");
12377 Ops[2] =
Builder.CreateBitCast(Ops[2], BitTy,
"vbsl");
12379 Ops[1] =
Builder.CreateAnd(Ops[0], Ops[1],
"vbsl");
12380 Ops[2] =
Builder.CreateAnd(
Builder.CreateNot(Ops[0]), Ops[2],
"vbsl");
12381 Ops[0] =
Builder.CreateOr(Ops[1], Ops[2],
"vbsl");
12382 return Builder.CreateBitCast(Ops[0], Ty);
12384 case NEON::BI__builtin_neon_vfma_lane_v:
12385 case NEON::BI__builtin_neon_vfmaq_lane_v: {
12388 Value *Addend = Ops[0];
12389 Value *Multiplicand = Ops[1];
12390 Value *LaneSource = Ops[2];
12391 Ops[0] = Multiplicand;
12392 Ops[1] = LaneSource;
12396 auto *SourceTy = BuiltinID == NEON::BI__builtin_neon_vfmaq_lane_v
12397 ? llvm::FixedVectorType::get(VTy->getElementType(),
12398 VTy->getNumElements() / 2)
12400 llvm::Constant *cst = cast<Constant>(Ops[3]);
12401 Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(), cst);
12402 Ops[1] =
Builder.CreateBitCast(Ops[1], SourceTy);
12403 Ops[1] =
Builder.CreateShuffleVector(Ops[1], Ops[1], SV,
"lane");
12406 Int =
Builder.getIsFPConstrained() ? Intrinsic::experimental_constrained_fma
12410 case NEON::BI__builtin_neon_vfma_laneq_v: {
12411 auto *VTy = cast<llvm::FixedVectorType>(Ty);
12413 if (VTy && VTy->getElementType() ==
DoubleTy) {
12416 llvm::FixedVectorType *VTy =
12418 Ops[2] =
Builder.CreateBitCast(Ops[2], VTy);
12419 Ops[2] =
Builder.CreateExtractElement(Ops[2], Ops[3],
"extract");
12422 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma,
12423 DoubleTy, {Ops[1], Ops[2], Ops[0]});
12426 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
12427 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
12429 auto *STy = llvm::FixedVectorType::get(VTy->getElementType(),
12430 VTy->getNumElements() * 2);
12431 Ops[2] =
Builder.CreateBitCast(Ops[2], STy);
12432 Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(),
12433 cast<ConstantInt>(Ops[3]));
12434 Ops[2] =
Builder.CreateShuffleVector(Ops[2], Ops[2], SV,
"lane");
12437 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
12438 {Ops[2], Ops[1], Ops[0]});
12440 case NEON::BI__builtin_neon_vfmaq_laneq_v: {
12441 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
12442 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
12444 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
12447 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
12448 {Ops[2], Ops[1], Ops[0]});
12450 case NEON::BI__builtin_neon_vfmah_lane_f16:
12451 case NEON::BI__builtin_neon_vfmas_lane_f32:
12452 case NEON::BI__builtin_neon_vfmah_laneq_f16:
12453 case NEON::BI__builtin_neon_vfmas_laneq_f32:
12454 case NEON::BI__builtin_neon_vfmad_lane_f64:
12455 case NEON::BI__builtin_neon_vfmad_laneq_f64: {
12458 Ops[2] =
Builder.CreateExtractElement(Ops[2], Ops[3],
"extract");
12460 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
12461 {Ops[1], Ops[2], Ops[0]});
12463 case NEON::BI__builtin_neon_vmull_v:
12465 Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull;
12466 if (
Type.isPoly())
Int = Intrinsic::aarch64_neon_pmull;
12468 case NEON::BI__builtin_neon_vmax_v:
12469 case NEON::BI__builtin_neon_vmaxq_v:
12471 Int = usgn ? Intrinsic::aarch64_neon_umax : Intrinsic::aarch64_neon_smax;
12472 if (Ty->isFPOrFPVectorTy())
Int = Intrinsic::aarch64_neon_fmax;
12474 case NEON::BI__builtin_neon_vmaxh_f16: {
12476 Int = Intrinsic::aarch64_neon_fmax;
12479 case NEON::BI__builtin_neon_vmin_v:
12480 case NEON::BI__builtin_neon_vminq_v:
12482 Int = usgn ? Intrinsic::aarch64_neon_umin : Intrinsic::aarch64_neon_smin;
12483 if (Ty->isFPOrFPVectorTy())
Int = Intrinsic::aarch64_neon_fmin;
12485 case NEON::BI__builtin_neon_vminh_f16: {
12487 Int = Intrinsic::aarch64_neon_fmin;
12490 case NEON::BI__builtin_neon_vabd_v:
12491 case NEON::BI__builtin_neon_vabdq_v:
12493 Int = usgn ? Intrinsic::aarch64_neon_uabd : Intrinsic::aarch64_neon_sabd;
12494 if (Ty->isFPOrFPVectorTy())
Int = Intrinsic::aarch64_neon_fabd;
12496 case NEON::BI__builtin_neon_vpadal_v:
12497 case NEON::BI__builtin_neon_vpadalq_v: {
12498 unsigned ArgElts = VTy->getNumElements();
12499 llvm::IntegerType *EltTy = cast<IntegerType>(VTy->getElementType());
12500 unsigned BitWidth = EltTy->getBitWidth();
12501 auto *ArgTy = llvm::FixedVectorType::get(
12502 llvm::IntegerType::get(
getLLVMContext(), BitWidth / 2), 2 * ArgElts);
12503 llvm::Type* Tys[2] = { VTy, ArgTy };
12504 Int = usgn ? Intrinsic::aarch64_neon_uaddlp : Intrinsic::aarch64_neon_saddlp;
12506 TmpOps.push_back(Ops[1]);
12509 llvm::Value *addend =
Builder.CreateBitCast(Ops[0], tmp->getType());
12510 return Builder.CreateAdd(tmp, addend);
12512 case NEON::BI__builtin_neon_vpmin_v:
12513 case NEON::BI__builtin_neon_vpminq_v:
12515 Int = usgn ? Intrinsic::aarch64_neon_uminp : Intrinsic::aarch64_neon_sminp;
12516 if (Ty->isFPOrFPVectorTy())
Int = Intrinsic::aarch64_neon_fminp;
12518 case NEON::BI__builtin_neon_vpmax_v:
12519 case NEON::BI__builtin_neon_vpmaxq_v:
12521 Int = usgn ? Intrinsic::aarch64_neon_umaxp : Intrinsic::aarch64_neon_smaxp;
12522 if (Ty->isFPOrFPVectorTy())
Int = Intrinsic::aarch64_neon_fmaxp;
12524 case NEON::BI__builtin_neon_vminnm_v:
12525 case NEON::BI__builtin_neon_vminnmq_v:
12526 Int = Intrinsic::aarch64_neon_fminnm;
12528 case NEON::BI__builtin_neon_vminnmh_f16:
12530 Int = Intrinsic::aarch64_neon_fminnm;
12532 case NEON::BI__builtin_neon_vmaxnm_v:
12533 case NEON::BI__builtin_neon_vmaxnmq_v:
12534 Int = Intrinsic::aarch64_neon_fmaxnm;
12536 case NEON::BI__builtin_neon_vmaxnmh_f16:
12538 Int = Intrinsic::aarch64_neon_fmaxnm;
12540 case NEON::BI__builtin_neon_vrecpss_f32: {
12545 case NEON::BI__builtin_neon_vrecpsd_f64:
12549 case NEON::BI__builtin_neon_vrecpsh_f16:
12553 case NEON::BI__builtin_neon_vqshrun_n_v:
12554 Int = Intrinsic::aarch64_neon_sqshrun;
12556 case NEON::BI__builtin_neon_vqrshrun_n_v:
12557 Int = Intrinsic::aarch64_neon_sqrshrun;
12559 case NEON::BI__builtin_neon_vqshrn_n_v:
12560 Int = usgn ? Intrinsic::aarch64_neon_uqshrn : Intrinsic::aarch64_neon_sqshrn;
12562 case NEON::BI__builtin_neon_vrshrn_n_v:
12563 Int = Intrinsic::aarch64_neon_rshrn;
12565 case NEON::BI__builtin_neon_vqrshrn_n_v:
12566 Int = usgn ? Intrinsic::aarch64_neon_uqrshrn : Intrinsic::aarch64_neon_sqrshrn;
12568 case NEON::BI__builtin_neon_vrndah_f16: {
12571 ? Intrinsic::experimental_constrained_round
12572 : Intrinsic::round;
12575 case NEON::BI__builtin_neon_vrnda_v:
12576 case NEON::BI__builtin_neon_vrndaq_v: {
12578 ? Intrinsic::experimental_constrained_round
12579 : Intrinsic::round;
12582 case NEON::BI__builtin_neon_vrndih_f16: {
12585 ? Intrinsic::experimental_constrained_nearbyint
12586 : Intrinsic::nearbyint;
12589 case NEON::BI__builtin_neon_vrndmh_f16: {
12592 ? Intrinsic::experimental_constrained_floor
12593 : Intrinsic::floor;
12596 case NEON::BI__builtin_neon_vrndm_v:
12597 case NEON::BI__builtin_neon_vrndmq_v: {
12599 ? Intrinsic::experimental_constrained_floor
12600 : Intrinsic::floor;
12603 case NEON::BI__builtin_neon_vrndnh_f16: {
12606 ? Intrinsic::experimental_constrained_roundeven
12607 : Intrinsic::roundeven;
12610 case NEON::BI__builtin_neon_vrndn_v:
12611 case NEON::BI__builtin_neon_vrndnq_v: {
12613 ? Intrinsic::experimental_constrained_roundeven
12614 : Intrinsic::roundeven;
12617 case NEON::BI__builtin_neon_vrndns_f32: {
12620 ? Intrinsic::experimental_constrained_roundeven
12621 : Intrinsic::roundeven;
12624 case NEON::BI__builtin_neon_vrndph_f16: {
12627 ? Intrinsic::experimental_constrained_ceil
12631 case NEON::BI__builtin_neon_vrndp_v:
12632 case NEON::BI__builtin_neon_vrndpq_v: {
12634 ? Intrinsic::experimental_constrained_ceil
12638 case NEON::BI__builtin_neon_vrndxh_f16: {
12641 ? Intrinsic::experimental_constrained_rint
12645 case NEON::BI__builtin_neon_vrndx_v:
12646 case NEON::BI__builtin_neon_vrndxq_v: {
12648 ? Intrinsic::experimental_constrained_rint
12652 case NEON::BI__builtin_neon_vrndh_f16: {
12655 ? Intrinsic::experimental_constrained_trunc
12656 : Intrinsic::trunc;
12659 case NEON::BI__builtin_neon_vrnd32x_f32:
12660 case NEON::BI__builtin_neon_vrnd32xq_f32:
12661 case NEON::BI__builtin_neon_vrnd32x_f64:
12662 case NEON::BI__builtin_neon_vrnd32xq_f64: {
12664 Int = Intrinsic::aarch64_neon_frint32x;
12667 case NEON::BI__builtin_neon_vrnd32z_f32:
12668 case NEON::BI__builtin_neon_vrnd32zq_f32:
12669 case NEON::BI__builtin_neon_vrnd32z_f64:
12670 case NEON::BI__builtin_neon_vrnd32zq_f64: {
12672 Int = Intrinsic::aarch64_neon_frint32z;
12675 case NEON::BI__builtin_neon_vrnd64x_f32:
12676 case NEON::BI__builtin_neon_vrnd64xq_f32:
12677 case NEON::BI__builtin_neon_vrnd64x_f64:
12678 case NEON::BI__builtin_neon_vrnd64xq_f64: {
12680 Int = Intrinsic::aarch64_neon_frint64x;
12683 case NEON::BI__builtin_neon_vrnd64z_f32:
12684 case NEON::BI__builtin_neon_vrnd64zq_f32:
12685 case NEON::BI__builtin_neon_vrnd64z_f64:
12686 case NEON::BI__builtin_neon_vrnd64zq_f64: {
12688 Int = Intrinsic::aarch64_neon_frint64z;
12691 case NEON::BI__builtin_neon_vrnd_v:
12692 case NEON::BI__builtin_neon_vrndq_v: {
12694 ? Intrinsic::experimental_constrained_trunc
12695 : Intrinsic::trunc;
12698 case NEON::BI__builtin_neon_vcvt_f64_v:
12699 case NEON::BI__builtin_neon_vcvtq_f64_v:
12700 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
12702 return usgn ?
Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
12703 :
Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
12704 case NEON::BI__builtin_neon_vcvt_f64_f32: {
12706 "unexpected vcvt_f64_f32 builtin");
12710 return Builder.CreateFPExt(Ops[0], Ty,
"vcvt");
12712 case NEON::BI__builtin_neon_vcvt_f32_f64: {
12714 "unexpected vcvt_f32_f64 builtin");
12718 return Builder.CreateFPTrunc(Ops[0], Ty,
"vcvt");
12720 case NEON::BI__builtin_neon_vcvt_s32_v:
12721 case NEON::BI__builtin_neon_vcvt_u32_v:
12722 case NEON::BI__builtin_neon_vcvt_s64_v:
12723 case NEON::BI__builtin_neon_vcvt_u64_v:
12724 case NEON::BI__builtin_neon_vcvt_s16_f16:
12725 case NEON::BI__builtin_neon_vcvt_u16_f16:
12726 case NEON::BI__builtin_neon_vcvtq_s32_v:
12727 case NEON::BI__builtin_neon_vcvtq_u32_v:
12728 case NEON::BI__builtin_neon_vcvtq_s64_v:
12729 case NEON::BI__builtin_neon_vcvtq_u64_v:
12730 case NEON::BI__builtin_neon_vcvtq_s16_f16:
12731 case NEON::BI__builtin_neon_vcvtq_u16_f16: {
12733 usgn ? Intrinsic::aarch64_neon_fcvtzu : Intrinsic::aarch64_neon_fcvtzs;
12737 case NEON::BI__builtin_neon_vcvta_s16_f16:
12738 case NEON::BI__builtin_neon_vcvta_u16_f16:
12739 case NEON::BI__builtin_neon_vcvta_s32_v:
12740 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
12741 case NEON::BI__builtin_neon_vcvtaq_s32_v:
12742 case NEON::BI__builtin_neon_vcvta_u32_v:
12743 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
12744 case NEON::BI__builtin_neon_vcvtaq_u32_v:
12745 case NEON::BI__builtin_neon_vcvta_s64_v:
12746 case NEON::BI__builtin_neon_vcvtaq_s64_v:
12747 case NEON::BI__builtin_neon_vcvta_u64_v:
12748 case NEON::BI__builtin_neon_vcvtaq_u64_v: {
12749 Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas;
12753 case NEON::BI__builtin_neon_vcvtm_s16_f16:
12754 case NEON::BI__builtin_neon_vcvtm_s32_v:
12755 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
12756 case NEON::BI__builtin_neon_vcvtmq_s32_v:
12757 case NEON::BI__builtin_neon_vcvtm_u16_f16:
12758 case NEON::BI__builtin_neon_vcvtm_u32_v:
12759 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
12760 case NEON::BI__builtin_neon_vcvtmq_u32_v:
12761 case NEON::BI__builtin_neon_vcvtm_s64_v:
12762 case NEON::BI__builtin_neon_vcvtmq_s64_v:
12763 case NEON::BI__builtin_neon_vcvtm_u64_v:
12764 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
12765 Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms;
12769 case NEON::BI__builtin_neon_vcvtn_s16_f16:
12770 case NEON::BI__builtin_neon_vcvtn_s32_v:
12771 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
12772 case NEON::BI__builtin_neon_vcvtnq_s32_v:
12773 case NEON::BI__builtin_neon_vcvtn_u16_f16:
12774 case NEON::BI__builtin_neon_vcvtn_u32_v:
12775 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
12776 case NEON::BI__builtin_neon_vcvtnq_u32_v:
12777 case NEON::BI__builtin_neon_vcvtn_s64_v:
12778 case NEON::BI__builtin_neon_vcvtnq_s64_v:
12779 case NEON::BI__builtin_neon_vcvtn_u64_v:
12780 case NEON::BI__builtin_neon_vcvtnq_u64_v: {
12781 Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns;
12785 case NEON::BI__builtin_neon_vcvtp_s16_f16:
12786 case NEON::BI__builtin_neon_vcvtp_s32_v:
12787 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
12788 case NEON::BI__builtin_neon_vcvtpq_s32_v:
12789 case NEON::BI__builtin_neon_vcvtp_u16_f16:
12790 case NEON::BI__builtin_neon_vcvtp_u32_v:
12791 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
12792 case NEON::BI__builtin_neon_vcvtpq_u32_v:
12793 case NEON::BI__builtin_neon_vcvtp_s64_v:
12794 case NEON::BI__builtin_neon_vcvtpq_s64_v:
12795 case NEON::BI__builtin_neon_vcvtp_u64_v:
12796 case NEON::BI__builtin_neon_vcvtpq_u64_v: {
12797 Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps;
12801 case NEON::BI__builtin_neon_vmulx_v:
12802 case NEON::BI__builtin_neon_vmulxq_v: {
12803 Int = Intrinsic::aarch64_neon_fmulx;
12806 case NEON::BI__builtin_neon_vmulxh_lane_f16:
12807 case NEON::BI__builtin_neon_vmulxh_laneq_f16: {
12811 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2],
"extract");
12813 Int = Intrinsic::aarch64_neon_fmulx;
12816 case NEON::BI__builtin_neon_vmul_lane_v:
12817 case NEON::BI__builtin_neon_vmul_laneq_v: {
12820 if (BuiltinID == NEON::BI__builtin_neon_vmul_laneq_v)
12823 llvm::FixedVectorType *VTy =
12825 Ops[1] =
Builder.CreateBitCast(Ops[1], VTy);
12826 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2],
"extract");
12830 case NEON::BI__builtin_neon_vnegd_s64:
12832 case NEON::BI__builtin_neon_vnegh_f16:
12834 case NEON::BI__builtin_neon_vpmaxnm_v:
12835 case NEON::BI__builtin_neon_vpmaxnmq_v: {
12836 Int = Intrinsic::aarch64_neon_fmaxnmp;
12839 case NEON::BI__builtin_neon_vpminnm_v:
12840 case NEON::BI__builtin_neon_vpminnmq_v: {
12841 Int = Intrinsic::aarch64_neon_fminnmp;
12844 case NEON::BI__builtin_neon_vsqrth_f16: {
12847 ? Intrinsic::experimental_constrained_sqrt
12851 case NEON::BI__builtin_neon_vsqrt_v:
12852 case NEON::BI__builtin_neon_vsqrtq_v: {
12854 ? Intrinsic::experimental_constrained_sqrt
12856 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
12859 case NEON::BI__builtin_neon_vrbit_v:
12860 case NEON::BI__builtin_neon_vrbitq_v: {
12861 Int = Intrinsic::bitreverse;
12864 case NEON::BI__builtin_neon_vaddv_u8:
12868 case NEON::BI__builtin_neon_vaddv_s8: {
12869 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
12871 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
12872 llvm::Type *Tys[2] = { Ty, VTy };
12877 case NEON::BI__builtin_neon_vaddv_u16:
12880 case NEON::BI__builtin_neon_vaddv_s16: {
12881 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
12883 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
12884 llvm::Type *Tys[2] = { Ty, VTy };
12889 case NEON::BI__builtin_neon_vaddvq_u8:
12892 case NEON::BI__builtin_neon_vaddvq_s8: {
12893 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
12895 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
12896 llvm::Type *Tys[2] = { Ty, VTy };
12901 case NEON::BI__builtin_neon_vaddvq_u16:
12904 case NEON::BI__builtin_neon_vaddvq_s16: {
12905 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
12907 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
12908 llvm::Type *Tys[2] = { Ty, VTy };
12913 case NEON::BI__builtin_neon_vmaxv_u8: {
12914 Int = Intrinsic::aarch64_neon_umaxv;
12916 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
12917 llvm::Type *Tys[2] = { Ty, VTy };
12922 case NEON::BI__builtin_neon_vmaxv_u16: {
12923 Int = Intrinsic::aarch64_neon_umaxv;
12925 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
12926 llvm::Type *Tys[2] = { Ty, VTy };
12931 case NEON::BI__builtin_neon_vmaxvq_u8: {
12932 Int = Intrinsic::aarch64_neon_umaxv;
12934 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
12935 llvm::Type *Tys[2] = { Ty, VTy };
12940 case NEON::BI__builtin_neon_vmaxvq_u16: {
12941 Int = Intrinsic::aarch64_neon_umaxv;
12943 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
12944 llvm::Type *Tys[2] = { Ty, VTy };
12949 case NEON::BI__builtin_neon_vmaxv_s8: {
12950 Int = Intrinsic::aarch64_neon_smaxv;
12952 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
12953 llvm::Type *Tys[2] = { Ty, VTy };
12958 case NEON::BI__builtin_neon_vmaxv_s16: {
12959 Int = Intrinsic::aarch64_neon_smaxv;
12961 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
12962 llvm::Type *Tys[2] = { Ty, VTy };
12967 case NEON::BI__builtin_neon_vmaxvq_s8: {
12968 Int = Intrinsic::aarch64_neon_smaxv;
12970 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
12971 llvm::Type *Tys[2] = { Ty, VTy };
12976 case NEON::BI__builtin_neon_vmaxvq_s16: {
12977 Int = Intrinsic::aarch64_neon_smaxv;
12979 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
12980 llvm::Type *Tys[2] = { Ty, VTy };
12985 case NEON::BI__builtin_neon_vmaxv_f16: {
12986 Int = Intrinsic::aarch64_neon_fmaxv;
12988 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
12989 llvm::Type *Tys[2] = { Ty, VTy };
12994 case NEON::BI__builtin_neon_vmaxvq_f16: {
12995 Int = Intrinsic::aarch64_neon_fmaxv;
12997 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
12998 llvm::Type *Tys[2] = { Ty, VTy };
13003 case NEON::BI__builtin_neon_vminv_u8: {
13004 Int = Intrinsic::aarch64_neon_uminv;
13006 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
13007 llvm::Type *Tys[2] = { Ty, VTy };
13012 case NEON::BI__builtin_neon_vminv_u16: {
13013 Int = Intrinsic::aarch64_neon_uminv;
13015 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
13016 llvm::Type *Tys[2] = { Ty, VTy };
13021 case NEON::BI__builtin_neon_vminvq_u8: {
13022 Int = Intrinsic::aarch64_neon_uminv;
13024 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
13025 llvm::Type *Tys[2] = { Ty, VTy };
13030 case NEON::BI__builtin_neon_vminvq_u16: {
13031 Int = Intrinsic::aarch64_neon_uminv;
13033 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
13034 llvm::Type *Tys[2] = { Ty, VTy };
13039 case NEON::BI__builtin_neon_vminv_s8: {
13040 Int = Intrinsic::aarch64_neon_sminv;
13042 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
13043 llvm::Type *Tys[2] = { Ty, VTy };
13048 case NEON::BI__builtin_neon_vminv_s16: {
13049 Int = Intrinsic::aarch64_neon_sminv;
13051 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
13052 llvm::Type *Tys[2] = { Ty, VTy };
13057 case NEON::BI__builtin_neon_vminvq_s8: {
13058 Int = Intrinsic::aarch64_neon_sminv;
13060 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
13061 llvm::Type *Tys[2] = { Ty, VTy };
13066 case NEON::BI__builtin_neon_vminvq_s16: {
13067 Int = Intrinsic::aarch64_neon_sminv;
13069 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
13070 llvm::Type *Tys[2] = { Ty, VTy };
13075 case NEON::BI__builtin_neon_vminv_f16: {
13076 Int = Intrinsic::aarch64_neon_fminv;
13078 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
13079 llvm::Type *Tys[2] = { Ty, VTy };
13084 case NEON::BI__builtin_neon_vminvq_f16: {
13085 Int = Intrinsic::aarch64_neon_fminv;
13087 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
13088 llvm::Type *Tys[2] = { Ty, VTy };
13093 case NEON::BI__builtin_neon_vmaxnmv_f16: {
13094 Int = Intrinsic::aarch64_neon_fmaxnmv;
13096 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
13097 llvm::Type *Tys[2] = { Ty, VTy };
13102 case NEON::BI__builtin_neon_vmaxnmvq_f16: {
13103 Int = Intrinsic::aarch64_neon_fmaxnmv;
13105 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
13106 llvm::Type *Tys[2] = { Ty, VTy };
13111 case NEON::BI__builtin_neon_vminnmv_f16: {
13112 Int = Intrinsic::aarch64_neon_fminnmv;
13114 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
13115 llvm::Type *Tys[2] = { Ty, VTy };
13120 case NEON::BI__builtin_neon_vminnmvq_f16: {
13121 Int = Intrinsic::aarch64_neon_fminnmv;
13123 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
13124 llvm::Type *Tys[2] = { Ty, VTy };
13129 case NEON::BI__builtin_neon_vmul_n_f64: {
13132 return Builder.CreateFMul(Ops[0], RHS);
13134 case NEON::BI__builtin_neon_vaddlv_u8: {
13135 Int = Intrinsic::aarch64_neon_uaddlv;
13137 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
13138 llvm::Type *Tys[2] = { Ty, VTy };
13143 case NEON::BI__builtin_neon_vaddlv_u16: {
13144 Int = Intrinsic::aarch64_neon_uaddlv;
13146 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
13147 llvm::Type *Tys[2] = { Ty, VTy };
13151 case NEON::BI__builtin_neon_vaddlvq_u8: {
13152 Int = Intrinsic::aarch64_neon_uaddlv;
13154 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
13155 llvm::Type *Tys[2] = { Ty, VTy };
13160 case NEON::BI__builtin_neon_vaddlvq_u16: {
13161 Int = Intrinsic::aarch64_neon_uaddlv;
13163 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
13164 llvm::Type *Tys[2] = { Ty, VTy };
13168 case NEON::BI__builtin_neon_vaddlv_s8: {
13169 Int = Intrinsic::aarch64_neon_saddlv;
13171 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
13172 llvm::Type *Tys[2] = { Ty, VTy };
13177 case NEON::BI__builtin_neon_vaddlv_s16: {
13178 Int = Intrinsic::aarch64_neon_saddlv;
13180 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
13181 llvm::Type *Tys[2] = { Ty, VTy };
13185 case NEON::BI__builtin_neon_vaddlvq_s8: {
13186 Int = Intrinsic::aarch64_neon_saddlv;
13188 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
13189 llvm::Type *Tys[2] = { Ty, VTy };
13194 case NEON::BI__builtin_neon_vaddlvq_s16: {
13195 Int = Intrinsic::aarch64_neon_saddlv;
13197 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
13198 llvm::Type *Tys[2] = { Ty, VTy };
13202 case NEON::BI__builtin_neon_vsri_n_v:
13203 case NEON::BI__builtin_neon_vsriq_n_v: {
13204 Int = Intrinsic::aarch64_neon_vsri;
13208 case NEON::BI__builtin_neon_vsli_n_v:
13209 case NEON::BI__builtin_neon_vsliq_n_v: {
13210 Int = Intrinsic::aarch64_neon_vsli;
13214 case NEON::BI__builtin_neon_vsra_n_v:
13215 case NEON::BI__builtin_neon_vsraq_n_v:
13216 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
13218 return Builder.CreateAdd(Ops[0], Ops[1]);
13219 case NEON::BI__builtin_neon_vrsra_n_v:
13220 case NEON::BI__builtin_neon_vrsraq_n_v: {
13221 Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl;
13223 TmpOps.push_back(Ops[1]);
13224 TmpOps.push_back(Ops[2]);
13226 llvm::Value *tmp =
EmitNeonCall(F, TmpOps,
"vrshr_n", 1,
true);
13227 Ops[0] =
Builder.CreateBitCast(Ops[0], VTy);
13228 return Builder.CreateAdd(Ops[0], tmp);
13230 case NEON::BI__builtin_neon_vld1_v:
13231 case NEON::BI__builtin_neon_vld1q_v: {
13234 case NEON::BI__builtin_neon_vst1_v:
13235 case NEON::BI__builtin_neon_vst1q_v:
13236 Ops[1] =
Builder.CreateBitCast(Ops[1], VTy);
13238 case NEON::BI__builtin_neon_vld1_lane_v:
13239 case NEON::BI__builtin_neon_vld1q_lane_v: {
13240 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13243 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vld1_lane");
13245 case NEON::BI__builtin_neon_vldap1_lane_s64:
13246 case NEON::BI__builtin_neon_vldap1q_lane_s64: {
13247 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13249 VTy->getElementType(), Ops[0], PtrOp0.
getAlignment());
13250 LI->setAtomic(llvm::AtomicOrdering::Acquire);
13252 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vldap1_lane");
13254 case NEON::BI__builtin_neon_vld1_dup_v:
13255 case NEON::BI__builtin_neon_vld1q_dup_v: {
13256 Value *
V = PoisonValue::get(Ty);
13259 llvm::Constant *CI = ConstantInt::get(
Int32Ty, 0);
13260 Ops[0] =
Builder.CreateInsertElement(
V, Ops[0], CI);
13263 case NEON::BI__builtin_neon_vst1_lane_v:
13264 case NEON::BI__builtin_neon_vst1q_lane_v:
13265 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13266 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2]);
13268 case NEON::BI__builtin_neon_vstl1_lane_s64:
13269 case NEON::BI__builtin_neon_vstl1q_lane_s64: {
13270 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13271 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2]);
13272 llvm::StoreInst *SI =
13274 SI->setAtomic(llvm::AtomicOrdering::Release);
13277 case NEON::BI__builtin_neon_vld2_v:
13278 case NEON::BI__builtin_neon_vld2q_v: {
13281 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld2");
13284 case NEON::BI__builtin_neon_vld3_v:
13285 case NEON::BI__builtin_neon_vld3q_v: {
13288 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld3");
13291 case NEON::BI__builtin_neon_vld4_v:
13292 case NEON::BI__builtin_neon_vld4q_v: {
13295 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld4");
13298 case NEON::BI__builtin_neon_vld2_dup_v:
13299 case NEON::BI__builtin_neon_vld2q_dup_v: {
13302 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld2");
13305 case NEON::BI__builtin_neon_vld3_dup_v:
13306 case NEON::BI__builtin_neon_vld3q_dup_v: {
13309 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld3");
13312 case NEON::BI__builtin_neon_vld4_dup_v:
13313 case NEON::BI__builtin_neon_vld4q_dup_v: {
13316 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld4");
13319 case NEON::BI__builtin_neon_vld2_lane_v:
13320 case NEON::BI__builtin_neon_vld2q_lane_v: {
13321 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
13323 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
13324 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13325 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13330 case NEON::BI__builtin_neon_vld3_lane_v:
13331 case NEON::BI__builtin_neon_vld3q_lane_v: {
13332 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
13334 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
13335 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13336 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13337 Ops[3] =
Builder.CreateBitCast(Ops[3], Ty);
13342 case NEON::BI__builtin_neon_vld4_lane_v:
13343 case NEON::BI__builtin_neon_vld4q_lane_v: {
13344 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
13346 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
13347 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13348 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13349 Ops[3] =
Builder.CreateBitCast(Ops[3], Ty);
13350 Ops[4] =
Builder.CreateBitCast(Ops[4], Ty);
13355 case NEON::BI__builtin_neon_vst2_v:
13356 case NEON::BI__builtin_neon_vst2q_v: {
13357 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13358 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() };
13362 case NEON::BI__builtin_neon_vst2_lane_v:
13363 case NEON::BI__builtin_neon_vst2q_lane_v: {
13364 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13366 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
13370 case NEON::BI__builtin_neon_vst3_v:
13371 case NEON::BI__builtin_neon_vst3q_v: {
13372 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13373 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
13377 case NEON::BI__builtin_neon_vst3_lane_v:
13378 case NEON::BI__builtin_neon_vst3q_lane_v: {
13379 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13381 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
13385 case NEON::BI__builtin_neon_vst4_v:
13386 case NEON::BI__builtin_neon_vst4q_v: {
13387 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13388 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
13392 case NEON::BI__builtin_neon_vst4_lane_v:
13393 case NEON::BI__builtin_neon_vst4q_lane_v: {
13394 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13396 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() };
13400 case NEON::BI__builtin_neon_vtrn_v:
13401 case NEON::BI__builtin_neon_vtrnq_v: {
13402 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13403 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13404 Value *SV =
nullptr;
13406 for (
unsigned vi = 0; vi != 2; ++vi) {
13408 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
13409 Indices.push_back(i+vi);
13410 Indices.push_back(i+e+vi);
13412 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
13413 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vtrn");
13418 case NEON::BI__builtin_neon_vuzp_v:
13419 case NEON::BI__builtin_neon_vuzpq_v: {
13420 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13421 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13422 Value *SV =
nullptr;
13424 for (
unsigned vi = 0; vi != 2; ++vi) {
13426 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
13427 Indices.push_back(2*i+vi);
13429 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
13430 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vuzp");
13435 case NEON::BI__builtin_neon_vzip_v:
13436 case NEON::BI__builtin_neon_vzipq_v: {
13437 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13438 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13439 Value *SV =
nullptr;
13441 for (
unsigned vi = 0; vi != 2; ++vi) {
13443 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
13444 Indices.push_back((i + vi*e) >> 1);
13445 Indices.push_back(((i + vi*e) >> 1)+e);
13447 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
13448 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vzip");
13453 case NEON::BI__builtin_neon_vqtbl1q_v: {
13457 case NEON::BI__builtin_neon_vqtbl2q_v: {
13461 case NEON::BI__builtin_neon_vqtbl3q_v: {
13465 case NEON::BI__builtin_neon_vqtbl4q_v: {
13469 case NEON::BI__builtin_neon_vqtbx1q_v: {
13473 case NEON::BI__builtin_neon_vqtbx2q_v: {
13477 case NEON::BI__builtin_neon_vqtbx3q_v: {
13481 case NEON::BI__builtin_neon_vqtbx4q_v: {
13485 case NEON::BI__builtin_neon_vsqadd_v:
13486 case NEON::BI__builtin_neon_vsqaddq_v: {
13487 Int = Intrinsic::aarch64_neon_usqadd;
13490 case NEON::BI__builtin_neon_vuqadd_v:
13491 case NEON::BI__builtin_neon_vuqaddq_v: {
13492 Int = Intrinsic::aarch64_neon_suqadd;
13500 assert((BuiltinID == BPF::BI__builtin_preserve_field_info ||
13501 BuiltinID == BPF::BI__builtin_btf_type_id ||
13502 BuiltinID == BPF::BI__builtin_preserve_type_info ||
13503 BuiltinID == BPF::BI__builtin_preserve_enum_value) &&
13504 "unexpected BPF builtin");
13509 static uint32_t BuiltinSeqNum;
13511 switch (BuiltinID) {
13513 llvm_unreachable(
"Unexpected BPF builtin");
13514 case BPF::BI__builtin_preserve_field_info: {
13515 const Expr *Arg =
E->getArg(0);
13520 "using __builtin_preserve_field_info() without -g");
13533 Value *InfoKind = ConstantInt::get(
Int64Ty,
C->getSExtValue());
13536 llvm::Function *FnGetFieldInfo = llvm::Intrinsic::getDeclaration(
13537 &
CGM.
getModule(), llvm::Intrinsic::bpf_preserve_field_info,
13538 {FieldAddr->getType()});
13539 return Builder.CreateCall(FnGetFieldInfo, {FieldAddr, InfoKind});
13541 case BPF::BI__builtin_btf_type_id:
13542 case BPF::BI__builtin_preserve_type_info: {
13548 const Expr *Arg0 =
E->getArg(0);
13553 Value *FlagValue = ConstantInt::get(
Int64Ty, Flag->getSExtValue());
13554 Value *SeqNumVal = ConstantInt::get(
Int32Ty, BuiltinSeqNum++);
13556 llvm::Function *FnDecl;
13557 if (BuiltinID == BPF::BI__builtin_btf_type_id)
13558 FnDecl = llvm::Intrinsic::getDeclaration(
13559 &
CGM.
getModule(), llvm::Intrinsic::bpf_btf_type_id, {});
13561 FnDecl = llvm::Intrinsic::getDeclaration(
13562 &
CGM.
getModule(), llvm::Intrinsic::bpf_preserve_type_info, {});
13563 CallInst *
Fn =
Builder.CreateCall(FnDecl, {SeqNumVal, FlagValue});
13564 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
13567 case BPF::BI__builtin_preserve_enum_value: {
13573 const Expr *Arg0 =
E->getArg(0);
13578 const auto *UO = cast<UnaryOperator>(Arg0->
IgnoreParens());
13579 const auto *CE = cast<CStyleCastExpr>(UO->getSubExpr());
13580 const auto *DR = cast<DeclRefExpr>(CE->getSubExpr());
13581 const auto *Enumerator = cast<EnumConstantDecl>(DR->getDecl());
13583 auto InitVal = Enumerator->getInitVal();
13584 std::string InitValStr;
13585 if (InitVal.isNegative() || InitVal >
uint64_t(INT64_MAX))
13586 InitValStr = std::to_string(InitVal.getSExtValue());
13588 InitValStr = std::to_string(InitVal.getZExtValue());
13589 std::string EnumStr = Enumerator->getNameAsString() +
":" + InitValStr;
13590 Value *EnumStrVal =
Builder.CreateGlobalStringPtr(EnumStr);
13593 Value *FlagValue = ConstantInt::get(
Int64Ty, Flag->getSExtValue());
13594 Value *SeqNumVal = ConstantInt::get(
Int32Ty, BuiltinSeqNum++);
13596 llvm::Function *IntrinsicFn = llvm::Intrinsic::getDeclaration(
13597 &
CGM.
getModule(), llvm::Intrinsic::bpf_preserve_enum_value, {});
13599 Builder.CreateCall(IntrinsicFn, {SeqNumVal, EnumStrVal, FlagValue});
13600 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
13608 assert((Ops.size() & (Ops.size() - 1)) == 0 &&
13609 "Not a power-of-two sized vector!");
13610 bool AllConstants =
true;
13611 for (
unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i)
13612 AllConstants &= isa<Constant>(Ops[i]);
13615 if (AllConstants) {
13617 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
13618 CstOps.push_back(cast<Constant>(Ops[i]));
13619 return llvm::ConstantVector::get(CstOps);
13624 llvm::FixedVectorType::get(Ops[0]->getType(), Ops.size()));
13626 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
13634 unsigned NumElts) {
13636 auto *MaskTy = llvm::FixedVectorType::get(
13638 cast<IntegerType>(Mask->
getType())->getBitWidth());
13639 Value *MaskVec = CGF.
Builder.CreateBitCast(Mask, MaskTy);
13645 for (
unsigned i = 0; i != NumElts; ++i)
13647 MaskVec = CGF.
Builder.CreateShuffleVector(
13648 MaskVec, MaskVec,
ArrayRef(Indices, NumElts),
"extract");
13655 Value *Ptr = Ops[0];
13659 cast<llvm::FixedVectorType>(Ops[1]->getType())->getNumElements());
13661 return CGF.
Builder.CreateMaskedStore(Ops[1], Ptr, Alignment, MaskVec);
13666 llvm::Type *Ty = Ops[1]->getType();
13667 Value *Ptr = Ops[0];
13670 CGF, Ops[2], cast<llvm::FixedVectorType>(Ty)->getNumElements());
13672 return CGF.
Builder.CreateMaskedLoad(Ty, Ptr, Alignment, MaskVec, Ops[1]);
13677 auto *ResultTy = cast<llvm::VectorType>(Ops[1]->getType());
13678 Value *Ptr = Ops[0];
13681 CGF, Ops[2], cast<FixedVectorType>(ResultTy)->getNumElements());
13683 llvm::Function *F = CGF.
CGM.
getIntrinsic(Intrinsic::masked_expandload,
13685 return CGF.
Builder.CreateCall(F, { Ptr, MaskVec, Ops[1] });
13691 auto *ResultTy = cast<llvm::FixedVectorType>(Ops[1]->getType());
13695 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
13696 : Intrinsic::x86_avx512_mask_expand;
13698 return CGF.
Builder.CreateCall(F, { Ops[0], Ops[1], MaskVec });
13703 auto *ResultTy = cast<llvm::FixedVectorType>(Ops[1]->getType());
13704 Value *Ptr = Ops[0];
13708 llvm::Function *F = CGF.
CGM.
getIntrinsic(Intrinsic::masked_compressstore,
13710 return CGF.
Builder.CreateCall(F, { Ops[1], Ptr, MaskVec });
13715 bool InvertLHS =
false) {
13716 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
13721 LHS = CGF.
Builder.CreateNot(LHS);
13723 return CGF.
Builder.CreateBitCast(CGF.
Builder.CreateBinOp(Opc, LHS, RHS),
13724 Ops[0]->getType());
13728 Value *Amt,
bool IsRight) {
13729 llvm::Type *Ty = Op0->
getType();
13735 unsigned NumElts = cast<llvm::FixedVectorType>(Ty)->getNumElements();
13736 Amt = CGF.
Builder.CreateIntCast(Amt, Ty->getScalarType(),
false);
13737 Amt = CGF.
Builder.CreateVectorSplat(NumElts, Amt);
13740 unsigned IID = IsRight ? Intrinsic::fshr : Intrinsic::fshl;
13742 return CGF.
Builder.CreateCall(F, {Op0, Op1, Amt});
13747 Value *Op0 = Ops[0];
13748 Value *Op1 = Ops[1];
13749 llvm::Type *Ty = Op0->
getType();
13750 uint64_t Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
13752 CmpInst::Predicate Pred;
13755 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
13758 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
13761 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
13764 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
13767 Pred = ICmpInst::ICMP_EQ;
13770 Pred = ICmpInst::ICMP_NE;
13773 return llvm::Constant::getNullValue(Ty);
13775 return llvm::Constant::getAllOnesValue(Ty);
13777 llvm_unreachable(
"Unexpected XOP vpcom/vpcomu predicate");
13789 if (
const auto *
C = dyn_cast<Constant>(Mask))
13790 if (
C->isAllOnesValue())
13794 CGF, Mask, cast<llvm::FixedVectorType>(Op0->
getType())->getNumElements());
13796 return CGF.
Builder.CreateSelect(Mask, Op0, Op1);
13802 if (
const auto *
C = dyn_cast<Constant>(Mask))
13803 if (
C->isAllOnesValue())
13806 auto *MaskTy = llvm::FixedVectorType::get(
13807 CGF.
Builder.getInt1Ty(), Mask->
getType()->getIntegerBitWidth());
13808 Mask = CGF.
Builder.CreateBitCast(Mask, MaskTy);
13809 Mask = CGF.
Builder.CreateExtractElement(Mask, (uint64_t)0);
13810 return CGF.
Builder.CreateSelect(Mask, Op0, Op1);
13814 unsigned NumElts,
Value *MaskIn) {
13816 const auto *
C = dyn_cast<Constant>(MaskIn);
13817 if (!
C || !
C->isAllOnesValue())
13823 for (
unsigned i = 0; i != NumElts; ++i)
13825 for (
unsigned i = NumElts; i != 8; ++i)
13826 Indices[i] = i % NumElts + NumElts;
13827 Cmp = CGF.
Builder.CreateShuffleVector(
13828 Cmp, llvm::Constant::getNullValue(Cmp->getType()), Indices);
13831 return CGF.
Builder.CreateBitCast(Cmp,
13833 std::max(NumElts, 8U)));
13838 assert((Ops.size() == 2 || Ops.size() == 4) &&
13839 "Unexpected number of arguments");
13841 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
13845 Cmp = Constant::getNullValue(
13846 llvm::FixedVectorType::get(CGF.
Builder.getInt1Ty(), NumElts));
13847 }
else if (CC == 7) {
13848 Cmp = Constant::getAllOnesValue(
13849 llvm::FixedVectorType::get(CGF.
Builder.getInt1Ty(), NumElts));
13851 ICmpInst::Predicate Pred;
13853 default: llvm_unreachable(
"Unknown condition code");
13854 case 0: Pred = ICmpInst::ICMP_EQ;
break;
13855 case 1: Pred =
Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
break;
13856 case 2: Pred =
Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
break;
13857 case 4: Pred = ICmpInst::ICMP_NE;
break;
13858 case 5: Pred =
Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
break;
13859 case 6: Pred =
Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
break;
13861 Cmp = CGF.
Builder.CreateICmp(Pred, Ops[0], Ops[1]);
13864 Value *MaskIn =
nullptr;
13865 if (Ops.size() == 4)
13872 Value *Zero = Constant::getNullValue(In->getType());
13878 unsigned Rnd = cast<llvm::ConstantInt>(Ops[3])->getZExtValue();
13879 llvm::Type *Ty = Ops[1]->getType();
13883 Intrinsic::ID IID = IsSigned ? Intrinsic::x86_avx512_sitofp_round
13884 : Intrinsic::x86_avx512_uitofp_round;
13886 Res = CGF.
Builder.CreateCall(F, { Ops[0], Ops[3] });
13888 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
13889 Res = IsSigned ? CGF.
Builder.CreateSIToFP(Ops[0], Ty)
13890 : CGF.
Builder.CreateUIToFP(Ops[0], Ty);
13901 bool Subtract =
false;
13902 Intrinsic::ID IID = Intrinsic::not_intrinsic;
13903 switch (BuiltinID) {
13905 case clang::X86::BI__builtin_ia32_vfmsubph512_mask3:
13908 case clang::X86::BI__builtin_ia32_vfmaddph512_mask:
13909 case clang::X86::BI__builtin_ia32_vfmaddph512_maskz:
13910 case clang::X86::BI__builtin_ia32_vfmaddph512_mask3:
13911 IID = llvm::Intrinsic::x86_avx512fp16_vfmadd_ph_512;
13913 case clang::X86::BI__builtin_ia32_vfmsubaddph512_mask3:
13916 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask:
13917 case clang::X86::BI__builtin_ia32_vfmaddsubph512_maskz:
13918 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask3:
13919 IID = llvm::Intrinsic::x86_avx512fp16_vfmaddsub_ph_512;
13921 case clang::X86::BI__builtin_ia32_vfmsubps512_mask3:
13924 case clang::X86::BI__builtin_ia32_vfmaddps512_mask:
13925 case clang::X86::BI__builtin_ia32_vfmaddps512_maskz:
13926 case clang::X86::BI__builtin_ia32_vfmaddps512_mask3:
13927 IID = llvm::Intrinsic::x86_avx512_vfmadd_ps_512;
break;
13928 case clang::X86::BI__builtin_ia32_vfmsubpd512_mask3:
13931 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask:
13932 case clang::X86::BI__builtin_ia32_vfmaddpd512_maskz:
13933 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask3:
13934 IID = llvm::Intrinsic::x86_avx512_vfmadd_pd_512;
break;
13935 case clang::X86::BI__builtin_ia32_vfmsubaddps512_mask3:
13938 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask:
13939 case clang::X86::BI__builtin_ia32_vfmaddsubps512_maskz:
13940 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask3:
13941 IID = llvm::Intrinsic::x86_avx512_vfmaddsub_ps_512;
13943 case clang::X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
13946 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask:
13947 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
13948 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
13949 IID = llvm::Intrinsic::x86_avx512_vfmaddsub_pd_512;
13963 if (IID != Intrinsic::not_intrinsic &&
13964 (cast<llvm::ConstantInt>(Ops.back())->getZExtValue() != (uint64_t)4 ||
13967 Res = CGF.
Builder.CreateCall(Intr, {A, B,
C, Ops.back() });
13969 llvm::Type *Ty = A->
getType();
13971 if (CGF.
Builder.getIsFPConstrained()) {
13972 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
13973 FMA = CGF.
CGM.
getIntrinsic(Intrinsic::experimental_constrained_fma, Ty);
13974 Res = CGF.
Builder.CreateConstrainedFPCall(FMA, {A, B,
C});
13977 Res = CGF.
Builder.CreateCall(FMA, {A, B,
C});
13982 Value *MaskFalseVal =
nullptr;
13983 switch (BuiltinID) {
13984 case clang::X86::BI__builtin_ia32_vfmaddph512_mask:
13985 case clang::X86::BI__builtin_ia32_vfmaddps512_mask:
13986 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask:
13987 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask:
13988 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask:
13989 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask:
13990 MaskFalseVal = Ops[0];
13992 case clang::X86::BI__builtin_ia32_vfmaddph512_maskz:
13993 case clang::X86::BI__builtin_ia32_vfmaddps512_maskz:
13994 case clang::X86::BI__builtin_ia32_vfmaddpd512_maskz:
13995 case clang::X86::BI__builtin_ia32_vfmaddsubph512_maskz:
13996 case clang::X86::BI__builtin_ia32_vfmaddsubps512_maskz:
13997 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
13998 MaskFalseVal = Constant::getNullValue(Ops[0]->getType());
14000 case clang::X86::BI__builtin_ia32_vfmsubph512_mask3:
14001 case clang::X86::BI__builtin_ia32_vfmaddph512_mask3:
14002 case clang::X86::BI__builtin_ia32_vfmsubps512_mask3:
14003 case clang::X86::BI__builtin_ia32_vfmaddps512_mask3:
14004 case clang::X86::BI__builtin_ia32_vfmsubpd512_mask3:
14005 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask3:
14006 case clang::X86::BI__builtin_ia32_vfmsubaddph512_mask3:
14007 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask3:
14008 case clang::X86::BI__builtin_ia32_vfmsubaddps512_mask3:
14009 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask3:
14010 case clang::X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
14011 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
14012 MaskFalseVal = Ops[2];
14024 bool ZeroMask =
false,
unsigned PTIdx = 0,
14025 bool NegAcc =
false) {
14027 if (Ops.size() > 4)
14028 Rnd = cast<llvm::ConstantInt>(Ops[4])->getZExtValue();
14031 Ops[2] = CGF.
Builder.CreateFNeg(Ops[2]);
14033 Ops[0] = CGF.
Builder.CreateExtractElement(Ops[0], (uint64_t)0);
14034 Ops[1] = CGF.
Builder.CreateExtractElement(Ops[1], (uint64_t)0);
14035 Ops[2] = CGF.
Builder.CreateExtractElement(Ops[2], (uint64_t)0);
14040 switch (Ops[0]->getType()->getPrimitiveSizeInBits()) {
14042 IID = Intrinsic::x86_avx512fp16_vfmadd_f16;
14045 IID = Intrinsic::x86_avx512_vfmadd_f32;
14048 IID = Intrinsic::x86_avx512_vfmadd_f64;
14051 llvm_unreachable(
"Unexpected size");
14054 {Ops[0], Ops[1], Ops[2], Ops[4]});
14055 }
else if (CGF.
Builder.getIsFPConstrained()) {
14056 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
14058 Intrinsic::experimental_constrained_fma, Ops[0]->getType());
14059 Res = CGF.
Builder.CreateConstrainedFPCall(FMA, Ops.slice(0, 3));
14062 Res = CGF.
Builder.CreateCall(FMA, Ops.slice(0, 3));
14065 if (Ops.size() > 3) {
14066 Value *PassThru = ZeroMask ? Constant::getNullValue(Res->
getType())
14072 if (NegAcc && PTIdx == 2)
14073 PassThru = CGF.
Builder.CreateExtractElement(Upper, (uint64_t)0);
14077 return CGF.
Builder.CreateInsertElement(Upper, Res, (uint64_t)0);
14082 llvm::Type *Ty = Ops[0]->getType();
14084 Ty = llvm::FixedVectorType::get(CGF.
Int64Ty,
14085 Ty->getPrimitiveSizeInBits() / 64);
14091 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
14092 LHS = CGF.
Builder.CreateShl(LHS, ShiftAmt);
14093 LHS = CGF.
Builder.CreateAShr(LHS, ShiftAmt);
14094 RHS = CGF.
Builder.CreateShl(RHS, ShiftAmt);
14095 RHS = CGF.
Builder.CreateAShr(RHS, ShiftAmt);
14098 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
14099 LHS = CGF.
Builder.CreateAnd(LHS, Mask);
14100 RHS = CGF.
Builder.CreateAnd(RHS, Mask);
14103 return CGF.
Builder.CreateMul(LHS, RHS);
14111 llvm::Type *Ty = Ops[0]->getType();
14113 unsigned VecWidth = Ty->getPrimitiveSizeInBits();
14114 unsigned EltWidth = Ty->getScalarSizeInBits();
14116 if (VecWidth == 128 && EltWidth == 32)
14117 IID = Intrinsic::x86_avx512_pternlog_d_128;
14118 else if (VecWidth == 256 && EltWidth == 32)
14119 IID = Intrinsic::x86_avx512_pternlog_d_256;
14120 else if (VecWidth == 512 && EltWidth == 32)
14121 IID = Intrinsic::x86_avx512_pternlog_d_512;
14122 else if (VecWidth == 128 && EltWidth == 64)
14123 IID = Intrinsic::x86_avx512_pternlog_q_128;
14124 else if (VecWidth == 256 && EltWidth == 64)
14125 IID = Intrinsic::x86_avx512_pternlog_q_256;
14126 else if (VecWidth == 512 && EltWidth == 64)
14127 IID = Intrinsic::x86_avx512_pternlog_q_512;
14129 llvm_unreachable(
"Unexpected intrinsic");
14133 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty) : Ops[0];
14138 llvm::Type *DstTy) {
14139 unsigned NumberOfElements =
14140 cast<llvm::FixedVectorType>(DstTy)->getNumElements();
14142 return CGF.
Builder.CreateSExt(Mask, DstTy,
"vpmovm2");
14147 StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
14148 return EmitX86CpuIs(CPUStr);
14154 llvm::Type *DstTy) {
14155 assert((Ops.size() == 1 || Ops.size() == 3 || Ops.size() == 4) &&
14156 "Unknown cvtph2ps intrinsic");
14159 if (Ops.size() == 4 && cast<llvm::ConstantInt>(Ops[3])->getZExtValue() != 4) {
14162 return CGF.
Builder.CreateCall(F, {Ops[0], Ops[1], Ops[2], Ops[3]});
14165 unsigned NumDstElts = cast<llvm::FixedVectorType>(DstTy)->getNumElements();
14166 Value *Src = Ops[0];
14170 cast<llvm::FixedVectorType>(Src->
getType())->getNumElements()) {
14171 assert(NumDstElts == 4 &&
"Unexpected vector size");
14176 auto *HalfTy = llvm::FixedVectorType::get(
14178 Src = CGF.
Builder.CreateBitCast(Src, HalfTy);
14181 Value *Res = CGF.
Builder.CreateFPExt(Src, DstTy,
"cvtph2ps");
14183 if (Ops.size() >= 3)
14188Value *CodeGenFunction::EmitX86CpuIs(StringRef CPUStr) {
14199 llvm::ArrayType::get(
Int32Ty, 1));
14203 cast<llvm::GlobalValue>(CpuModel)->setDSOLocal(
true);
14209 std::tie(Index,
Value) = StringSwitch<std::pair<unsigned, unsigned>>(CPUStr)
14211 .Case(STRING, {0u,
static_cast<unsigned>(llvm::X86::ENUM)})
14213 .Case(
ALIAS, {1u,
static_cast<unsigned>(llvm::X86::ENUM)})
14215 .Case(STR, {1u,
static_cast<unsigned>(llvm::X86::ENUM)})
14217 .Case(
ALIAS, {2u,
static_cast<unsigned>(llvm::X86::ENUM)})
14219 .Case(STR, {2u,
static_cast<unsigned>(llvm::X86::ENUM)})
14220#include
"llvm/TargetParser/X86TargetParser.def"
14222 assert(
Value != 0 &&
"Invalid CPUStr passed to CpuIs");
14225 llvm::Value *Idxs[] = {ConstantInt::get(
Int32Ty, 0),
14226 ConstantInt::get(
Int32Ty, Index)};
14232 return Builder.CreateICmpEQ(CpuValue,
14238 StringRef FeatureStr = cast<StringLiteral>(FeatureExpr)->getString();
14239 if (!
getContext().getTargetInfo().validateCpuSupports(FeatureStr))
14241 return EmitX86CpuSupports(FeatureStr);
14245 return EmitX86CpuSupports(llvm::X86::getCpuSupportsMask(FeatureStrs));
14249CodeGenFunction::EmitX86CpuSupports(std::array<uint32_t, 4> FeatureMask) {
14251 if (FeatureMask[0] != 0) {
14259 llvm::ArrayType::get(
Int32Ty, 1));
14263 cast<llvm::GlobalValue>(CpuModel)->setDSOLocal(
true);
14280 llvm::Type *ATy = llvm::ArrayType::get(
Int32Ty, 3);
14281 llvm::Constant *CpuFeatures2 =
14283 cast<llvm::GlobalValue>(CpuFeatures2)->setDSOLocal(
true);
14284 for (
int i = 1; i != 4; ++i) {
14285 const uint32_t M = FeatureMask[i];
14302Value *CodeGenFunction::EmitAArch64CpuInit() {
14303 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy,
false);
14304 llvm::FunctionCallee
Func =
14306 cast<llvm::GlobalValue>(
Func.getCallee())->setDSOLocal(
true);
14307 cast<llvm::GlobalValue>(
Func.getCallee())
14308 ->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
14313 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy,
false);
14314 llvm::FunctionCallee
Func =
14316 auto *CalleeGV = cast<llvm::GlobalValue>(
Func.getCallee());
14317 CalleeGV->setDSOLocal(
true);
14318 CalleeGV->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
14322Value *CodeGenFunction::EmitX86CpuInit() {
14323 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy,
14325 llvm::FunctionCallee
Func =
14327 cast<llvm::GlobalValue>(
Func.getCallee())->setDSOLocal(
true);
14328 cast<llvm::GlobalValue>(
Func.getCallee())
14329 ->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
14333Value *CodeGenFunction::EmitAArch64CpuSupports(
const CallExpr *
E) {
14335 StringRef ArgStr = cast<StringLiteral>(ArgExpr)->getString();
14337 ArgStr.split(Features,
"+");
14338 for (
auto &Feature : Features) {
14339 Feature = Feature.trim();
14340 if (!llvm::AArch64::parseFMVExtension(Feature))
14342 if (Feature !=
"default")
14343 Features.push_back(Feature);
14345 return EmitAArch64CpuSupports(Features);
14350 uint64_t FeaturesMask = llvm::AArch64::getCpuSupportsMask(FeaturesStrs);
14352 if (FeaturesMask != 0) {
14357 llvm::Type *STy = llvm::StructType::get(
Int64Ty);
14358 llvm::Constant *AArch64CPUFeatures =
14360 cast<llvm::GlobalValue>(AArch64CPUFeatures)->setDSOLocal(
true);
14362 STy, AArch64CPUFeatures,
14377 StringRef FeatureStr = cast<StringLiteral>(FeatureExpr)->getString();
14378 if (!
getContext().getTargetInfo().validateCpuSupports(FeatureStr))
14384 llvm::ArrayType *ArrayOfInt64Ty = llvm::ArrayType::get(
Int64Ty, 1);
14385 llvm::Type *StructTy = llvm::StructType::get(
Int32Ty, ArrayOfInt64Ty);
14386 llvm::Constant *RISCVFeaturesBits =
14388 auto *GV = cast<llvm::GlobalValue>(RISCVFeaturesBits);
14389 GV->setDSOLocal(
true);
14391 auto LoadFeatureBit = [&](
unsigned Index) {
14393 Value *IndexVal = llvm::ConstantInt::get(
Int32Ty, Index);
14394 llvm::Value *GEPIndices[] = {
Builder.getInt32(0),
Builder.getInt32(1),
14398 Value *FeaturesBit =
14400 return FeaturesBit;
14403 int BitPos = RISCVISAInfo::getRISCVFeaturesBitPosition(FeatureStr);
14404 assert(BitPos != -1 &&
"validation should have rejected this feature");
14406 Value *Bitset =
Builder.CreateAnd(LoadFeatureBit(0), MaskV);
14407 return Builder.CreateICmpEQ(Bitset, MaskV);
14412 if (BuiltinID == Builtin::BI__builtin_cpu_is)
14413 return EmitX86CpuIs(
E);
14414 if (BuiltinID == Builtin::BI__builtin_cpu_supports)
14415 return EmitX86CpuSupports(
E);
14416 if (BuiltinID == Builtin::BI__builtin_cpu_init)
14417 return EmitX86CpuInit();
14425 bool IsMaskFCmp =
false;
14426 bool IsConjFMA =
false;
14429 unsigned ICEArguments = 0;
14434 for (
unsigned i = 0, e =
E->getNumArgs(); i != e; i++) {
14444 auto getCmpIntrinsicCall = [
this, &Ops](Intrinsic::ID
ID,
unsigned Imm) {
14445 Ops.push_back(llvm::ConstantInt::get(
Int8Ty, Imm));
14447 return Builder.CreateCall(F, Ops);
14455 auto getVectorFCmpIR = [
this, &Ops,
E](CmpInst::Predicate Pred,
14456 bool IsSignaling) {
14457 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
14460 Cmp =
Builder.CreateFCmpS(Pred, Ops[0], Ops[1]);
14462 Cmp =
Builder.CreateFCmp(Pred, Ops[0], Ops[1]);
14463 llvm::VectorType *FPVecTy = cast<llvm::VectorType>(Ops[0]->getType());
14464 llvm::VectorType *IntVecTy = llvm::VectorType::getInteger(FPVecTy);
14466 return Builder.CreateBitCast(Sext, FPVecTy);
14469 switch (BuiltinID) {
14470 default:
return nullptr;
14471 case X86::BI_mm_prefetch: {
14473 ConstantInt *
C = cast<ConstantInt>(Ops[1]);
14474 Value *RW = ConstantInt::get(
Int32Ty, (
C->getZExtValue() >> 2) & 0x1);
14475 Value *Locality = ConstantInt::get(
Int32Ty,
C->getZExtValue() & 0x3);
14480 case X86::BI_mm_clflush: {
14484 case X86::BI_mm_lfence: {
14487 case X86::BI_mm_mfence: {
14490 case X86::BI_mm_sfence: {
14493 case X86::BI_mm_pause: {
14496 case X86::BI__rdtsc: {
14499 case X86::BI__builtin_ia32_rdtscp: {
14505 case X86::BI__builtin_ia32_lzcnt_u16:
14506 case X86::BI__builtin_ia32_lzcnt_u32:
14507 case X86::BI__builtin_ia32_lzcnt_u64: {
14511 case X86::BI__builtin_ia32_tzcnt_u16:
14512 case X86::BI__builtin_ia32_tzcnt_u32:
14513 case X86::BI__builtin_ia32_tzcnt_u64: {
14517 case X86::BI__builtin_ia32_undef128:
14518 case X86::BI__builtin_ia32_undef256:
14519 case X86::BI__builtin_ia32_undef512:
14526 case X86::BI__builtin_ia32_vec_ext_v4hi:
14527 case X86::BI__builtin_ia32_vec_ext_v16qi:
14528 case X86::BI__builtin_ia32_vec_ext_v8hi:
14529 case X86::BI__builtin_ia32_vec_ext_v4si:
14530 case X86::BI__builtin_ia32_vec_ext_v4sf:
14531 case X86::BI__builtin_ia32_vec_ext_v2di:
14532 case X86::BI__builtin_ia32_vec_ext_v32qi:
14533 case X86::BI__builtin_ia32_vec_ext_v16hi:
14534 case X86::BI__builtin_ia32_vec_ext_v8si:
14535 case X86::BI__builtin_ia32_vec_ext_v4di: {
14537 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
14538 uint64_t Index = cast<ConstantInt>(Ops[1])->getZExtValue();
14539 Index &= NumElts - 1;
14542 return Builder.CreateExtractElement(Ops[0], Index);
14544 case X86::BI__builtin_ia32_vec_set_v4hi:
14545 case X86::BI__builtin_ia32_vec_set_v16qi:
14546 case X86::BI__builtin_ia32_vec_set_v8hi:
14547 case X86::BI__builtin_ia32_vec_set_v4si:
14548 case X86::BI__builtin_ia32_vec_set_v2di:
14549 case X86::BI__builtin_ia32_vec_set_v32qi:
14550 case X86::BI__builtin_ia32_vec_set_v16hi:
14551 case X86::BI__builtin_ia32_vec_set_v8si:
14552 case X86::BI__builtin_ia32_vec_set_v4di: {
14554 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
14555 unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue();
14556 Index &= NumElts - 1;
14559 return Builder.CreateInsertElement(Ops[0], Ops[1], Index);
14561 case X86::BI_mm_setcsr:
14562 case X86::BI__builtin_ia32_ldmxcsr: {
14568 case X86::BI_mm_getcsr:
14569 case X86::BI__builtin_ia32_stmxcsr: {
14575 case X86::BI__builtin_ia32_xsave:
14576 case X86::BI__builtin_ia32_xsave64:
14577 case X86::BI__builtin_ia32_xrstor:
14578 case X86::BI__builtin_ia32_xrstor64:
14579 case X86::BI__builtin_ia32_xsaveopt:
14580 case X86::BI__builtin_ia32_xsaveopt64:
14581 case X86::BI__builtin_ia32_xrstors:
14582 case X86::BI__builtin_ia32_xrstors64:
14583 case X86::BI__builtin_ia32_xsavec:
14584 case X86::BI__builtin_ia32_xsavec64:
14585 case X86::BI__builtin_ia32_xsaves:
14586 case X86::BI__builtin_ia32_xsaves64:
14587 case X86::BI__builtin_ia32_xsetbv:
14588 case X86::BI_xsetbv: {
14590#define INTRINSIC_X86_XSAVE_ID(NAME) \
14591 case X86::BI__builtin_ia32_##NAME: \
14592 ID = Intrinsic::x86_##NAME; \
14594 switch (BuiltinID) {
14595 default: llvm_unreachable(
"Unsupported intrinsic!");
14609 case X86::BI_xsetbv:
14610 ID = Intrinsic::x86_xsetbv;
14613#undef INTRINSIC_X86_XSAVE_ID
14618 Ops.push_back(Mlo);
14621 case X86::BI__builtin_ia32_xgetbv:
14622 case X86::BI_xgetbv:
14624 case X86::BI__builtin_ia32_storedqudi128_mask:
14625 case X86::BI__builtin_ia32_storedqusi128_mask:
14626 case X86::BI__builtin_ia32_storedquhi128_mask:
14627 case X86::BI__builtin_ia32_storedquqi128_mask:
14628 case X86::BI__builtin_ia32_storeupd128_mask:
14629 case X86::BI__builtin_ia32_storeups128_mask:
14630 case X86::BI__builtin_ia32_storedqudi256_mask:
14631 case X86::BI__builtin_ia32_storedqusi256_mask:
14632 case X86::BI__builtin_ia32_storedquhi256_mask:
14633 case X86::BI__builtin_ia32_storedquqi256_mask:
14634 case X86::BI__builtin_ia32_storeupd256_mask:
14635 case X86::BI__builtin_ia32_storeups256_mask:
14636 case X86::BI__builtin_ia32_storedqudi512_mask:
14637 case X86::BI__builtin_ia32_storedqusi512_mask:
14638 case X86::BI__builtin_ia32_storedquhi512_mask:
14639 case X86::BI__builtin_ia32_storedquqi512_mask:
14640 case X86::BI__builtin_ia32_storeupd512_mask:
14641 case X86::BI__builtin_ia32_storeups512_mask:
14644 case X86::BI__builtin_ia32_storesh128_mask:
14645 case X86::BI__builtin_ia32_storess128_mask:
14646 case X86::BI__builtin_ia32_storesd128_mask:
14649 case X86::BI__builtin_ia32_vpopcntb_128:
14650 case X86::BI__builtin_ia32_vpopcntd_128:
14651 case X86::BI__builtin_ia32_vpopcntq_128:
14652 case X86::BI__builtin_ia32_vpopcntw_128:
14653 case X86::BI__builtin_ia32_vpopcntb_256:
14654 case X86::BI__builtin_ia32_vpopcntd_256:
14655 case X86::BI__builtin_ia32_vpopcntq_256:
14656 case X86::BI__builtin_ia32_vpopcntw_256:
14657 case X86::BI__builtin_ia32_vpopcntb_512:
14658 case X86::BI__builtin_ia32_vpopcntd_512:
14659 case X86::BI__builtin_ia32_vpopcntq_512:
14660 case X86::BI__builtin_ia32_vpopcntw_512: {
14663 return Builder.CreateCall(F, Ops);
14665 case X86::BI__builtin_ia32_cvtmask2b128:
14666 case X86::BI__builtin_ia32_cvtmask2b256:
14667 case X86::BI__builtin_ia32_cvtmask2b512:
14668 case X86::BI__builtin_ia32_cvtmask2w128:
14669 case X86::BI__builtin_ia32_cvtmask2w256:
14670 case X86::BI__builtin_ia32_cvtmask2w512:
14671 case X86::BI__builtin_ia32_cvtmask2d128:
14672 case X86::BI__builtin_ia32_cvtmask2d256:
14673 case X86::BI__builtin_ia32_cvtmask2d512:
14674 case X86::BI__builtin_ia32_cvtmask2q128:
14675 case X86::BI__builtin_ia32_cvtmask2q256:
14676 case X86::BI__builtin_ia32_cvtmask2q512:
14679 case X86::BI__builtin_ia32_cvtb2mask128:
14680 case X86::BI__builtin_ia32_cvtb2mask256:
14681 case X86::BI__builtin_ia32_cvtb2mask512:
14682 case X86::BI__builtin_ia32_cvtw2mask128:
14683 case X86::BI__builtin_ia32_cvtw2mask256:
14684 case X86::BI__builtin_ia32_cvtw2mask512:
14685 case X86::BI__builtin_ia32_cvtd2mask128:
14686 case X86::BI__builtin_ia32_cvtd2mask256:
14687 case X86::BI__builtin_ia32_cvtd2mask512:
14688 case X86::BI__builtin_ia32_cvtq2mask128:
14689 case X86::BI__builtin_ia32_cvtq2mask256:
14690 case X86::BI__builtin_ia32_cvtq2mask512:
14693 case X86::BI__builtin_ia32_cvtdq2ps512_mask:
14694 case X86::BI__builtin_ia32_cvtqq2ps512_mask:
14695 case X86::BI__builtin_ia32_cvtqq2pd512_mask:
14696 case X86::BI__builtin_ia32_vcvtw2ph512_mask:
14697 case X86::BI__builtin_ia32_vcvtdq2ph512_mask:
14698 case X86::BI__builtin_ia32_vcvtqq2ph512_mask:
14700 case X86::BI__builtin_ia32_cvtudq2ps512_mask:
14701 case X86::BI__builtin_ia32_cvtuqq2ps512_mask:
14702 case X86::BI__builtin_ia32_cvtuqq2pd512_mask:
14703 case X86::BI__builtin_ia32_vcvtuw2ph512_mask:
14704 case X86::BI__builtin_ia32_vcvtudq2ph512_mask:
14705 case X86::BI__builtin_ia32_vcvtuqq2ph512_mask:
14708 case X86::BI__builtin_ia32_vfmaddss3:
14709 case X86::BI__builtin_ia32_vfmaddsd3:
14710 case X86::BI__builtin_ia32_vfmaddsh3_mask:
14711 case X86::BI__builtin_ia32_vfmaddss3_mask:
14712 case X86::BI__builtin_ia32_vfmaddsd3_mask:
14714 case X86::BI__builtin_ia32_vfmaddss:
14715 case X86::BI__builtin_ia32_vfmaddsd:
14717 Constant::getNullValue(Ops[0]->getType()));
14718 case X86::BI__builtin_ia32_vfmaddsh3_maskz:
14719 case X86::BI__builtin_ia32_vfmaddss3_maskz:
14720 case X86::BI__builtin_ia32_vfmaddsd3_maskz:
14722 case X86::BI__builtin_ia32_vfmaddsh3_mask3:
14723 case X86::BI__builtin_ia32_vfmaddss3_mask3:
14724 case X86::BI__builtin_ia32_vfmaddsd3_mask3:
14726 case X86::BI__builtin_ia32_vfmsubsh3_mask3:
14727 case X86::BI__builtin_ia32_vfmsubss3_mask3:
14728 case X86::BI__builtin_ia32_vfmsubsd3_mask3:
14731 case X86::BI__builtin_ia32_vfmaddph:
14732 case X86::BI__builtin_ia32_vfmaddps:
14733 case X86::BI__builtin_ia32_vfmaddpd:
14734 case X86::BI__builtin_ia32_vfmaddph256:
14735 case X86::BI__builtin_ia32_vfmaddps256:
14736 case X86::BI__builtin_ia32_vfmaddpd256:
14737 case X86::BI__builtin_ia32_vfmaddph512_mask:
14738 case X86::BI__builtin_ia32_vfmaddph512_maskz:
14739 case X86::BI__builtin_ia32_vfmaddph512_mask3:
14740 case X86::BI__builtin_ia32_vfmaddps512_mask:
14741 case X86::BI__builtin_ia32_vfmaddps512_maskz:
14742 case X86::BI__builtin_ia32_vfmaddps512_mask3:
14743 case X86::BI__builtin_ia32_vfmsubps512_mask3:
14744 case X86::BI__builtin_ia32_vfmaddpd512_mask:
14745 case X86::BI__builtin_ia32_vfmaddpd512_maskz:
14746 case X86::BI__builtin_ia32_vfmaddpd512_mask3:
14747 case X86::BI__builtin_ia32_vfmsubpd512_mask3:
14748 case X86::BI__builtin_ia32_vfmsubph512_mask3:
14750 case X86::BI__builtin_ia32_vfmaddsubph512_mask:
14751 case X86::BI__builtin_ia32_vfmaddsubph512_maskz:
14752 case X86::BI__builtin_ia32_vfmaddsubph512_mask3:
14753 case X86::BI__builtin_ia32_vfmsubaddph512_mask3:
14754 case X86::BI__builtin_ia32_vfmaddsubps512_mask:
14755 case X86::BI__builtin_ia32_vfmaddsubps512_maskz:
14756 case X86::BI__builtin_ia32_vfmaddsubps512_mask3:
14757 case X86::BI__builtin_ia32_vfmsubaddps512_mask3:
14758 case X86::BI__builtin_ia32_vfmaddsubpd512_mask:
14759 case X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
14760 case X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
14761 case X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
14764 case X86::BI__builtin_ia32_movdqa32store128_mask:
14765 case X86::BI__builtin_ia32_movdqa64store128_mask:
14766 case X86::BI__builtin_ia32_storeaps128_mask:
14767 case X86::BI__builtin_ia32_storeapd128_mask:
14768 case X86::BI__builtin_ia32_movdqa32store256_mask:
14769 case X86::BI__builtin_ia32_movdqa64store256_mask:
14770 case X86::BI__builtin_ia32_storeaps256_mask:
14771 case X86::BI__builtin_ia32_storeapd256_mask:
14772 case X86::BI__builtin_ia32_movdqa32store512_mask:
14773 case X86::BI__builtin_ia32_movdqa64store512_mask:
14774 case X86::BI__builtin_ia32_storeaps512_mask:
14775 case X86::BI__builtin_ia32_storeapd512_mask:
14780 case X86::BI__builtin_ia32_loadups128_mask:
14781 case X86::BI__builtin_ia32_loadups256_mask:
14782 case X86::BI__builtin_ia32_loadups512_mask:
14783 case X86::BI__builtin_ia32_loadupd128_mask:
14784 case X86::BI__builtin_ia32_loadupd256_mask:
14785 case X86::BI__builtin_ia32_loadupd512_mask:
14786 case X86::BI__builtin_ia32_loaddquqi128_mask:
14787 case X86::BI__builtin_ia32_loaddquqi256_mask:
14788 case X86::BI__builtin_ia32_loaddquqi512_mask:
14789 case X86::BI__builtin_ia32_loaddquhi128_mask:
14790 case X86::BI__builtin_ia32_loaddquhi256_mask:
14791 case X86::BI__builtin_ia32_loaddquhi512_mask:
14792 case X86::BI__builtin_ia32_loaddqusi128_mask:
14793 case X86::BI__builtin_ia32_loaddqusi256_mask:
14794 case X86::BI__builtin_ia32_loaddqusi512_mask:
14795 case X86::BI__builtin_ia32_loaddqudi128_mask:
14796 case X86::BI__builtin_ia32_loaddqudi256_mask:
14797 case X86::BI__builtin_ia32_loaddqudi512_mask:
14800 case X86::BI__builtin_ia32_loadsh128_mask:
14801 case X86::BI__builtin_ia32_loadss128_mask:
14802 case X86::BI__builtin_ia32_loadsd128_mask:
14805 case X86::BI__builtin_ia32_loadaps128_mask:
14806 case X86::BI__builtin_ia32_loadaps256_mask:
14807 case X86::BI__builtin_ia32_loadaps512_mask:
14808 case X86::BI__builtin_ia32_loadapd128_mask:
14809 case X86::BI__builtin_ia32_loadapd256_mask:
14810 case X86::BI__builtin_ia32_loadapd512_mask:
14811 case X86::BI__builtin_ia32_movdqa32load128_mask:
14812 case X86::BI__builtin_ia32_movdqa32load256_mask:
14813 case X86::BI__builtin_ia32_movdqa32load512_mask:
14814 case X86::BI__builtin_ia32_movdqa64load128_mask:
14815 case X86::BI__builtin_ia32_movdqa64load256_mask:
14816 case X86::BI__builtin_ia32_movdqa64load512_mask:
14821 case X86::BI__builtin_ia32_expandloaddf128_mask:
14822 case X86::BI__builtin_ia32_expandloaddf256_mask:
14823 case X86::BI__builtin_ia32_expandloaddf512_mask:
14824 case X86::BI__builtin_ia32_expandloadsf128_mask:
14825 case X86::BI__builtin_ia32_expandloadsf256_mask:
14826 case X86::BI__builtin_ia32_expandloadsf512_mask:
14827 case X86::BI__builtin_ia32_expandloaddi128_mask:
14828 case X86::BI__builtin_ia32_expandloaddi256_mask:
14829 case X86::BI__builtin_ia32_expandloaddi512_mask:
14830 case X86::BI__builtin_ia32_expandloadsi128_mask:
14831 case X86::BI__builtin_ia32_expandloadsi256_mask:
14832 case X86::BI__builtin_ia32_expandloadsi512_mask:
14833 case X86::BI__builtin_ia32_expandloadhi128_mask:
14834 case X86::BI__builtin_ia32_expandloadhi256_mask:
14835 case X86::BI__builtin_ia32_expandloadhi512_mask:
14836 case X86::BI__builtin_ia32_expandloadqi128_mask:
14837 case X86::BI__builtin_ia32_expandloadqi256_mask:
14838 case X86::BI__builtin_ia32_expandloadqi512_mask:
14841 case X86::BI__builtin_ia32_compressstoredf128_mask:
14842 case X86::BI__builtin_ia32_compressstoredf256_mask:
14843 case X86::BI__builtin_ia32_compressstoredf512_mask:
14844 case X86::BI__builtin_ia32_compressstoresf128_mask:
14845 case X86::BI__builtin_ia32_compressstoresf256_mask:
14846 case X86::BI__builtin_ia32_compressstoresf512_mask:
14847 case X86::BI__builtin_ia32_compressstoredi128_mask:
14848 case X86::BI__builtin_ia32_compressstoredi256_mask:
14849 case X86::BI__builtin_ia32_compressstoredi512_mask:
14850 case X86::BI__builtin_ia32_compressstoresi128_mask:
14851 case X86::BI__builtin_ia32_compressstoresi256_mask:
14852 case X86::BI__builtin_ia32_compressstoresi512_mask:
14853 case X86::BI__builtin_ia32_compressstorehi128_mask:
14854 case X86::BI__builtin_ia32_compressstorehi256_mask:
14855 case X86::BI__builtin_ia32_compressstorehi512_mask:
14856 case X86::BI__builtin_ia32_compressstoreqi128_mask:
14857 case X86::BI__builtin_ia32_compressstoreqi256_mask:
14858 case X86::BI__builtin_ia32_compressstoreqi512_mask:
14861 case X86::BI__builtin_ia32_expanddf128_mask:
14862 case X86::BI__builtin_ia32_expanddf256_mask:
14863 case X86::BI__builtin_ia32_expanddf512_mask:
14864 case X86::BI__builtin_ia32_expandsf128_mask:
14865 case X86::BI__builtin_ia32_expandsf256_mask:
14866 case X86::BI__builtin_ia32_expandsf512_mask:
14867 case X86::BI__builtin_ia32_expanddi128_mask:
14868 case X86::BI__builtin_ia32_expanddi256_mask:
14869 case X86::BI__builtin_ia32_expanddi512_mask:
14870 case X86::BI__builtin_ia32_expandsi128_mask:
14871 case X86::BI__builtin_ia32_expandsi256_mask:
14872 case X86::BI__builtin_ia32_expandsi512_mask:
14873 case X86::BI__builtin_ia32_expandhi128_mask:
14874 case X86::BI__builtin_ia32_expandhi256_mask:
14875 case X86::BI__builtin_ia32_expandhi512_mask:
14876 case X86::BI__builtin_ia32_expandqi128_mask:
14877 case X86::BI__builtin_ia32_expandqi256_mask:
14878 case X86::BI__builtin_ia32_expandqi512_mask:
14881 case X86::BI__builtin_ia32_compressdf128_mask:
14882 case X86::BI__builtin_ia32_compressdf256_mask:
14883 case X86::BI__builtin_ia32_compressdf512_mask:
14884 case X86::BI__builtin_ia32_compresssf128_mask:
14885 case X86::BI__builtin_ia32_compresssf256_mask:
14886 case X86::BI__builtin_ia32_compresssf512_mask:
14887 case X86::BI__builtin_ia32_compressdi128_mask:
14888 case X86::BI__builtin_ia32_compressdi256_mask:
14889 case X86::BI__builtin_ia32_compressdi512_mask:
14890 case X86::BI__builtin_ia32_compresssi128_mask:
14891 case X86::BI__builtin_ia32_compresssi256_mask:
14892 case X86::BI__builtin_ia32_compresssi512_mask:
14893 case X86::BI__builtin_ia32_compresshi128_mask:
14894 case X86::BI__builtin_ia32_compresshi256_mask:
14895 case X86::BI__builtin_ia32_compresshi512_mask:
14896 case X86::BI__builtin_ia32_compressqi128_mask:
14897 case X86::BI__builtin_ia32_compressqi256_mask:
14898 case X86::BI__builtin_ia32_compressqi512_mask:
14901 case X86::BI__builtin_ia32_gather3div2df:
14902 case X86::BI__builtin_ia32_gather3div2di:
14903 case X86::BI__builtin_ia32_gather3div4df:
14904 case X86::BI__builtin_ia32_gather3div4di:
14905 case X86::BI__builtin_ia32_gather3div4sf:
14906 case X86::BI__builtin_ia32_gather3div4si:
14907 case X86::BI__builtin_ia32_gather3div8sf:
14908 case X86::BI__builtin_ia32_gather3div8si:
14909 case X86::BI__builtin_ia32_gather3siv2df:
14910 case X86::BI__builtin_ia32_gather3siv2di:
14911 case X86::BI__builtin_ia32_gather3siv4df:
14912 case X86::BI__builtin_ia32_gather3siv4di:
14913 case X86::BI__builtin_ia32_gather3siv4sf:
14914 case X86::BI__builtin_ia32_gather3siv4si:
14915 case X86::BI__builtin_ia32_gather3siv8sf:
14916 case X86::BI__builtin_ia32_gather3siv8si:
14917 case X86::BI__builtin_ia32_gathersiv8df:
14918 case X86::BI__builtin_ia32_gathersiv16sf:
14919 case X86::BI__builtin_ia32_gatherdiv8df:
14920 case X86::BI__builtin_ia32_gatherdiv16sf:
14921 case X86::BI__builtin_ia32_gathersiv8di:
14922 case X86::BI__builtin_ia32_gathersiv16si:
14923 case X86::BI__builtin_ia32_gatherdiv8di:
14924 case X86::BI__builtin_ia32_gatherdiv16si: {
14926 switch (BuiltinID) {
14927 default: llvm_unreachable(
"Unexpected builtin");
14928 case X86::BI__builtin_ia32_gather3div2df:
14929 IID = Intrinsic::x86_avx512_mask_gather3div2_df;
14931 case X86::BI__builtin_ia32_gather3div2di:
14932 IID = Intrinsic::x86_avx512_mask_gather3div2_di;
14934 case X86::BI__builtin_ia32_gather3div4df:
14935 IID = Intrinsic::x86_avx512_mask_gather3div4_df;
14937 case X86::BI__builtin_ia32_gather3div4di:
14938 IID = Intrinsic::x86_avx512_mask_gather3div4_di;
14940 case X86::BI__builtin_ia32_gather3div4sf:
14941 IID = Intrinsic::x86_avx512_mask_gather3div4_sf;
14943 case X86::BI__builtin_ia32_gather3div4si:
14944 IID = Intrinsic::x86_avx512_mask_gather3div4_si;
14946 case X86::BI__builtin_ia32_gather3div8sf:
14947 IID = Intrinsic::x86_avx512_mask_gather3div8_sf;
14949 case X86::BI__builtin_ia32_gather3div8si:
14950 IID = Intrinsic::x86_avx512_mask_gather3div8_si;
14952 case X86::BI__builtin_ia32_gather3siv2df:
14953 IID = Intrinsic::x86_avx512_mask_gather3siv2_df;
14955 case X86::BI__builtin_ia32_gather3siv2di:
14956 IID = Intrinsic::x86_avx512_mask_gather3siv2_di;
14958 case X86::BI__builtin_ia32_gather3siv4df:
14959 IID = Intrinsic::x86_avx512_mask_gather3siv4_df;
14961 case X86::BI__builtin_ia32_gather3siv4di:
14962 IID = Intrinsic::x86_avx512_mask_gather3siv4_di;
14964 case X86::BI__builtin_ia32_gather3siv4sf:
14965 IID = Intrinsic::x86_avx512_mask_gather3siv4_sf;
14967 case X86::BI__builtin_ia32_gather3siv4si:
14968 IID = Intrinsic::x86_avx512_mask_gather3siv4_si;
14970 case X86::BI__builtin_ia32_gather3siv8sf:
14971 IID = Intrinsic::x86_avx512_mask_gather3siv8_sf;
14973 case X86::BI__builtin_ia32_gather3siv8si:
14974 IID = Intrinsic::x86_avx512_mask_gather3siv8_si;
14976 case X86::BI__builtin_ia32_gathersiv8df:
14977 IID = Intrinsic::x86_avx512_mask_gather_dpd_512;
14979 case X86::BI__builtin_ia32_gathersiv16sf:
14980 IID = Intrinsic::x86_avx512_mask_gather_dps_512;
14982 case X86::BI__builtin_ia32_gatherdiv8df:
14983 IID = Intrinsic::x86_avx512_mask_gather_qpd_512;
14985 case X86::BI__builtin_ia32_gatherdiv16sf:
14986 IID = Intrinsic::x86_avx512_mask_gather_qps_512;
14988 case X86::BI__builtin_ia32_gathersiv8di:
14989 IID = Intrinsic::x86_avx512_mask_gather_dpq_512;
14991 case X86::BI__builtin_ia32_gathersiv16si:
14992 IID = Intrinsic::x86_avx512_mask_gather_dpi_512;
14994 case X86::BI__builtin_ia32_gatherdiv8di:
14995 IID = Intrinsic::x86_avx512_mask_gather_qpq_512;
14997 case X86::BI__builtin_ia32_gatherdiv16si:
14998 IID = Intrinsic::x86_avx512_mask_gather_qpi_512;
15002 unsigned MinElts = std::min(
15003 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(),
15004 cast<llvm::FixedVectorType>(Ops[2]->getType())->getNumElements());
15007 return Builder.CreateCall(Intr, Ops);
15010 case X86::BI__builtin_ia32_scattersiv8df:
15011 case X86::BI__builtin_ia32_scattersiv16sf:
15012 case X86::BI__builtin_ia32_scatterdiv8df:
15013 case X86::BI__builtin_ia32_scatterdiv16sf:
15014 case X86::BI__builtin_ia32_scattersiv8di:
15015 case X86::BI__builtin_ia32_scattersiv16si:
15016 case X86::BI__builtin_ia32_scatterdiv8di:
15017 case X86::BI__builtin_ia32_scatterdiv16si:
15018 case X86::BI__builtin_ia32_scatterdiv2df:
15019 case X86::BI__builtin_ia32_scatterdiv2di:
15020 case X86::BI__builtin_ia32_scatterdiv4df:
15021 case X86::BI__builtin_ia32_scatterdiv4di:
15022 case X86::BI__builtin_ia32_scatterdiv4sf:
15023 case X86::BI__builtin_ia32_scatterdiv4si:
15024 case X86::BI__builtin_ia32_scatterdiv8sf:
15025 case X86::BI__builtin_ia32_scatterdiv8si:
15026 case X86::BI__builtin_ia32_scattersiv2df:
15027 case X86::BI__builtin_ia32_scattersiv2di:
15028 case X86::BI__builtin_ia32_scattersiv4df:
15029 case X86::BI__builtin_ia32_scattersiv4di:
15030 case X86::BI__builtin_ia32_scattersiv4sf:
15031 case X86::BI__builtin_ia32_scattersiv4si:
15032 case X86::BI__builtin_ia32_scattersiv8sf:
15033 case X86::BI__builtin_ia32_scattersiv8si: {
15035 switch (BuiltinID) {
15036 default: llvm_unreachable(
"Unexpected builtin");
15037 case X86::BI__builtin_ia32_scattersiv8df:
15038 IID = Intrinsic::x86_avx512_mask_scatter_dpd_512;
15040 case X86::BI__builtin_ia32_scattersiv16sf:
15041 IID = Intrinsic::x86_avx512_mask_scatter_dps_512;
15043 case X86::BI__builtin_ia32_scatterdiv8df:
15044 IID = Intrinsic::x86_avx512_mask_scatter_qpd_512;
15046 case X86::BI__builtin_ia32_scatterdiv16sf:
15047 IID = Intrinsic::x86_avx512_mask_scatter_qps_512;
15049 case X86::BI__builtin_ia32_scattersiv8di:
15050 IID = Intrinsic::x86_avx512_mask_scatter_dpq_512;
15052 case X86::BI__builtin_ia32_scattersiv16si:
15053 IID = Intrinsic::x86_avx512_mask_scatter_dpi_512;
15055 case X86::BI__builtin_ia32_scatterdiv8di:
15056 IID = Intrinsic::x86_avx512_mask_scatter_qpq_512;
15058 case X86::BI__builtin_ia32_scatterdiv16si:
15059 IID = Intrinsic::x86_avx512_mask_scatter_qpi_512;
15061 case X86::BI__builtin_ia32_scatterdiv2df:
15062 IID = Intrinsic::x86_avx512_mask_scatterdiv2_df;
15064 case X86::BI__builtin_ia32_scatterdiv2di:
15065 IID = Intrinsic::x86_avx512_mask_scatterdiv2_di;
15067 case X86::BI__builtin_ia32_scatterdiv4df:
15068 IID = Intrinsic::x86_avx512_mask_scatterdiv4_df;
15070 case X86::BI__builtin_ia32_scatterdiv4di:
15071 IID = Intrinsic::x86_avx512_mask_scatterdiv4_di;
15073 case X86::BI__builtin_ia32_scatterdiv4sf:
15074 IID = Intrinsic::x86_avx512_mask_scatterdiv4_sf;
15076 case X86::BI__builtin_ia32_scatterdiv4si:
15077 IID = Intrinsic::x86_avx512_mask_scatterdiv4_si;
15079 case X86::BI__builtin_ia32_scatterdiv8sf:
15080 IID = Intrinsic::x86_avx512_mask_scatterdiv8_sf;
15082 case X86::BI__builtin_ia32_scatterdiv8si:
15083 IID = Intrinsic::x86_avx512_mask_scatterdiv8_si;
15085 case X86::BI__builtin_ia32_scattersiv2df:
15086 IID = Intrinsic::x86_avx512_mask_scattersiv2_df;
15088 case X86::BI__builtin_ia32_scattersiv2di:
15089 IID = Intrinsic::x86_avx512_mask_scattersiv2_di;
15091 case X86::BI__builtin_ia32_scattersiv4df:
15092 IID = Intrinsic::x86_avx512_mask_scattersiv4_df;
15094 case X86::BI__builtin_ia32_scattersiv4di:
15095 IID = Intrinsic::x86_avx512_mask_scattersiv4_di;
15097 case X86::BI__builtin_ia32_scattersiv4sf:
15098 IID = Intrinsic::x86_avx512_mask_scattersiv4_sf;
15100 case X86::BI__builtin_ia32_scattersiv4si:
15101 IID = Intrinsic::x86_avx512_mask_scattersiv4_si;
15103 case X86::BI__builtin_ia32_scattersiv8sf:
15104 IID = Intrinsic::x86_avx512_mask_scattersiv8_sf;
15106 case X86::BI__builtin_ia32_scattersiv8si:
15107 IID = Intrinsic::x86_avx512_mask_scattersiv8_si;
15111 unsigned MinElts = std::min(
15112 cast<llvm::FixedVectorType>(Ops[2]->getType())->getNumElements(),
15113 cast<llvm::FixedVectorType>(Ops[3]->getType())->getNumElements());
15116 return Builder.CreateCall(Intr, Ops);
15119 case X86::BI__builtin_ia32_vextractf128_pd256:
15120 case X86::BI__builtin_ia32_vextractf128_ps256:
15121 case X86::BI__builtin_ia32_vextractf128_si256:
15122 case X86::BI__builtin_ia32_extract128i256:
15123 case X86::BI__builtin_ia32_extractf64x4_mask:
15124 case X86::BI__builtin_ia32_extractf32x4_mask:
15125 case X86::BI__builtin_ia32_extracti64x4_mask:
15126 case X86::BI__builtin_ia32_extracti32x4_mask:
15127 case X86::BI__builtin_ia32_extractf32x8_mask:
15128 case X86::BI__builtin_ia32_extracti32x8_mask:
15129 case X86::BI__builtin_ia32_extractf32x4_256_mask:
15130 case X86::BI__builtin_ia32_extracti32x4_256_mask:
15131 case X86::BI__builtin_ia32_extractf64x2_256_mask:
15132 case X86::BI__builtin_ia32_extracti64x2_256_mask:
15133 case X86::BI__builtin_ia32_extractf64x2_512_mask:
15134 case X86::BI__builtin_ia32_extracti64x2_512_mask: {
15136 unsigned NumElts = DstTy->getNumElements();
15137 unsigned SrcNumElts =
15138 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15139 unsigned SubVectors = SrcNumElts / NumElts;
15140 unsigned Index = cast<ConstantInt>(Ops[1])->getZExtValue();
15141 assert(llvm::isPowerOf2_32(SubVectors) &&
"Expected power of 2 subvectors");
15142 Index &= SubVectors - 1;
15146 for (
unsigned i = 0; i != NumElts; ++i)
15147 Indices[i] = i + Index;
15152 if (Ops.size() == 4)
15157 case X86::BI__builtin_ia32_vinsertf128_pd256:
15158 case X86::BI__builtin_ia32_vinsertf128_ps256:
15159 case X86::BI__builtin_ia32_vinsertf128_si256:
15160 case X86::BI__builtin_ia32_insert128i256:
15161 case X86::BI__builtin_ia32_insertf64x4:
15162 case X86::BI__builtin_ia32_insertf32x4:
15163 case X86::BI__builtin_ia32_inserti64x4:
15164 case X86::BI__builtin_ia32_inserti32x4:
15165 case X86::BI__builtin_ia32_insertf32x8:
15166 case X86::BI__builtin_ia32_inserti32x8:
15167 case X86::BI__builtin_ia32_insertf32x4_256:
15168 case X86::BI__builtin_ia32_inserti32x4_256:
15169 case X86::BI__builtin_ia32_insertf64x2_256:
15170 case X86::BI__builtin_ia32_inserti64x2_256:
15171 case X86::BI__builtin_ia32_insertf64x2_512:
15172 case X86::BI__builtin_ia32_inserti64x2_512: {
15173 unsigned DstNumElts =
15174 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15175 unsigned SrcNumElts =
15176 cast<llvm::FixedVectorType>(Ops[1]->getType())->getNumElements();
15177 unsigned SubVectors = DstNumElts / SrcNumElts;
15178 unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue();
15179 assert(llvm::isPowerOf2_32(SubVectors) &&
"Expected power of 2 subvectors");
15180 Index &= SubVectors - 1;
15181 Index *= SrcNumElts;
15184 for (
unsigned i = 0; i != DstNumElts; ++i)
15185 Indices[i] = (i >= SrcNumElts) ? SrcNumElts + (i % SrcNumElts) : i;
15188 Ops[1],
ArrayRef(Indices, DstNumElts),
"widen");
15190 for (
unsigned i = 0; i != DstNumElts; ++i) {
15191 if (i >= Index && i < (Index + SrcNumElts))
15192 Indices[i] = (i - Index) + DstNumElts;
15197 return Builder.CreateShuffleVector(Ops[0], Op1,
15198 ArrayRef(Indices, DstNumElts),
"insert");
15200 case X86::BI__builtin_ia32_pmovqd512_mask:
15201 case X86::BI__builtin_ia32_pmovwb512_mask: {
15202 Value *Res =
Builder.CreateTrunc(Ops[0], Ops[1]->getType());
15205 case X86::BI__builtin_ia32_pmovdb512_mask:
15206 case X86::BI__builtin_ia32_pmovdw512_mask:
15207 case X86::BI__builtin_ia32_pmovqw512_mask: {
15208 if (
const auto *
C = dyn_cast<Constant>(Ops[2]))
15209 if (
C->isAllOnesValue())
15210 return Builder.CreateTrunc(Ops[0], Ops[1]->getType());
15213 switch (BuiltinID) {
15214 default: llvm_unreachable(
"Unsupported intrinsic!");
15215 case X86::BI__builtin_ia32_pmovdb512_mask:
15216 IID = Intrinsic::x86_avx512_mask_pmov_db_512;
15218 case X86::BI__builtin_ia32_pmovdw512_mask:
15219 IID = Intrinsic::x86_avx512_mask_pmov_dw_512;
15221 case X86::BI__builtin_ia32_pmovqw512_mask:
15222 IID = Intrinsic::x86_avx512_mask_pmov_qw_512;
15227 return Builder.CreateCall(Intr, Ops);
15229 case X86::BI__builtin_ia32_pblendw128:
15230 case X86::BI__builtin_ia32_blendpd:
15231 case X86::BI__builtin_ia32_blendps:
15232 case X86::BI__builtin_ia32_blendpd256:
15233 case X86::BI__builtin_ia32_blendps256:
15234 case X86::BI__builtin_ia32_pblendw256:
15235 case X86::BI__builtin_ia32_pblendd128:
15236 case X86::BI__builtin_ia32_pblendd256: {
15238 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15239 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
15244 for (
unsigned i = 0; i != NumElts; ++i)
15245 Indices[i] = ((Imm >> (i % 8)) & 0x1) ? NumElts + i : i;
15247 return Builder.CreateShuffleVector(Ops[0], Ops[1],
15248 ArrayRef(Indices, NumElts),
"blend");
15250 case X86::BI__builtin_ia32_pshuflw:
15251 case X86::BI__builtin_ia32_pshuflw256:
15252 case X86::BI__builtin_ia32_pshuflw512: {
15253 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
15254 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15255 unsigned NumElts = Ty->getNumElements();
15258 Imm = (Imm & 0xff) * 0x01010101;
15261 for (
unsigned l = 0; l != NumElts; l += 8) {
15262 for (
unsigned i = 0; i != 4; ++i) {
15263 Indices[l + i] = l + (Imm & 3);
15266 for (
unsigned i = 4; i != 8; ++i)
15267 Indices[l + i] = l + i;
15270 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
15273 case X86::BI__builtin_ia32_pshufhw:
15274 case X86::BI__builtin_ia32_pshufhw256:
15275 case X86::BI__builtin_ia32_pshufhw512: {
15276 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
15277 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15278 unsigned NumElts = Ty->getNumElements();
15281 Imm = (Imm & 0xff) * 0x01010101;
15284 for (
unsigned l = 0; l != NumElts; l += 8) {
15285 for (
unsigned i = 0; i != 4; ++i)
15286 Indices[l + i] = l + i;
15287 for (
unsigned i = 4; i != 8; ++i) {
15288 Indices[l + i] = l + 4 + (Imm & 3);
15293 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
15296 case X86::BI__builtin_ia32_pshufd:
15297 case X86::BI__builtin_ia32_pshufd256:
15298 case X86::BI__builtin_ia32_pshufd512:
15299 case X86::BI__builtin_ia32_vpermilpd:
15300 case X86::BI__builtin_ia32_vpermilps:
15301 case X86::BI__builtin_ia32_vpermilpd256:
15302 case X86::BI__builtin_ia32_vpermilps256:
15303 case X86::BI__builtin_ia32_vpermilpd512:
15304 case X86::BI__builtin_ia32_vpermilps512: {
15305 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
15306 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15307 unsigned NumElts = Ty->getNumElements();
15308 unsigned NumLanes = Ty->getPrimitiveSizeInBits() / 128;
15309 unsigned NumLaneElts = NumElts / NumLanes;
15312 Imm = (Imm & 0xff) * 0x01010101;
15315 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
15316 for (
unsigned i = 0; i != NumLaneElts; ++i) {
15317 Indices[i + l] = (Imm % NumLaneElts) + l;
15318 Imm /= NumLaneElts;
15322 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
15325 case X86::BI__builtin_ia32_shufpd:
15326 case X86::BI__builtin_ia32_shufpd256:
15327 case X86::BI__builtin_ia32_shufpd512:
15328 case X86::BI__builtin_ia32_shufps:
15329 case X86::BI__builtin_ia32_shufps256:
15330 case X86::BI__builtin_ia32_shufps512: {
15331 uint32_t Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
15332 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15333 unsigned NumElts = Ty->getNumElements();
15334 unsigned NumLanes = Ty->getPrimitiveSizeInBits() / 128;
15335 unsigned NumLaneElts = NumElts / NumLanes;
15338 Imm = (Imm & 0xff) * 0x01010101;
15341 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
15342 for (
unsigned i = 0; i != NumLaneElts; ++i) {
15343 unsigned Index = Imm % NumLaneElts;
15344 Imm /= NumLaneElts;
15345 if (i >= (NumLaneElts / 2))
15347 Indices[l + i] = l + Index;
15351 return Builder.CreateShuffleVector(Ops[0], Ops[1],
15352 ArrayRef(Indices, NumElts),
"shufp");
15354 case X86::BI__builtin_ia32_permdi256:
15355 case X86::BI__builtin_ia32_permdf256:
15356 case X86::BI__builtin_ia32_permdi512:
15357 case X86::BI__builtin_ia32_permdf512: {
15358 unsigned Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
15359 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15360 unsigned NumElts = Ty->getNumElements();
15364 for (
unsigned l = 0; l != NumElts; l += 4)
15365 for (
unsigned i = 0; i != 4; ++i)
15366 Indices[l + i] = l + ((Imm >> (2 * i)) & 0x3);
15368 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
15371 case X86::BI__builtin_ia32_palignr128:
15372 case X86::BI__builtin_ia32_palignr256:
15373 case X86::BI__builtin_ia32_palignr512: {
15374 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff;
15377 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15378 assert(NumElts % 16 == 0);
15382 if (ShiftVal >= 32)
15387 if (ShiftVal > 16) {
15390 Ops[0] = llvm::Constant::getNullValue(Ops[0]->getType());
15395 for (
unsigned l = 0; l != NumElts; l += 16) {
15396 for (
unsigned i = 0; i != 16; ++i) {
15397 unsigned Idx = ShiftVal + i;
15399 Idx += NumElts - 16;
15400 Indices[l + i] = Idx + l;
15404 return Builder.CreateShuffleVector(Ops[1], Ops[0],
15405 ArrayRef(Indices, NumElts),
"palignr");
15407 case X86::BI__builtin_ia32_alignd128:
15408 case X86::BI__builtin_ia32_alignd256:
15409 case X86::BI__builtin_ia32_alignd512:
15410 case X86::BI__builtin_ia32_alignq128:
15411 case X86::BI__builtin_ia32_alignq256:
15412 case X86::BI__builtin_ia32_alignq512: {
15414 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15415 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff;
15418 ShiftVal &= NumElts - 1;
15421 for (
unsigned i = 0; i != NumElts; ++i)
15422 Indices[i] = i + ShiftVal;
15424 return Builder.CreateShuffleVector(Ops[1], Ops[0],
15425 ArrayRef(Indices, NumElts),
"valign");
15427 case X86::BI__builtin_ia32_shuf_f32x4_256:
15428 case X86::BI__builtin_ia32_shuf_f64x2_256:
15429 case X86::BI__builtin_ia32_shuf_i32x4_256:
15430 case X86::BI__builtin_ia32_shuf_i64x2_256:
15431 case X86::BI__builtin_ia32_shuf_f32x4:
15432 case X86::BI__builtin_ia32_shuf_f64x2:
15433 case X86::BI__builtin_ia32_shuf_i32x4:
15434 case X86::BI__builtin_ia32_shuf_i64x2: {
15435 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
15436 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15437 unsigned NumElts = Ty->getNumElements();
15438 unsigned NumLanes = Ty->getPrimitiveSizeInBits() == 512 ? 4 : 2;
15439 unsigned NumLaneElts = NumElts / NumLanes;
15442 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
15443 unsigned Index = (Imm % NumLanes) * NumLaneElts;
15445 if (l >= (NumElts / 2))
15447 for (
unsigned i = 0; i != NumLaneElts; ++i) {
15448 Indices[l + i] = Index + i;
15452 return Builder.CreateShuffleVector(Ops[0], Ops[1],
15453 ArrayRef(Indices, NumElts),
"shuf");
15456 case X86::BI__builtin_ia32_vperm2f128_pd256:
15457 case X86::BI__builtin_ia32_vperm2f128_ps256:
15458 case X86::BI__builtin_ia32_vperm2f128_si256:
15459 case X86::BI__builtin_ia32_permti256: {
15460 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
15462 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15471 for (
unsigned l = 0; l != 2; ++l) {
15473 if (Imm & (1 << ((l * 4) + 3)))
15474 OutOps[l] = llvm::ConstantAggregateZero::get(Ops[0]->getType());
15475 else if (Imm & (1 << ((l * 4) + 1)))
15476 OutOps[l] = Ops[1];
15478 OutOps[l] = Ops[0];
15480 for (
unsigned i = 0; i != NumElts/2; ++i) {
15482 unsigned Idx = (l * NumElts) + i;
15485 if (Imm & (1 << (l * 4)))
15487 Indices[(l * (NumElts/2)) + i] = Idx;
15491 return Builder.CreateShuffleVector(OutOps[0], OutOps[1],
15492 ArrayRef(Indices, NumElts),
"vperm");
15495 case X86::BI__builtin_ia32_pslldqi128_byteshift:
15496 case X86::BI__builtin_ia32_pslldqi256_byteshift:
15497 case X86::BI__builtin_ia32_pslldqi512_byteshift: {
15498 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
15499 auto *ResultType = cast<llvm::FixedVectorType>(Ops[0]->getType());
15501 unsigned NumElts = ResultType->getNumElements() * 8;
15504 if (ShiftVal >= 16)
15505 return llvm::Constant::getNullValue(ResultType);
15509 for (
unsigned l = 0; l != NumElts; l += 16) {
15510 for (
unsigned i = 0; i != 16; ++i) {
15511 unsigned Idx = NumElts + i - ShiftVal;
15512 if (Idx < NumElts) Idx -= NumElts - 16;
15513 Indices[l + i] = Idx + l;
15517 auto *VecTy = llvm::FixedVectorType::get(
Int8Ty, NumElts);
15519 Value *
Zero = llvm::Constant::getNullValue(VecTy);
15521 Zero, Cast,
ArrayRef(Indices, NumElts),
"pslldq");
15522 return Builder.CreateBitCast(SV, Ops[0]->getType(),
"cast");
15524 case X86::BI__builtin_ia32_psrldqi128_byteshift:
15525 case X86::BI__builtin_ia32_psrldqi256_byteshift:
15526 case X86::BI__builtin_ia32_psrldqi512_byteshift: {
15527 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
15528 auto *ResultType = cast<llvm::FixedVectorType>(Ops[0]->getType());
15530 unsigned NumElts = ResultType->getNumElements() * 8;
15533 if (ShiftVal >= 16)
15534 return llvm::Constant::getNullValue(ResultType);
15538 for (
unsigned l = 0; l != NumElts; l += 16) {
15539 for (
unsigned i = 0; i != 16; ++i) {
15540 unsigned Idx = i + ShiftVal;
15541 if (Idx >= 16) Idx += NumElts - 16;
15542 Indices[l + i] = Idx + l;
15546 auto *VecTy = llvm::FixedVectorType::get(
Int8Ty, NumElts);
15548 Value *
Zero = llvm::Constant::getNullValue(VecTy);
15550 Cast, Zero,
ArrayRef(Indices, NumElts),
"psrldq");
15551 return Builder.CreateBitCast(SV, ResultType,
"cast");
15553 case X86::BI__builtin_ia32_kshiftliqi:
15554 case X86::BI__builtin_ia32_kshiftlihi:
15555 case X86::BI__builtin_ia32_kshiftlisi:
15556 case X86::BI__builtin_ia32_kshiftlidi: {
15557 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
15558 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15560 if (ShiftVal >= NumElts)
15561 return llvm::Constant::getNullValue(Ops[0]->getType());
15566 for (
unsigned i = 0; i != NumElts; ++i)
15567 Indices[i] = NumElts + i - ShiftVal;
15569 Value *
Zero = llvm::Constant::getNullValue(
In->getType());
15571 Zero, In,
ArrayRef(Indices, NumElts),
"kshiftl");
15572 return Builder.CreateBitCast(SV, Ops[0]->getType());
15574 case X86::BI__builtin_ia32_kshiftriqi:
15575 case X86::BI__builtin_ia32_kshiftrihi:
15576 case X86::BI__builtin_ia32_kshiftrisi:
15577 case X86::BI__builtin_ia32_kshiftridi: {
15578 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
15579 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15581 if (ShiftVal >= NumElts)
15582 return llvm::Constant::getNullValue(Ops[0]->getType());
15587 for (
unsigned i = 0; i != NumElts; ++i)
15588 Indices[i] = i + ShiftVal;
15590 Value *
Zero = llvm::Constant::getNullValue(
In->getType());
15592 In, Zero,
ArrayRef(Indices, NumElts),
"kshiftr");
15593 return Builder.CreateBitCast(SV, Ops[0]->getType());
15595 case X86::BI__builtin_ia32_movnti:
15596 case X86::BI__builtin_ia32_movnti64:
15597 case X86::BI__builtin_ia32_movntsd:
15598 case X86::BI__builtin_ia32_movntss: {
15599 llvm::MDNode *
Node = llvm::MDNode::get(
15602 Value *Ptr = Ops[0];
15603 Value *Src = Ops[1];
15606 if (BuiltinID == X86::BI__builtin_ia32_movntsd ||
15607 BuiltinID == X86::BI__builtin_ia32_movntss)
15608 Src =
Builder.CreateExtractElement(Src, (uint64_t)0,
"extract");
15612 SI->setMetadata(llvm::LLVMContext::MD_nontemporal,
Node);
15613 SI->setAlignment(llvm::Align(1));
15617 case X86::BI__builtin_ia32_vprotb:
15618 case X86::BI__builtin_ia32_vprotw:
15619 case X86::BI__builtin_ia32_vprotd:
15620 case X86::BI__builtin_ia32_vprotq:
15621 case X86::BI__builtin_ia32_vprotbi:
15622 case X86::BI__builtin_ia32_vprotwi:
15623 case X86::BI__builtin_ia32_vprotdi:
15624 case X86::BI__builtin_ia32_vprotqi:
15625 case X86::BI__builtin_ia32_prold128:
15626 case X86::BI__builtin_ia32_prold256:
15627 case X86::BI__builtin_ia32_prold512:
15628 case X86::BI__builtin_ia32_prolq128:
15629 case X86::BI__builtin_ia32_prolq256:
15630 case X86::BI__builtin_ia32_prolq512:
15631 case X86::BI__builtin_ia32_prolvd128:
15632 case X86::BI__builtin_ia32_prolvd256:
15633 case X86::BI__builtin_ia32_prolvd512:
15634 case X86::BI__builtin_ia32_prolvq128:
15635 case X86::BI__builtin_ia32_prolvq256:
15636 case X86::BI__builtin_ia32_prolvq512:
15638 case X86::BI__builtin_ia32_prord128:
15639 case X86::BI__builtin_ia32_prord256:
15640 case X86::BI__builtin_ia32_prord512:
15641 case X86::BI__builtin_ia32_prorq128:
15642 case X86::BI__builtin_ia32_prorq256:
15643 case X86::BI__builtin_ia32_prorq512:
15644 case X86::BI__builtin_ia32_prorvd128:
15645 case X86::BI__builtin_ia32_prorvd256:
15646 case X86::BI__builtin_ia32_prorvd512:
15647 case X86::BI__builtin_ia32_prorvq128:
15648 case X86::BI__builtin_ia32_prorvq256:
15649 case X86::BI__builtin_ia32_prorvq512:
15651 case X86::BI__builtin_ia32_selectb_128:
15652 case X86::BI__builtin_ia32_selectb_256:
15653 case X86::BI__builtin_ia32_selectb_512:
15654 case X86::BI__builtin_ia32_selectw_128:
15655 case X86::BI__builtin_ia32_selectw_256:
15656 case X86::BI__builtin_ia32_selectw_512:
15657 case X86::BI__builtin_ia32_selectd_128:
15658 case X86::BI__builtin_ia32_selectd_256:
15659 case X86::BI__builtin_ia32_selectd_512:
15660 case X86::BI__builtin_ia32_selectq_128:
15661 case X86::BI__builtin_ia32_selectq_256:
15662 case X86::BI__builtin_ia32_selectq_512:
15663 case X86::BI__builtin_ia32_selectph_128:
15664 case X86::BI__builtin_ia32_selectph_256:
15665 case X86::BI__builtin_ia32_selectph_512:
15666 case X86::BI__builtin_ia32_selectpbf_128:
15667 case X86::BI__builtin_ia32_selectpbf_256:
15668 case X86::BI__builtin_ia32_selectpbf_512:
15669 case X86::BI__builtin_ia32_selectps_128:
15670 case X86::BI__builtin_ia32_selectps_256:
15671 case X86::BI__builtin_ia32_selectps_512:
15672 case X86::BI__builtin_ia32_selectpd_128:
15673 case X86::BI__builtin_ia32_selectpd_256:
15674 case X86::BI__builtin_ia32_selectpd_512:
15676 case X86::BI__builtin_ia32_selectsh_128:
15677 case X86::BI__builtin_ia32_selectsbf_128:
15678 case X86::BI__builtin_ia32_selectss_128:
15679 case X86::BI__builtin_ia32_selectsd_128: {
15680 Value *A =
Builder.CreateExtractElement(Ops[1], (uint64_t)0);
15681 Value *B =
Builder.CreateExtractElement(Ops[2], (uint64_t)0);
15683 return Builder.CreateInsertElement(Ops[1], A, (uint64_t)0);
15685 case X86::BI__builtin_ia32_cmpb128_mask:
15686 case X86::BI__builtin_ia32_cmpb256_mask:
15687 case X86::BI__builtin_ia32_cmpb512_mask:
15688 case X86::BI__builtin_ia32_cmpw128_mask:
15689 case X86::BI__builtin_ia32_cmpw256_mask:
15690 case X86::BI__builtin_ia32_cmpw512_mask:
15691 case X86::BI__builtin_ia32_cmpd128_mask:
15692 case X86::BI__builtin_ia32_cmpd256_mask:
15693 case X86::BI__builtin_ia32_cmpd512_mask:
15694 case X86::BI__builtin_ia32_cmpq128_mask:
15695 case X86::BI__builtin_ia32_cmpq256_mask:
15696 case X86::BI__builtin_ia32_cmpq512_mask: {
15697 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
15700 case X86::BI__builtin_ia32_ucmpb128_mask:
15701 case X86::BI__builtin_ia32_ucmpb256_mask:
15702 case X86::BI__builtin_ia32_ucmpb512_mask:
15703 case X86::BI__builtin_ia32_ucmpw128_mask:
15704 case X86::BI__builtin_ia32_ucmpw256_mask:
15705 case X86::BI__builtin_ia32_ucmpw512_mask:
15706 case X86::BI__builtin_ia32_ucmpd128_mask:
15707 case X86::BI__builtin_ia32_ucmpd256_mask:
15708 case X86::BI__builtin_ia32_ucmpd512_mask:
15709 case X86::BI__builtin_ia32_ucmpq128_mask:
15710 case X86::BI__builtin_ia32_ucmpq256_mask:
15711 case X86::BI__builtin_ia32_ucmpq512_mask: {
15712 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
15715 case X86::BI__builtin_ia32_vpcomb:
15716 case X86::BI__builtin_ia32_vpcomw:
15717 case X86::BI__builtin_ia32_vpcomd:
15718 case X86::BI__builtin_ia32_vpcomq:
15720 case X86::BI__builtin_ia32_vpcomub:
15721 case X86::BI__builtin_ia32_vpcomuw:
15722 case X86::BI__builtin_ia32_vpcomud:
15723 case X86::BI__builtin_ia32_vpcomuq:
15726 case X86::BI__builtin_ia32_kortestcqi:
15727 case X86::BI__builtin_ia32_kortestchi:
15728 case X86::BI__builtin_ia32_kortestcsi:
15729 case X86::BI__builtin_ia32_kortestcdi: {
15731 Value *
C = llvm::Constant::getAllOnesValue(Ops[0]->getType());
15735 case X86::BI__builtin_ia32_kortestzqi:
15736 case X86::BI__builtin_ia32_kortestzhi:
15737 case X86::BI__builtin_ia32_kortestzsi:
15738 case X86::BI__builtin_ia32_kortestzdi: {
15740 Value *
C = llvm::Constant::getNullValue(Ops[0]->getType());
15745 case X86::BI__builtin_ia32_ktestcqi:
15746 case X86::BI__builtin_ia32_ktestzqi:
15747 case X86::BI__builtin_ia32_ktestchi:
15748 case X86::BI__builtin_ia32_ktestzhi:
15749 case X86::BI__builtin_ia32_ktestcsi:
15750 case X86::BI__builtin_ia32_ktestzsi:
15751 case X86::BI__builtin_ia32_ktestcdi:
15752 case X86::BI__builtin_ia32_ktestzdi: {
15754 switch (BuiltinID) {
15755 default: llvm_unreachable(
"Unsupported intrinsic!");
15756 case X86::BI__builtin_ia32_ktestcqi:
15757 IID = Intrinsic::x86_avx512_ktestc_b;
15759 case X86::BI__builtin_ia32_ktestzqi:
15760 IID = Intrinsic::x86_avx512_ktestz_b;
15762 case X86::BI__builtin_ia32_ktestchi:
15763 IID = Intrinsic::x86_avx512_ktestc_w;
15765 case X86::BI__builtin_ia32_ktestzhi:
15766 IID = Intrinsic::x86_avx512_ktestz_w;
15768 case X86::BI__builtin_ia32_ktestcsi:
15769 IID = Intrinsic::x86_avx512_ktestc_d;
15771 case X86::BI__builtin_ia32_ktestzsi:
15772 IID = Intrinsic::x86_avx512_ktestz_d;
15774 case X86::BI__builtin_ia32_ktestcdi:
15775 IID = Intrinsic::x86_avx512_ktestc_q;
15777 case X86::BI__builtin_ia32_ktestzdi:
15778 IID = Intrinsic::x86_avx512_ktestz_q;
15782 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15786 return Builder.CreateCall(Intr, {LHS, RHS});
15789 case X86::BI__builtin_ia32_kaddqi:
15790 case X86::BI__builtin_ia32_kaddhi:
15791 case X86::BI__builtin_ia32_kaddsi:
15792 case X86::BI__builtin_ia32_kadddi: {
15794 switch (BuiltinID) {
15795 default: llvm_unreachable(
"Unsupported intrinsic!");
15796 case X86::BI__builtin_ia32_kaddqi:
15797 IID = Intrinsic::x86_avx512_kadd_b;
15799 case X86::BI__builtin_ia32_kaddhi:
15800 IID = Intrinsic::x86_avx512_kadd_w;
15802 case X86::BI__builtin_ia32_kaddsi:
15803 IID = Intrinsic::x86_avx512_kadd_d;
15805 case X86::BI__builtin_ia32_kadddi:
15806 IID = Intrinsic::x86_avx512_kadd_q;
15810 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15815 return Builder.CreateBitCast(Res, Ops[0]->getType());
15817 case X86::BI__builtin_ia32_kandqi:
15818 case X86::BI__builtin_ia32_kandhi:
15819 case X86::BI__builtin_ia32_kandsi:
15820 case X86::BI__builtin_ia32_kanddi:
15822 case X86::BI__builtin_ia32_kandnqi:
15823 case X86::BI__builtin_ia32_kandnhi:
15824 case X86::BI__builtin_ia32_kandnsi:
15825 case X86::BI__builtin_ia32_kandndi:
15827 case X86::BI__builtin_ia32_korqi:
15828 case X86::BI__builtin_ia32_korhi:
15829 case X86::BI__builtin_ia32_korsi:
15830 case X86::BI__builtin_ia32_kordi:
15832 case X86::BI__builtin_ia32_kxnorqi:
15833 case X86::BI__builtin_ia32_kxnorhi:
15834 case X86::BI__builtin_ia32_kxnorsi:
15835 case X86::BI__builtin_ia32_kxnordi:
15837 case X86::BI__builtin_ia32_kxorqi:
15838 case X86::BI__builtin_ia32_kxorhi:
15839 case X86::BI__builtin_ia32_kxorsi:
15840 case X86::BI__builtin_ia32_kxordi:
15842 case X86::BI__builtin_ia32_knotqi:
15843 case X86::BI__builtin_ia32_knothi:
15844 case X86::BI__builtin_ia32_knotsi:
15845 case X86::BI__builtin_ia32_knotdi: {
15846 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15849 Ops[0]->getType());
15851 case X86::BI__builtin_ia32_kmovb:
15852 case X86::BI__builtin_ia32_kmovw:
15853 case X86::BI__builtin_ia32_kmovd:
15854 case X86::BI__builtin_ia32_kmovq: {
15858 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15860 return Builder.CreateBitCast(Res, Ops[0]->getType());
15863 case X86::BI__builtin_ia32_kunpckdi:
15864 case X86::BI__builtin_ia32_kunpcksi:
15865 case X86::BI__builtin_ia32_kunpckhi: {
15866 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15870 for (
unsigned i = 0; i != NumElts; ++i)
15875 LHS =
Builder.CreateShuffleVector(LHS, LHS,
ArrayRef(Indices, NumElts / 2));
15876 RHS =
Builder.CreateShuffleVector(RHS, RHS,
ArrayRef(Indices, NumElts / 2));
15881 return Builder.CreateBitCast(Res, Ops[0]->getType());
15884 case X86::BI__builtin_ia32_vplzcntd_128:
15885 case X86::BI__builtin_ia32_vplzcntd_256:
15886 case X86::BI__builtin_ia32_vplzcntd_512:
15887 case X86::BI__builtin_ia32_vplzcntq_128:
15888 case X86::BI__builtin_ia32_vplzcntq_256:
15889 case X86::BI__builtin_ia32_vplzcntq_512: {
15893 case X86::BI__builtin_ia32_sqrtss:
15894 case X86::BI__builtin_ia32_sqrtsd: {
15895 Value *A =
Builder.CreateExtractElement(Ops[0], (uint64_t)0);
15897 if (
Builder.getIsFPConstrained()) {
15898 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
15901 A =
Builder.CreateConstrainedFPCall(F, {A});
15904 A =
Builder.CreateCall(F, {A});
15906 return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0);
15908 case X86::BI__builtin_ia32_sqrtsh_round_mask:
15909 case X86::BI__builtin_ia32_sqrtsd_round_mask:
15910 case X86::BI__builtin_ia32_sqrtss_round_mask: {
15911 unsigned CC = cast<llvm::ConstantInt>(Ops[4])->getZExtValue();
15917 switch (BuiltinID) {
15919 llvm_unreachable(
"Unsupported intrinsic!");
15920 case X86::BI__builtin_ia32_sqrtsh_round_mask:
15921 IID = Intrinsic::x86_avx512fp16_mask_sqrt_sh;
15923 case X86::BI__builtin_ia32_sqrtsd_round_mask:
15924 IID = Intrinsic::x86_avx512_mask_sqrt_sd;
15926 case X86::BI__builtin_ia32_sqrtss_round_mask:
15927 IID = Intrinsic::x86_avx512_mask_sqrt_ss;
15932 Value *A =
Builder.CreateExtractElement(Ops[1], (uint64_t)0);
15934 if (
Builder.getIsFPConstrained()) {
15935 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
15938 A =
Builder.CreateConstrainedFPCall(F, A);
15941 A =
Builder.CreateCall(F, A);
15943 Value *Src =
Builder.CreateExtractElement(Ops[2], (uint64_t)0);
15945 return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0);
15947 case X86::BI__builtin_ia32_sqrtpd256:
15948 case X86::BI__builtin_ia32_sqrtpd:
15949 case X86::BI__builtin_ia32_sqrtps256:
15950 case X86::BI__builtin_ia32_sqrtps:
15951 case X86::BI__builtin_ia32_sqrtph256:
15952 case X86::BI__builtin_ia32_sqrtph:
15953 case X86::BI__builtin_ia32_sqrtph512:
15954 case X86::BI__builtin_ia32_sqrtps512:
15955 case X86::BI__builtin_ia32_sqrtpd512: {
15956 if (Ops.size() == 2) {
15957 unsigned CC = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
15963 switch (BuiltinID) {
15965 llvm_unreachable(
"Unsupported intrinsic!");
15966 case X86::BI__builtin_ia32_sqrtph512:
15967 IID = Intrinsic::x86_avx512fp16_sqrt_ph_512;
15969 case X86::BI__builtin_ia32_sqrtps512:
15970 IID = Intrinsic::x86_avx512_sqrt_ps_512;
15972 case X86::BI__builtin_ia32_sqrtpd512:
15973 IID = Intrinsic::x86_avx512_sqrt_pd_512;
15979 if (
Builder.getIsFPConstrained()) {
15980 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
15982 Ops[0]->getType());
15983 return Builder.CreateConstrainedFPCall(F, Ops[0]);
15986 return Builder.CreateCall(F, Ops[0]);
15990 case X86::BI__builtin_ia32_pmuludq128:
15991 case X86::BI__builtin_ia32_pmuludq256:
15992 case X86::BI__builtin_ia32_pmuludq512:
15995 case X86::BI__builtin_ia32_pmuldq128:
15996 case X86::BI__builtin_ia32_pmuldq256:
15997 case X86::BI__builtin_ia32_pmuldq512:
16000 case X86::BI__builtin_ia32_pternlogd512_mask:
16001 case X86::BI__builtin_ia32_pternlogq512_mask:
16002 case X86::BI__builtin_ia32_pternlogd128_mask:
16003 case X86::BI__builtin_ia32_pternlogd256_mask:
16004 case X86::BI__builtin_ia32_pternlogq128_mask:
16005 case X86::BI__builtin_ia32_pternlogq256_mask:
16008 case X86::BI__builtin_ia32_pternlogd512_maskz:
16009 case X86::BI__builtin_ia32_pternlogq512_maskz:
16010 case X86::BI__builtin_ia32_pternlogd128_maskz:
16011 case X86::BI__builtin_ia32_pternlogd256_maskz:
16012 case X86::BI__builtin_ia32_pternlogq128_maskz:
16013 case X86::BI__builtin_ia32_pternlogq256_maskz:
16016 case X86::BI__builtin_ia32_vpshldd128:
16017 case X86::BI__builtin_ia32_vpshldd256:
16018 case X86::BI__builtin_ia32_vpshldd512:
16019 case X86::BI__builtin_ia32_vpshldq128:
16020 case X86::BI__builtin_ia32_vpshldq256:
16021 case X86::BI__builtin_ia32_vpshldq512:
16022 case X86::BI__builtin_ia32_vpshldw128:
16023 case X86::BI__builtin_ia32_vpshldw256:
16024 case X86::BI__builtin_ia32_vpshldw512:
16027 case X86::BI__builtin_ia32_vpshrdd128:
16028 case X86::BI__builtin_ia32_vpshrdd256:
16029 case X86::BI__builtin_ia32_vpshrdd512:
16030 case X86::BI__builtin_ia32_vpshrdq128:
16031 case X86::BI__builtin_ia32_vpshrdq256:
16032 case X86::BI__builtin_ia32_vpshrdq512:
16033 case X86::BI__builtin_ia32_vpshrdw128:
16034 case X86::BI__builtin_ia32_vpshrdw256:
16035 case X86::BI__builtin_ia32_vpshrdw512:
16039 case X86::BI__builtin_ia32_vpshldvd128:
16040 case X86::BI__builtin_ia32_vpshldvd256:
16041 case X86::BI__builtin_ia32_vpshldvd512:
16042 case X86::BI__builtin_ia32_vpshldvq128:
16043 case X86::BI__builtin_ia32_vpshldvq256:
16044 case X86::BI__builtin_ia32_vpshldvq512:
16045 case X86::BI__builtin_ia32_vpshldvw128:
16046 case X86::BI__builtin_ia32_vpshldvw256:
16047 case X86::BI__builtin_ia32_vpshldvw512:
16050 case X86::BI__builtin_ia32_vpshrdvd128:
16051 case X86::BI__builtin_ia32_vpshrdvd256:
16052 case X86::BI__builtin_ia32_vpshrdvd512:
16053 case X86::BI__builtin_ia32_vpshrdvq128:
16054 case X86::BI__builtin_ia32_vpshrdvq256:
16055 case X86::BI__builtin_ia32_vpshrdvq512:
16056 case X86::BI__builtin_ia32_vpshrdvw128:
16057 case X86::BI__builtin_ia32_vpshrdvw256:
16058 case X86::BI__builtin_ia32_vpshrdvw512:
16063 case X86::BI__builtin_ia32_reduce_fadd_pd512:
16064 case X86::BI__builtin_ia32_reduce_fadd_ps512:
16065 case X86::BI__builtin_ia32_reduce_fadd_ph512:
16066 case X86::BI__builtin_ia32_reduce_fadd_ph256:
16067 case X86::BI__builtin_ia32_reduce_fadd_ph128: {
16070 IRBuilder<>::FastMathFlagGuard FMFGuard(
Builder);
16071 Builder.getFastMathFlags().setAllowReassoc();
16072 return Builder.CreateCall(F, {Ops[0], Ops[1]});
16074 case X86::BI__builtin_ia32_reduce_fmul_pd512:
16075 case X86::BI__builtin_ia32_reduce_fmul_ps512:
16076 case X86::BI__builtin_ia32_reduce_fmul_ph512:
16077 case X86::BI__builtin_ia32_reduce_fmul_ph256:
16078 case X86::BI__builtin_ia32_reduce_fmul_ph128: {
16081 IRBuilder<>::FastMathFlagGuard FMFGuard(
Builder);
16082 Builder.getFastMathFlags().setAllowReassoc();
16083 return Builder.CreateCall(F, {Ops[0], Ops[1]});
16085 case X86::BI__builtin_ia32_reduce_fmax_pd512:
16086 case X86::BI__builtin_ia32_reduce_fmax_ps512:
16087 case X86::BI__builtin_ia32_reduce_fmax_ph512:
16088 case X86::BI__builtin_ia32_reduce_fmax_ph256:
16089 case X86::BI__builtin_ia32_reduce_fmax_ph128: {
16092 IRBuilder<>::FastMathFlagGuard FMFGuard(
Builder);
16093 Builder.getFastMathFlags().setNoNaNs();
16094 return Builder.CreateCall(F, {Ops[0]});
16096 case X86::BI__builtin_ia32_reduce_fmin_pd512:
16097 case X86::BI__builtin_ia32_reduce_fmin_ps512:
16098 case X86::BI__builtin_ia32_reduce_fmin_ph512:
16099 case X86::BI__builtin_ia32_reduce_fmin_ph256:
16100 case X86::BI__builtin_ia32_reduce_fmin_ph128: {
16103 IRBuilder<>::FastMathFlagGuard FMFGuard(
Builder);
16104 Builder.getFastMathFlags().setNoNaNs();
16105 return Builder.CreateCall(F, {Ops[0]});
16108 case X86::BI__builtin_ia32_rdrand16_step:
16109 case X86::BI__builtin_ia32_rdrand32_step:
16110 case X86::BI__builtin_ia32_rdrand64_step:
16111 case X86::BI__builtin_ia32_rdseed16_step:
16112 case X86::BI__builtin_ia32_rdseed32_step:
16113 case X86::BI__builtin_ia32_rdseed64_step: {
16115 switch (BuiltinID) {
16116 default: llvm_unreachable(
"Unsupported intrinsic!");
16117 case X86::BI__builtin_ia32_rdrand16_step:
16118 ID = Intrinsic::x86_rdrand_16;
16120 case X86::BI__builtin_ia32_rdrand32_step:
16121 ID = Intrinsic::x86_rdrand_32;
16123 case X86::BI__builtin_ia32_rdrand64_step:
16124 ID = Intrinsic::x86_rdrand_64;
16126 case X86::BI__builtin_ia32_rdseed16_step:
16127 ID = Intrinsic::x86_rdseed_16;
16129 case X86::BI__builtin_ia32_rdseed32_step:
16130 ID = Intrinsic::x86_rdseed_32;
16132 case X86::BI__builtin_ia32_rdseed64_step:
16133 ID = Intrinsic::x86_rdseed_64;
16142 case X86::BI__builtin_ia32_addcarryx_u32:
16143 case X86::BI__builtin_ia32_addcarryx_u64:
16144 case X86::BI__builtin_ia32_subborrow_u32:
16145 case X86::BI__builtin_ia32_subborrow_u64: {
16147 switch (BuiltinID) {
16148 default: llvm_unreachable(
"Unsupported intrinsic!");
16149 case X86::BI__builtin_ia32_addcarryx_u32:
16150 IID = Intrinsic::x86_addcarry_32;
16152 case X86::BI__builtin_ia32_addcarryx_u64:
16153 IID = Intrinsic::x86_addcarry_64;
16155 case X86::BI__builtin_ia32_subborrow_u32:
16156 IID = Intrinsic::x86_subborrow_32;
16158 case X86::BI__builtin_ia32_subborrow_u64:
16159 IID = Intrinsic::x86_subborrow_64;
16164 { Ops[0], Ops[1], Ops[2] });
16170 case X86::BI__builtin_ia32_fpclassps128_mask:
16171 case X86::BI__builtin_ia32_fpclassps256_mask:
16172 case X86::BI__builtin_ia32_fpclassps512_mask:
16173 case X86::BI__builtin_ia32_fpclassph128_mask:
16174 case X86::BI__builtin_ia32_fpclassph256_mask:
16175 case X86::BI__builtin_ia32_fpclassph512_mask:
16176 case X86::BI__builtin_ia32_fpclasspd128_mask:
16177 case X86::BI__builtin_ia32_fpclasspd256_mask:
16178 case X86::BI__builtin_ia32_fpclasspd512_mask: {
16180 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16181 Value *MaskIn = Ops[2];
16182 Ops.erase(&Ops[2]);
16185 switch (BuiltinID) {
16186 default: llvm_unreachable(
"Unsupported intrinsic!");
16187 case X86::BI__builtin_ia32_fpclassph128_mask:
16188 ID = Intrinsic::x86_avx512fp16_fpclass_ph_128;
16190 case X86::BI__builtin_ia32_fpclassph256_mask:
16191 ID = Intrinsic::x86_avx512fp16_fpclass_ph_256;
16193 case X86::BI__builtin_ia32_fpclassph512_mask:
16194 ID = Intrinsic::x86_avx512fp16_fpclass_ph_512;
16196 case X86::BI__builtin_ia32_fpclassps128_mask:
16197 ID = Intrinsic::x86_avx512_fpclass_ps_128;
16199 case X86::BI__builtin_ia32_fpclassps256_mask:
16200 ID = Intrinsic::x86_avx512_fpclass_ps_256;
16202 case X86::BI__builtin_ia32_fpclassps512_mask:
16203 ID = Intrinsic::x86_avx512_fpclass_ps_512;
16205 case X86::BI__builtin_ia32_fpclasspd128_mask:
16206 ID = Intrinsic::x86_avx512_fpclass_pd_128;
16208 case X86::BI__builtin_ia32_fpclasspd256_mask:
16209 ID = Intrinsic::x86_avx512_fpclass_pd_256;
16211 case X86::BI__builtin_ia32_fpclasspd512_mask:
16212 ID = Intrinsic::x86_avx512_fpclass_pd_512;
16220 case X86::BI__builtin_ia32_vp2intersect_q_512:
16221 case X86::BI__builtin_ia32_vp2intersect_q_256:
16222 case X86::BI__builtin_ia32_vp2intersect_q_128:
16223 case X86::BI__builtin_ia32_vp2intersect_d_512:
16224 case X86::BI__builtin_ia32_vp2intersect_d_256:
16225 case X86::BI__builtin_ia32_vp2intersect_d_128: {
16227 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16230 switch (BuiltinID) {
16231 default: llvm_unreachable(
"Unsupported intrinsic!");
16232 case X86::BI__builtin_ia32_vp2intersect_q_512:
16233 ID = Intrinsic::x86_avx512_vp2intersect_q_512;
16235 case X86::BI__builtin_ia32_vp2intersect_q_256:
16236 ID = Intrinsic::x86_avx512_vp2intersect_q_256;
16238 case X86::BI__builtin_ia32_vp2intersect_q_128:
16239 ID = Intrinsic::x86_avx512_vp2intersect_q_128;
16241 case X86::BI__builtin_ia32_vp2intersect_d_512:
16242 ID = Intrinsic::x86_avx512_vp2intersect_d_512;
16244 case X86::BI__builtin_ia32_vp2intersect_d_256:
16245 ID = Intrinsic::x86_avx512_vp2intersect_d_256;
16247 case X86::BI__builtin_ia32_vp2intersect_d_128:
16248 ID = Intrinsic::x86_avx512_vp2intersect_d_128;
16262 case X86::BI__builtin_ia32_vpmultishiftqb128:
16263 case X86::BI__builtin_ia32_vpmultishiftqb256:
16264 case X86::BI__builtin_ia32_vpmultishiftqb512: {
16266 switch (BuiltinID) {
16267 default: llvm_unreachable(
"Unsupported intrinsic!");
16268 case X86::BI__builtin_ia32_vpmultishiftqb128:
16269 ID = Intrinsic::x86_avx512_pmultishift_qb_128;
16271 case X86::BI__builtin_ia32_vpmultishiftqb256:
16272 ID = Intrinsic::x86_avx512_pmultishift_qb_256;
16274 case X86::BI__builtin_ia32_vpmultishiftqb512:
16275 ID = Intrinsic::x86_avx512_pmultishift_qb_512;
16282 case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
16283 case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
16284 case X86::BI__builtin_ia32_vpshufbitqmb512_mask: {
16286 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16287 Value *MaskIn = Ops[2];
16288 Ops.erase(&Ops[2]);
16291 switch (BuiltinID) {
16292 default: llvm_unreachable(
"Unsupported intrinsic!");
16293 case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
16294 ID = Intrinsic::x86_avx512_vpshufbitqmb_128;
16296 case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
16297 ID = Intrinsic::x86_avx512_vpshufbitqmb_256;
16299 case X86::BI__builtin_ia32_vpshufbitqmb512_mask:
16300 ID = Intrinsic::x86_avx512_vpshufbitqmb_512;
16309 case X86::BI__builtin_ia32_cmpeqps:
16310 case X86::BI__builtin_ia32_cmpeqpd:
16311 return getVectorFCmpIR(CmpInst::FCMP_OEQ,
false);
16312 case X86::BI__builtin_ia32_cmpltps:
16313 case X86::BI__builtin_ia32_cmpltpd:
16314 return getVectorFCmpIR(CmpInst::FCMP_OLT,
true);
16315 case X86::BI__builtin_ia32_cmpleps:
16316 case X86::BI__builtin_ia32_cmplepd:
16317 return getVectorFCmpIR(CmpInst::FCMP_OLE,
true);
16318 case X86::BI__builtin_ia32_cmpunordps:
16319 case X86::BI__builtin_ia32_cmpunordpd:
16320 return getVectorFCmpIR(CmpInst::FCMP_UNO,
false);
16321 case X86::BI__builtin_ia32_cmpneqps:
16322 case X86::BI__builtin_ia32_cmpneqpd:
16323 return getVectorFCmpIR(CmpInst::FCMP_UNE,
false);
16324 case X86::BI__builtin_ia32_cmpnltps:
16325 case X86::BI__builtin_ia32_cmpnltpd:
16326 return getVectorFCmpIR(CmpInst::FCMP_UGE,
true);
16327 case X86::BI__builtin_ia32_cmpnleps:
16328 case X86::BI__builtin_ia32_cmpnlepd:
16329 return getVectorFCmpIR(CmpInst::FCMP_UGT,
true);
16330 case X86::BI__builtin_ia32_cmpordps:
16331 case X86::BI__builtin_ia32_cmpordpd:
16332 return getVectorFCmpIR(CmpInst::FCMP_ORD,
false);
16333 case X86::BI__builtin_ia32_cmpph128_mask:
16334 case X86::BI__builtin_ia32_cmpph256_mask:
16335 case X86::BI__builtin_ia32_cmpph512_mask:
16336 case X86::BI__builtin_ia32_cmpps128_mask:
16337 case X86::BI__builtin_ia32_cmpps256_mask:
16338 case X86::BI__builtin_ia32_cmpps512_mask:
16339 case X86::BI__builtin_ia32_cmppd128_mask:
16340 case X86::BI__builtin_ia32_cmppd256_mask:
16341 case X86::BI__builtin_ia32_cmppd512_mask:
16344 case X86::BI__builtin_ia32_cmpps:
16345 case X86::BI__builtin_ia32_cmpps256:
16346 case X86::BI__builtin_ia32_cmppd:
16347 case X86::BI__builtin_ia32_cmppd256: {
16355 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x1f;
16360 FCmpInst::Predicate Pred;
16364 switch (CC & 0xf) {
16365 case 0x00: Pred = FCmpInst::FCMP_OEQ; IsSignaling =
false;
break;
16366 case 0x01: Pred = FCmpInst::FCMP_OLT; IsSignaling =
true;
break;
16367 case 0x02: Pred = FCmpInst::FCMP_OLE; IsSignaling =
true;
break;
16368 case 0x03: Pred = FCmpInst::FCMP_UNO; IsSignaling =
false;
break;
16369 case 0x04: Pred = FCmpInst::FCMP_UNE; IsSignaling =
false;
break;
16370 case 0x05: Pred = FCmpInst::FCMP_UGE; IsSignaling =
true;
break;
16371 case 0x06: Pred = FCmpInst::FCMP_UGT; IsSignaling =
true;
break;
16372 case 0x07: Pred = FCmpInst::FCMP_ORD; IsSignaling =
false;
break;
16373 case 0x08: Pred = FCmpInst::FCMP_UEQ; IsSignaling =
false;
break;
16374 case 0x09: Pred = FCmpInst::FCMP_ULT; IsSignaling =
true;
break;
16375 case 0x0a: Pred = FCmpInst::FCMP_ULE; IsSignaling =
true;
break;
16376 case 0x0b: Pred = FCmpInst::FCMP_FALSE; IsSignaling =
false;
break;
16377 case 0x0c: Pred = FCmpInst::FCMP_ONE; IsSignaling =
false;
break;
16378 case 0x0d: Pred = FCmpInst::FCMP_OGE; IsSignaling =
true;
break;
16379 case 0x0e: Pred = FCmpInst::FCMP_OGT; IsSignaling =
true;
break;
16380 case 0x0f: Pred = FCmpInst::FCMP_TRUE; IsSignaling =
false;
break;
16381 default: llvm_unreachable(
"Unhandled CC");
16386 IsSignaling = !IsSignaling;
16393 if (
Builder.getIsFPConstrained() &&
16394 (Pred == FCmpInst::FCMP_TRUE || Pred == FCmpInst::FCMP_FALSE ||
16398 switch (BuiltinID) {
16399 default: llvm_unreachable(
"Unexpected builtin");
16400 case X86::BI__builtin_ia32_cmpps:
16401 IID = Intrinsic::x86_sse_cmp_ps;
16403 case X86::BI__builtin_ia32_cmpps256:
16404 IID = Intrinsic::x86_avx_cmp_ps_256;
16406 case X86::BI__builtin_ia32_cmppd:
16407 IID = Intrinsic::x86_sse2_cmp_pd;
16409 case X86::BI__builtin_ia32_cmppd256:
16410 IID = Intrinsic::x86_avx_cmp_pd_256;
16412 case X86::BI__builtin_ia32_cmpph128_mask:
16413 IID = Intrinsic::x86_avx512fp16_mask_cmp_ph_128;
16415 case X86::BI__builtin_ia32_cmpph256_mask:
16416 IID = Intrinsic::x86_avx512fp16_mask_cmp_ph_256;
16418 case X86::BI__builtin_ia32_cmpph512_mask:
16419 IID = Intrinsic::x86_avx512fp16_mask_cmp_ph_512;
16421 case X86::BI__builtin_ia32_cmpps512_mask:
16422 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
16424 case X86::BI__builtin_ia32_cmppd512_mask:
16425 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
16427 case X86::BI__builtin_ia32_cmpps128_mask:
16428 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
16430 case X86::BI__builtin_ia32_cmpps256_mask:
16431 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
16433 case X86::BI__builtin_ia32_cmppd128_mask:
16434 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
16436 case X86::BI__builtin_ia32_cmppd256_mask:
16437 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
16444 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16450 return Builder.CreateCall(Intr, Ops);
16461 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16464 Cmp =
Builder.CreateFCmpS(Pred, Ops[0], Ops[1]);
16466 Cmp =
Builder.CreateFCmp(Pred, Ops[0], Ops[1]);
16470 return getVectorFCmpIR(Pred, IsSignaling);
16474 case X86::BI__builtin_ia32_cmpeqss:
16475 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 0);
16476 case X86::BI__builtin_ia32_cmpltss:
16477 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 1);
16478 case X86::BI__builtin_ia32_cmpless:
16479 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 2);
16480 case X86::BI__builtin_ia32_cmpunordss:
16481 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 3);
16482 case X86::BI__builtin_ia32_cmpneqss:
16483 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 4);
16484 case X86::BI__builtin_ia32_cmpnltss:
16485 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 5);
16486 case X86::BI__builtin_ia32_cmpnless:
16487 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 6);
16488 case X86::BI__builtin_ia32_cmpordss:
16489 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 7);
16490 case X86::BI__builtin_ia32_cmpeqsd:
16491 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 0);
16492 case X86::BI__builtin_ia32_cmpltsd:
16493 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 1);
16494 case X86::BI__builtin_ia32_cmplesd:
16495 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 2);
16496 case X86::BI__builtin_ia32_cmpunordsd:
16497 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 3);
16498 case X86::BI__builtin_ia32_cmpneqsd:
16499 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 4);
16500 case X86::BI__builtin_ia32_cmpnltsd:
16501 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 5);
16502 case X86::BI__builtin_ia32_cmpnlesd:
16503 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 6);
16504 case X86::BI__builtin_ia32_cmpordsd:
16505 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 7);
16508 case X86::BI__builtin_ia32_vcvtph2ps:
16509 case X86::BI__builtin_ia32_vcvtph2ps256:
16510 case X86::BI__builtin_ia32_vcvtph2ps_mask:
16511 case X86::BI__builtin_ia32_vcvtph2ps256_mask:
16512 case X86::BI__builtin_ia32_vcvtph2ps512_mask: {
16513 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
16518 case X86::BI__builtin_ia32_cvtneps2bf16_128_mask: {
16521 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements());
16522 Intrinsic::ID IID = Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128;
16525 case X86::BI__builtin_ia32_cvtsbf162ss_32:
16528 case X86::BI__builtin_ia32_cvtneps2bf16_256_mask:
16529 case X86::BI__builtin_ia32_cvtneps2bf16_512_mask: {
16531 switch (BuiltinID) {
16532 default: llvm_unreachable(
"Unsupported intrinsic!");
16533 case X86::BI__builtin_ia32_cvtneps2bf16_256_mask:
16534 IID = Intrinsic::x86_avx512bf16_cvtneps2bf16_256;
16536 case X86::BI__builtin_ia32_cvtneps2bf16_512_mask:
16537 IID = Intrinsic::x86_avx512bf16_cvtneps2bf16_512;
16544 case X86::BI__cpuid:
16545 case X86::BI__cpuidex: {
16547 Value *SubFuncId = BuiltinID == X86::BI__cpuidex
16551 llvm::StructType *CpuidRetTy =
16553 llvm::FunctionType *FTy =
16556 StringRef
Asm, Constraints;
16557 if (
getTarget().getTriple().getArch() == llvm::Triple::x86) {
16559 Constraints =
"={ax},={bx},={cx},={dx},{ax},{cx}";
16562 Asm =
"xchgq %rbx, ${1:q}\n"
16564 "xchgq %rbx, ${1:q}";
16565 Constraints =
"={ax},=r,={cx},={dx},0,2";
16568 llvm::InlineAsm *IA = llvm::InlineAsm::get(FTy,
Asm, Constraints,
16570 Value *IACall =
Builder.CreateCall(IA, {FuncId, SubFuncId});
16573 for (
unsigned i = 0; i < 4; i++) {
16574 Value *Extracted =
Builder.CreateExtractValue(IACall, i);
16584 case X86::BI__emul:
16585 case X86::BI__emulu: {
16587 bool isSigned = (BuiltinID == X86::BI__emul);
16590 return Builder.CreateMul(LHS, RHS,
"", !isSigned, isSigned);
16592 case X86::BI__mulh:
16593 case X86::BI__umulh:
16594 case X86::BI_mul128:
16595 case X86::BI_umul128: {
16597 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
16599 bool IsSigned = (BuiltinID == X86::BI__mulh || BuiltinID == X86::BI_mul128);
16600 Value *LHS =
Builder.CreateIntCast(Ops[0], Int128Ty, IsSigned);
16601 Value *RHS =
Builder.CreateIntCast(Ops[1], Int128Ty, IsSigned);
16603 Value *MulResult, *HigherBits;
16605 MulResult =
Builder.CreateNSWMul(LHS, RHS);
16606 HigherBits =
Builder.CreateAShr(MulResult, 64);
16608 MulResult =
Builder.CreateNUWMul(LHS, RHS);
16609 HigherBits =
Builder.CreateLShr(MulResult, 64);
16611 HigherBits =
Builder.CreateIntCast(HigherBits, ResType, IsSigned);
16613 if (BuiltinID == X86::BI__mulh || BuiltinID == X86::BI__umulh)
16618 return Builder.CreateIntCast(MulResult, ResType, IsSigned);
16621 case X86::BI__faststorefence: {
16622 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
16623 llvm::SyncScope::System);
16625 case X86::BI__shiftleft128:
16626 case X86::BI__shiftright128: {
16628 BuiltinID == X86::BI__shiftleft128 ? Intrinsic::fshl : Intrinsic::fshr,
16633 std::swap(Ops[0], Ops[1]);
16635 return Builder.CreateCall(F, Ops);
16637 case X86::BI_ReadWriteBarrier:
16638 case X86::BI_ReadBarrier:
16639 case X86::BI_WriteBarrier: {
16640 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
16641 llvm::SyncScope::SingleThread);
16644 case X86::BI_AddressOfReturnAddress: {
16647 return Builder.CreateCall(F);
16649 case X86::BI__stosb: {
16657 case X86::BI__int2c: {
16659 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy,
false);
16660 llvm::InlineAsm *IA =
16661 llvm::InlineAsm::get(FTy,
"int $$0x2c",
"",
true);
16662 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
16664 llvm::Attribute::NoReturn);
16665 llvm::CallInst *CI =
Builder.CreateCall(IA);
16666 CI->setAttributes(NoReturnAttr);
16669 case X86::BI__readfsbyte:
16670 case X86::BI__readfsword:
16671 case X86::BI__readfsdword:
16672 case X86::BI__readfsqword: {
16678 Load->setVolatile(
true);
16681 case X86::BI__readgsbyte:
16682 case X86::BI__readgsword:
16683 case X86::BI__readgsdword:
16684 case X86::BI__readgsqword: {
16690 Load->setVolatile(
true);
16693 case X86::BI__builtin_ia32_encodekey128_u32: {
16694 Intrinsic::ID IID = Intrinsic::x86_encodekey128;
16698 for (
int i = 0; i < 3; ++i) {
16706 case X86::BI__builtin_ia32_encodekey256_u32: {
16707 Intrinsic::ID IID = Intrinsic::x86_encodekey256;
16712 for (
int i = 0; i < 4; ++i) {
16720 case X86::BI__builtin_ia32_aesenc128kl_u8:
16721 case X86::BI__builtin_ia32_aesdec128kl_u8:
16722 case X86::BI__builtin_ia32_aesenc256kl_u8:
16723 case X86::BI__builtin_ia32_aesdec256kl_u8: {
16725 StringRef BlockName;
16726 switch (BuiltinID) {
16728 llvm_unreachable(
"Unexpected builtin");
16729 case X86::BI__builtin_ia32_aesenc128kl_u8:
16730 IID = Intrinsic::x86_aesenc128kl;
16731 BlockName =
"aesenc128kl";
16733 case X86::BI__builtin_ia32_aesdec128kl_u8:
16734 IID = Intrinsic::x86_aesdec128kl;
16735 BlockName =
"aesdec128kl";
16737 case X86::BI__builtin_ia32_aesenc256kl_u8:
16738 IID = Intrinsic::x86_aesenc256kl;
16739 BlockName =
"aesenc256kl";
16741 case X86::BI__builtin_ia32_aesdec256kl_u8:
16742 IID = Intrinsic::x86_aesdec256kl;
16743 BlockName =
"aesdec256kl";
16749 BasicBlock *NoError =
16757 Builder.CreateCondBr(Succ, NoError, Error);
16759 Builder.SetInsertPoint(NoError);
16763 Builder.SetInsertPoint(Error);
16764 Constant *
Zero = llvm::Constant::getNullValue(Out->getType());
16771 case X86::BI__builtin_ia32_aesencwide128kl_u8:
16772 case X86::BI__builtin_ia32_aesdecwide128kl_u8:
16773 case X86::BI__builtin_ia32_aesencwide256kl_u8:
16774 case X86::BI__builtin_ia32_aesdecwide256kl_u8: {
16776 StringRef BlockName;
16777 switch (BuiltinID) {
16778 case X86::BI__builtin_ia32_aesencwide128kl_u8:
16779 IID = Intrinsic::x86_aesencwide128kl;
16780 BlockName =
"aesencwide128kl";
16782 case X86::BI__builtin_ia32_aesdecwide128kl_u8:
16783 IID = Intrinsic::x86_aesdecwide128kl;
16784 BlockName =
"aesdecwide128kl";
16786 case X86::BI__builtin_ia32_aesencwide256kl_u8:
16787 IID = Intrinsic::x86_aesencwide256kl;
16788 BlockName =
"aesencwide256kl";
16790 case X86::BI__builtin_ia32_aesdecwide256kl_u8:
16791 IID = Intrinsic::x86_aesdecwide256kl;
16792 BlockName =
"aesdecwide256kl";
16796 llvm::Type *Ty = FixedVectorType::get(
Builder.getInt64Ty(), 2);
16799 for (
int i = 0; i != 8; ++i) {
16800 Value *Ptr =
Builder.CreateConstGEP1_32(Ty, Ops[1], i);
16806 BasicBlock *NoError =
16813 Builder.CreateCondBr(Succ, NoError, Error);
16815 Builder.SetInsertPoint(NoError);
16816 for (
int i = 0; i != 8; ++i) {
16823 Builder.SetInsertPoint(Error);
16824 for (
int i = 0; i != 8; ++i) {
16826 Constant *
Zero = llvm::Constant::getNullValue(Out->getType());
16827 Value *Ptr =
Builder.CreateConstGEP1_32(Out->getType(), Ops[0], i);
16835 case X86::BI__builtin_ia32_vfcmaddcph512_mask:
16838 case X86::BI__builtin_ia32_vfmaddcph512_mask: {
16839 Intrinsic::ID IID = IsConjFMA
16840 ? Intrinsic::x86_avx512fp16_mask_vfcmadd_cph_512
16841 : Intrinsic::x86_avx512fp16_mask_vfmadd_cph_512;
16845 case X86::BI__builtin_ia32_vfcmaddcsh_round_mask:
16848 case X86::BI__builtin_ia32_vfmaddcsh_round_mask: {
16849 Intrinsic::ID IID = IsConjFMA ? Intrinsic::x86_avx512fp16_mask_vfcmadd_csh
16850 : Intrinsic::x86_avx512fp16_mask_vfmadd_csh;
16855 case X86::BI__builtin_ia32_vfcmaddcsh_round_mask3:
16858 case X86::BI__builtin_ia32_vfmaddcsh_round_mask3: {
16859 Intrinsic::ID IID = IsConjFMA ? Intrinsic::x86_avx512fp16_mask_vfcmadd_csh
16860 : Intrinsic::x86_avx512fp16_mask_vfmadd_csh;
16862 static constexpr int Mask[] = {0, 5, 6, 7};
16863 return Builder.CreateShuffleVector(
Call, Ops[2], Mask);
16865 case X86::BI__builtin_ia32_prefetchi:
16868 {Ops[0], llvm::ConstantInt::get(Int32Ty, 0), Ops[1],
16869 llvm::ConstantInt::get(Int32Ty, 0)});
16887 Intrinsic::ID
ID = Intrinsic::not_intrinsic;
16889#include "llvm/TargetParser/PPCTargetParser.def"
16890 auto GenAIXPPCBuiltinCpuExpr = [&](
unsigned SupportMethod,
unsigned FieldIdx,
16891 unsigned Mask, CmpInst::Predicate CompOp,
16892 unsigned OpValue) ->
Value * {
16893 if (SupportMethod == BUILTIN_PPC_FALSE)
16896 if (SupportMethod == BUILTIN_PPC_TRUE)
16899 assert(SupportMethod <= SYS_CALL &&
"Invalid value for SupportMethod.");
16901 llvm::Value *FieldValue =
nullptr;
16902 if (SupportMethod == USE_SYS_CONF) {
16903 llvm::Type *STy = llvm::StructType::get(PPC_SYSTEMCONFIG_TYPE);
16904 llvm::Constant *SysConf =
16908 llvm::Value *Idxs[] = {ConstantInt::get(
Int32Ty, 0),
16909 ConstantInt::get(
Int32Ty, FieldIdx)};
16914 }
else if (SupportMethod == SYS_CALL) {
16915 llvm::FunctionType *FTy =
16917 llvm::FunctionCallee
Func =
16923 assert(FieldValue &&
16924 "SupportMethod value is not defined in PPCTargetParser.def.");
16927 FieldValue =
Builder.CreateAnd(FieldValue, Mask);
16929 llvm::Type *ValueType = FieldValue->getType();
16930 bool IsValueType64Bit = ValueType->isIntegerTy(64);
16932 (IsValueType64Bit || ValueType->isIntegerTy(32)) &&
16933 "Only 32/64-bit integers are supported in GenAIXPPCBuiltinCpuExpr().");
16936 CompOp, FieldValue,
16937 ConstantInt::get(IsValueType64Bit ?
Int64Ty :
Int32Ty, OpValue));
16940 switch (BuiltinID) {
16941 default:
return nullptr;
16943 case Builtin::BI__builtin_cpu_is: {
16945 StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
16948 unsigned LinuxSupportMethod, LinuxIDValue, AIXSupportMethod, AIXIDValue;
16949 typedef std::tuple<unsigned, unsigned, unsigned, unsigned>
CPUInfo;
16951 std::tie(LinuxSupportMethod, LinuxIDValue, AIXSupportMethod, AIXIDValue) =
16952 static_cast<CPUInfo>(StringSwitch<CPUInfo>(CPUStr)
16953#define PPC_CPU(NAME, Linux_SUPPORT_METHOD, LinuxID, AIX_SUPPORT_METHOD, \
16955 .Case(NAME, {Linux_SUPPORT_METHOD, LinuxID, AIX_SUPPORT_METHOD, AIXID})
16956#include "llvm/TargetParser/PPCTargetParser.def"
16957 .Default({BUILTIN_PPC_UNSUPPORTED, 0,
16958 BUILTIN_PPC_UNSUPPORTED, 0}));
16960 if (Triple.isOSAIX()) {
16961 assert((AIXSupportMethod != BUILTIN_PPC_UNSUPPORTED) &&
16962 "Invalid CPU name. Missed by SemaChecking?");
16963 return GenAIXPPCBuiltinCpuExpr(AIXSupportMethod, AIX_SYSCON_IMPL_IDX, 0,
16964 ICmpInst::ICMP_EQ, AIXIDValue);
16967 assert(Triple.isOSLinux() &&
16968 "__builtin_cpu_is() is only supported for AIX and Linux.");
16970 assert((LinuxSupportMethod != BUILTIN_PPC_UNSUPPORTED) &&
16971 "Invalid CPU name. Missed by SemaChecking?");
16973 if (LinuxSupportMethod == BUILTIN_PPC_FALSE)
16976 Value *Op0 = llvm::ConstantInt::get(
Int32Ty, PPC_FAWORD_CPUID);
16978 Value *TheCall =
Builder.CreateCall(F, {Op0},
"cpu_is");
16979 return Builder.CreateICmpEQ(TheCall,
16980 llvm::ConstantInt::get(
Int32Ty, LinuxIDValue));
16982 case Builtin::BI__builtin_cpu_supports: {
16985 StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
16986 if (Triple.isOSAIX()) {
16987 unsigned SupportMethod, FieldIdx, Mask,
Value;
16988 CmpInst::Predicate CompOp;
16992 std::tie(SupportMethod, FieldIdx, Mask, CompOp,
Value) =
16993 static_cast<CPUSupportType
>(StringSwitch<CPUSupportType>(CPUStr)
16994#define PPC_AIX_FEATURE(NAME, DESC, SUPPORT_METHOD, INDEX, MASK, COMP_OP, \
16996 .Case(NAME, {SUPPORT_METHOD, INDEX, MASK, COMP_OP, VALUE})
16997#include "llvm/TargetParser/PPCTargetParser.def"
16998 .Default({BUILTIN_PPC_FALSE, 0, 0,
16999 CmpInst::Predicate(), 0}));
17000 return GenAIXPPCBuiltinCpuExpr(SupportMethod, FieldIdx, Mask, CompOp,
17004 assert(Triple.isOSLinux() &&
17005 "__builtin_cpu_supports() is only supported for AIX and Linux.");
17006 unsigned FeatureWord;
17008 std::tie(FeatureWord, BitMask) =
17009 StringSwitch<std::pair<unsigned, unsigned>>(CPUStr)
17010#define
PPC_LNX_FEATURE(Name, Description, EnumName, Bitmask, FA_WORD) \
17011 .Case(Name, {FA_WORD, Bitmask})
17012#include
"llvm/TargetParser/PPCTargetParser.def"
17016 Value *Op0 = llvm::ConstantInt::get(
Int32Ty, FeatureWord);
17018 Value *TheCall =
Builder.CreateCall(F, {Op0},
"cpu_supports");
17020 Builder.CreateAnd(TheCall, llvm::ConstantInt::get(
Int32Ty, BitMask));
17021 return Builder.CreateICmpNE(Mask, llvm::Constant::getNullValue(
Int32Ty));
17022#undef PPC_FAWORD_HWCAP
17023#undef PPC_FAWORD_HWCAP2
17024#undef PPC_FAWORD_CPUID
17029 case PPC::BI__builtin_ppc_get_timebase:
17033 case PPC::BI__builtin_altivec_lvx:
17034 case PPC::BI__builtin_altivec_lvxl:
17035 case PPC::BI__builtin_altivec_lvebx:
17036 case PPC::BI__builtin_altivec_lvehx:
17037 case PPC::BI__builtin_altivec_lvewx:
17038 case PPC::BI__builtin_altivec_lvsl:
17039 case PPC::BI__builtin_altivec_lvsr:
17040 case PPC::BI__builtin_vsx_lxvd2x:
17041 case PPC::BI__builtin_vsx_lxvw4x:
17042 case PPC::BI__builtin_vsx_lxvd2x_be:
17043 case PPC::BI__builtin_vsx_lxvw4x_be:
17044 case PPC::BI__builtin_vsx_lxvl:
17045 case PPC::BI__builtin_vsx_lxvll:
17050 if (!(BuiltinID == PPC::BI__builtin_vsx_lxvl ||
17051 BuiltinID == PPC::BI__builtin_vsx_lxvll)) {
17056 switch (BuiltinID) {
17057 default: llvm_unreachable(
"Unsupported ld/lvsl/lvsr intrinsic!");
17058 case PPC::BI__builtin_altivec_lvx:
17059 ID = Intrinsic::ppc_altivec_lvx;
17061 case PPC::BI__builtin_altivec_lvxl:
17062 ID = Intrinsic::ppc_altivec_lvxl;
17064 case PPC::BI__builtin_altivec_lvebx:
17065 ID = Intrinsic::ppc_altivec_lvebx;
17067 case PPC::BI__builtin_altivec_lvehx:
17068 ID = Intrinsic::ppc_altivec_lvehx;
17070 case PPC::BI__builtin_altivec_lvewx:
17071 ID = Intrinsic::ppc_altivec_lvewx;
17073 case PPC::BI__builtin_altivec_lvsl:
17074 ID = Intrinsic::ppc_altivec_lvsl;
17076 case PPC::BI__builtin_altivec_lvsr:
17077 ID = Intrinsic::ppc_altivec_lvsr;
17079 case PPC::BI__builtin_vsx_lxvd2x:
17080 ID = Intrinsic::ppc_vsx_lxvd2x;
17082 case PPC::BI__builtin_vsx_lxvw4x:
17083 ID = Intrinsic::ppc_vsx_lxvw4x;
17085 case PPC::BI__builtin_vsx_lxvd2x_be:
17086 ID = Intrinsic::ppc_vsx_lxvd2x_be;
17088 case PPC::BI__builtin_vsx_lxvw4x_be:
17089 ID = Intrinsic::ppc_vsx_lxvw4x_be;
17091 case PPC::BI__builtin_vsx_lxvl:
17092 ID = Intrinsic::ppc_vsx_lxvl;
17094 case PPC::BI__builtin_vsx_lxvll:
17095 ID = Intrinsic::ppc_vsx_lxvll;
17099 return Builder.CreateCall(F, Ops,
"");
17103 case PPC::BI__builtin_altivec_stvx:
17104 case PPC::BI__builtin_altivec_stvxl:
17105 case PPC::BI__builtin_altivec_stvebx:
17106 case PPC::BI__builtin_altivec_stvehx:
17107 case PPC::BI__builtin_altivec_stvewx:
17108 case PPC::BI__builtin_vsx_stxvd2x:
17109 case PPC::BI__builtin_vsx_stxvw4x:
17110 case PPC::BI__builtin_vsx_stxvd2x_be:
17111 case PPC::BI__builtin_vsx_stxvw4x_be:
17112 case PPC::BI__builtin_vsx_stxvl:
17113 case PPC::BI__builtin_vsx_stxvll:
17119 if (!(BuiltinID == PPC::BI__builtin_vsx_stxvl ||
17120 BuiltinID == PPC::BI__builtin_vsx_stxvll)) {
17125 switch (BuiltinID) {
17126 default: llvm_unreachable(
"Unsupported st intrinsic!");
17127 case PPC::BI__builtin_altivec_stvx:
17128 ID = Intrinsic::ppc_altivec_stvx;
17130 case PPC::BI__builtin_altivec_stvxl:
17131 ID = Intrinsic::ppc_altivec_stvxl;
17133 case PPC::BI__builtin_altivec_stvebx:
17134 ID = Intrinsic::ppc_altivec_stvebx;
17136 case PPC::BI__builtin_altivec_stvehx:
17137 ID = Intrinsic::ppc_altivec_stvehx;
17139 case PPC::BI__builtin_altivec_stvewx:
17140 ID = Intrinsic::ppc_altivec_stvewx;
17142 case PPC::BI__builtin_vsx_stxvd2x:
17143 ID = Intrinsic::ppc_vsx_stxvd2x;
17145 case PPC::BI__builtin_vsx_stxvw4x:
17146 ID = Intrinsic::ppc_vsx_stxvw4x;
17148 case PPC::BI__builtin_vsx_stxvd2x_be:
17149 ID = Intrinsic::ppc_vsx_stxvd2x_be;
17151 case PPC::BI__builtin_vsx_stxvw4x_be:
17152 ID = Intrinsic::ppc_vsx_stxvw4x_be;
17154 case PPC::BI__builtin_vsx_stxvl:
17155 ID = Intrinsic::ppc_vsx_stxvl;
17157 case PPC::BI__builtin_vsx_stxvll:
17158 ID = Intrinsic::ppc_vsx_stxvll;
17162 return Builder.CreateCall(F, Ops,
"");
17164 case PPC::BI__builtin_vsx_ldrmb: {
17170 int64_t NumBytes = cast<ConstantInt>(Op1)->getZExtValue();
17175 if (NumBytes == 16) {
17183 for (
int Idx = 0; Idx < 16; Idx++)
17184 RevMask.push_back(15 - Idx);
17185 return Builder.CreateShuffleVector(LD, LD, RevMask);
17189 llvm::Function *Lvs =
CGM.
getIntrinsic(IsLE ? Intrinsic::ppc_altivec_lvsr
17190 : Intrinsic::ppc_altivec_lvsl);
17191 llvm::Function *Vperm =
CGM.
getIntrinsic(Intrinsic::ppc_altivec_vperm);
17193 Int8Ty, Op0, ConstantInt::get(Op1->
getType(), NumBytes - 1));
17195 Value *HiLd =
Builder.CreateCall(Lvx, HiMem,
"ld.hi");
17198 Op0 = IsLE ? HiLd : LoLd;
17199 Op1 = IsLE ? LoLd : HiLd;
17200 Value *AllElts =
Builder.CreateCall(Vperm, {Op0, Op1, Mask1},
"shuffle1");
17201 Constant *
Zero = llvm::Constant::getNullValue(IsLE ? ResTy : AllElts->
getType());
17205 for (
int Idx = 0; Idx < 16; Idx++) {
17206 int Val = (NumBytes - Idx - 1 >= 0) ? (NumBytes - Idx - 1)
17207 : 16 - (NumBytes - Idx);
17208 Consts.push_back(Val);
17210 return Builder.CreateShuffleVector(
Builder.CreateBitCast(AllElts, ResTy),
17214 for (
int Idx = 0; Idx < 16; Idx++)
17215 Consts.push_back(
Builder.getInt8(NumBytes + Idx));
17216 Value *Mask2 = ConstantVector::get(Consts);
17217 return Builder.CreateBitCast(
17218 Builder.CreateCall(Vperm, {Zero, AllElts, Mask2},
"shuffle2"), ResTy);
17220 case PPC::BI__builtin_vsx_strmb: {
17224 int64_t NumBytes = cast<ConstantInt>(Op1)->getZExtValue();
17226 auto StoreSubVec = [&](
unsigned Width,
unsigned Offset,
unsigned EltNo) {
17230 Value *StVec = Op2;
17233 for (
int Idx = 0; Idx < 16; Idx++)
17234 RevMask.push_back(15 - Idx);
17235 StVec =
Builder.CreateShuffleVector(Op2, Op2, RevMask);
17241 unsigned NumElts = 0;
17244 llvm_unreachable(
"width for stores must be a power of 2");
17263 Op2, llvm::FixedVectorType::get(ConvTy, NumElts));
17266 Value *Elt =
Builder.CreateExtractElement(Vec, EltNo);
17267 if (IsLE && Width > 1) {
17269 Elt =
Builder.CreateCall(F, Elt);
17274 unsigned Stored = 0;
17275 unsigned RemainingBytes = NumBytes;
17277 if (NumBytes == 16)
17278 return StoreSubVec(16, 0, 0);
17279 if (NumBytes >= 8) {
17280 Result = StoreSubVec(8, NumBytes - 8, IsLE ? 0 : 1);
17281 RemainingBytes -= 8;
17284 if (RemainingBytes >= 4) {
17285 Result = StoreSubVec(4, NumBytes - Stored - 4,
17286 IsLE ? (Stored >> 2) : 3 - (Stored >> 2));
17287 RemainingBytes -= 4;
17290 if (RemainingBytes >= 2) {
17291 Result = StoreSubVec(2, NumBytes - Stored - 2,
17292 IsLE ? (Stored >> 1) : 7 - (Stored >> 1));
17293 RemainingBytes -= 2;
17296 if (RemainingBytes)
17298 StoreSubVec(1, NumBytes - Stored - 1, IsLE ? Stored : 15 - Stored);
17302 case PPC::BI__builtin_vsx_xvsqrtsp:
17303 case PPC::BI__builtin_vsx_xvsqrtdp: {
17306 if (
Builder.getIsFPConstrained()) {
17308 Intrinsic::experimental_constrained_sqrt, ResultType);
17309 return Builder.CreateConstrainedFPCall(F,
X);
17316 case PPC::BI__builtin_altivec_vclzb:
17317 case PPC::BI__builtin_altivec_vclzh:
17318 case PPC::BI__builtin_altivec_vclzw:
17319 case PPC::BI__builtin_altivec_vclzd: {
17322 Value *Undef = ConstantInt::get(
Builder.getInt1Ty(),
false);
17324 return Builder.CreateCall(F, {
X, Undef});
17326 case PPC::BI__builtin_altivec_vctzb:
17327 case PPC::BI__builtin_altivec_vctzh:
17328 case PPC::BI__builtin_altivec_vctzw:
17329 case PPC::BI__builtin_altivec_vctzd: {
17332 Value *Undef = ConstantInt::get(
Builder.getInt1Ty(),
false);
17334 return Builder.CreateCall(F, {
X, Undef});
17336 case PPC::BI__builtin_altivec_vinsd:
17337 case PPC::BI__builtin_altivec_vinsw:
17338 case PPC::BI__builtin_altivec_vinsd_elt:
17339 case PPC::BI__builtin_altivec_vinsw_elt: {
17345 bool IsUnaligned = (BuiltinID == PPC::BI__builtin_altivec_vinsw ||
17346 BuiltinID == PPC::BI__builtin_altivec_vinsd);
17348 bool Is32bit = (BuiltinID == PPC::BI__builtin_altivec_vinsw ||
17349 BuiltinID == PPC::BI__builtin_altivec_vinsw_elt);
17352 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
17354 "Third Arg to vinsw/vinsd intrinsic must be a constant integer!");
17358 int ValidMaxValue = 0;
17360 ValidMaxValue = (Is32bit) ? 12 : 8;
17362 ValidMaxValue = (Is32bit) ? 3 : 1;
17365 int64_t ConstArg = ArgCI->getSExtValue();
17368 std::string RangeErrMsg = IsUnaligned ?
"byte" :
"element";
17369 RangeErrMsg +=
" number " + llvm::to_string(ConstArg);
17370 RangeErrMsg +=
" is outside of the valid range [0, ";
17371 RangeErrMsg += llvm::to_string(ValidMaxValue) +
"]";
17374 if (ConstArg < 0 || ConstArg > ValidMaxValue)
17378 if (!IsUnaligned) {
17379 ConstArg *= Is32bit ? 4 : 8;
17382 ConstArg = (Is32bit ? 12 : 8) - ConstArg;
17385 ID = Is32bit ? Intrinsic::ppc_altivec_vinsw : Intrinsic::ppc_altivec_vinsd;
17386 Op2 = ConstantInt::getSigned(
Int32Ty, ConstArg);
17390 ?
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int32Ty, 4))
17392 llvm::FixedVectorType::get(
Int64Ty, 2));
17393 return Builder.CreateBitCast(
17396 case PPC::BI__builtin_altivec_vpopcntb:
17397 case PPC::BI__builtin_altivec_vpopcnth:
17398 case PPC::BI__builtin_altivec_vpopcntw:
17399 case PPC::BI__builtin_altivec_vpopcntd: {
17405 case PPC::BI__builtin_altivec_vadduqm:
17406 case PPC::BI__builtin_altivec_vsubuqm: {
17409 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
17410 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(Int128Ty, 1));
17411 Op1 =
Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(Int128Ty, 1));
17412 if (BuiltinID == PPC::BI__builtin_altivec_vadduqm)
17413 return Builder.CreateAdd(Op0, Op1,
"vadduqm");
17415 return Builder.CreateSub(Op0, Op1,
"vsubuqm");
17417 case PPC::BI__builtin_altivec_vaddcuq_c:
17418 case PPC::BI__builtin_altivec_vsubcuq_c: {
17422 llvm::Type *V1I128Ty = llvm::FixedVectorType::get(
17424 Ops.push_back(
Builder.CreateBitCast(Op0, V1I128Ty));
17425 Ops.push_back(
Builder.CreateBitCast(Op1, V1I128Ty));
17426 ID = (BuiltinID == PPC::BI__builtin_altivec_vaddcuq_c)
17427 ? Intrinsic::ppc_altivec_vaddcuq
17428 : Intrinsic::ppc_altivec_vsubcuq;
17431 case PPC::BI__builtin_altivec_vaddeuqm_c:
17432 case PPC::BI__builtin_altivec_vaddecuq_c:
17433 case PPC::BI__builtin_altivec_vsubeuqm_c:
17434 case PPC::BI__builtin_altivec_vsubecuq_c: {
17439 llvm::Type *V1I128Ty = llvm::FixedVectorType::get(
17441 Ops.push_back(
Builder.CreateBitCast(Op0, V1I128Ty));
17442 Ops.push_back(
Builder.CreateBitCast(Op1, V1I128Ty));
17443 Ops.push_back(
Builder.CreateBitCast(Op2, V1I128Ty));
17444 switch (BuiltinID) {
17446 llvm_unreachable(
"Unsupported intrinsic!");
17447 case PPC::BI__builtin_altivec_vaddeuqm_c:
17448 ID = Intrinsic::ppc_altivec_vaddeuqm;
17450 case PPC::BI__builtin_altivec_vaddecuq_c:
17451 ID = Intrinsic::ppc_altivec_vaddecuq;
17453 case PPC::BI__builtin_altivec_vsubeuqm_c:
17454 ID = Intrinsic::ppc_altivec_vsubeuqm;
17456 case PPC::BI__builtin_altivec_vsubecuq_c:
17457 ID = Intrinsic::ppc_altivec_vsubecuq;
17462 case PPC::BI__builtin_ppc_rldimi:
17463 case PPC::BI__builtin_ppc_rlwimi: {
17470 if (BuiltinID == PPC::BI__builtin_ppc_rldimi &&
17480 ? Intrinsic::ppc_rldimi
17481 : Intrinsic::ppc_rlwimi),
17482 {Op0, Op1, Op2, Op3});
17484 case PPC::BI__builtin_ppc_rlwnm: {
17491 case PPC::BI__builtin_ppc_poppar4:
17492 case PPC::BI__builtin_ppc_poppar8: {
17494 llvm::Type *ArgType = Op0->
getType();
17500 if (
Result->getType() != ResultType)
17505 case PPC::BI__builtin_ppc_cmpb: {
17508 if (
getTarget().getTriple().isPPC64()) {
17511 return Builder.CreateCall(F, {Op0, Op1},
"cmpb");
17531 Constant *ShiftAmt = ConstantInt::get(
Int64Ty, 32);
17540 Value *ResHi =
Builder.CreateShl(ResHiShift, ShiftAmt);
17541 return Builder.CreateOr(ResLo, ResHi);
17544 case PPC::BI__builtin_vsx_xvcpsgnsp:
17545 case PPC::BI__builtin_vsx_xvcpsgndp: {
17549 ID = Intrinsic::copysign;
17551 return Builder.CreateCall(F, {
X, Y});
17554 case PPC::BI__builtin_vsx_xvrspip:
17555 case PPC::BI__builtin_vsx_xvrdpip:
17556 case PPC::BI__builtin_vsx_xvrdpim:
17557 case PPC::BI__builtin_vsx_xvrspim:
17558 case PPC::BI__builtin_vsx_xvrdpi:
17559 case PPC::BI__builtin_vsx_xvrspi:
17560 case PPC::BI__builtin_vsx_xvrdpic:
17561 case PPC::BI__builtin_vsx_xvrspic:
17562 case PPC::BI__builtin_vsx_xvrdpiz:
17563 case PPC::BI__builtin_vsx_xvrspiz: {
17566 if (BuiltinID == PPC::BI__builtin_vsx_xvrdpim ||
17567 BuiltinID == PPC::BI__builtin_vsx_xvrspim)
17569 ? Intrinsic::experimental_constrained_floor
17570 : Intrinsic::floor;
17571 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpi ||
17572 BuiltinID == PPC::BI__builtin_vsx_xvrspi)
17574 ? Intrinsic::experimental_constrained_round
17575 : Intrinsic::round;
17576 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpic ||
17577 BuiltinID == PPC::BI__builtin_vsx_xvrspic)
17579 ? Intrinsic::experimental_constrained_rint
17581 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpip ||
17582 BuiltinID == PPC::BI__builtin_vsx_xvrspip)
17584 ? Intrinsic::experimental_constrained_ceil
17586 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpiz ||
17587 BuiltinID == PPC::BI__builtin_vsx_xvrspiz)
17589 ? Intrinsic::experimental_constrained_trunc
17590 : Intrinsic::trunc;
17592 return Builder.getIsFPConstrained() ?
Builder.CreateConstrainedFPCall(F,
X)
17597 case PPC::BI__builtin_vsx_xvabsdp:
17598 case PPC::BI__builtin_vsx_xvabssp: {
17606 case PPC::BI__builtin_ppc_recipdivf:
17607 case PPC::BI__builtin_ppc_recipdivd:
17608 case PPC::BI__builtin_ppc_rsqrtf:
17609 case PPC::BI__builtin_ppc_rsqrtd: {
17610 FastMathFlags FMF =
Builder.getFastMathFlags();
17611 Builder.getFastMathFlags().setFast();
17615 if (BuiltinID == PPC::BI__builtin_ppc_recipdivf ||
17616 BuiltinID == PPC::BI__builtin_ppc_recipdivd) {
17619 Builder.getFastMathFlags() &= (FMF);
17622 auto *One = ConstantFP::get(ResultType, 1.0);
17625 Builder.getFastMathFlags() &= (FMF);
17628 case PPC::BI__builtin_ppc_alignx: {
17631 ConstantInt *AlignmentCI = cast<ConstantInt>(Op0);
17632 if (AlignmentCI->getValue().ugt(llvm::Value::MaximumAlignment))
17633 AlignmentCI = ConstantInt::get(AlignmentCI->getIntegerType(),
17634 llvm::Value::MaximumAlignment);
17638 AlignmentCI,
nullptr);
17641 case PPC::BI__builtin_ppc_rdlam: {
17645 llvm::Type *Ty = Op0->
getType();
17646 Value *ShiftAmt =
Builder.CreateIntCast(Op1, Ty,
false);
17648 Value *Rotate =
Builder.CreateCall(F, {Op0, Op0, ShiftAmt});
17649 return Builder.CreateAnd(Rotate, Op2);
17651 case PPC::BI__builtin_ppc_load2r: {
17658 case PPC::BI__builtin_ppc_fnmsub:
17659 case PPC::BI__builtin_ppc_fnmsubs:
17660 case PPC::BI__builtin_vsx_xvmaddadp:
17661 case PPC::BI__builtin_vsx_xvmaddasp:
17662 case PPC::BI__builtin_vsx_xvnmaddadp:
17663 case PPC::BI__builtin_vsx_xvnmaddasp:
17664 case PPC::BI__builtin_vsx_xvmsubadp:
17665 case PPC::BI__builtin_vsx_xvmsubasp:
17666 case PPC::BI__builtin_vsx_xvnmsubadp:
17667 case PPC::BI__builtin_vsx_xvnmsubasp: {
17673 if (
Builder.getIsFPConstrained())
17674 F =
CGM.
getIntrinsic(Intrinsic::experimental_constrained_fma, ResultType);
17677 switch (BuiltinID) {
17678 case PPC::BI__builtin_vsx_xvmaddadp:
17679 case PPC::BI__builtin_vsx_xvmaddasp:
17680 if (
Builder.getIsFPConstrained())
17681 return Builder.CreateConstrainedFPCall(F, {
X, Y, Z});
17683 return Builder.CreateCall(F, {
X, Y, Z});
17684 case PPC::BI__builtin_vsx_xvnmaddadp:
17685 case PPC::BI__builtin_vsx_xvnmaddasp:
17686 if (
Builder.getIsFPConstrained())
17688 Builder.CreateConstrainedFPCall(F, {X, Y, Z}),
"neg");
17690 return Builder.CreateFNeg(
Builder.CreateCall(F, {X, Y, Z}),
"neg");
17691 case PPC::BI__builtin_vsx_xvmsubadp:
17692 case PPC::BI__builtin_vsx_xvmsubasp:
17693 if (
Builder.getIsFPConstrained())
17694 return Builder.CreateConstrainedFPCall(
17695 F, {
X, Y,
Builder.CreateFNeg(Z,
"neg")});
17698 case PPC::BI__builtin_ppc_fnmsub:
17699 case PPC::BI__builtin_ppc_fnmsubs:
17700 case PPC::BI__builtin_vsx_xvnmsubadp:
17701 case PPC::BI__builtin_vsx_xvnmsubasp:
17702 if (
Builder.getIsFPConstrained())
17704 Builder.CreateConstrainedFPCall(
17705 F, {X, Y, Builder.CreateFNeg(Z,
"neg")}),
17711 llvm_unreachable(
"Unknown FMA operation");
17715 case PPC::BI__builtin_vsx_insertword: {
17723 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
17725 "Third arg to xxinsertw intrinsic must be constant integer");
17727 int64_t Index = std::clamp(ArgCI->getSExtValue(), (int64_t)0, MaxIndex);
17734 std::swap(Op0, Op1);
17738 Op1 =
Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(
Int64Ty, 2));
17742 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int64Ty, 2));
17746 Index = MaxIndex - Index;
17750 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int32Ty, 4));
17751 Op2 = ConstantInt::getSigned(
Int32Ty, Index);
17752 return Builder.CreateCall(F, {Op0, Op1, Op2});
17755 case PPC::BI__builtin_vsx_extractuword: {
17758 llvm::Function *F =
CGM.
getIntrinsic(Intrinsic::ppc_vsx_xxextractuw);
17761 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int64Ty, 2));
17765 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op1);
17767 "Second Arg to xxextractuw intrinsic must be a constant integer!");
17769 int64_t Index = std::clamp(ArgCI->getSExtValue(), (int64_t)0, MaxIndex);
17773 Index = MaxIndex - Index;
17774 Op1 = ConstantInt::getSigned(
Int32Ty, Index);
17779 Value *ShuffleCall =
17781 return ShuffleCall;
17783 Op1 = ConstantInt::getSigned(
Int32Ty, Index);
17784 return Builder.CreateCall(F, {Op0, Op1});
17788 case PPC::BI__builtin_vsx_xxpermdi: {
17792 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
17793 assert(ArgCI &&
"Third arg must be constant integer!");
17795 unsigned Index = ArgCI->getZExtValue();
17796 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int64Ty, 2));
17797 Op1 =
Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(
Int64Ty, 2));
17802 int ElemIdx0 = (Index & 2) >> 1;
17803 int ElemIdx1 = 2 + (Index & 1);
17805 int ShuffleElts[2] = {ElemIdx0, ElemIdx1};
17806 Value *ShuffleCall =
Builder.CreateShuffleVector(Op0, Op1, ShuffleElts);
17809 return Builder.CreateBitCast(ShuffleCall, RetTy);
17812 case PPC::BI__builtin_vsx_xxsldwi: {
17816 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
17817 assert(ArgCI &&
"Third argument must be a compile time constant");
17818 unsigned Index = ArgCI->getZExtValue() & 0x3;
17819 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int32Ty, 4));
17820 Op1 =
Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(
Int32Ty, 4));
17831 ElemIdx0 = (8 - Index) % 8;
17832 ElemIdx1 = (9 - Index) % 8;
17833 ElemIdx2 = (10 - Index) % 8;
17834 ElemIdx3 = (11 - Index) % 8;
17838 ElemIdx1 = Index + 1;
17839 ElemIdx2 = Index + 2;
17840 ElemIdx3 = Index + 3;
17843 int ShuffleElts[4] = {ElemIdx0, ElemIdx1, ElemIdx2, ElemIdx3};
17844 Value *ShuffleCall =
Builder.CreateShuffleVector(Op0, Op1, ShuffleElts);
17847 return Builder.CreateBitCast(ShuffleCall, RetTy);
17850 case PPC::BI__builtin_pack_vector_int128: {
17854 Value *PoisonValue =
17855 llvm::PoisonValue::get(llvm::FixedVectorType::get(Op0->
getType(), 2));
17857 PoisonValue, Op0, (uint64_t)(isLittleEndian ? 1 : 0));
17858 Res =
Builder.CreateInsertElement(Res, Op1,
17859 (uint64_t)(isLittleEndian ? 0 : 1));
17863 case PPC::BI__builtin_unpack_vector_int128: {
17866 ConstantInt *Index = cast<ConstantInt>(Op1);
17872 ConstantInt::get(Index->getIntegerType(), 1 - Index->getZExtValue());
17874 return Builder.CreateExtractElement(Unpacked, Index);
17877 case PPC::BI__builtin_ppc_sthcx: {
17881 return Builder.CreateCall(F, {Op0, Op1});
17890#define CUSTOM_BUILTIN(Name, Intr, Types, Accumulate, Feature) \
17891 case PPC::BI__builtin_##Name:
17892#include "clang/Basic/BuiltinsPPC.def"
17895 for (
unsigned i = 0, e =
E->getNumArgs(); i != e; i++)
17905 if (BuiltinID == PPC::BI__builtin_mma_disassemble_acc ||
17906 BuiltinID == PPC::BI__builtin_vsx_disassemble_pair ||
17907 BuiltinID == PPC::BI__builtin_mma_disassemble_pair) {
17908 unsigned NumVecs = 2;
17909 auto Intrinsic = Intrinsic::ppc_vsx_disassemble_pair;
17910 if (BuiltinID == PPC::BI__builtin_mma_disassemble_acc) {
17912 Intrinsic = Intrinsic::ppc_mma_disassemble_acc;
17918 llvm::Type *VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
17919 Value *Ptr = Ops[0];
17920 for (
unsigned i=0; i<NumVecs; i++) {
17922 llvm::ConstantInt* Index = llvm::ConstantInt::get(
IntTy, i);
17928 if (BuiltinID == PPC::BI__builtin_vsx_build_pair ||
17929 BuiltinID == PPC::BI__builtin_mma_build_acc) {
17937 std::reverse(Ops.begin() + 1, Ops.end());
17940 switch (BuiltinID) {
17941 #define CUSTOM_BUILTIN(Name, Intr, Types, Acc, Feature) \
17942 case PPC::BI__builtin_##Name: \
17943 ID = Intrinsic::ppc_##Intr; \
17944 Accumulate = Acc; \
17946 #include "clang/Basic/BuiltinsPPC.def"
17948 if (BuiltinID == PPC::BI__builtin_vsx_lxvp ||
17949 BuiltinID == PPC::BI__builtin_vsx_stxvp ||
17950 BuiltinID == PPC::BI__builtin_mma_lxvp ||
17951 BuiltinID == PPC::BI__builtin_mma_stxvp) {
17952 if (BuiltinID == PPC::BI__builtin_vsx_lxvp ||
17953 BuiltinID == PPC::BI__builtin_mma_lxvp) {
17960 return Builder.CreateCall(F, Ops,
"");
17966 CallOps.push_back(Acc);
17968 for (
unsigned i=1; i<Ops.size(); i++)
17969 CallOps.push_back(Ops[i]);
17975 case PPC::BI__builtin_ppc_compare_and_swap:
17976 case PPC::BI__builtin_ppc_compare_and_swaplp: {
17985 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Monotonic,
true);
17993 Value *LoadedVal = Pair.first.getScalarVal();
17997 case PPC::BI__builtin_ppc_fetch_and_add:
17998 case PPC::BI__builtin_ppc_fetch_and_addlp: {
18000 llvm::AtomicOrdering::Monotonic);
18002 case PPC::BI__builtin_ppc_fetch_and_and:
18003 case PPC::BI__builtin_ppc_fetch_and_andlp: {
18005 llvm::AtomicOrdering::Monotonic);
18008 case PPC::BI__builtin_ppc_fetch_and_or:
18009 case PPC::BI__builtin_ppc_fetch_and_orlp: {
18011 llvm::AtomicOrdering::Monotonic);
18013 case PPC::BI__builtin_ppc_fetch_and_swap:
18014 case PPC::BI__builtin_ppc_fetch_and_swaplp: {
18016 llvm::AtomicOrdering::Monotonic);
18018 case PPC::BI__builtin_ppc_ldarx:
18019 case PPC::BI__builtin_ppc_lwarx:
18020 case PPC::BI__builtin_ppc_lharx:
18021 case PPC::BI__builtin_ppc_lbarx:
18023 case PPC::BI__builtin_ppc_mfspr: {
18029 return Builder.CreateCall(F, {Op0});
18031 case PPC::BI__builtin_ppc_mtspr: {
18038 return Builder.CreateCall(F, {Op0, Op1});
18040 case PPC::BI__builtin_ppc_popcntb: {
18042 llvm::Type *ArgType = ArgValue->
getType();
18044 return Builder.CreateCall(F, {ArgValue},
"popcntb");
18046 case PPC::BI__builtin_ppc_mtfsf: {
18056 case PPC::BI__builtin_ppc_swdiv_nochk:
18057 case PPC::BI__builtin_ppc_swdivs_nochk: {
18060 FastMathFlags FMF =
Builder.getFastMathFlags();
18061 Builder.getFastMathFlags().setFast();
18062 Value *FDiv =
Builder.CreateFDiv(Op0, Op1,
"swdiv_nochk");
18063 Builder.getFastMathFlags() &= (FMF);
18066 case PPC::BI__builtin_ppc_fric:
18068 *
this,
E, Intrinsic::rint,
18069 Intrinsic::experimental_constrained_rint))
18071 case PPC::BI__builtin_ppc_frim:
18072 case PPC::BI__builtin_ppc_frims:
18074 *
this,
E, Intrinsic::floor,
18075 Intrinsic::experimental_constrained_floor))
18077 case PPC::BI__builtin_ppc_frin:
18078 case PPC::BI__builtin_ppc_frins:
18080 *
this,
E, Intrinsic::round,
18081 Intrinsic::experimental_constrained_round))
18083 case PPC::BI__builtin_ppc_frip:
18084 case PPC::BI__builtin_ppc_frips:
18086 *
this,
E, Intrinsic::ceil,
18087 Intrinsic::experimental_constrained_ceil))
18089 case PPC::BI__builtin_ppc_friz:
18090 case PPC::BI__builtin_ppc_frizs:
18092 *
this,
E, Intrinsic::trunc,
18093 Intrinsic::experimental_constrained_trunc))
18095 case PPC::BI__builtin_ppc_fsqrt:
18096 case PPC::BI__builtin_ppc_fsqrts:
18098 *
this,
E, Intrinsic::sqrt,
18099 Intrinsic::experimental_constrained_sqrt))
18101 case PPC::BI__builtin_ppc_test_data_class: {
18106 {Op0, Op1},
"test_data_class");
18108 case PPC::BI__builtin_ppc_maxfe: {
18114 {Op0, Op1, Op2, Op3});
18116 case PPC::BI__builtin_ppc_maxfl: {
18122 {Op0, Op1, Op2, Op3});
18124 case PPC::BI__builtin_ppc_maxfs: {
18130 {Op0, Op1, Op2, Op3});
18132 case PPC::BI__builtin_ppc_minfe: {
18138 {Op0, Op1, Op2, Op3});
18140 case PPC::BI__builtin_ppc_minfl: {
18146 {Op0, Op1, Op2, Op3});
18148 case PPC::BI__builtin_ppc_minfs: {
18154 {Op0, Op1, Op2, Op3});
18156 case PPC::BI__builtin_ppc_swdiv:
18157 case PPC::BI__builtin_ppc_swdivs: {
18160 return Builder.CreateFDiv(Op0, Op1,
"swdiv");
18162 case PPC::BI__builtin_ppc_set_fpscr_rn:
18164 {EmitScalarExpr(E->getArg(0))});
18165 case PPC::BI__builtin_ppc_mffs:
18178 Attribute::getWithDereferenceableBytes(
Call->getContext(), 64));
18179 Call->addRetAttr(Attribute::getWithAlignment(
Call->getContext(), Align(4)));
18183 auto *RetTy = cast<llvm::PointerType>(CGF.
ConvertType(BuiltinRetType));
18184 if (RetTy ==
Call->getType())
18193 Attribute::getWithDereferenceableBytes(
Call->getContext(), 256));
18194 Call->addRetAttr(Attribute::getWithAlignment(
Call->getContext(), Align(8)));
18209 llvm::LoadInst *LD;
18213 if (Cov == CodeObjectVersionKind::COV_None) {
18214 StringRef Name =
"__oclc_ABI_version";
18215 auto *ABIVersionC = CGF.
CGM.
getModule().getNamedGlobal(Name);
18217 ABIVersionC =
new llvm::GlobalVariable(
18219 llvm::GlobalValue::ExternalLinkage,
nullptr, Name,
nullptr,
18220 llvm::GlobalVariable::NotThreadLocal,
18231 llvm::ConstantInt::get(CGF.
Int32Ty, CodeObjectVersionKind::COV_5));
18235 CGF.
Int8Ty, EmitAMDGPUImplicitArgPtr(CGF), 12 + Index * 2);
18239 CGF.
Int8Ty, EmitAMDGPUDispatchPtr(CGF), 4 + Index * 2);
18241 auto Result = CGF.
Builder.CreateSelect(IsCOV5, ImplicitGEP, DispatchGEP);
18245 Value *GEP =
nullptr;
18246 if (Cov >= CodeObjectVersionKind::COV_5) {
18248 GEP = CGF.
Builder.CreateConstGEP1_32(
18249 CGF.
Int8Ty, EmitAMDGPUImplicitArgPtr(CGF), 12 + Index * 2);
18252 GEP = CGF.
Builder.CreateConstGEP1_32(
18253 CGF.
Int8Ty, EmitAMDGPUDispatchPtr(CGF), 4 + Index * 2);
18260 llvm::MDNode *RNode = MDHelper.createRange(
APInt(16, 1),
18262 LD->setMetadata(llvm::LLVMContext::MD_range, RNode);
18263 LD->setMetadata(llvm::LLVMContext::MD_noundef,
18265 LD->setMetadata(llvm::LLVMContext::MD_invariant_load,
18272 const unsigned XOffset = 12;
18273 auto *DP = EmitAMDGPUDispatchPtr(CGF);
18275 auto *Offset = llvm::ConstantInt::get(CGF.
Int32Ty, XOffset + Index * 4);
18279 LD->setMetadata(llvm::LLVMContext::MD_invariant_load,
18292 llvm::AtomicOrdering &AO,
18293 llvm::SyncScope::ID &SSID) {
18294 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
18297 assert(llvm::isValidAtomicOrderingCABI(ord));
18298 switch (
static_cast<llvm::AtomicOrderingCABI
>(ord)) {
18299 case llvm::AtomicOrderingCABI::acquire:
18300 case llvm::AtomicOrderingCABI::consume:
18301 AO = llvm::AtomicOrdering::Acquire;
18303 case llvm::AtomicOrderingCABI::release:
18304 AO = llvm::AtomicOrdering::Release;
18306 case llvm::AtomicOrderingCABI::acq_rel:
18307 AO = llvm::AtomicOrdering::AcquireRelease;
18309 case llvm::AtomicOrderingCABI::seq_cst:
18310 AO = llvm::AtomicOrdering::SequentiallyConsistent;
18312 case llvm::AtomicOrderingCABI::relaxed:
18313 AO = llvm::AtomicOrdering::Monotonic;
18319 if (llvm::getConstantStringInfo(
Scope, scp)) {
18325 int scope = cast<llvm::ConstantInt>(
Scope)->getZExtValue();
18328 SSID = llvm::SyncScope::System;
18340 SSID = llvm::SyncScope::SingleThread;
18343 SSID = llvm::SyncScope::System;
18351 llvm::Value *Arg =
nullptr;
18352 if ((ICEArguments & (1 << Idx)) == 0) {
18357 std::optional<llvm::APSInt>
Result =
18359 assert(
Result &&
"Expected argument to be a constant");
18367 switch (elementCount) {
18369 return Intrinsic::dx_dot2;
18371 return Intrinsic::dx_dot3;
18373 return Intrinsic::dx_dot4;
18377 return Intrinsic::dx_sdot;
18380 return Intrinsic::dx_udot;
18388 switch (BuiltinID) {
18389 case Builtin::BI__builtin_hlsl_elementwise_all: {
18391 return Builder.CreateIntrinsic(
18396 case Builtin::BI__builtin_hlsl_elementwise_any: {
18398 return Builder.CreateIntrinsic(
18403 case Builtin::BI__builtin_hlsl_elementwise_clamp: {
18409 bool IsUnsigned =
false;
18411 Ty = VecTy->getElementType();
18413 return Builder.CreateIntrinsic(
18415 IsUnsigned ? Intrinsic::dx_uclamp : Intrinsic::dx_clamp,
18418 case Builtin::BI__builtin_hlsl_dot: {
18421 llvm::Type *T0 = Op0->
getType();
18422 llvm::Type *T1 = Op1->
getType();
18423 if (!T0->isVectorTy() && !T1->isVectorTy()) {
18424 if (T0->isFloatingPointTy())
18425 return Builder.CreateFMul(Op0, Op1,
"dx.dot");
18427 if (T0->isIntegerTy())
18428 return Builder.CreateMul(Op0, Op1,
"dx.dot");
18432 "Scalar dot product is only supported on ints and floats.");
18435 assert(T0->isVectorTy() && T1->isVectorTy() &&
18436 "Dot product of vector and scalar is not supported.");
18439 assert(T0->getScalarType() == T1->getScalarType() &&
18440 "Dot product of vectors need the same element types.");
18443 [[maybe_unused]]
auto *VecTy1 =
18447 "Dot product requires vectors to be of the same size.");
18449 return Builder.CreateIntrinsic(
18450 T0->getScalarType(),
18452 VecTy0->getNumElements()),
18455 case Builtin::BI__builtin_hlsl_lerp: {
18460 llvm_unreachable(
"lerp operand must have a float representation");
18461 return Builder.CreateIntrinsic(
18465 case Builtin::BI__builtin_hlsl_elementwise_frac: {
18468 llvm_unreachable(
"frac operand must have a float representation");
18469 return Builder.CreateIntrinsic(
18473case Builtin::BI__builtin_hlsl_elementwise_isinf: {
18475 llvm::Type *Xty = Op0->
getType();
18476 llvm::Type *retType = llvm::Type::getInt1Ty(this->
getLLVMContext());
18477 if (Xty->isVectorTy()) {
18479 retType = llvm::VectorType::get(
18480 retType, ElementCount::getFixed(XVecTy->getNumElements()));
18483 llvm_unreachable(
"isinf operand must have a float representation");
18484 return Builder.CreateIntrinsic(retType, Intrinsic::dx_isinf,
18487 case Builtin::BI__builtin_hlsl_mad: {
18492 return Builder.CreateIntrinsic(
18493 M->
getType(), Intrinsic::fmuladd,
18498 return Builder.CreateIntrinsic(
18499 M->
getType(), Intrinsic::dx_imad,
18503 return Builder.CreateNSWAdd(Mul, B);
18507 return Builder.CreateIntrinsic(
18508 M->
getType(), Intrinsic::dx_umad,
18512 return Builder.CreateNUWAdd(Mul, B);
18514 case Builtin::BI__builtin_hlsl_elementwise_rcp: {
18517 llvm_unreachable(
"rcp operand must have a float representation");
18518 llvm::Type *Ty = Op0->
getType();
18519 llvm::Type *EltTy = Ty->getScalarType();
18520 Constant *One = Ty->isVectorTy()
18521 ? ConstantVector::getSplat(
18522 ElementCount::getFixed(
18523 cast<FixedVectorType>(Ty)->getNumElements()),
18524 ConstantFP::get(EltTy, 1.0))
18525 : ConstantFP::get(EltTy, 1.0);
18526 return Builder.CreateFDiv(One, Op0,
"hlsl.rcp");
18528 case Builtin::BI__builtin_hlsl_elementwise_rsqrt: {
18531 llvm_unreachable(
"rsqrt operand must have a float representation");
18532 return Builder.CreateIntrinsic(
18536 case Builtin::BI__builtin_hlsl_wave_get_lane_index: {
18538 llvm::FunctionType::get(
IntTy, {},
false),
"__hlsl_wave_get_lane_index",
18547 constexpr const char *
Tag =
"amdgpu-as";
18549 LLVMContext &Ctx = Inst->getContext();
18551 for (
unsigned K = 2; K <
E->getNumArgs(); ++K) {
18554 if (llvm::getConstantStringInfo(
V, AS)) {
18555 MMRAs.push_back({
Tag, AS});
18560 "expected an address space name as a string literal");
18564 MMRAs.erase(llvm::unique(MMRAs), MMRAs.end());
18565 Inst->setMetadata(LLVMContext::MD_mmra, MMRAMetadata::getMD(Ctx, MMRAs));
18570 llvm::AtomicOrdering AO = llvm::AtomicOrdering::SequentiallyConsistent;
18571 llvm::SyncScope::ID SSID;
18572 switch (BuiltinID) {
18573 case AMDGPU::BI__builtin_amdgcn_div_scale:
18574 case AMDGPU::BI__builtin_amdgcn_div_scalef: {
18587 llvm::Value *Tmp =
Builder.CreateCall(Callee, {
X, Y, Z});
18590 llvm::Value *Flag =
Builder.CreateExtractValue(Tmp, 1);
18594 llvm::Value *FlagExt =
Builder.CreateZExt(Flag, RealFlagType);
18598 case AMDGPU::BI__builtin_amdgcn_div_fmas:
18599 case AMDGPU::BI__builtin_amdgcn_div_fmasf: {
18607 llvm::Value *Src3ToBool =
Builder.CreateIsNotNull(Src3);
18608 return Builder.CreateCall(F, {Src0, Src1, Src2, Src3ToBool});
18611 case AMDGPU::BI__builtin_amdgcn_ds_swizzle:
18612 return emitBuiltinWithOneOverloadedType<2>(*
this,
E,
18613 Intrinsic::amdgcn_ds_swizzle);
18614 case AMDGPU::BI__builtin_amdgcn_mov_dpp8:
18615 return emitBuiltinWithOneOverloadedType<2>(*
this,
E,
18616 Intrinsic::amdgcn_mov_dpp8);
18617 case AMDGPU::BI__builtin_amdgcn_mov_dpp:
18618 case AMDGPU::BI__builtin_amdgcn_update_dpp: {
18622 unsigned ICEArguments = 0;
18626 for (
unsigned I = 0; I !=
E->getNumArgs(); ++I) {
18629 assert(Args.size() == 5 || Args.size() == 6);
18630 if (Args.size() == 5)
18631 Args.insert(Args.begin(), llvm::PoisonValue::get(Args[0]->getType()));
18634 return Builder.CreateCall(F, Args);
18636 case AMDGPU::BI__builtin_amdgcn_permlane16:
18637 case AMDGPU::BI__builtin_amdgcn_permlanex16:
18638 return emitBuiltinWithOneOverloadedType<6>(
18640 BuiltinID == AMDGPU::BI__builtin_amdgcn_permlane16
18641 ? Intrinsic::amdgcn_permlane16
18642 : Intrinsic::amdgcn_permlanex16);
18643 case AMDGPU::BI__builtin_amdgcn_permlane64:
18644 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
18645 Intrinsic::amdgcn_permlane64);
18646 case AMDGPU::BI__builtin_amdgcn_readlane:
18647 return emitBuiltinWithOneOverloadedType<2>(*
this,
E,
18648 Intrinsic::amdgcn_readlane);
18649 case AMDGPU::BI__builtin_amdgcn_readfirstlane:
18650 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
18651 Intrinsic::amdgcn_readfirstlane);
18652 case AMDGPU::BI__builtin_amdgcn_div_fixup:
18653 case AMDGPU::BI__builtin_amdgcn_div_fixupf:
18654 case AMDGPU::BI__builtin_amdgcn_div_fixuph:
18655 return emitBuiltinWithOneOverloadedType<3>(*
this,
E,
18656 Intrinsic::amdgcn_div_fixup);
18657 case AMDGPU::BI__builtin_amdgcn_trig_preop:
18658 case AMDGPU::BI__builtin_amdgcn_trig_preopf:
18660 case AMDGPU::BI__builtin_amdgcn_rcp:
18661 case AMDGPU::BI__builtin_amdgcn_rcpf:
18662 case AMDGPU::BI__builtin_amdgcn_rcph:
18663 return emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::amdgcn_rcp);
18664 case AMDGPU::BI__builtin_amdgcn_sqrt:
18665 case AMDGPU::BI__builtin_amdgcn_sqrtf:
18666 case AMDGPU::BI__builtin_amdgcn_sqrth:
18667 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
18668 Intrinsic::amdgcn_sqrt);
18669 case AMDGPU::BI__builtin_amdgcn_rsq:
18670 case AMDGPU::BI__builtin_amdgcn_rsqf:
18671 case AMDGPU::BI__builtin_amdgcn_rsqh:
18672 return emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::amdgcn_rsq);
18673 case AMDGPU::BI__builtin_amdgcn_rsq_clamp:
18674 case AMDGPU::BI__builtin_amdgcn_rsq_clampf:
18675 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
18676 Intrinsic::amdgcn_rsq_clamp);
18677 case AMDGPU::BI__builtin_amdgcn_sinf:
18678 case AMDGPU::BI__builtin_amdgcn_sinh:
18679 return emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::amdgcn_sin);
18680 case AMDGPU::BI__builtin_amdgcn_cosf:
18681 case AMDGPU::BI__builtin_amdgcn_cosh:
18682 return emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::amdgcn_cos);
18683 case AMDGPU::BI__builtin_amdgcn_dispatch_ptr:
18684 return EmitAMDGPUDispatchPtr(*
this,
E);
18685 case AMDGPU::BI__builtin_amdgcn_logf:
18686 return emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::amdgcn_log);
18687 case AMDGPU::BI__builtin_amdgcn_exp2f:
18688 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
18689 Intrinsic::amdgcn_exp2);
18690 case AMDGPU::BI__builtin_amdgcn_log_clampf:
18691 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
18692 Intrinsic::amdgcn_log_clamp);
18693 case AMDGPU::BI__builtin_amdgcn_ldexp:
18694 case AMDGPU::BI__builtin_amdgcn_ldexpf: {
18697 llvm::Function *F =
18698 CGM.
getIntrinsic(Intrinsic::ldexp, {Src0->getType(), Src1->getType()});
18699 return Builder.CreateCall(F, {Src0, Src1});
18701 case AMDGPU::BI__builtin_amdgcn_ldexph: {
18706 llvm::Function *F =
18710 case AMDGPU::BI__builtin_amdgcn_frexp_mant:
18711 case AMDGPU::BI__builtin_amdgcn_frexp_mantf:
18712 case AMDGPU::BI__builtin_amdgcn_frexp_manth:
18713 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
18714 Intrinsic::amdgcn_frexp_mant);
18715 case AMDGPU::BI__builtin_amdgcn_frexp_exp:
18716 case AMDGPU::BI__builtin_amdgcn_frexp_expf: {
18720 return Builder.CreateCall(F, Src0);
18722 case AMDGPU::BI__builtin_amdgcn_frexp_exph: {
18726 return Builder.CreateCall(F, Src0);
18728 case AMDGPU::BI__builtin_amdgcn_fract:
18729 case AMDGPU::BI__builtin_amdgcn_fractf:
18730 case AMDGPU::BI__builtin_amdgcn_fracth:
18731 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
18732 Intrinsic::amdgcn_fract);
18733 case AMDGPU::BI__builtin_amdgcn_lerp:
18734 return emitBuiltinWithOneOverloadedType<3>(*
this,
E,
18735 Intrinsic::amdgcn_lerp);
18736 case AMDGPU::BI__builtin_amdgcn_ubfe:
18737 return emitBuiltinWithOneOverloadedType<3>(*
this,
E,
18738 Intrinsic::amdgcn_ubfe);
18739 case AMDGPU::BI__builtin_amdgcn_sbfe:
18740 return emitBuiltinWithOneOverloadedType<3>(*
this,
E,
18741 Intrinsic::amdgcn_sbfe);
18742 case AMDGPU::BI__builtin_amdgcn_ballot_w32:
18743 case AMDGPU::BI__builtin_amdgcn_ballot_w64: {
18747 return Builder.CreateCall(F, { Src });
18749 case AMDGPU::BI__builtin_amdgcn_uicmp:
18750 case AMDGPU::BI__builtin_amdgcn_uicmpl:
18751 case AMDGPU::BI__builtin_amdgcn_sicmp:
18752 case AMDGPU::BI__builtin_amdgcn_sicmpl: {
18759 {
Builder.getInt64Ty(), Src0->getType() });
18760 return Builder.CreateCall(F, { Src0, Src1, Src2 });
18762 case AMDGPU::BI__builtin_amdgcn_fcmp:
18763 case AMDGPU::BI__builtin_amdgcn_fcmpf: {
18770 {
Builder.getInt64Ty(), Src0->getType() });
18771 return Builder.CreateCall(F, { Src0, Src1, Src2 });
18773 case AMDGPU::BI__builtin_amdgcn_class:
18774 case AMDGPU::BI__builtin_amdgcn_classf:
18775 case AMDGPU::BI__builtin_amdgcn_classh:
18777 case AMDGPU::BI__builtin_amdgcn_fmed3f:
18778 case AMDGPU::BI__builtin_amdgcn_fmed3h:
18779 return emitBuiltinWithOneOverloadedType<3>(*
this,
E,
18780 Intrinsic::amdgcn_fmed3);
18781 case AMDGPU::BI__builtin_amdgcn_ds_append:
18782 case AMDGPU::BI__builtin_amdgcn_ds_consume: {
18783 Intrinsic::ID Intrin = BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_append ?
18784 Intrinsic::amdgcn_ds_append : Intrinsic::amdgcn_ds_consume;
18789 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f64:
18790 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f32:
18791 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2f16:
18792 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmin_f64:
18793 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmax_f64:
18794 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f64:
18795 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmin_f64:
18796 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmax_f64:
18797 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f32:
18798 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2f16: {
18801 switch (BuiltinID) {
18802 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f32:
18804 IID = Intrinsic::amdgcn_global_atomic_fadd;
18806 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2f16:
18807 ArgTy = llvm::FixedVectorType::get(
18809 IID = Intrinsic::amdgcn_global_atomic_fadd;
18811 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f64:
18812 IID = Intrinsic::amdgcn_global_atomic_fadd;
18814 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmin_f64:
18815 IID = Intrinsic::amdgcn_global_atomic_fmin;
18817 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmax_f64:
18818 IID = Intrinsic::amdgcn_global_atomic_fmax;
18820 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f64:
18821 IID = Intrinsic::amdgcn_flat_atomic_fadd;
18823 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmin_f64:
18824 IID = Intrinsic::amdgcn_flat_atomic_fmin;
18826 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmax_f64:
18827 IID = Intrinsic::amdgcn_flat_atomic_fmax;
18829 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f32:
18831 IID = Intrinsic::amdgcn_flat_atomic_fadd;
18833 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2f16:
18834 ArgTy = llvm::FixedVectorType::get(
18836 IID = Intrinsic::amdgcn_flat_atomic_fadd;
18841 llvm::Function *F =
18843 return Builder.CreateCall(F, {Addr, Val});
18845 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2bf16:
18846 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2bf16: {
18848 switch (BuiltinID) {
18849 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2bf16:
18850 IID = Intrinsic::amdgcn_global_atomic_fadd_v2bf16;
18852 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2bf16:
18853 IID = Intrinsic::amdgcn_flat_atomic_fadd_v2bf16;
18859 return Builder.CreateCall(F, {Addr, Val});
18861 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_i32:
18862 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_v2i32:
18863 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4i16:
18864 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4f16:
18865 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4bf16:
18866 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16:
18867 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8f16:
18868 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8bf16: {
18871 switch (BuiltinID) {
18872 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_i32:
18873 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_v2i32:
18874 IID = Intrinsic::amdgcn_global_load_tr_b64;
18876 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4i16:
18877 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4f16:
18878 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4bf16:
18879 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16:
18880 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8f16:
18881 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8bf16:
18882 IID = Intrinsic::amdgcn_global_load_tr_b128;
18888 return Builder.CreateCall(F, {Addr});
18890 case AMDGPU::BI__builtin_amdgcn_get_fpenv: {
18893 return Builder.CreateCall(F);
18895 case AMDGPU::BI__builtin_amdgcn_set_fpenv: {
18901 case AMDGPU::BI__builtin_amdgcn_read_exec:
18903 case AMDGPU::BI__builtin_amdgcn_read_exec_lo:
18905 case AMDGPU::BI__builtin_amdgcn_read_exec_hi:
18907 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray:
18908 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_h:
18909 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_l:
18910 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_lh: {
18920 RayOrigin =
Builder.CreateShuffleVector(RayOrigin, RayOrigin,
18924 RayInverseDir =
Builder.CreateShuffleVector(RayInverseDir, RayInverseDir,
18928 {NodePtr->getType(), RayDir->getType()});
18929 return Builder.CreateCall(F, {NodePtr, RayExtent, RayOrigin, RayDir,
18930 RayInverseDir, TextureDescr});
18933 case AMDGPU::BI__builtin_amdgcn_ds_bvh_stack_rtn: {
18935 for (
int i = 0, e =
E->getNumArgs(); i != e; ++i)
18943 Value *I0 =
Builder.CreateInsertElement(PoisonValue::get(RetTy), Rtn,
18945 return Builder.CreateInsertElement(I0, A, 1);
18948 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32:
18949 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w32:
18950 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64:
18951 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w64:
18952 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32:
18953 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w32:
18954 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64:
18955 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w64:
18956 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32:
18957 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64:
18958 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32:
18959 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64:
18960 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32:
18961 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64:
18962 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32:
18963 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64:
18964 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32_gfx12:
18965 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64_gfx12:
18966 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32_gfx12:
18967 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64_gfx12:
18968 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32_gfx12:
18969 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64_gfx12:
18970 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12:
18971 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64_gfx12:
18972 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32_gfx12:
18973 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64_gfx12:
18974 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12:
18975 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64_gfx12:
18976 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w32_gfx12:
18977 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w64_gfx12:
18978 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w32_gfx12:
18979 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w64_gfx12:
18980 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w32_gfx12:
18981 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w64_gfx12:
18982 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w32_gfx12:
18983 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w64_gfx12:
18984 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w32_gfx12:
18985 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w64_gfx12:
18986 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w32:
18987 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w64:
18988 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w32:
18989 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w64:
18990 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w32:
18991 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w64:
18992 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w32:
18993 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w64:
18994 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w32:
18995 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w64:
18996 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w32:
18997 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w64:
18998 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w32:
18999 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w64:
19000 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w32:
19001 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w64:
19002 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w32:
19003 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w64:
19004 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w32:
19005 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w64:
19006 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w32:
19007 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w64: {
19020 bool AppendFalseForOpselArg =
false;
19021 unsigned BuiltinWMMAOp;
19023 switch (BuiltinID) {
19024 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32:
19025 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64:
19026 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12:
19027 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64_gfx12:
19028 ArgsForMatchingMatrixTypes = {2, 0};
19029 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_f16;
19031 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32:
19032 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64:
19033 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32_gfx12:
19034 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64_gfx12:
19035 ArgsForMatchingMatrixTypes = {2, 0};
19036 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_bf16;
19038 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32_gfx12:
19039 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64_gfx12:
19040 AppendFalseForOpselArg =
true;
19042 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32:
19043 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64:
19044 ArgsForMatchingMatrixTypes = {2, 0};
19045 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f16_16x16x16_f16;
19047 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32_gfx12:
19048 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64_gfx12:
19049 AppendFalseForOpselArg =
true;
19051 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32:
19052 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64:
19053 ArgsForMatchingMatrixTypes = {2, 0};
19054 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_bf16_16x16x16_bf16;
19056 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w32:
19057 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w64:
19058 ArgsForMatchingMatrixTypes = {2, 0};
19059 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f16_16x16x16_f16_tied;
19061 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w32:
19062 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w64:
19063 ArgsForMatchingMatrixTypes = {2, 0};
19064 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_bf16_16x16x16_bf16_tied;
19066 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32:
19067 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64:
19068 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12:
19069 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64_gfx12:
19070 ArgsForMatchingMatrixTypes = {4, 1};
19071 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x16_iu8;
19073 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32:
19074 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64:
19075 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32_gfx12:
19076 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64_gfx12:
19077 ArgsForMatchingMatrixTypes = {4, 1};
19078 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x16_iu4;
19080 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w32_gfx12:
19081 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w64_gfx12:
19082 ArgsForMatchingMatrixTypes = {2, 0};
19083 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_fp8_fp8;
19085 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w32_gfx12:
19086 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w64_gfx12:
19087 ArgsForMatchingMatrixTypes = {2, 0};
19088 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_fp8_bf8;
19090 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w32_gfx12:
19091 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w64_gfx12:
19092 ArgsForMatchingMatrixTypes = {2, 0};
19093 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_bf8_fp8;
19095 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w32_gfx12:
19096 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w64_gfx12:
19097 ArgsForMatchingMatrixTypes = {2, 0};
19098 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_bf8_bf8;
19100 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w32_gfx12:
19101 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w64_gfx12:
19102 ArgsForMatchingMatrixTypes = {4, 1};
19103 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x32_iu4;
19105 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w32:
19106 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w64:
19107 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
19108 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_f16;
19110 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w32:
19111 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w64:
19112 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
19113 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_bf16;
19115 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w32:
19116 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w64:
19117 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
19118 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f16_16x16x32_f16;
19120 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w32:
19121 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w64:
19122 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
19123 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_bf16_16x16x32_bf16;
19125 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w32:
19126 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w64:
19127 ArgsForMatchingMatrixTypes = {4, 1, 3, 5};
19128 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_i32_16x16x32_iu8;
19130 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w32:
19131 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w64:
19132 ArgsForMatchingMatrixTypes = {4, 1, 3, 5};
19133 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_i32_16x16x32_iu4;
19135 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w32:
19136 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w64:
19137 ArgsForMatchingMatrixTypes = {4, 1, 3, 5};
19138 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_i32_16x16x64_iu4;
19140 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w32:
19141 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w64:
19142 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
19143 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_fp8;
19145 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w32:
19146 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w64:
19147 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
19148 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_bf8;
19150 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w32:
19151 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w64:
19152 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
19153 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_fp8;
19155 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w32:
19156 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w64:
19157 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
19158 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_bf8;
19163 for (
int i = 0, e =
E->getNumArgs(); i != e; ++i)
19165 if (AppendFalseForOpselArg)
19166 Args.push_back(
Builder.getFalse());
19169 for (
auto ArgIdx : ArgsForMatchingMatrixTypes)
19170 ArgTypes.push_back(Args[ArgIdx]->getType());
19173 return Builder.CreateCall(F, Args);
19177 case AMDGPU::BI__builtin_amdgcn_workitem_id_x:
19179 case AMDGPU::BI__builtin_amdgcn_workitem_id_y:
19181 case AMDGPU::BI__builtin_amdgcn_workitem_id_z:
19185 case AMDGPU::BI__builtin_amdgcn_workgroup_size_x:
19186 return EmitAMDGPUWorkGroupSize(*
this, 0);
19187 case AMDGPU::BI__builtin_amdgcn_workgroup_size_y:
19188 return EmitAMDGPUWorkGroupSize(*
this, 1);
19189 case AMDGPU::BI__builtin_amdgcn_workgroup_size_z:
19190 return EmitAMDGPUWorkGroupSize(*
this, 2);
19193 case AMDGPU::BI__builtin_amdgcn_grid_size_x:
19194 return EmitAMDGPUGridSize(*
this, 0);
19195 case AMDGPU::BI__builtin_amdgcn_grid_size_y:
19196 return EmitAMDGPUGridSize(*
this, 1);
19197 case AMDGPU::BI__builtin_amdgcn_grid_size_z:
19198 return EmitAMDGPUGridSize(*
this, 2);
19201 case AMDGPU::BI__builtin_r600_recipsqrt_ieee:
19202 case AMDGPU::BI__builtin_r600_recipsqrt_ieeef:
19203 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
19204 Intrinsic::r600_recipsqrt_ieee);
19205 case AMDGPU::BI__builtin_r600_read_tidig_x:
19207 case AMDGPU::BI__builtin_r600_read_tidig_y:
19209 case AMDGPU::BI__builtin_r600_read_tidig_z:
19211 case AMDGPU::BI__builtin_amdgcn_alignbit: {
19216 return Builder.CreateCall(F, { Src0, Src1, Src2 });
19218 case AMDGPU::BI__builtin_amdgcn_fence: {
19221 FenceInst *Fence =
Builder.CreateFence(AO, SSID);
19222 if (
E->getNumArgs() > 2)
19226 case AMDGPU::BI__builtin_amdgcn_atomic_inc32:
19227 case AMDGPU::BI__builtin_amdgcn_atomic_inc64:
19228 case AMDGPU::BI__builtin_amdgcn_atomic_dec32:
19229 case AMDGPU::BI__builtin_amdgcn_atomic_dec64:
19230 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f64:
19231 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f32:
19232 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2f16:
19233 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2bf16:
19234 case AMDGPU::BI__builtin_amdgcn_ds_faddf:
19235 case AMDGPU::BI__builtin_amdgcn_ds_fminf:
19236 case AMDGPU::BI__builtin_amdgcn_ds_fmaxf: {
19237 llvm::AtomicRMWInst::BinOp BinOp;
19238 switch (BuiltinID) {
19239 case AMDGPU::BI__builtin_amdgcn_atomic_inc32:
19240 case AMDGPU::BI__builtin_amdgcn_atomic_inc64:
19241 BinOp = llvm::AtomicRMWInst::UIncWrap;
19243 case AMDGPU::BI__builtin_amdgcn_atomic_dec32:
19244 case AMDGPU::BI__builtin_amdgcn_atomic_dec64:
19245 BinOp = llvm::AtomicRMWInst::UDecWrap;
19247 case AMDGPU::BI__builtin_amdgcn_ds_faddf:
19248 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f64:
19249 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f32:
19250 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2f16:
19251 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2bf16:
19252 BinOp = llvm::AtomicRMWInst::FAdd;
19254 case AMDGPU::BI__builtin_amdgcn_ds_fminf:
19255 BinOp = llvm::AtomicRMWInst::FMin;
19257 case AMDGPU::BI__builtin_amdgcn_ds_fmaxf:
19258 BinOp = llvm::AtomicRMWInst::FMax;
19264 llvm::Type *OrigTy = Val->
getType();
19269 if (BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_faddf ||
19270 BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_fminf ||
19271 BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_fmaxf) {
19281 if (
E->getNumArgs() >= 4) {
19287 SSID = llvm::SyncScope::System;
19288 AO = AtomicOrdering::SequentiallyConsistent;
19291 if (BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2bf16) {
19292 llvm::Type *V2BF16Ty = FixedVectorType::get(
19293 llvm::Type::getBFloatTy(
Builder.getContext()), 2);
19294 Val =
Builder.CreateBitCast(Val, V2BF16Ty);
19298 llvm::AtomicRMWInst *RMW =
19301 RMW->setVolatile(
true);
19302 return Builder.CreateBitCast(RMW, OrigTy);
19304 case AMDGPU::BI__builtin_amdgcn_s_sendmsg_rtn:
19305 case AMDGPU::BI__builtin_amdgcn_s_sendmsg_rtnl: {
19311 return Builder.CreateCall(F, {Arg});
19313 case AMDGPU::BI__builtin_amdgcn_make_buffer_rsrc:
19314 return emitBuiltinWithOneOverloadedType<4>(
19315 *
this,
E, Intrinsic::amdgcn_make_buffer_rsrc);
19316 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b8:
19317 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b16:
19318 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b32:
19319 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b64:
19320 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b96:
19321 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b128:
19322 return emitBuiltinWithOneOverloadedType<5>(
19323 *
this,
E, Intrinsic::amdgcn_raw_ptr_buffer_store);
19324 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b8:
19325 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b16:
19326 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b32:
19327 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b64:
19328 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b96:
19329 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b128: {
19330 llvm::Type *RetTy =
nullptr;
19331 switch (BuiltinID) {
19332 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b8:
19335 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b16:
19338 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b32:
19341 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b64:
19342 RetTy = llvm::FixedVectorType::get(
Int32Ty, 2);
19344 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b96:
19345 RetTy = llvm::FixedVectorType::get(
Int32Ty, 3);
19347 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b128:
19348 RetTy = llvm::FixedVectorType::get(
Int32Ty, 4);
19366 unsigned IntrinsicID,
19368 unsigned NumArgs =
E->getNumArgs() - 1;
19370 for (
unsigned I = 0; I < NumArgs; ++I)
19382 switch (BuiltinID) {
19383 case SystemZ::BI__builtin_tbegin: {
19385 Value *Control = llvm::ConstantInt::get(
Int32Ty, 0xff0c);
19387 return Builder.CreateCall(F, {TDB, Control});
19389 case SystemZ::BI__builtin_tbegin_nofloat: {
19391 Value *Control = llvm::ConstantInt::get(
Int32Ty, 0xff0c);
19393 return Builder.CreateCall(F, {TDB, Control});
19395 case SystemZ::BI__builtin_tbeginc: {
19397 Value *Control = llvm::ConstantInt::get(
Int32Ty, 0xff08);
19399 return Builder.CreateCall(F, {TDB, Control});
19401 case SystemZ::BI__builtin_tabort: {
19406 case SystemZ::BI__builtin_non_tx_store: {
19418 case SystemZ::BI__builtin_s390_vpopctb:
19419 case SystemZ::BI__builtin_s390_vpopcth:
19420 case SystemZ::BI__builtin_s390_vpopctf:
19421 case SystemZ::BI__builtin_s390_vpopctg: {
19428 case SystemZ::BI__builtin_s390_vclzb:
19429 case SystemZ::BI__builtin_s390_vclzh:
19430 case SystemZ::BI__builtin_s390_vclzf:
19431 case SystemZ::BI__builtin_s390_vclzg: {
19434 Value *Undef = ConstantInt::get(
Builder.getInt1Ty(),
false);
19436 return Builder.CreateCall(F, {
X, Undef});
19439 case SystemZ::BI__builtin_s390_vctzb:
19440 case SystemZ::BI__builtin_s390_vctzh:
19441 case SystemZ::BI__builtin_s390_vctzf:
19442 case SystemZ::BI__builtin_s390_vctzg: {
19445 Value *Undef = ConstantInt::get(
Builder.getInt1Ty(),
false);
19447 return Builder.CreateCall(F, {
X, Undef});
19450 case SystemZ::BI__builtin_s390_verllb:
19451 case SystemZ::BI__builtin_s390_verllh:
19452 case SystemZ::BI__builtin_s390_verllf:
19453 case SystemZ::BI__builtin_s390_verllg: {
19458 unsigned NumElts = cast<llvm::FixedVectorType>(ResultType)->getNumElements();
19459 Amt =
Builder.CreateIntCast(Amt, ResultType->getScalarType(),
false);
19460 Amt =
Builder.CreateVectorSplat(NumElts, Amt);
19462 return Builder.CreateCall(F, { Src, Src, Amt });
19465 case SystemZ::BI__builtin_s390_verllvb:
19466 case SystemZ::BI__builtin_s390_verllvh:
19467 case SystemZ::BI__builtin_s390_verllvf:
19468 case SystemZ::BI__builtin_s390_verllvg: {
19473 return Builder.CreateCall(F, { Src, Src, Amt });
19476 case SystemZ::BI__builtin_s390_vfsqsb:
19477 case SystemZ::BI__builtin_s390_vfsqdb: {
19480 if (
Builder.getIsFPConstrained()) {
19482 return Builder.CreateConstrainedFPCall(F, {
X });
19488 case SystemZ::BI__builtin_s390_vfmasb:
19489 case SystemZ::BI__builtin_s390_vfmadb: {
19494 if (
Builder.getIsFPConstrained()) {
19496 return Builder.CreateConstrainedFPCall(F, {
X, Y, Z});
19499 return Builder.CreateCall(F, {
X, Y, Z});
19502 case SystemZ::BI__builtin_s390_vfmssb:
19503 case SystemZ::BI__builtin_s390_vfmsdb: {
19508 if (
Builder.getIsFPConstrained()) {
19510 return Builder.CreateConstrainedFPCall(F, {
X, Y,
Builder.CreateFNeg(Z,
"neg")});
19516 case SystemZ::BI__builtin_s390_vfnmasb:
19517 case SystemZ::BI__builtin_s390_vfnmadb: {
19522 if (
Builder.getIsFPConstrained()) {
19524 return Builder.CreateFNeg(
Builder.CreateConstrainedFPCall(F, {X, Y, Z}),
"neg");
19527 return Builder.CreateFNeg(
Builder.CreateCall(F, {X, Y, Z}),
"neg");
19530 case SystemZ::BI__builtin_s390_vfnmssb:
19531 case SystemZ::BI__builtin_s390_vfnmsdb: {
19536 if (
Builder.getIsFPConstrained()) {
19539 return Builder.CreateFNeg(
Builder.CreateConstrainedFPCall(F, {X, Y, NegZ}));
19546 case SystemZ::BI__builtin_s390_vflpsb:
19547 case SystemZ::BI__builtin_s390_vflpdb: {
19553 case SystemZ::BI__builtin_s390_vflnsb:
19554 case SystemZ::BI__builtin_s390_vflndb: {
19560 case SystemZ::BI__builtin_s390_vfisb:
19561 case SystemZ::BI__builtin_s390_vfidb: {
19569 Intrinsic::ID
ID = Intrinsic::not_intrinsic;
19571 switch (M4.getZExtValue()) {
19574 switch (M5.getZExtValue()) {
19576 case 0:
ID = Intrinsic::rint;
19577 CI = Intrinsic::experimental_constrained_rint;
break;
19581 switch (M5.getZExtValue()) {
19583 case 0:
ID = Intrinsic::nearbyint;
19584 CI = Intrinsic::experimental_constrained_nearbyint;
break;
19585 case 1:
ID = Intrinsic::round;
19586 CI = Intrinsic::experimental_constrained_round;
break;
19587 case 5:
ID = Intrinsic::trunc;
19588 CI = Intrinsic::experimental_constrained_trunc;
break;
19589 case 6:
ID = Intrinsic::ceil;
19590 CI = Intrinsic::experimental_constrained_ceil;
break;
19591 case 7:
ID = Intrinsic::floor;
19592 CI = Intrinsic::experimental_constrained_floor;
break;
19596 if (ID != Intrinsic::not_intrinsic) {
19597 if (
Builder.getIsFPConstrained()) {
19599 return Builder.CreateConstrainedFPCall(F,
X);
19605 switch (BuiltinID) {
19606 case SystemZ::BI__builtin_s390_vfisb:
ID = Intrinsic::s390_vfisb;
break;
19607 case SystemZ::BI__builtin_s390_vfidb:
ID = Intrinsic::s390_vfidb;
break;
19608 default: llvm_unreachable(
"Unknown BuiltinID");
19613 return Builder.CreateCall(F, {
X, M4Value, M5Value});
19615 case SystemZ::BI__builtin_s390_vfmaxsb:
19616 case SystemZ::BI__builtin_s390_vfmaxdb: {
19624 Intrinsic::ID
ID = Intrinsic::not_intrinsic;
19626 switch (M4.getZExtValue()) {
19628 case 4:
ID = Intrinsic::maxnum;
19629 CI = Intrinsic::experimental_constrained_maxnum;
break;
19631 if (ID != Intrinsic::not_intrinsic) {
19632 if (
Builder.getIsFPConstrained()) {
19634 return Builder.CreateConstrainedFPCall(F, {
X, Y});
19637 return Builder.CreateCall(F, {
X, Y});
19640 switch (BuiltinID) {
19641 case SystemZ::BI__builtin_s390_vfmaxsb:
ID = Intrinsic::s390_vfmaxsb;
break;
19642 case SystemZ::BI__builtin_s390_vfmaxdb:
ID = Intrinsic::s390_vfmaxdb;
break;
19643 default: llvm_unreachable(
"Unknown BuiltinID");
19647 return Builder.CreateCall(F, {
X, Y, M4Value});
19649 case SystemZ::BI__builtin_s390_vfminsb:
19650 case SystemZ::BI__builtin_s390_vfmindb: {
19658 Intrinsic::ID
ID = Intrinsic::not_intrinsic;
19660 switch (M4.getZExtValue()) {
19662 case 4:
ID = Intrinsic::minnum;
19663 CI = Intrinsic::experimental_constrained_minnum;
break;
19665 if (ID != Intrinsic::not_intrinsic) {
19666 if (
Builder.getIsFPConstrained()) {
19668 return Builder.CreateConstrainedFPCall(F, {
X, Y});
19671 return Builder.CreateCall(F, {
X, Y});
19674 switch (BuiltinID) {
19675 case SystemZ::BI__builtin_s390_vfminsb:
ID = Intrinsic::s390_vfminsb;
break;
19676 case SystemZ::BI__builtin_s390_vfmindb:
ID = Intrinsic::s390_vfmindb;
break;
19677 default: llvm_unreachable(
"Unknown BuiltinID");
19681 return Builder.CreateCall(F, {
X, Y, M4Value});
19684 case SystemZ::BI__builtin_s390_vlbrh:
19685 case SystemZ::BI__builtin_s390_vlbrf:
19686 case SystemZ::BI__builtin_s390_vlbrg: {
19695#define INTRINSIC_WITH_CC(NAME) \
19696 case SystemZ::BI__builtin_##NAME: \
19697 return EmitSystemZIntrinsicWithCC(*this, Intrinsic::NAME, E)
19776#undef INTRINSIC_WITH_CC
19785struct NVPTXMmaLdstInfo {
19786 unsigned NumResults;
19792#define MMA_INTR(geom_op_type, layout) \
19793 Intrinsic::nvvm_wmma_##geom_op_type##_##layout##_stride
19794#define MMA_LDST(n, geom_op_type) \
19795 { n, MMA_INTR(geom_op_type, col), MMA_INTR(geom_op_type, row) }
19797static NVPTXMmaLdstInfo getNVPTXMmaLdstInfo(
unsigned BuiltinID) {
19798 switch (BuiltinID) {
19800 case NVPTX::BI__hmma_m16n16k16_ld_a:
19801 return MMA_LDST(8, m16n16k16_load_a_f16);
19802 case NVPTX::BI__hmma_m16n16k16_ld_b:
19803 return MMA_LDST(8, m16n16k16_load_b_f16);
19804 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
19805 return MMA_LDST(4, m16n16k16_load_c_f16);
19806 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
19807 return MMA_LDST(8, m16n16k16_load_c_f32);
19808 case NVPTX::BI__hmma_m32n8k16_ld_a:
19809 return MMA_LDST(8, m32n8k16_load_a_f16);
19810 case NVPTX::BI__hmma_m32n8k16_ld_b:
19811 return MMA_LDST(8, m32n8k16_load_b_f16);
19812 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
19813 return MMA_LDST(4, m32n8k16_load_c_f16);
19814 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
19815 return MMA_LDST(8, m32n8k16_load_c_f32);
19816 case NVPTX::BI__hmma_m8n32k16_ld_a:
19817 return MMA_LDST(8, m8n32k16_load_a_f16);
19818 case NVPTX::BI__hmma_m8n32k16_ld_b:
19819 return MMA_LDST(8, m8n32k16_load_b_f16);
19820 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
19821 return MMA_LDST(4, m8n32k16_load_c_f16);
19822 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
19823 return MMA_LDST(8, m8n32k16_load_c_f32);
19826 case NVPTX::BI__imma_m16n16k16_ld_a_s8:
19827 return MMA_LDST(2, m16n16k16_load_a_s8);
19828 case NVPTX::BI__imma_m16n16k16_ld_a_u8:
19829 return MMA_LDST(2, m16n16k16_load_a_u8);
19830 case NVPTX::BI__imma_m16n16k16_ld_b_s8:
19831 return MMA_LDST(2, m16n16k16_load_b_s8);
19832 case NVPTX::BI__imma_m16n16k16_ld_b_u8:
19833 return MMA_LDST(2, m16n16k16_load_b_u8);
19834 case NVPTX::BI__imma_m16n16k16_ld_c:
19835 return MMA_LDST(8, m16n16k16_load_c_s32);
19836 case NVPTX::BI__imma_m32n8k16_ld_a_s8:
19837 return MMA_LDST(4, m32n8k16_load_a_s8);
19838 case NVPTX::BI__imma_m32n8k16_ld_a_u8:
19839 return MMA_LDST(4, m32n8k16_load_a_u8);
19840 case NVPTX::BI__imma_m32n8k16_ld_b_s8:
19841 return MMA_LDST(1, m32n8k16_load_b_s8);
19842 case NVPTX::BI__imma_m32n8k16_ld_b_u8:
19843 return MMA_LDST(1, m32n8k16_load_b_u8);
19844 case NVPTX::BI__imma_m32n8k16_ld_c:
19845 return MMA_LDST(8, m32n8k16_load_c_s32);
19846 case NVPTX::BI__imma_m8n32k16_ld_a_s8:
19847 return MMA_LDST(1, m8n32k16_load_a_s8);
19848 case NVPTX::BI__imma_m8n32k16_ld_a_u8:
19849 return MMA_LDST(1, m8n32k16_load_a_u8);
19850 case NVPTX::BI__imma_m8n32k16_ld_b_s8:
19851 return MMA_LDST(4, m8n32k16_load_b_s8);
19852 case NVPTX::BI__imma_m8n32k16_ld_b_u8:
19853 return MMA_LDST(4, m8n32k16_load_b_u8);
19854 case NVPTX::BI__imma_m8n32k16_ld_c:
19855 return MMA_LDST(8, m8n32k16_load_c_s32);
19859 case NVPTX::BI__imma_m8n8k32_ld_a_s4:
19860 return {1, 0,
MMA_INTR(m8n8k32_load_a_s4, row)};
19861 case NVPTX::BI__imma_m8n8k32_ld_a_u4:
19862 return {1, 0,
MMA_INTR(m8n8k32_load_a_u4, row)};
19863 case NVPTX::BI__imma_m8n8k32_ld_b_s4:
19864 return {1,
MMA_INTR(m8n8k32_load_b_s4, col), 0};
19865 case NVPTX::BI__imma_m8n8k32_ld_b_u4:
19866 return {1,
MMA_INTR(m8n8k32_load_b_u4, col), 0};
19867 case NVPTX::BI__imma_m8n8k32_ld_c:
19868 return MMA_LDST(2, m8n8k32_load_c_s32);
19869 case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
19870 return {1, 0,
MMA_INTR(m8n8k128_load_a_b1, row)};
19871 case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
19872 return {1,
MMA_INTR(m8n8k128_load_b_b1, col), 0};
19873 case NVPTX::BI__bmma_m8n8k128_ld_c:
19874 return MMA_LDST(2, m8n8k128_load_c_s32);
19877 case NVPTX::BI__dmma_m8n8k4_ld_a:
19878 return MMA_LDST(1, m8n8k4_load_a_f64);
19879 case NVPTX::BI__dmma_m8n8k4_ld_b:
19880 return MMA_LDST(1, m8n8k4_load_b_f64);
19881 case NVPTX::BI__dmma_m8n8k4_ld_c:
19882 return MMA_LDST(2, m8n8k4_load_c_f64);
19885 case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
19886 return MMA_LDST(4, m16n16k16_load_a_bf16);
19887 case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
19888 return MMA_LDST(4, m16n16k16_load_b_bf16);
19889 case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
19890 return MMA_LDST(2, m8n32k16_load_a_bf16);
19891 case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
19892 return MMA_LDST(8, m8n32k16_load_b_bf16);
19893 case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
19894 return MMA_LDST(8, m32n8k16_load_a_bf16);
19895 case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
19896 return MMA_LDST(2, m32n8k16_load_b_bf16);
19897 case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
19898 return MMA_LDST(4, m16n16k8_load_a_tf32);
19899 case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
19900 return MMA_LDST(4, m16n16k8_load_b_tf32);
19901 case NVPTX::BI__mma_tf32_m16n16k8_ld_c:
19902 return MMA_LDST(8, m16n16k8_load_c_f32);
19908 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
19909 return MMA_LDST(4, m16n16k16_store_d_f16);
19910 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
19911 return MMA_LDST(8, m16n16k16_store_d_f32);
19912 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
19913 return MMA_LDST(4, m32n8k16_store_d_f16);
19914 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
19915 return MMA_LDST(8, m32n8k16_store_d_f32);
19916 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
19917 return MMA_LDST(4, m8n32k16_store_d_f16);
19918 case NVPTX::BI__hmma_m8n32k16_st_c_f32:
19919 return MMA_LDST(8, m8n32k16_store_d_f32);
19924 case NVPTX::BI__imma_m16n16k16_st_c_i32:
19925 return MMA_LDST(8, m16n16k16_store_d_s32);
19926 case NVPTX::BI__imma_m32n8k16_st_c_i32:
19927 return MMA_LDST(8, m32n8k16_store_d_s32);
19928 case NVPTX::BI__imma_m8n32k16_st_c_i32:
19929 return MMA_LDST(8, m8n32k16_store_d_s32);
19930 case NVPTX::BI__imma_m8n8k32_st_c_i32:
19931 return MMA_LDST(2, m8n8k32_store_d_s32);
19932 case NVPTX::BI__bmma_m8n8k128_st_c_i32:
19933 return MMA_LDST(2, m8n8k128_store_d_s32);
19936 case NVPTX::BI__dmma_m8n8k4_st_c_f64:
19937 return MMA_LDST(2, m8n8k4_store_d_f64);
19940 case NVPTX::BI__mma_m16n16k8_st_c_f32:
19941 return MMA_LDST(8, m16n16k8_store_d_f32);
19944 llvm_unreachable(
"Unknown MMA builtin");
19951struct NVPTXMmaInfo {
19960 std::array<unsigned, 8> Variants;
19962 unsigned getMMAIntrinsic(
int Layout,
bool Satf) {
19963 unsigned Index = Layout + 4 * Satf;
19964 if (Index >= Variants.size())
19966 return Variants[Index];
19972static NVPTXMmaInfo getNVPTXMmaInfo(
unsigned BuiltinID) {
19974#define MMA_VARIANTS(geom, type) \
19975 Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type, \
19976 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type, \
19977 Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type, \
19978 Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type
19979#define MMA_SATF_VARIANTS(geom, type) \
19980 MMA_VARIANTS(geom, type), \
19981 Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type##_satfinite, \
19982 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type##_satfinite, \
19983 Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type##_satfinite, \
19984 Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type##_satfinite
19986#define MMA_VARIANTS_I4(geom, type) \
19988 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type, \
19992 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type##_satfinite, \
19996#define MMA_VARIANTS_B1_XOR(geom, type) \
19998 Intrinsic::nvvm_wmma_##geom##_mma_xor_popc_row_col_##type, \
20005#define MMA_VARIANTS_B1_AND(geom, type) \
20007 Intrinsic::nvvm_wmma_##geom##_mma_and_popc_row_col_##type, \
20015 switch (BuiltinID) {
20019 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
20021 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
20023 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
20025 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
20027 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
20029 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
20031 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
20033 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
20035 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
20037 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
20039 case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
20041 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
20045 case NVPTX::BI__imma_m16n16k16_mma_s8:
20047 case NVPTX::BI__imma_m16n16k16_mma_u8:
20049 case NVPTX::BI__imma_m32n8k16_mma_s8:
20051 case NVPTX::BI__imma_m32n8k16_mma_u8:
20053 case NVPTX::BI__imma_m8n32k16_mma_s8:
20055 case NVPTX::BI__imma_m8n32k16_mma_u8:
20059 case NVPTX::BI__imma_m8n8k32_mma_s4:
20061 case NVPTX::BI__imma_m8n8k32_mma_u4:
20063 case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1:
20065 case NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1:
20069 case NVPTX::BI__dmma_m8n8k4_mma_f64:
20073 case NVPTX::BI__mma_bf16_m16n16k16_mma_f32:
20074 return {4, 4, 8, 8, {{
MMA_VARIANTS(m16n16k16, bf16)}}};
20075 case NVPTX::BI__mma_bf16_m8n32k16_mma_f32:
20076 return {2, 8, 8, 8, {{
MMA_VARIANTS(m8n32k16, bf16)}}};
20077 case NVPTX::BI__mma_bf16_m32n8k16_mma_f32:
20078 return {8, 2, 8, 8, {{
MMA_VARIANTS(m32n8k16, bf16)}}};
20079 case NVPTX::BI__mma_tf32_m16n16k8_mma_f32:
20080 return {4, 4, 8, 8, {{
MMA_VARIANTS(m16n16k8, tf32)}}};
20082 llvm_unreachable(
"Unexpected builtin ID.");
20085#undef MMA_SATF_VARIANTS
20086#undef MMA_VARIANTS_I4
20087#undef MMA_VARIANTS_B1_AND
20088#undef MMA_VARIANTS_B1_XOR
20097 return CGF.
Builder.CreateCall(
20099 {Ptr, ConstantInt::get(CGF.Builder.getInt32Ty(), Align.getQuantity())});
20105 llvm::Type *ElemTy =
20107 return CGF.
Builder.CreateCall(
20109 {Ptr, CGF.EmitScalarExpr(E->getArg(1))});
20112static Value *MakeCpAsync(
unsigned IntrinsicID,
unsigned IntrinsicIDS,
20115 return E->getNumArgs() == 3
20117 {CGF.EmitScalarExpr(E->getArg(0)),
20118 CGF.EmitScalarExpr(E->getArg(1)),
20119 CGF.EmitScalarExpr(E->getArg(2))})
20121 {CGF.EmitScalarExpr(E->getArg(0)),
20122 CGF.EmitScalarExpr(E->getArg(1))});
20125static Value *MakeHalfType(
unsigned IntrinsicID,
unsigned BuiltinID,
20128 if (!(
C.getLangOpts().NativeHalfType ||
20129 !
C.getTargetInfo().useFP16ConversionIntrinsics())) {
20131 " requires native half type support.");
20135 if (IntrinsicID == Intrinsic::nvvm_ldg_global_f ||
20136 IntrinsicID == Intrinsic::nvvm_ldu_global_f)
20137 return MakeLdgLdu(IntrinsicID, CGF,
E);
20141 auto *FTy = F->getFunctionType();
20142 unsigned ICEArguments = 0;
20144 C.GetBuiltinType(BuiltinID, Error, &ICEArguments);
20146 for (
unsigned i = 0, e =
E->getNumArgs(); i != e; ++i) {
20147 assert((ICEArguments & (1 << i)) == 0);
20149 auto *PTy = FTy->getParamType(i);
20150 if (PTy != ArgValue->
getType())
20151 ArgValue = CGF.
Builder.CreateBitCast(ArgValue, PTy);
20152 Args.push_back(ArgValue);
20155 return CGF.
Builder.CreateCall(F, Args);
20161 switch (BuiltinID) {
20162 case NVPTX::BI__nvvm_atom_add_gen_i:
20163 case NVPTX::BI__nvvm_atom_add_gen_l:
20164 case NVPTX::BI__nvvm_atom_add_gen_ll:
20167 case NVPTX::BI__nvvm_atom_sub_gen_i:
20168 case NVPTX::BI__nvvm_atom_sub_gen_l:
20169 case NVPTX::BI__nvvm_atom_sub_gen_ll:
20172 case NVPTX::BI__nvvm_atom_and_gen_i:
20173 case NVPTX::BI__nvvm_atom_and_gen_l:
20174 case NVPTX::BI__nvvm_atom_and_gen_ll:
20177 case NVPTX::BI__nvvm_atom_or_gen_i:
20178 case NVPTX::BI__nvvm_atom_or_gen_l:
20179 case NVPTX::BI__nvvm_atom_or_gen_ll:
20182 case NVPTX::BI__nvvm_atom_xor_gen_i:
20183 case NVPTX::BI__nvvm_atom_xor_gen_l:
20184 case NVPTX::BI__nvvm_atom_xor_gen_ll:
20187 case NVPTX::BI__nvvm_atom_xchg_gen_i:
20188 case NVPTX::BI__nvvm_atom_xchg_gen_l:
20189 case NVPTX::BI__nvvm_atom_xchg_gen_ll:
20192 case NVPTX::BI__nvvm_atom_max_gen_i:
20193 case NVPTX::BI__nvvm_atom_max_gen_l:
20194 case NVPTX::BI__nvvm_atom_max_gen_ll:
20197 case NVPTX::BI__nvvm_atom_max_gen_ui:
20198 case NVPTX::BI__nvvm_atom_max_gen_ul:
20199 case NVPTX::BI__nvvm_atom_max_gen_ull:
20202 case NVPTX::BI__nvvm_atom_min_gen_i:
20203 case NVPTX::BI__nvvm_atom_min_gen_l:
20204 case NVPTX::BI__nvvm_atom_min_gen_ll:
20207 case NVPTX::BI__nvvm_atom_min_gen_ui:
20208 case NVPTX::BI__nvvm_atom_min_gen_ul:
20209 case NVPTX::BI__nvvm_atom_min_gen_ull:
20212 case NVPTX::BI__nvvm_atom_cas_gen_i:
20213 case NVPTX::BI__nvvm_atom_cas_gen_l:
20214 case NVPTX::BI__nvvm_atom_cas_gen_ll:
20219 case NVPTX::BI__nvvm_atom_add_gen_f:
20220 case NVPTX::BI__nvvm_atom_add_gen_d: {
20225 AtomicOrdering::SequentiallyConsistent);
20228 case NVPTX::BI__nvvm_atom_inc_gen_ui: {
20233 return Builder.CreateCall(FnALI32, {Ptr, Val});
20236 case NVPTX::BI__nvvm_atom_dec_gen_ui: {
20241 return Builder.CreateCall(FnALD32, {Ptr, Val});
20244 case NVPTX::BI__nvvm_ldg_c:
20245 case NVPTX::BI__nvvm_ldg_sc:
20246 case NVPTX::BI__nvvm_ldg_c2:
20247 case NVPTX::BI__nvvm_ldg_sc2:
20248 case NVPTX::BI__nvvm_ldg_c4:
20249 case NVPTX::BI__nvvm_ldg_sc4:
20250 case NVPTX::BI__nvvm_ldg_s:
20251 case NVPTX::BI__nvvm_ldg_s2:
20252 case NVPTX::BI__nvvm_ldg_s4:
20253 case NVPTX::BI__nvvm_ldg_i:
20254 case NVPTX::BI__nvvm_ldg_i2:
20255 case NVPTX::BI__nvvm_ldg_i4:
20256 case NVPTX::BI__nvvm_ldg_l:
20257 case NVPTX::BI__nvvm_ldg_l2:
20258 case NVPTX::BI__nvvm_ldg_ll:
20259 case NVPTX::BI__nvvm_ldg_ll2:
20260 case NVPTX::BI__nvvm_ldg_uc:
20261 case NVPTX::BI__nvvm_ldg_uc2:
20262 case NVPTX::BI__nvvm_ldg_uc4:
20263 case NVPTX::BI__nvvm_ldg_us:
20264 case NVPTX::BI__nvvm_ldg_us2:
20265 case NVPTX::BI__nvvm_ldg_us4:
20266 case NVPTX::BI__nvvm_ldg_ui:
20267 case NVPTX::BI__nvvm_ldg_ui2:
20268 case NVPTX::BI__nvvm_ldg_ui4:
20269 case NVPTX::BI__nvvm_ldg_ul:
20270 case NVPTX::BI__nvvm_ldg_ul2:
20271 case NVPTX::BI__nvvm_ldg_ull:
20272 case NVPTX::BI__nvvm_ldg_ull2:
20276 return MakeLdgLdu(Intrinsic::nvvm_ldg_global_i, *
this,
E);
20277 case NVPTX::BI__nvvm_ldg_f:
20278 case NVPTX::BI__nvvm_ldg_f2:
20279 case NVPTX::BI__nvvm_ldg_f4:
20280 case NVPTX::BI__nvvm_ldg_d:
20281 case NVPTX::BI__nvvm_ldg_d2:
20282 return MakeLdgLdu(Intrinsic::nvvm_ldg_global_f, *
this,
E);
20284 case NVPTX::BI__nvvm_ldu_c:
20285 case NVPTX::BI__nvvm_ldu_sc:
20286 case NVPTX::BI__nvvm_ldu_c2:
20287 case NVPTX::BI__nvvm_ldu_sc2:
20288 case NVPTX::BI__nvvm_ldu_c4:
20289 case NVPTX::BI__nvvm_ldu_sc4:
20290 case NVPTX::BI__nvvm_ldu_s:
20291 case NVPTX::BI__nvvm_ldu_s2:
20292 case NVPTX::BI__nvvm_ldu_s4:
20293 case NVPTX::BI__nvvm_ldu_i:
20294 case NVPTX::BI__nvvm_ldu_i2:
20295 case NVPTX::BI__nvvm_ldu_i4:
20296 case NVPTX::BI__nvvm_ldu_l:
20297 case NVPTX::BI__nvvm_ldu_l2:
20298 case NVPTX::BI__nvvm_ldu_ll:
20299 case NVPTX::BI__nvvm_ldu_ll2:
20300 case NVPTX::BI__nvvm_ldu_uc:
20301 case NVPTX::BI__nvvm_ldu_uc2:
20302 case NVPTX::BI__nvvm_ldu_uc4:
20303 case NVPTX::BI__nvvm_ldu_us:
20304 case NVPTX::BI__nvvm_ldu_us2:
20305 case NVPTX::BI__nvvm_ldu_us4:
20306 case NVPTX::BI__nvvm_ldu_ui:
20307 case NVPTX::BI__nvvm_ldu_ui2:
20308 case NVPTX::BI__nvvm_ldu_ui4:
20309 case NVPTX::BI__nvvm_ldu_ul:
20310 case NVPTX::BI__nvvm_ldu_ul2:
20311 case NVPTX::BI__nvvm_ldu_ull:
20312 case NVPTX::BI__nvvm_ldu_ull2:
20313 return MakeLdgLdu(Intrinsic::nvvm_ldu_global_i, *
this,
E);
20314 case NVPTX::BI__nvvm_ldu_f:
20315 case NVPTX::BI__nvvm_ldu_f2:
20316 case NVPTX::BI__nvvm_ldu_f4:
20317 case NVPTX::BI__nvvm_ldu_d:
20318 case NVPTX::BI__nvvm_ldu_d2:
20319 return MakeLdgLdu(Intrinsic::nvvm_ldu_global_f, *
this,
E);
20321 case NVPTX::BI__nvvm_atom_cta_add_gen_i:
20322 case NVPTX::BI__nvvm_atom_cta_add_gen_l:
20323 case NVPTX::BI__nvvm_atom_cta_add_gen_ll:
20324 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_cta, *
this,
E);
20325 case NVPTX::BI__nvvm_atom_sys_add_gen_i:
20326 case NVPTX::BI__nvvm_atom_sys_add_gen_l:
20327 case NVPTX::BI__nvvm_atom_sys_add_gen_ll:
20328 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_sys, *
this,
E);
20329 case NVPTX::BI__nvvm_atom_cta_add_gen_f:
20330 case NVPTX::BI__nvvm_atom_cta_add_gen_d:
20331 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_cta, *
this,
E);
20332 case NVPTX::BI__nvvm_atom_sys_add_gen_f:
20333 case NVPTX::BI__nvvm_atom_sys_add_gen_d:
20334 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_sys, *
this,
E);
20335 case NVPTX::BI__nvvm_atom_cta_xchg_gen_i:
20336 case NVPTX::BI__nvvm_atom_cta_xchg_gen_l:
20337 case NVPTX::BI__nvvm_atom_cta_xchg_gen_ll:
20338 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_cta, *
this,
E);
20339 case NVPTX::BI__nvvm_atom_sys_xchg_gen_i:
20340 case NVPTX::BI__nvvm_atom_sys_xchg_gen_l:
20341 case NVPTX::BI__nvvm_atom_sys_xchg_gen_ll:
20342 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_sys, *
this,
E);
20343 case NVPTX::BI__nvvm_atom_cta_max_gen_i:
20344 case NVPTX::BI__nvvm_atom_cta_max_gen_ui:
20345 case NVPTX::BI__nvvm_atom_cta_max_gen_l:
20346 case NVPTX::BI__nvvm_atom_cta_max_gen_ul:
20347 case NVPTX::BI__nvvm_atom_cta_max_gen_ll:
20348 case NVPTX::BI__nvvm_atom_cta_max_gen_ull:
20349 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_cta, *
this,
E);
20350 case NVPTX::BI__nvvm_atom_sys_max_gen_i:
20351 case NVPTX::BI__nvvm_atom_sys_max_gen_ui:
20352 case NVPTX::BI__nvvm_atom_sys_max_gen_l:
20353 case NVPTX::BI__nvvm_atom_sys_max_gen_ul:
20354 case NVPTX::BI__nvvm_atom_sys_max_gen_ll:
20355 case NVPTX::BI__nvvm_atom_sys_max_gen_ull:
20356 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_sys, *
this,
E);
20357 case NVPTX::BI__nvvm_atom_cta_min_gen_i:
20358 case NVPTX::BI__nvvm_atom_cta_min_gen_ui:
20359 case NVPTX::BI__nvvm_atom_cta_min_gen_l:
20360 case NVPTX::BI__nvvm_atom_cta_min_gen_ul:
20361 case NVPTX::BI__nvvm_atom_cta_min_gen_ll:
20362 case NVPTX::BI__nvvm_atom_cta_min_gen_ull:
20363 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_cta, *
this,
E);
20364 case NVPTX::BI__nvvm_atom_sys_min_gen_i:
20365 case NVPTX::BI__nvvm_atom_sys_min_gen_ui:
20366 case NVPTX::BI__nvvm_atom_sys_min_gen_l:
20367 case NVPTX::BI__nvvm_atom_sys_min_gen_ul:
20368 case NVPTX::BI__nvvm_atom_sys_min_gen_ll:
20369 case NVPTX::BI__nvvm_atom_sys_min_gen_ull:
20370 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_sys, *
this,
E);
20371 case NVPTX::BI__nvvm_atom_cta_inc_gen_ui:
20372 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_cta, *
this,
E);
20373 case NVPTX::BI__nvvm_atom_cta_dec_gen_ui:
20374 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_cta, *
this,
E);
20375 case NVPTX::BI__nvvm_atom_sys_inc_gen_ui:
20376 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_sys, *
this,
E);
20377 case NVPTX::BI__nvvm_atom_sys_dec_gen_ui:
20378 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_sys, *
this,
E);
20379 case NVPTX::BI__nvvm_atom_cta_and_gen_i:
20380 case NVPTX::BI__nvvm_atom_cta_and_gen_l:
20381 case NVPTX::BI__nvvm_atom_cta_and_gen_ll:
20382 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_cta, *
this,
E);
20383 case NVPTX::BI__nvvm_atom_sys_and_gen_i:
20384 case NVPTX::BI__nvvm_atom_sys_and_gen_l:
20385 case NVPTX::BI__nvvm_atom_sys_and_gen_ll:
20386 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_sys, *
this,
E);
20387 case NVPTX::BI__nvvm_atom_cta_or_gen_i:
20388 case NVPTX::BI__nvvm_atom_cta_or_gen_l:
20389 case NVPTX::BI__nvvm_atom_cta_or_gen_ll:
20390 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_cta, *
this,
E);
20391 case NVPTX::BI__nvvm_atom_sys_or_gen_i:
20392 case NVPTX::BI__nvvm_atom_sys_or_gen_l:
20393 case NVPTX::BI__nvvm_atom_sys_or_gen_ll:
20394 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_sys, *
this,
E);
20395 case NVPTX::BI__nvvm_atom_cta_xor_gen_i:
20396 case NVPTX::BI__nvvm_atom_cta_xor_gen_l:
20397 case NVPTX::BI__nvvm_atom_cta_xor_gen_ll:
20398 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_cta, *
this,
E);
20399 case NVPTX::BI__nvvm_atom_sys_xor_gen_i:
20400 case NVPTX::BI__nvvm_atom_sys_xor_gen_l:
20401 case NVPTX::BI__nvvm_atom_sys_xor_gen_ll:
20402 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_sys, *
this,
E);
20403 case NVPTX::BI__nvvm_atom_cta_cas_gen_i:
20404 case NVPTX::BI__nvvm_atom_cta_cas_gen_l:
20405 case NVPTX::BI__nvvm_atom_cta_cas_gen_ll: {
20407 llvm::Type *ElemTy =
20411 Intrinsic::nvvm_atomic_cas_gen_i_cta, {ElemTy, Ptr->getType()}),
20412 {Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))});
20414 case NVPTX::BI__nvvm_atom_sys_cas_gen_i:
20415 case NVPTX::BI__nvvm_atom_sys_cas_gen_l:
20416 case NVPTX::BI__nvvm_atom_sys_cas_gen_ll: {
20418 llvm::Type *ElemTy =
20422 Intrinsic::nvvm_atomic_cas_gen_i_sys, {ElemTy, Ptr->getType()}),
20423 {Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))});
20425 case NVPTX::BI__nvvm_match_all_sync_i32p:
20426 case NVPTX::BI__nvvm_match_all_sync_i64p: {
20432 ? Intrinsic::nvvm_match_all_sync_i32p
20433 : Intrinsic::nvvm_match_all_sync_i64p),
20438 return Builder.CreateExtractValue(ResultPair, 0);
20442 case NVPTX::BI__hmma_m16n16k16_ld_a:
20443 case NVPTX::BI__hmma_m16n16k16_ld_b:
20444 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
20445 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
20446 case NVPTX::BI__hmma_m32n8k16_ld_a:
20447 case NVPTX::BI__hmma_m32n8k16_ld_b:
20448 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
20449 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
20450 case NVPTX::BI__hmma_m8n32k16_ld_a:
20451 case NVPTX::BI__hmma_m8n32k16_ld_b:
20452 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
20453 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
20455 case NVPTX::BI__imma_m16n16k16_ld_a_s8:
20456 case NVPTX::BI__imma_m16n16k16_ld_a_u8:
20457 case NVPTX::BI__imma_m16n16k16_ld_b_s8:
20458 case NVPTX::BI__imma_m16n16k16_ld_b_u8:
20459 case NVPTX::BI__imma_m16n16k16_ld_c:
20460 case NVPTX::BI__imma_m32n8k16_ld_a_s8:
20461 case NVPTX::BI__imma_m32n8k16_ld_a_u8:
20462 case NVPTX::BI__imma_m32n8k16_ld_b_s8:
20463 case NVPTX::BI__imma_m32n8k16_ld_b_u8:
20464 case NVPTX::BI__imma_m32n8k16_ld_c:
20465 case NVPTX::BI__imma_m8n32k16_ld_a_s8:
20466 case NVPTX::BI__imma_m8n32k16_ld_a_u8:
20467 case NVPTX::BI__imma_m8n32k16_ld_b_s8:
20468 case NVPTX::BI__imma_m8n32k16_ld_b_u8:
20469 case NVPTX::BI__imma_m8n32k16_ld_c:
20471 case NVPTX::BI__imma_m8n8k32_ld_a_s4:
20472 case NVPTX::BI__imma_m8n8k32_ld_a_u4:
20473 case NVPTX::BI__imma_m8n8k32_ld_b_s4:
20474 case NVPTX::BI__imma_m8n8k32_ld_b_u4:
20475 case NVPTX::BI__imma_m8n8k32_ld_c:
20476 case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
20477 case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
20478 case NVPTX::BI__bmma_m8n8k128_ld_c:
20480 case NVPTX::BI__dmma_m8n8k4_ld_a:
20481 case NVPTX::BI__dmma_m8n8k4_ld_b:
20482 case NVPTX::BI__dmma_m8n8k4_ld_c:
20484 case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
20485 case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
20486 case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
20487 case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
20488 case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
20489 case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
20490 case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
20491 case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
20492 case NVPTX::BI__mma_tf32_m16n16k8_ld_c: {
20496 std::optional<llvm::APSInt> isColMajorArg =
20498 if (!isColMajorArg)
20500 bool isColMajor = isColMajorArg->getSExtValue();
20501 NVPTXMmaLdstInfo II = getNVPTXMmaLdstInfo(BuiltinID);
20502 unsigned IID = isColMajor ? II.IID_col : II.IID_row;
20510 assert(II.NumResults);
20511 if (II.NumResults == 1) {
20515 for (
unsigned i = 0; i < II.NumResults; ++i) {
20520 llvm::ConstantInt::get(
IntTy, i)),
20527 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
20528 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
20529 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
20530 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
20531 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
20532 case NVPTX::BI__hmma_m8n32k16_st_c_f32:
20533 case NVPTX::BI__imma_m16n16k16_st_c_i32:
20534 case NVPTX::BI__imma_m32n8k16_st_c_i32:
20535 case NVPTX::BI__imma_m8n32k16_st_c_i32:
20536 case NVPTX::BI__imma_m8n8k32_st_c_i32:
20537 case NVPTX::BI__bmma_m8n8k128_st_c_i32:
20538 case NVPTX::BI__dmma_m8n8k4_st_c_f64:
20539 case NVPTX::BI__mma_m16n16k8_st_c_f32: {
20543 std::optional<llvm::APSInt> isColMajorArg =
20545 if (!isColMajorArg)
20547 bool isColMajor = isColMajorArg->getSExtValue();
20548 NVPTXMmaLdstInfo II = getNVPTXMmaLdstInfo(BuiltinID);
20549 unsigned IID = isColMajor ? II.IID_col : II.IID_row;
20554 llvm::Type *ParamType = Intrinsic->getFunctionType()->getParamType(1);
20556 for (
unsigned i = 0; i < II.NumResults; ++i) {
20560 llvm::ConstantInt::get(
IntTy, i)),
20562 Values.push_back(
Builder.CreateBitCast(
V, ParamType));
20564 Values.push_back(Ldm);
20571 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
20572 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
20573 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
20574 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
20575 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
20576 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
20577 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
20578 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
20579 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
20580 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
20581 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
20582 case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
20583 case NVPTX::BI__imma_m16n16k16_mma_s8:
20584 case NVPTX::BI__imma_m16n16k16_mma_u8:
20585 case NVPTX::BI__imma_m32n8k16_mma_s8:
20586 case NVPTX::BI__imma_m32n8k16_mma_u8:
20587 case NVPTX::BI__imma_m8n32k16_mma_s8:
20588 case NVPTX::BI__imma_m8n32k16_mma_u8:
20589 case NVPTX::BI__imma_m8n8k32_mma_s4:
20590 case NVPTX::BI__imma_m8n8k32_mma_u4:
20591 case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1:
20592 case NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1:
20593 case NVPTX::BI__dmma_m8n8k4_mma_f64:
20594 case NVPTX::BI__mma_bf16_m16n16k16_mma_f32:
20595 case NVPTX::BI__mma_bf16_m8n32k16_mma_f32:
20596 case NVPTX::BI__mma_bf16_m32n8k16_mma_f32:
20597 case NVPTX::BI__mma_tf32_m16n16k8_mma_f32: {
20602 std::optional<llvm::APSInt> LayoutArg =
20606 int Layout = LayoutArg->getSExtValue();
20607 if (Layout < 0 || Layout > 3)
20609 llvm::APSInt SatfArg;
20610 if (BuiltinID == NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1 ||
20611 BuiltinID == NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1)
20613 else if (std::optional<llvm::APSInt> OptSatfArg =
20615 SatfArg = *OptSatfArg;
20618 bool Satf = SatfArg.getSExtValue();
20619 NVPTXMmaInfo MI = getNVPTXMmaInfo(BuiltinID);
20620 unsigned IID = MI.getMMAIntrinsic(Layout, Satf);
20626 llvm::Type *AType = Intrinsic->getFunctionType()->getParamType(0);
20628 for (
unsigned i = 0; i < MI.NumEltsA; ++i) {
20632 llvm::ConstantInt::get(
IntTy, i)),
20634 Values.push_back(
Builder.CreateBitCast(
V, AType));
20637 llvm::Type *BType = Intrinsic->getFunctionType()->getParamType(MI.NumEltsA);
20638 for (
unsigned i = 0; i < MI.NumEltsB; ++i) {
20642 llvm::ConstantInt::get(
IntTy, i)),
20644 Values.push_back(
Builder.CreateBitCast(
V, BType));
20647 llvm::Type *CType =
20648 Intrinsic->getFunctionType()->getParamType(MI.NumEltsA + MI.NumEltsB);
20649 for (
unsigned i = 0; i < MI.NumEltsC; ++i) {
20653 llvm::ConstantInt::get(
IntTy, i)),
20655 Values.push_back(
Builder.CreateBitCast(
V, CType));
20659 for (
unsigned i = 0; i < MI.NumEltsD; ++i)
20663 llvm::ConstantInt::get(
IntTy, i)),
20668 case NVPTX::BI__nvvm_ex2_approx_f16:
20669 return MakeHalfType(Intrinsic::nvvm_ex2_approx_f16, BuiltinID,
E, *
this);
20670 case NVPTX::BI__nvvm_ex2_approx_f16x2:
20671 return MakeHalfType(Intrinsic::nvvm_ex2_approx_f16x2, BuiltinID,
E, *
this);
20672 case NVPTX::BI__nvvm_ff2f16x2_rn:
20673 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rn, BuiltinID,
E, *
this);
20674 case NVPTX::BI__nvvm_ff2f16x2_rn_relu:
20675 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rn_relu, BuiltinID,
E, *
this);
20676 case NVPTX::BI__nvvm_ff2f16x2_rz:
20677 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rz, BuiltinID,
E, *
this);
20678 case NVPTX::BI__nvvm_ff2f16x2_rz_relu:
20679 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rz_relu, BuiltinID,
E, *
this);
20680 case NVPTX::BI__nvvm_fma_rn_f16:
20681 return MakeHalfType(Intrinsic::nvvm_fma_rn_f16, BuiltinID,
E, *
this);
20682 case NVPTX::BI__nvvm_fma_rn_f16x2:
20683 return MakeHalfType(Intrinsic::nvvm_fma_rn_f16x2, BuiltinID,
E, *
this);
20684 case NVPTX::BI__nvvm_fma_rn_ftz_f16:
20685 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_f16, BuiltinID,
E, *
this);
20686 case NVPTX::BI__nvvm_fma_rn_ftz_f16x2:
20687 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_f16x2, BuiltinID,
E, *
this);
20688 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16:
20689 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_relu_f16, BuiltinID,
E,
20691 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16x2:
20692 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_relu_f16x2, BuiltinID,
E,
20694 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16:
20695 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_sat_f16, BuiltinID,
E,
20697 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16x2:
20698 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_sat_f16x2, BuiltinID,
E,
20700 case NVPTX::BI__nvvm_fma_rn_relu_f16:
20701 return MakeHalfType(Intrinsic::nvvm_fma_rn_relu_f16, BuiltinID,
E, *
this);
20702 case NVPTX::BI__nvvm_fma_rn_relu_f16x2:
20703 return MakeHalfType(Intrinsic::nvvm_fma_rn_relu_f16x2, BuiltinID,
E, *
this);
20704 case NVPTX::BI__nvvm_fma_rn_sat_f16:
20705 return MakeHalfType(Intrinsic::nvvm_fma_rn_sat_f16, BuiltinID,
E, *
this);
20706 case NVPTX::BI__nvvm_fma_rn_sat_f16x2:
20707 return MakeHalfType(Intrinsic::nvvm_fma_rn_sat_f16x2, BuiltinID,
E, *
this);
20708 case NVPTX::BI__nvvm_fmax_f16:
20709 return MakeHalfType(Intrinsic::nvvm_fmax_f16, BuiltinID,
E, *
this);
20710 case NVPTX::BI__nvvm_fmax_f16x2:
20711 return MakeHalfType(Intrinsic::nvvm_fmax_f16x2, BuiltinID,
E, *
this);
20712 case NVPTX::BI__nvvm_fmax_ftz_f16:
20713 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_f16, BuiltinID,
E, *
this);
20714 case NVPTX::BI__nvvm_fmax_ftz_f16x2:
20715 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_f16x2, BuiltinID,
E, *
this);
20716 case NVPTX::BI__nvvm_fmax_ftz_nan_f16:
20717 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_f16, BuiltinID,
E, *
this);
20718 case NVPTX::BI__nvvm_fmax_ftz_nan_f16x2:
20719 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_f16x2, BuiltinID,
E,
20721 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16:
20722 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f16, BuiltinID,
20724 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16x2:
20725 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f16x2,
20726 BuiltinID,
E, *
this);
20727 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16:
20728 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_xorsign_abs_f16, BuiltinID,
E,
20730 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16x2:
20731 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_xorsign_abs_f16x2, BuiltinID,
20733 case NVPTX::BI__nvvm_fmax_nan_f16:
20734 return MakeHalfType(Intrinsic::nvvm_fmax_nan_f16, BuiltinID,
E, *
this);
20735 case NVPTX::BI__nvvm_fmax_nan_f16x2:
20736 return MakeHalfType(Intrinsic::nvvm_fmax_nan_f16x2, BuiltinID,
E, *
this);
20737 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16:
20738 return MakeHalfType(Intrinsic::nvvm_fmax_nan_xorsign_abs_f16, BuiltinID,
E,
20740 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16x2:
20741 return MakeHalfType(Intrinsic::nvvm_fmax_nan_xorsign_abs_f16x2, BuiltinID,
20743 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16:
20744 return MakeHalfType(Intrinsic::nvvm_fmax_xorsign_abs_f16, BuiltinID,
E,
20746 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16x2:
20747 return MakeHalfType(Intrinsic::nvvm_fmax_xorsign_abs_f16x2, BuiltinID,
E,
20749 case NVPTX::BI__nvvm_fmin_f16:
20750 return MakeHalfType(Intrinsic::nvvm_fmin_f16, BuiltinID,
E, *
this);
20751 case NVPTX::BI__nvvm_fmin_f16x2:
20752 return MakeHalfType(Intrinsic::nvvm_fmin_f16x2, BuiltinID,
E, *
this);
20753 case NVPTX::BI__nvvm_fmin_ftz_f16:
20754 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_f16, BuiltinID,
E, *
this);
20755 case NVPTX::BI__nvvm_fmin_ftz_f16x2:
20756 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_f16x2, BuiltinID,
E, *
this);
20757 case NVPTX::BI__nvvm_fmin_ftz_nan_f16:
20758 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_f16, BuiltinID,
E, *
this);
20759 case NVPTX::BI__nvvm_fmin_ftz_nan_f16x2:
20760 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_f16x2, BuiltinID,
E,
20762 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16:
20763 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f16, BuiltinID,
20765 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16x2:
20766 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f16x2,
20767 BuiltinID,
E, *
this);
20768 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16:
20769 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_xorsign_abs_f16, BuiltinID,
E,
20771 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16x2:
20772 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_xorsign_abs_f16x2, BuiltinID,
20774 case NVPTX::BI__nvvm_fmin_nan_f16:
20775 return MakeHalfType(Intrinsic::nvvm_fmin_nan_f16, BuiltinID,
E, *
this);
20776 case NVPTX::BI__nvvm_fmin_nan_f16x2:
20777 return MakeHalfType(Intrinsic::nvvm_fmin_nan_f16x2, BuiltinID,
E, *
this);
20778 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16:
20779 return MakeHalfType(Intrinsic::nvvm_fmin_nan_xorsign_abs_f16, BuiltinID,
E,
20781 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16x2:
20782 return MakeHalfType(Intrinsic::nvvm_fmin_nan_xorsign_abs_f16x2, BuiltinID,
20784 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16:
20785 return MakeHalfType(Intrinsic::nvvm_fmin_xorsign_abs_f16, BuiltinID,
E,
20787 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16x2:
20788 return MakeHalfType(Intrinsic::nvvm_fmin_xorsign_abs_f16x2, BuiltinID,
E,
20790 case NVPTX::BI__nvvm_ldg_h:
20791 return MakeHalfType(Intrinsic::nvvm_ldg_global_f, BuiltinID,
E, *
this);
20792 case NVPTX::BI__nvvm_ldg_h2:
20793 return MakeHalfType(Intrinsic::nvvm_ldg_global_f, BuiltinID,
E, *
this);
20794 case NVPTX::BI__nvvm_ldu_h:
20795 return MakeHalfType(Intrinsic::nvvm_ldu_global_f, BuiltinID,
E, *
this);
20796 case NVPTX::BI__nvvm_ldu_h2: {
20797 return MakeHalfType(Intrinsic::nvvm_ldu_global_f, BuiltinID,
E, *
this);
20799 case NVPTX::BI__nvvm_cp_async_ca_shared_global_4:
20800 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_4,
20801 Intrinsic::nvvm_cp_async_ca_shared_global_4_s, *
this,
E,
20803 case NVPTX::BI__nvvm_cp_async_ca_shared_global_8:
20804 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_8,
20805 Intrinsic::nvvm_cp_async_ca_shared_global_8_s, *
this,
E,
20807 case NVPTX::BI__nvvm_cp_async_ca_shared_global_16:
20808 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_16,
20809 Intrinsic::nvvm_cp_async_ca_shared_global_16_s, *
this,
E,
20811 case NVPTX::BI__nvvm_cp_async_cg_shared_global_16:
20812 return MakeCpAsync(Intrinsic::nvvm_cp_async_cg_shared_global_16,
20813 Intrinsic::nvvm_cp_async_cg_shared_global_16_s, *
this,
E,
20815 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_x:
20818 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_y:
20821 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_z:
20824 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_w:
20827 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_x:
20830 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_y:
20833 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_z:
20836 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_w:
20839 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_x:
20842 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_y:
20845 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_z:
20848 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_w:
20851 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_x:
20854 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_y:
20857 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_z:
20860 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_w:
20863 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctarank:
20866 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctarank:
20869 case NVPTX::BI__nvvm_is_explicit_cluster:
20872 case NVPTX::BI__nvvm_isspacep_shared_cluster:
20876 case NVPTX::BI__nvvm_mapa:
20879 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
20880 case NVPTX::BI__nvvm_mapa_shared_cluster:
20883 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
20884 case NVPTX::BI__nvvm_getctarank:
20888 case NVPTX::BI__nvvm_getctarank_shared_cluster:
20892 case NVPTX::BI__nvvm_barrier_cluster_arrive:
20895 case NVPTX::BI__nvvm_barrier_cluster_arrive_relaxed:
20898 case NVPTX::BI__nvvm_barrier_cluster_wait:
20901 case NVPTX::BI__nvvm_fence_sc_cluster:
20910struct BuiltinAlignArgs {
20911 llvm::Value *Src =
nullptr;
20912 llvm::Type *SrcType =
nullptr;
20913 llvm::Value *Alignment =
nullptr;
20914 llvm::Value *Mask =
nullptr;
20915 llvm::IntegerType *IntType =
nullptr;
20923 SrcType = Src->getType();
20924 if (SrcType->isPointerTy()) {
20925 IntType = IntegerType::get(
20929 assert(SrcType->isIntegerTy());
20930 IntType = cast<llvm::IntegerType>(SrcType);
20933 Alignment = CGF.
Builder.CreateZExtOrTrunc(Alignment, IntType,
"alignment");
20934 auto *One = llvm::ConstantInt::get(IntType, 1);
20935 Mask = CGF.
Builder.CreateSub(Alignment, One,
"mask");
20942 BuiltinAlignArgs Args(
E, *
this);
20943 llvm::Value *SrcAddress = Args.Src;
20944 if (Args.SrcType->isPointerTy())
20946 Builder.CreateBitOrPointerCast(Args.Src, Args.IntType,
"src_addr");
20948 Builder.CreateAnd(SrcAddress, Args.Mask,
"set_bits"),
20949 llvm::Constant::getNullValue(Args.IntType),
"is_aligned"));
20956 BuiltinAlignArgs Args(
E, *
this);
20957 llvm::Value *SrcForMask = Args.Src;
20963 if (Args.Src->getType()->isPointerTy()) {
20973 SrcForMask =
Builder.CreateAdd(SrcForMask, Args.Mask,
"over_boundary");
20977 llvm::Value *InvertedMask =
Builder.CreateNot(Args.Mask,
"inverted_mask");
20978 llvm::Value *
Result =
nullptr;
20979 if (Args.Src->getType()->isPointerTy()) {
20981 Intrinsic::ptrmask, {Args.SrcType, Args.IntType},
20982 {SrcForMask, InvertedMask},
nullptr,
"aligned_result");
20984 Result =
Builder.CreateAnd(SrcForMask, InvertedMask,
"aligned_result");
20986 assert(
Result->getType() == Args.SrcType);
20992 switch (BuiltinID) {
20993 case WebAssembly::BI__builtin_wasm_memory_size: {
20998 return Builder.CreateCall(Callee, I);
21000 case WebAssembly::BI__builtin_wasm_memory_grow: {
21006 return Builder.CreateCall(Callee, Args);
21008 case WebAssembly::BI__builtin_wasm_tls_size: {
21011 return Builder.CreateCall(Callee);
21013 case WebAssembly::BI__builtin_wasm_tls_align: {
21016 return Builder.CreateCall(Callee);
21018 case WebAssembly::BI__builtin_wasm_tls_base: {
21020 return Builder.CreateCall(Callee);
21022 case WebAssembly::BI__builtin_wasm_throw: {
21026 return Builder.CreateCall(Callee, {
Tag, Obj});
21028 case WebAssembly::BI__builtin_wasm_rethrow: {
21030 return Builder.CreateCall(Callee);
21032 case WebAssembly::BI__builtin_wasm_memory_atomic_wait32: {
21039 case WebAssembly::BI__builtin_wasm_memory_atomic_wait64: {
21046 case WebAssembly::BI__builtin_wasm_memory_atomic_notify: {
21050 return Builder.CreateCall(Callee, {Addr, Count});
21052 case WebAssembly::BI__builtin_wasm_trunc_s_i32_f32:
21053 case WebAssembly::BI__builtin_wasm_trunc_s_i32_f64:
21054 case WebAssembly::BI__builtin_wasm_trunc_s_i64_f32:
21055 case WebAssembly::BI__builtin_wasm_trunc_s_i64_f64: {
21060 return Builder.CreateCall(Callee, {Src});
21062 case WebAssembly::BI__builtin_wasm_trunc_u_i32_f32:
21063 case WebAssembly::BI__builtin_wasm_trunc_u_i32_f64:
21064 case WebAssembly::BI__builtin_wasm_trunc_u_i64_f32:
21065 case WebAssembly::BI__builtin_wasm_trunc_u_i64_f64: {
21070 return Builder.CreateCall(Callee, {Src});
21072 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32_f32:
21073 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32_f64:
21074 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i64_f32:
21075 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i64_f64:
21076 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32x4_f32x4: {
21081 return Builder.CreateCall(Callee, {Src});
21083 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32_f32:
21084 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32_f64:
21085 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i64_f32:
21086 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i64_f64:
21087 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32x4_f32x4: {
21092 return Builder.CreateCall(Callee, {Src});
21094 case WebAssembly::BI__builtin_wasm_min_f32:
21095 case WebAssembly::BI__builtin_wasm_min_f64:
21096 case WebAssembly::BI__builtin_wasm_min_f16x8:
21097 case WebAssembly::BI__builtin_wasm_min_f32x4:
21098 case WebAssembly::BI__builtin_wasm_min_f64x2: {
21103 return Builder.CreateCall(Callee, {LHS, RHS});
21105 case WebAssembly::BI__builtin_wasm_max_f32:
21106 case WebAssembly::BI__builtin_wasm_max_f64:
21107 case WebAssembly::BI__builtin_wasm_max_f16x8:
21108 case WebAssembly::BI__builtin_wasm_max_f32x4:
21109 case WebAssembly::BI__builtin_wasm_max_f64x2: {
21114 return Builder.CreateCall(Callee, {LHS, RHS});
21116 case WebAssembly::BI__builtin_wasm_pmin_f16x8:
21117 case WebAssembly::BI__builtin_wasm_pmin_f32x4:
21118 case WebAssembly::BI__builtin_wasm_pmin_f64x2: {
21123 return Builder.CreateCall(Callee, {LHS, RHS});
21125 case WebAssembly::BI__builtin_wasm_pmax_f16x8:
21126 case WebAssembly::BI__builtin_wasm_pmax_f32x4:
21127 case WebAssembly::BI__builtin_wasm_pmax_f64x2: {
21132 return Builder.CreateCall(Callee, {LHS, RHS});
21134 case WebAssembly::BI__builtin_wasm_ceil_f32x4:
21135 case WebAssembly::BI__builtin_wasm_floor_f32x4:
21136 case WebAssembly::BI__builtin_wasm_trunc_f32x4:
21137 case WebAssembly::BI__builtin_wasm_nearest_f32x4:
21138 case WebAssembly::BI__builtin_wasm_ceil_f64x2:
21139 case WebAssembly::BI__builtin_wasm_floor_f64x2:
21140 case WebAssembly::BI__builtin_wasm_trunc_f64x2:
21141 case WebAssembly::BI__builtin_wasm_nearest_f64x2: {
21143 switch (BuiltinID) {
21144 case WebAssembly::BI__builtin_wasm_ceil_f32x4:
21145 case WebAssembly::BI__builtin_wasm_ceil_f64x2:
21146 IntNo = Intrinsic::ceil;
21148 case WebAssembly::BI__builtin_wasm_floor_f32x4:
21149 case WebAssembly::BI__builtin_wasm_floor_f64x2:
21150 IntNo = Intrinsic::floor;
21152 case WebAssembly::BI__builtin_wasm_trunc_f32x4:
21153 case WebAssembly::BI__builtin_wasm_trunc_f64x2:
21154 IntNo = Intrinsic::trunc;
21156 case WebAssembly::BI__builtin_wasm_nearest_f32x4:
21157 case WebAssembly::BI__builtin_wasm_nearest_f64x2:
21158 IntNo = Intrinsic::nearbyint;
21161 llvm_unreachable(
"unexpected builtin ID");
21167 case WebAssembly::BI__builtin_wasm_ref_null_extern: {
21169 return Builder.CreateCall(Callee);
21171 case WebAssembly::BI__builtin_wasm_ref_null_func: {
21173 return Builder.CreateCall(Callee);
21175 case WebAssembly::BI__builtin_wasm_swizzle_i8x16: {
21179 return Builder.CreateCall(Callee, {Src, Indices});
21181 case WebAssembly::BI__builtin_wasm_add_sat_s_i8x16:
21182 case WebAssembly::BI__builtin_wasm_add_sat_u_i8x16:
21183 case WebAssembly::BI__builtin_wasm_add_sat_s_i16x8:
21184 case WebAssembly::BI__builtin_wasm_add_sat_u_i16x8:
21185 case WebAssembly::BI__builtin_wasm_sub_sat_s_i8x16:
21186 case WebAssembly::BI__builtin_wasm_sub_sat_u_i8x16:
21187 case WebAssembly::BI__builtin_wasm_sub_sat_s_i16x8:
21188 case WebAssembly::BI__builtin_wasm_sub_sat_u_i16x8: {
21190 switch (BuiltinID) {
21191 case WebAssembly::BI__builtin_wasm_add_sat_s_i8x16:
21192 case WebAssembly::BI__builtin_wasm_add_sat_s_i16x8:
21193 IntNo = Intrinsic::sadd_sat;
21195 case WebAssembly::BI__builtin_wasm_add_sat_u_i8x16:
21196 case WebAssembly::BI__builtin_wasm_add_sat_u_i16x8:
21197 IntNo = Intrinsic::uadd_sat;
21199 case WebAssembly::BI__builtin_wasm_sub_sat_s_i8x16:
21200 case WebAssembly::BI__builtin_wasm_sub_sat_s_i16x8:
21201 IntNo = Intrinsic::wasm_sub_sat_signed;
21203 case WebAssembly::BI__builtin_wasm_sub_sat_u_i8x16:
21204 case WebAssembly::BI__builtin_wasm_sub_sat_u_i16x8:
21205 IntNo = Intrinsic::wasm_sub_sat_unsigned;
21208 llvm_unreachable(
"unexpected builtin ID");
21213 return Builder.CreateCall(Callee, {LHS, RHS});
21215 case WebAssembly::BI__builtin_wasm_abs_i8x16:
21216 case WebAssembly::BI__builtin_wasm_abs_i16x8:
21217 case WebAssembly::BI__builtin_wasm_abs_i32x4:
21218 case WebAssembly::BI__builtin_wasm_abs_i64x2: {
21221 Constant *
Zero = llvm::Constant::getNullValue(Vec->
getType());
21222 Value *ICmp =
Builder.CreateICmpSLT(Vec, Zero,
"abscond");
21223 return Builder.CreateSelect(ICmp, Neg, Vec,
"abs");
21225 case WebAssembly::BI__builtin_wasm_min_s_i8x16:
21226 case WebAssembly::BI__builtin_wasm_min_u_i8x16:
21227 case WebAssembly::BI__builtin_wasm_max_s_i8x16:
21228 case WebAssembly::BI__builtin_wasm_max_u_i8x16:
21229 case WebAssembly::BI__builtin_wasm_min_s_i16x8:
21230 case WebAssembly::BI__builtin_wasm_min_u_i16x8:
21231 case WebAssembly::BI__builtin_wasm_max_s_i16x8:
21232 case WebAssembly::BI__builtin_wasm_max_u_i16x8:
21233 case WebAssembly::BI__builtin_wasm_min_s_i32x4:
21234 case WebAssembly::BI__builtin_wasm_min_u_i32x4:
21235 case WebAssembly::BI__builtin_wasm_max_s_i32x4:
21236 case WebAssembly::BI__builtin_wasm_max_u_i32x4: {
21240 switch (BuiltinID) {
21241 case WebAssembly::BI__builtin_wasm_min_s_i8x16:
21242 case WebAssembly::BI__builtin_wasm_min_s_i16x8:
21243 case WebAssembly::BI__builtin_wasm_min_s_i32x4:
21244 ICmp =
Builder.CreateICmpSLT(LHS, RHS);
21246 case WebAssembly::BI__builtin_wasm_min_u_i8x16:
21247 case WebAssembly::BI__builtin_wasm_min_u_i16x8:
21248 case WebAssembly::BI__builtin_wasm_min_u_i32x4:
21249 ICmp =
Builder.CreateICmpULT(LHS, RHS);
21251 case WebAssembly::BI__builtin_wasm_max_s_i8x16:
21252 case WebAssembly::BI__builtin_wasm_max_s_i16x8:
21253 case WebAssembly::BI__builtin_wasm_max_s_i32x4:
21254 ICmp =
Builder.CreateICmpSGT(LHS, RHS);
21256 case WebAssembly::BI__builtin_wasm_max_u_i8x16:
21257 case WebAssembly::BI__builtin_wasm_max_u_i16x8:
21258 case WebAssembly::BI__builtin_wasm_max_u_i32x4:
21259 ICmp =
Builder.CreateICmpUGT(LHS, RHS);
21262 llvm_unreachable(
"unexpected builtin ID");
21264 return Builder.CreateSelect(ICmp, LHS, RHS);
21266 case WebAssembly::BI__builtin_wasm_avgr_u_i8x16:
21267 case WebAssembly::BI__builtin_wasm_avgr_u_i16x8: {
21272 return Builder.CreateCall(Callee, {LHS, RHS});
21274 case WebAssembly::BI__builtin_wasm_q15mulr_sat_s_i16x8: {
21278 return Builder.CreateCall(Callee, {LHS, RHS});
21280 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_s_i16x8:
21281 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_u_i16x8:
21282 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_s_i32x4:
21283 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_u_i32x4: {
21286 switch (BuiltinID) {
21287 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_s_i16x8:
21288 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_s_i32x4:
21289 IntNo = Intrinsic::wasm_extadd_pairwise_signed;
21291 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_u_i16x8:
21292 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_u_i32x4:
21293 IntNo = Intrinsic::wasm_extadd_pairwise_unsigned;
21296 llvm_unreachable(
"unexpected builtin ID");
21300 return Builder.CreateCall(Callee, Vec);
21302 case WebAssembly::BI__builtin_wasm_bitselect: {
21308 return Builder.CreateCall(Callee, {V1, V2,
C});
21310 case WebAssembly::BI__builtin_wasm_dot_s_i32x4_i16x8: {
21314 return Builder.CreateCall(Callee, {LHS, RHS});
21316 case WebAssembly::BI__builtin_wasm_popcnt_i8x16: {
21320 return Builder.CreateCall(Callee, {Vec});
21322 case WebAssembly::BI__builtin_wasm_any_true_v128:
21323 case WebAssembly::BI__builtin_wasm_all_true_i8x16:
21324 case WebAssembly::BI__builtin_wasm_all_true_i16x8:
21325 case WebAssembly::BI__builtin_wasm_all_true_i32x4:
21326 case WebAssembly::BI__builtin_wasm_all_true_i64x2: {
21328 switch (BuiltinID) {
21329 case WebAssembly::BI__builtin_wasm_any_true_v128:
21330 IntNo = Intrinsic::wasm_anytrue;
21332 case WebAssembly::BI__builtin_wasm_all_true_i8x16:
21333 case WebAssembly::BI__builtin_wasm_all_true_i16x8:
21334 case WebAssembly::BI__builtin_wasm_all_true_i32x4:
21335 case WebAssembly::BI__builtin_wasm_all_true_i64x2:
21336 IntNo = Intrinsic::wasm_alltrue;
21339 llvm_unreachable(
"unexpected builtin ID");
21343 return Builder.CreateCall(Callee, {Vec});
21345 case WebAssembly::BI__builtin_wasm_bitmask_i8x16:
21346 case WebAssembly::BI__builtin_wasm_bitmask_i16x8:
21347 case WebAssembly::BI__builtin_wasm_bitmask_i32x4:
21348 case WebAssembly::BI__builtin_wasm_bitmask_i64x2: {
21352 return Builder.CreateCall(Callee, {Vec});
21354 case WebAssembly::BI__builtin_wasm_abs_f32x4:
21355 case WebAssembly::BI__builtin_wasm_abs_f64x2: {
21358 return Builder.CreateCall(Callee, {Vec});
21360 case WebAssembly::BI__builtin_wasm_sqrt_f32x4:
21361 case WebAssembly::BI__builtin_wasm_sqrt_f64x2: {
21364 return Builder.CreateCall(Callee, {Vec});
21366 case WebAssembly::BI__builtin_wasm_narrow_s_i8x16_i16x8:
21367 case WebAssembly::BI__builtin_wasm_narrow_u_i8x16_i16x8:
21368 case WebAssembly::BI__builtin_wasm_narrow_s_i16x8_i32x4:
21369 case WebAssembly::BI__builtin_wasm_narrow_u_i16x8_i32x4: {
21373 switch (BuiltinID) {
21374 case WebAssembly::BI__builtin_wasm_narrow_s_i8x16_i16x8:
21375 case WebAssembly::BI__builtin_wasm_narrow_s_i16x8_i32x4:
21376 IntNo = Intrinsic::wasm_narrow_signed;
21378 case WebAssembly::BI__builtin_wasm_narrow_u_i8x16_i16x8:
21379 case WebAssembly::BI__builtin_wasm_narrow_u_i16x8_i32x4:
21380 IntNo = Intrinsic::wasm_narrow_unsigned;
21383 llvm_unreachable(
"unexpected builtin ID");
21387 return Builder.CreateCall(Callee, {Low, High});
21389 case WebAssembly::BI__builtin_wasm_trunc_sat_s_zero_f64x2_i32x4:
21390 case WebAssembly::BI__builtin_wasm_trunc_sat_u_zero_f64x2_i32x4: {
21393 switch (BuiltinID) {
21394 case WebAssembly::BI__builtin_wasm_trunc_sat_s_zero_f64x2_i32x4:
21395 IntNo = Intrinsic::fptosi_sat;
21397 case WebAssembly::BI__builtin_wasm_trunc_sat_u_zero_f64x2_i32x4:
21398 IntNo = Intrinsic::fptoui_sat;
21401 llvm_unreachable(
"unexpected builtin ID");
21403 llvm::Type *SrcT = Vec->
getType();
21404 llvm::Type *TruncT = SrcT->getWithNewType(
Builder.getInt32Ty());
21407 Value *Splat = Constant::getNullValue(TruncT);
21410 case WebAssembly::BI__builtin_wasm_shuffle_i8x16: {
21415 while (OpIdx < 18) {
21416 std::optional<llvm::APSInt> LaneConst =
21418 assert(LaneConst &&
"Constant arg isn't actually constant?");
21419 Ops[OpIdx++] = llvm::ConstantInt::get(
getLLVMContext(), *LaneConst);
21422 return Builder.CreateCall(Callee, Ops);
21424 case WebAssembly::BI__builtin_wasm_relaxed_madd_f16x8:
21425 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f16x8:
21426 case WebAssembly::BI__builtin_wasm_relaxed_madd_f32x4:
21427 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f32x4:
21428 case WebAssembly::BI__builtin_wasm_relaxed_madd_f64x2:
21429 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f64x2: {
21434 switch (BuiltinID) {
21435 case WebAssembly::BI__builtin_wasm_relaxed_madd_f16x8:
21436 case WebAssembly::BI__builtin_wasm_relaxed_madd_f32x4:
21437 case WebAssembly::BI__builtin_wasm_relaxed_madd_f64x2:
21438 IntNo = Intrinsic::wasm_relaxed_madd;
21440 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f16x8:
21441 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f32x4:
21442 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f64x2:
21443 IntNo = Intrinsic::wasm_relaxed_nmadd;
21446 llvm_unreachable(
"unexpected builtin ID");
21449 return Builder.CreateCall(Callee, {A, B,
C});
21451 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i8x16:
21452 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i16x8:
21453 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i32x4:
21454 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i64x2: {
21460 return Builder.CreateCall(Callee, {A, B,
C});
21462 case WebAssembly::BI__builtin_wasm_relaxed_swizzle_i8x16: {
21466 return Builder.CreateCall(Callee, {Src, Indices});
21468 case WebAssembly::BI__builtin_wasm_relaxed_min_f32x4:
21469 case WebAssembly::BI__builtin_wasm_relaxed_max_f32x4:
21470 case WebAssembly::BI__builtin_wasm_relaxed_min_f64x2:
21471 case WebAssembly::BI__builtin_wasm_relaxed_max_f64x2: {
21475 switch (BuiltinID) {
21476 case WebAssembly::BI__builtin_wasm_relaxed_min_f32x4:
21477 case WebAssembly::BI__builtin_wasm_relaxed_min_f64x2:
21478 IntNo = Intrinsic::wasm_relaxed_min;
21480 case WebAssembly::BI__builtin_wasm_relaxed_max_f32x4:
21481 case WebAssembly::BI__builtin_wasm_relaxed_max_f64x2:
21482 IntNo = Intrinsic::wasm_relaxed_max;
21485 llvm_unreachable(
"unexpected builtin ID");
21488 return Builder.CreateCall(Callee, {LHS, RHS});
21490 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_i32x4_f32x4:
21491 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_i32x4_f32x4:
21492 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_zero_i32x4_f64x2:
21493 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_zero_i32x4_f64x2: {
21496 switch (BuiltinID) {
21497 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_i32x4_f32x4:
21498 IntNo = Intrinsic::wasm_relaxed_trunc_signed;
21500 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_i32x4_f32x4:
21501 IntNo = Intrinsic::wasm_relaxed_trunc_unsigned;
21503 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_zero_i32x4_f64x2:
21504 IntNo = Intrinsic::wasm_relaxed_trunc_signed_zero;
21506 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_zero_i32x4_f64x2:
21507 IntNo = Intrinsic::wasm_relaxed_trunc_unsigned_zero;
21510 llvm_unreachable(
"unexpected builtin ID");
21513 return Builder.CreateCall(Callee, {Vec});
21515 case WebAssembly::BI__builtin_wasm_relaxed_q15mulr_s_i16x8: {
21519 return Builder.CreateCall(Callee, {LHS, RHS});
21521 case WebAssembly::BI__builtin_wasm_relaxed_dot_i8x16_i7x16_s_i16x8: {
21526 return Builder.CreateCall(Callee, {LHS, RHS});
21528 case WebAssembly::BI__builtin_wasm_relaxed_dot_i8x16_i7x16_add_s_i32x4: {
21533 CGM.
getIntrinsic(Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed);
21534 return Builder.CreateCall(Callee, {LHS, RHS, Acc});
21536 case WebAssembly::BI__builtin_wasm_relaxed_dot_bf16x8_add_f32_f32x4: {
21542 return Builder.CreateCall(Callee, {LHS, RHS, Acc});
21544 case WebAssembly::BI__builtin_wasm_loadf16_f32: {
21547 return Builder.CreateCall(Callee, {Addr});
21549 case WebAssembly::BI__builtin_wasm_storef16_f32: {
21553 return Builder.CreateCall(Callee, {Val, Addr});
21555 case WebAssembly::BI__builtin_wasm_splat_f16x8: {
21558 return Builder.CreateCall(Callee, {Val});
21560 case WebAssembly::BI__builtin_wasm_extract_lane_f16x8: {
21566 case WebAssembly::BI__builtin_wasm_replace_lane_f16x8: {
21573 case WebAssembly::BI__builtin_wasm_table_get: {
21584 "Unexpected reference type for __builtin_wasm_table_get");
21585 return Builder.CreateCall(Callee, {Table, Index});
21587 case WebAssembly::BI__builtin_wasm_table_set: {
21599 "Unexpected reference type for __builtin_wasm_table_set");
21600 return Builder.CreateCall(Callee, {Table, Index, Val});
21602 case WebAssembly::BI__builtin_wasm_table_size: {
21608 case WebAssembly::BI__builtin_wasm_table_grow: {
21621 "Unexpected reference type for __builtin_wasm_table_grow");
21623 return Builder.CreateCall(Callee, {Table, Val, NElems});
21625 case WebAssembly::BI__builtin_wasm_table_fill: {
21639 "Unexpected reference type for __builtin_wasm_table_fill");
21641 return Builder.CreateCall(Callee, {Table, Index, Val, NElems});
21643 case WebAssembly::BI__builtin_wasm_table_copy: {
21653 return Builder.CreateCall(Callee, {TableX, TableY, SrcIdx, DstIdx, NElems});
21660static std::pair<Intrinsic::ID, unsigned>
21663 unsigned BuiltinID;
21664 Intrinsic::ID IntrinsicID;
21667 static Info Infos[] = {
21668#define CUSTOM_BUILTIN_MAPPING(x,s) \
21669 { Hexagon::BI__builtin_HEXAGON_##x, Intrinsic::hexagon_##x, s },
21701#include "clang/Basic/BuiltinsHexagonMapCustomDep.def"
21702#undef CUSTOM_BUILTIN_MAPPING
21705 auto CmpInfo = [] (Info A, Info B) {
return A.BuiltinID < B.BuiltinID; };
21706 static const bool SortOnce = (llvm::sort(Infos, CmpInfo),
true);
21709 const Info *F = llvm::lower_bound(Infos, Info{BuiltinID, 0, 0}, CmpInfo);
21710 if (F == std::end(Infos) || F->BuiltinID != BuiltinID)
21711 return {Intrinsic::not_intrinsic, 0};
21713 return {F->IntrinsicID, F->VecLen};
21722 auto MakeCircOp = [
this,
E](
unsigned IntID,
bool IsLoad) {
21736 for (
unsigned i = 1, e =
E->getNumArgs(); i != e; ++i)
21742 llvm::Value *NewBase = IsLoad ?
Builder.CreateExtractValue(
Result, 1)
21746 llvm::Value *RetVal =
21756 auto MakeBrevLd = [
this,
E](
unsigned IntID, llvm::Type *DestTy) {
21773 CGM.
getIntrinsic(IntID), {BaseAddress, EmitScalarExpr(E->getArg(2))});
21776 llvm::Value *DestVal =
Builder.CreateExtractValue(
Result, 0);
21781 DestVal =
Builder.CreateTrunc(DestVal, DestTy);
21788 auto V2Q = [
this, VecLen] (llvm::Value *Vec) {
21789 Intrinsic::ID
ID = VecLen == 128 ? Intrinsic::hexagon_V6_vandvrt_128B
21790 : Intrinsic::hexagon_V6_vandvrt;
21792 {Vec,
Builder.getInt32(-1)});
21794 auto Q2V = [
this, VecLen] (llvm::Value *Pred) {
21795 Intrinsic::ID
ID = VecLen == 128 ? Intrinsic::hexagon_V6_vandqrt_128B
21796 : Intrinsic::hexagon_V6_vandqrt;
21798 {Pred,
Builder.getInt32(-1)});
21801 switch (BuiltinID) {
21805 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarry:
21806 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarry_128B:
21807 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarry:
21808 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarry_128B: {
21815 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1)), PredIn});
21817 llvm::Value *PredOut =
Builder.CreateExtractValue(
Result, 1);
21825 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarryo:
21826 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarryo_128B:
21827 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarryo:
21828 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarryo_128B: {
21834 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
21836 llvm::Value *PredOut =
Builder.CreateExtractValue(
Result, 1);
21842 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstoreq:
21843 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorenq:
21844 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentq:
21845 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentnq:
21846 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstoreq_128B:
21847 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorenq_128B:
21848 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentq_128B:
21849 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentnq_128B: {
21851 const Expr *PredOp =
E->getArg(0);
21853 if (
auto *Cast = dyn_cast<ImplicitCastExpr>(PredOp)) {
21854 if (
Cast->getCastKind() == CK_BitCast)
21855 PredOp =
Cast->getSubExpr();
21858 for (
int i = 1, e =
E->getNumArgs(); i != e; ++i)
21863 case Hexagon::BI__builtin_HEXAGON_L2_loadrub_pci:
21864 case Hexagon::BI__builtin_HEXAGON_L2_loadrb_pci:
21865 case Hexagon::BI__builtin_HEXAGON_L2_loadruh_pci:
21866 case Hexagon::BI__builtin_HEXAGON_L2_loadrh_pci:
21867 case Hexagon::BI__builtin_HEXAGON_L2_loadri_pci:
21868 case Hexagon::BI__builtin_HEXAGON_L2_loadrd_pci:
21869 case Hexagon::BI__builtin_HEXAGON_L2_loadrub_pcr:
21870 case Hexagon::BI__builtin_HEXAGON_L2_loadrb_pcr:
21871 case Hexagon::BI__builtin_HEXAGON_L2_loadruh_pcr:
21872 case Hexagon::BI__builtin_HEXAGON_L2_loadrh_pcr:
21873 case Hexagon::BI__builtin_HEXAGON_L2_loadri_pcr:
21874 case Hexagon::BI__builtin_HEXAGON_L2_loadrd_pcr:
21875 return MakeCircOp(ID,
true);
21876 case Hexagon::BI__builtin_HEXAGON_S2_storerb_pci:
21877 case Hexagon::BI__builtin_HEXAGON_S2_storerh_pci:
21878 case Hexagon::BI__builtin_HEXAGON_S2_storerf_pci:
21879 case Hexagon::BI__builtin_HEXAGON_S2_storeri_pci:
21880 case Hexagon::BI__builtin_HEXAGON_S2_storerd_pci:
21881 case Hexagon::BI__builtin_HEXAGON_S2_storerb_pcr:
21882 case Hexagon::BI__builtin_HEXAGON_S2_storerh_pcr:
21883 case Hexagon::BI__builtin_HEXAGON_S2_storerf_pcr:
21884 case Hexagon::BI__builtin_HEXAGON_S2_storeri_pcr:
21885 case Hexagon::BI__builtin_HEXAGON_S2_storerd_pcr:
21886 return MakeCircOp(ID,
false);
21887 case Hexagon::BI__builtin_brev_ldub:
21888 return MakeBrevLd(Intrinsic::hexagon_L2_loadrub_pbr,
Int8Ty);
21889 case Hexagon::BI__builtin_brev_ldb:
21890 return MakeBrevLd(Intrinsic::hexagon_L2_loadrb_pbr,
Int8Ty);
21891 case Hexagon::BI__builtin_brev_lduh:
21892 return MakeBrevLd(Intrinsic::hexagon_L2_loadruh_pbr,
Int16Ty);
21893 case Hexagon::BI__builtin_brev_ldh:
21894 return MakeBrevLd(Intrinsic::hexagon_L2_loadrh_pbr,
Int16Ty);
21895 case Hexagon::BI__builtin_brev_ldw:
21896 return MakeBrevLd(Intrinsic::hexagon_L2_loadri_pbr,
Int32Ty);
21897 case Hexagon::BI__builtin_brev_ldd:
21898 return MakeBrevLd(Intrinsic::hexagon_L2_loadrd_pbr,
Int64Ty);
21908 if (BuiltinID == Builtin::BI__builtin_cpu_supports)
21910 if (BuiltinID == Builtin::BI__builtin_cpu_init)
21917 unsigned ICEArguments = 0;
21925 if (BuiltinID == RISCVVector::BI__builtin_rvv_vget_v ||
21926 BuiltinID == RISCVVector::BI__builtin_rvv_vset_v)
21927 ICEArguments = 1 << 1;
21932 if (BuiltinID == RISCV::BI__builtin_riscv_ntl_load)
21933 ICEArguments |= (1 << 1);
21934 if (BuiltinID == RISCV::BI__builtin_riscv_ntl_store)
21935 ICEArguments |= (1 << 2);
21937 for (
unsigned i = 0, e =
E->getNumArgs(); i != e; i++) {
21942 Ops.push_back(AggValue);
21948 Intrinsic::ID
ID = Intrinsic::not_intrinsic;
21952 constexpr unsigned RVV_VTA = 0x1;
21953 constexpr unsigned RVV_VMA = 0x2;
21954 int PolicyAttrs = 0;
21955 bool IsMasked =
false;
21959 switch (BuiltinID) {
21960 default: llvm_unreachable(
"unexpected builtin ID");
21961 case RISCV::BI__builtin_riscv_orc_b_32:
21962 case RISCV::BI__builtin_riscv_orc_b_64:
21963 case RISCV::BI__builtin_riscv_clz_32:
21964 case RISCV::BI__builtin_riscv_clz_64:
21965 case RISCV::BI__builtin_riscv_ctz_32:
21966 case RISCV::BI__builtin_riscv_ctz_64:
21967 case RISCV::BI__builtin_riscv_clmul_32:
21968 case RISCV::BI__builtin_riscv_clmul_64:
21969 case RISCV::BI__builtin_riscv_clmulh_32:
21970 case RISCV::BI__builtin_riscv_clmulh_64:
21971 case RISCV::BI__builtin_riscv_clmulr_32:
21972 case RISCV::BI__builtin_riscv_clmulr_64:
21973 case RISCV::BI__builtin_riscv_xperm4_32:
21974 case RISCV::BI__builtin_riscv_xperm4_64:
21975 case RISCV::BI__builtin_riscv_xperm8_32:
21976 case RISCV::BI__builtin_riscv_xperm8_64:
21977 case RISCV::BI__builtin_riscv_brev8_32:
21978 case RISCV::BI__builtin_riscv_brev8_64:
21979 case RISCV::BI__builtin_riscv_zip_32:
21980 case RISCV::BI__builtin_riscv_unzip_32: {
21981 switch (BuiltinID) {
21982 default: llvm_unreachable(
"unexpected builtin ID");
21984 case RISCV::BI__builtin_riscv_orc_b_32:
21985 case RISCV::BI__builtin_riscv_orc_b_64:
21986 ID = Intrinsic::riscv_orc_b;
21988 case RISCV::BI__builtin_riscv_clz_32:
21989 case RISCV::BI__builtin_riscv_clz_64: {
21992 if (
Result->getType() != ResultType)
21997 case RISCV::BI__builtin_riscv_ctz_32:
21998 case RISCV::BI__builtin_riscv_ctz_64: {
22001 if (
Result->getType() != ResultType)
22008 case RISCV::BI__builtin_riscv_clmul_32:
22009 case RISCV::BI__builtin_riscv_clmul_64:
22010 ID = Intrinsic::riscv_clmul;
22012 case RISCV::BI__builtin_riscv_clmulh_32:
22013 case RISCV::BI__builtin_riscv_clmulh_64:
22014 ID = Intrinsic::riscv_clmulh;
22016 case RISCV::BI__builtin_riscv_clmulr_32:
22017 case RISCV::BI__builtin_riscv_clmulr_64:
22018 ID = Intrinsic::riscv_clmulr;
22022 case RISCV::BI__builtin_riscv_xperm8_32:
22023 case RISCV::BI__builtin_riscv_xperm8_64:
22024 ID = Intrinsic::riscv_xperm8;
22026 case RISCV::BI__builtin_riscv_xperm4_32:
22027 case RISCV::BI__builtin_riscv_xperm4_64:
22028 ID = Intrinsic::riscv_xperm4;
22032 case RISCV::BI__builtin_riscv_brev8_32:
22033 case RISCV::BI__builtin_riscv_brev8_64:
22034 ID = Intrinsic::riscv_brev8;
22036 case RISCV::BI__builtin_riscv_zip_32:
22037 ID = Intrinsic::riscv_zip;
22039 case RISCV::BI__builtin_riscv_unzip_32:
22040 ID = Intrinsic::riscv_unzip;
22044 IntrinsicTypes = {ResultType};
22051 case RISCV::BI__builtin_riscv_sha256sig0:
22052 ID = Intrinsic::riscv_sha256sig0;
22054 case RISCV::BI__builtin_riscv_sha256sig1:
22055 ID = Intrinsic::riscv_sha256sig1;
22057 case RISCV::BI__builtin_riscv_sha256sum0:
22058 ID = Intrinsic::riscv_sha256sum0;
22060 case RISCV::BI__builtin_riscv_sha256sum1:
22061 ID = Intrinsic::riscv_sha256sum1;
22065 case RISCV::BI__builtin_riscv_sm4ks:
22066 ID = Intrinsic::riscv_sm4ks;
22068 case RISCV::BI__builtin_riscv_sm4ed:
22069 ID = Intrinsic::riscv_sm4ed;
22073 case RISCV::BI__builtin_riscv_sm3p0:
22074 ID = Intrinsic::riscv_sm3p0;
22076 case RISCV::BI__builtin_riscv_sm3p1:
22077 ID = Intrinsic::riscv_sm3p1;
22081 case RISCV::BI__builtin_riscv_ntl_load: {
22083 unsigned DomainVal = 5;
22084 if (Ops.size() == 2)
22085 DomainVal = cast<ConstantInt>(Ops[1])->getZExtValue();
22087 llvm::MDNode *RISCVDomainNode = llvm::MDNode::get(
22089 llvm::ConstantAsMetadata::get(
Builder.getInt32(DomainVal)));
22090 llvm::MDNode *NontemporalNode = llvm::MDNode::get(
22094 if(ResTy->isScalableTy()) {
22095 const ScalableVectorType *SVTy = cast<ScalableVectorType>(ResTy);
22096 llvm::Type *ScalarTy = ResTy->getScalarType();
22097 Width = ScalarTy->getPrimitiveSizeInBits() *
22098 SVTy->getElementCount().getKnownMinValue();
22100 Width = ResTy->getPrimitiveSizeInBits();
22104 Load->setMetadata(llvm::LLVMContext::MD_nontemporal, NontemporalNode);
22105 Load->setMetadata(
CGM.
getModule().getMDKindID(
"riscv-nontemporal-domain"),
22110 case RISCV::BI__builtin_riscv_ntl_store: {
22111 unsigned DomainVal = 5;
22112 if (Ops.size() == 3)
22113 DomainVal = cast<ConstantInt>(Ops[2])->getZExtValue();
22115 llvm::MDNode *RISCVDomainNode = llvm::MDNode::get(
22117 llvm::ConstantAsMetadata::get(
Builder.getInt32(DomainVal)));
22118 llvm::MDNode *NontemporalNode = llvm::MDNode::get(
22122 Store->setMetadata(llvm::LLVMContext::MD_nontemporal, NontemporalNode);
22130#include "clang/Basic/riscv_vector_builtin_cg.inc"
22132#include "clang/Basic/riscv_sifive_vector_builtin_cg.inc"
22135 assert(ID != Intrinsic::not_intrinsic);
22138 return Builder.CreateCall(F, Ops,
"");
Defines the clang::ASTContext interface.
#define PPC_LNX_FEATURE(NAME, DESC, ENUMNAME, ENUMVAL, HWCAPN)
static constexpr SparcCPUInfo CPUInfo[]
#define X86_CPU_SUBTYPE(ENUM, STR)
#define X86_CPU_SUBTYPE_ALIAS(ENUM, ALIAS)
#define X86_VENDOR(ENUM, STRING)
#define X86_CPU_TYPE_ALIAS(ENUM, ALIAS)
#define X86_CPU_TYPE(ENUM, STR)
static constexpr Builtin::Info BuiltinInfo[]
static void Accumulate(SMap &SM, CFGBlock *B)
static Value * EmitSpecialRegisterBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Type *RegisterType, llvm::Type *ValueType, SpecialRegisterAccessKind AccessKind, StringRef SysReg="")
static llvm::Value * ARMMVEVectorReinterpret(CGBuilderTy &Builder, CodeGenFunction *CGF, llvm::Value *V, llvm::Type *DestType)
static Value * MakeBinaryAtomicValue(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E, AtomicOrdering Ordering=AtomicOrdering::SequentiallyConsistent)
Utility to insert an atomic instruction based on Intrinsic::ID and the expression node.
static char bitActionToX86BTCode(BitTest::ActionKind A)
#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier)
static Value * EmitAtomicCmpXchg128ForMSIntrin(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering SuccessOrdering)
#define INTRINSIC_X86_XSAVE_ID(NAME)
static CanQualType getOSLogArgType(ASTContext &C, int Size)
Get the argument type for arguments to os_log_helper.
static Value * EmitOverflowCheckedAbs(CodeGenFunction &CGF, const CallExpr *E, bool SanitizeOverflow)
static llvm::VectorType * GetFloatNeonType(CodeGenFunction *CGF, NeonTypeFlags IntTypeFlags)
static Value * tryUseTestFPKind(CodeGenFunction &CGF, unsigned BuiltinID, Value *V)
static llvm::Value * MVEImmediateShr(CGBuilderTy &Builder, llvm::Value *V, uint32_t Shift, bool Unsigned)
static bool areBOSTypesCompatible(int From, int To)
Checks if using the result of __builtin_object_size(p, From) in place of __builtin_object_size(p,...
static llvm::Value * SignOrZeroExtend(CGBuilderTy &Builder, llvm::Value *V, llvm::Type *T, bool Unsigned)
static void InsertExplicitZeroOperand(CGBuilderTy &Builder, llvm::Type *Ty, SmallVectorImpl< Value * > &Ops)
static const ARMVectorIntrinsicInfo AArch64SMEIntrinsicMap[]
static Value * EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E, SmallVectorImpl< Value * > &Ops, llvm::Triple::ArchType Arch)
#define MMA_VARIANTS_B1_AND(geom, type)
static void swapCommutativeSMEOperands(unsigned BuiltinID, SmallVectorImpl< Value * > &Ops)
static bool AArch64SISDIntrinsicsProvenSorted
static Value * EmitX86CompressExpand(CodeGenFunction &CGF, ArrayRef< Value * > Ops, bool IsCompress)
static const ARMVectorIntrinsicInfo AArch64SVEIntrinsicMap[]
static bool HasExtraNeonArgument(unsigned BuiltinID)
Return true if BuiltinID is an overloaded Neon intrinsic with an extra argument that specifies the ve...
static bool TypeRequiresBuiltinLaunderImp(const ASTContext &Ctx, QualType Ty, llvm::SmallPtrSetImpl< const Decl * > &Seen)
static Value * EmitAtomicIncrementValue(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering Ordering=AtomicOrdering::SequentiallyConsistent)
static std::pair< Intrinsic::ID, unsigned > getIntrinsicForHexagonNonClangBuiltin(unsigned BuiltinID)
static Value * emitRangedBuiltin(CodeGenFunction &CGF, unsigned IntrinsicID, int low, int high)
#define MMA_INTR(geom_op_type, layout)
static Value * EmitX86MaskedCompare(CodeGenFunction &CGF, unsigned CC, bool Signed, ArrayRef< Value * > Ops)
static Value * emitUnaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier)
static bool AArch64SVEIntrinsicsProvenSorted
static RValue EmitMSVCRTSetJmp(CodeGenFunction &CGF, MSVCSetJmpKind SJKind, const CallExpr *E)
MSVC handles setjmp a bit differently on different platforms.
static const ARMVectorIntrinsicInfo * findARMVectorIntrinsicInMap(ArrayRef< ARMVectorIntrinsicInfo > IntrinsicMap, unsigned BuiltinID, bool &MapProvenSorted)
static Value * EmitScalarFMAExpr(CodeGenFunction &CGF, const CallExpr *E, MutableArrayRef< Value * > Ops, Value *Upper, bool ZeroMask=false, unsigned PTIdx=0, bool NegAcc=false)
#define MUTATE_LDBL(func)
static Value * EmitX86ExpandLoad(CodeGenFunction &CGF, ArrayRef< Value * > Ops)
static unsigned CountCountedByAttrs(const RecordDecl *RD)
static Value * emitMaybeConstrainedFPToIntRoundBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
static Value * EmitX86MaskedLoad(CodeGenFunction &CGF, ArrayRef< Value * > Ops, Align Alignment)
static bool TypeRequiresBuiltinLaunder(CodeGenModule &CGM, QualType Ty)
Determine if the specified type requires laundering by checking if it is a dynamic class type or cont...
static void InsertExplicitUndefOperand(CGBuilderTy &Builder, llvm::Type *Ty, SmallVectorImpl< Value * > &Ops)
static Value * EmitISOVolatileLoad(CodeGenFunction &CGF, const CallExpr *E)
static struct WidthAndSignedness EncompassingIntegerType(ArrayRef< struct WidthAndSignedness > Types)
static Integer GetIntegerConstantValue(const Expr *E, ASTContext &Context)
#define MMA_VARIANTS(geom, type)
static bool AArch64SMEIntrinsicsProvenSorted
static llvm::Value * VectorZip(CGBuilderTy &Builder, llvm::Value *V0, llvm::Value *V1)
static Value * EmitTargetArchBuiltinExpr(CodeGenFunction *CGF, unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
constexpr unsigned SVEBitsPerBlock
static std::optional< CodeGenFunction::MSVCIntrin > translateX86ToMsvcIntrin(unsigned BuiltinID)
static const std::pair< unsigned, unsigned > NEONEquivalentIntrinsicMap[]
#define NEONMAP0(NameBase)
static Value * EmitX86MaskedStore(CodeGenFunction &CGF, ArrayRef< Value * > Ops, Align Alignment)
static Value * emitBinaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E, Instruction::BinaryOps Op, bool Invert=false)
Utility to insert an atomic instruction based Intrinsic::ID and the expression node,...
static Value * EmitX86Muldq(CodeGenFunction &CGF, bool IsSigned, ArrayRef< Value * > Ops)
static llvm::Value * ARMMVEVectorElementReverse(CGBuilderTy &Builder, llvm::Value *V, unsigned ReverseWidth)
#define MMA_SATF_VARIANTS(geom, type)
static std::optional< CodeGenFunction::MSVCIntrin > translateAarch64ToMsvcIntrin(unsigned BuiltinID)
static std::optional< CodeGenFunction::MSVCIntrin > translateArmToMsvcIntrin(unsigned BuiltinID)
Value * emitBuiltinWithOneOverloadedType(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, llvm::StringRef Name="")
static llvm::Value * EmitBitTestIntrinsic(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E)
Emit a _bittest* intrinsic.
static const ARMVectorIntrinsicInfo ARMSIMDIntrinsicMap[]
static Value * EmitSignBit(CodeGenFunction &CGF, Value *V)
Emit the computation of the sign bit for a floating point value.
static Value * EmitFAbs(CodeGenFunction &CGF, Value *V)
EmitFAbs - Emit a call to @llvm.fabs().
#define CUSTOM_BUILTIN_MAPPING(x, s)
static Value * EmitX86CvtF16ToFloatExpr(CodeGenFunction &CGF, ArrayRef< Value * > Ops, llvm::Type *DstTy)
static bool isSpecialUnsignedMultiplySignedResult(unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info, WidthAndSignedness ResultInfo)
static llvm::Value * getDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType)
static llvm::Value * emitPPCLoadReserveIntrinsic(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E)
static llvm::Value * VectorUnzip(CGBuilderTy &Builder, llvm::Value *V, bool Odd)
static Value * EmitX86SExtMask(CodeGenFunction &CGF, Value *Op, llvm::Type *DstTy)
static Value * emitTernaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
static WidthAndSignedness getIntegerWidthAndSignedness(const clang::ASTContext &context, const clang::QualType Type)
static Value * EmitX86FunnelShift(CodeGenFunction &CGF, Value *Op0, Value *Op1, Value *Amt, bool IsRight)
static RValue EmitCheckedMixedSignMultiply(CodeGenFunction &CGF, const clang::Expr *Op1, WidthAndSignedness Op1Info, const clang::Expr *Op2, WidthAndSignedness Op2Info, const clang::Expr *ResultArg, QualType ResultQTy, WidthAndSignedness ResultInfo)
Emit a checked mixed-sign multiply.
static llvm::ScalableVectorType * getSVEVectorForElementType(llvm::Type *EltTy)
static unsigned mutateLongDoubleBuiltin(unsigned BuiltinID)
#define INTRINSIC_WITH_CC(NAME)
static llvm::FixedVectorType * GetNeonType(CodeGenFunction *CGF, NeonTypeFlags TypeFlags, bool HasLegalHalfType=true, bool V1Ty=false, bool AllowBFloatArgsAndRet=true)
static RValue EmitBinaryAtomic(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E)
static llvm::Value * ARMMVEConstantSplat(CGBuilderTy &Builder, llvm::Type *VT)
static Value * EmitX86MaskLogic(CodeGenFunction &CGF, Instruction::BinaryOps Opc, ArrayRef< Value * > Ops, bool InvertLHS=false)
static Value * EmitFromInt(CodeGenFunction &CGF, llvm::Value *V, QualType T, llvm::Type *ResultType)
static Value * EmitAMDGCNBallotForExec(CodeGenFunction &CGF, const CallExpr *E, llvm::Type *RegisterType, llvm::Type *ValueType, bool isExecHi)
static void initializeAlloca(CodeGenFunction &CGF, AllocaInst *AI, Value *Size, Align AlignmentInBytes)
static Value * EmitX86Select(CodeGenFunction &CGF, Value *Mask, Value *Op0, Value *Op1)
SpecialRegisterAccessKind
static Value * EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering SuccessOrdering=AtomicOrdering::SequentiallyConsistent)
This function should be invoked to emit atomic cmpxchg for Microsoft's _InterlockedCompareExchange* i...
static Address CheckAtomicAlignment(CodeGenFunction &CGF, const CallExpr *E)
static Value * EmitX86FMAExpr(CodeGenFunction &CGF, const CallExpr *E, ArrayRef< Value * > Ops, unsigned BuiltinID, bool IsAddSub)
static Value * getMaskVecValue(CodeGenFunction &CGF, Value *Mask, unsigned NumElts)
static bool isSpecialMixedSignMultiply(unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info, WidthAndSignedness ResultInfo)
Determine if a binop is a checked mixed-sign multiply we can specialize.
static Value * MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E, bool ReturnBool)
Utility to insert an atomic cmpxchg instruction.
static Value * emitBinaryExpMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Intrinsic::ID IntrinsicID, llvm::Intrinsic::ID ConstrainedIntrinsicID)
static Value * EmitToInt(CodeGenFunction &CGF, llvm::Value *V, QualType T, llvm::IntegerType *IntType)
Emit the conversions required to turn the given value into an integer of the given size.
static llvm::Value * ARMMVEVectorSplat(CGBuilderTy &Builder, llvm::Value *V)
static Value * EmitX86MaskedCompareResult(CodeGenFunction &CGF, Value *Cmp, unsigned NumElts, Value *MaskIn)
static Value * EmitX86CompressStore(CodeGenFunction &CGF, ArrayRef< Value * > Ops)
static bool NEONSIMDIntrinsicsProvenSorted
static const ARMVectorIntrinsicInfo AArch64SIMDIntrinsicMap[]
static Value * EmitNontemporalStore(CodeGenFunction &CGF, const CallExpr *E)
static llvm::Value * EmitOverflowIntrinsic(CodeGenFunction &CGF, const llvm::Intrinsic::ID IntrinsicID, llvm::Value *X, llvm::Value *Y, llvm::Value *&Carry)
Emit a call to llvm.
static Value * EmitCommonNeonSISDBuiltinExpr(CodeGenFunction &CGF, const ARMVectorIntrinsicInfo &SISDInfo, SmallVectorImpl< Value * > &Ops, const CallExpr *E)
static Value * emitFPIntBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
#define MMA_LDST(n, geom_op_type)
static Value * EmitX86vpcom(CodeGenFunction &CGF, ArrayRef< Value * > Ops, bool IsSigned)
static Value * emitFrexpBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Intrinsic::ID IntrinsicID)
@ _InterlockedExchangeAdd_rel
@ _InterlockedIncrement_acq
@ _InterlockedExchange_nf
@ _InterlockedIncrement_nf
@ _InterlockedExchange_acq
@ _InterlockedCompareExchange128_rel
@ _InterlockedCompareExchange128_acq
@ _InterlockedCompareExchange_acq
@ _InterlockedExchangeAdd_nf
@ _InterlockedCompareExchange_nf
@ _InterlockedDecrement_rel
@ _InterlockedExchangeSub
@ _InterlockedExchangeAdd_acq
@ _InterlockedIncrement_rel
@ _InterlockedCompareExchange128_nf
@ _InterlockedCompareExchange128
@ _InterlockedExchange_rel
@ _InterlockedDecrement_nf
@ _InterlockedExchangeAdd
@ _InterlockedDecrement_acq
@ _InterlockedCompareExchange_rel
static Value * EmitX86ConvertToMask(CodeGenFunction &CGF, Value *In)
static Value * EmitISOVolatileStore(CodeGenFunction &CGF, const CallExpr *E)
static Value * EmitSystemZIntrinsicWithCC(CodeGenFunction &CGF, unsigned IntrinsicID, const CallExpr *E)
Handle a SystemZ function in which the final argument is a pointer to an int that receives the post-i...
static RValue EmitHipStdParUnsupportedBuiltin(CodeGenFunction *CGF, const FunctionDecl *FD)
static llvm::Value * EmitX86BitTestIntrinsic(CodeGenFunction &CGF, BitTest BT, const CallExpr *E, Value *BitBase, Value *BitPos)
static RValue EmitCheckedUnsignedMultiplySignedResult(CodeGenFunction &CGF, const clang::Expr *Op1, WidthAndSignedness Op1Info, const clang::Expr *Op2, WidthAndSignedness Op2Info, const clang::Expr *ResultArg, QualType ResultQTy, WidthAndSignedness ResultInfo)
static Value * emitCallMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID, llvm::Type *Ty, ArrayRef< Value * > Args)
static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *FD, const CallExpr *E, llvm::Constant *calleeValue)
static Value * EmitNontemporalLoad(CodeGenFunction &CGF, const CallExpr *E)
static Value * EmitX86Ternlog(CodeGenFunction &CGF, bool ZeroMask, ArrayRef< Value * > Ops)
static llvm::AtomicOrdering getBitTestAtomicOrdering(BitTest::InterlockingKind I)
#define MMA_VARIANTS_B1_XOR(geom, type)
#define MMA_VARIANTS_I4(geom, type)
Intrinsic::ID getDotProductIntrinsic(QualType QT, int elementCount)
static Value * EmitX86ConvertIntToFp(CodeGenFunction &CGF, const CallExpr *E, ArrayRef< Value * > Ops, bool IsSigned)
static Value * packTBLDVectorList(CodeGenFunction &CGF, ArrayRef< Value * > Ops, Value *ExtOp, Value *IndexOp, llvm::Type *ResTy, unsigned IntID, const char *Name)
static Value * EmitAbs(CodeGenFunction &CGF, Value *ArgValue, bool HasNSW)
static Value * EmitX86ScalarSelect(CodeGenFunction &CGF, Value *Mask, Value *Op0, Value *Op1)
static Value * EmitAtomicDecrementValue(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering Ordering=AtomicOrdering::SequentiallyConsistent)
static bool AArch64SIMDIntrinsicsProvenSorted
static const ARMVectorIntrinsicInfo AArch64SISDIntrinsicMap[]
CodeGenFunction::ComplexPairTy ComplexPairTy
#define ALIAS(NAME, TOK, FLAGS)
llvm::MachO::Record Record
static std::string getName(const CallEvent &Call)
static std::string toString(const clang::SanitizerSet &Sanitizers)
Produce a string containing comma-separated names of sanitizers in Sanitizers set.
static QualType getPointeeType(const MemRegion *R)
Enumerates target-specific builtins in their own namespaces within namespace clang.
Defines the clang::TargetOptions class.
__DEVICE__ float modf(float __x, float *__iptr)
__DEVICE__ double nan(const char *)
__device__ __2f16 float __ockl_bool s
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
CharUnits getTypeAlignInChars(QualType T) const
Return the ABI-specified alignment of a (complete) type T, in characters.
unsigned getIntWidth(QualType T) const
const ASTRecordLayout & getASTRecordLayout(const RecordDecl *D) const
Get or compute information about the layout of the specified record (struct/union/class) D,...
Builtin::Context & BuiltinInfo
QualType getConstantArrayType(QualType EltTy, const llvm::APInt &ArySize, const Expr *SizeExpr, ArraySizeModifier ASM, unsigned IndexTypeQuals) const
Return the unique reference to the type for a constant array of the specified element type.
QualType getBaseElementType(const ArrayType *VAT) const
Return the innermost element type of an array type.
TypeInfo getTypeInfo(const Type *T) const
Get the size and alignment of the specified complete type in bits.
QualType getObjCIdType() const
Represents the Objective-CC id type.
bool hasSameUnqualifiedType(QualType T1, QualType T2) const
Determine whether the given types are equivalent after cvr-qualifiers have been removed.
const ArrayType * getAsArrayType(QualType T) const
Type Query functions.
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
QualType GetBuiltinType(unsigned ID, GetBuiltinTypeError &Error, unsigned *IntegerConstantArgs=nullptr) const
Return the type for the specified builtin.
const TargetInfo & getTargetInfo() const
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
unsigned getTargetAddressSpace(LangAS AS) const
@ GE_Missing_type
Missing a type.
ASTRecordLayout - This class contains layout information for one RecordDecl, which is a struct/union/...
CharUnits getSize() const
getSize - Get the record size in characters.
uint64_t getFieldOffset(unsigned FieldNo) const
getFieldOffset - Get the offset of the given field index, in bits.
Represents an array type, per C99 6.7.5.2 - Array Declarators.
QualType getElementType() const
bool isLibFunction(unsigned ID) const
Return true if this is a builtin for a libc/libm function, with a "__builtin_" prefix (e....
llvm::StringRef getName(unsigned ID) const
Return the identifier name for the specified builtin, e.g.
bool isConstWithoutErrnoAndExceptions(unsigned ID) const
Return true if this function has no side effects and doesn't read memory, except for possibly errno o...
bool isConstWithoutExceptions(unsigned ID) const
bool isConst(unsigned ID) const
Return true if this function has no side effects and doesn't read memory.
CallExpr - Represents a function call (C99 6.5.2.2, C++ [expr.call]).
CharUnits - This is an opaque type for sizes expressed in character units.
llvm::Align getAsAlign() const
getAsAlign - Returns Quantity as a valid llvm::Align, Beware llvm::Align assumes power of two 8-bit b...
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
static CharUnits One()
One - Construct a CharUnits quantity of one.
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
XRayInstrSet XRayInstrumentationBundle
Set of XRay instrumentation kinds to emit.
Like RawAddress, an abstract representation of an aligned address, but the pointer contained in this ...
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
Return the pointer contained in this class after authenticating it and adding offset to it if necessa...
CharUnits getAlignment() const
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
Address withAlignment(CharUnits NewAlignment) const
Return address with different alignment, but same pointer and element type.
llvm::PointerType * getType() const
Return the type of the pointer value.
Address getAddress() const
A scoped helper to set the current debug location to the specified location or preferred location of ...
static ApplyDebugLocation CreateArtificial(CodeGenFunction &CGF)
Apply TemporaryLocation if it is valid.
static ApplyDebugLocation CreateEmpty(CodeGenFunction &CGF)
Set the IRBuilder to not attach debug locations.
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
llvm::StoreInst * CreateAlignedStore(llvm::Value *Val, llvm::Value *Addr, CharUnits Align, bool IsVolatile=false)
Address CreateGEP(CodeGenFunction &CGF, Address Addr, llvm::Value *Index, const llvm::Twine &Name="")
llvm::CallInst * CreateMemMove(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
llvm::CallInst * CreateMemCpyInline(Address Dest, Address Src, uint64_t Size)
llvm::AtomicRMWInst * CreateAtomicRMW(llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val, llvm::AtomicOrdering Ordering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
llvm::CallInst * CreateMemSetInline(Address Dest, llvm::Value *Value, uint64_t Size)
llvm::StoreInst * CreateDefaultAlignedStore(llvm::Value *Val, llvm::Value *Addr, bool IsVolatile=false)
llvm::CallInst * CreateMemSet(Address Dest, llvm::Value *Value, llvm::Value *Size, bool IsVolatile=false)
llvm::AtomicCmpXchgInst * CreateAtomicCmpXchg(Address Addr, llvm::Value *Cmp, llvm::Value *New, llvm::AtomicOrdering SuccessOrdering, llvm::AtomicOrdering FailureOrdering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
Address CreateConstByteGEP(Address Addr, CharUnits Offset, const llvm::Twine &Name="")
Address CreateLaunderInvariantGroup(Address Addr)
llvm::CallInst * CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
llvm::LoadInst * CreateAlignedLoad(llvm::Type *Ty, llvm::Value *Addr, CharUnits Align, const llvm::Twine &Name="")
Address CreateAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
Address CreateConstInBoundsGEP(Address Addr, uint64_t Index, const llvm::Twine &Name="")
Given addr = T* ... produce name = getelementptr inbounds addr, i64 index where i64 is actually the t...
Address CreateInBoundsGEP(Address Addr, ArrayRef< llvm::Value * > IdxList, llvm::Type *ElementType, CharUnits Align, const Twine &Name="")
virtual std::string getDeviceSideName(const NamedDecl *ND)=0
Returns function or variable name on device side even if the current compilation is for host.
virtual llvm::GlobalVariable * getThrowInfo(QualType T)
All available information about a concrete callee.
static CGCallee forDirect(llvm::Constant *functionPtr, const CGCalleeInfo &abstractInfo=CGCalleeInfo())
llvm::DIType * getOrCreateStandaloneType(QualType Ty, SourceLocation Loc)
Emit standalone debug info for a type.
llvm::DILocation * CreateTrapFailureMessageFor(llvm::DebugLoc TrapLocation, StringRef Category, StringRef FailureMsg)
Create a debug location from TrapLocation that adds an artificial inline frame where the frame name i...
CGFunctionInfo - Class to encapsulate the information about a function definition.
virtual void EmitGCMemmoveCollectable(CodeGen::CodeGenFunction &CGF, Address DestPtr, Address SrcPtr, llvm::Value *Size)=0
EnqueuedBlockInfo emitOpenCLEnqueuedBlock(CodeGenFunction &CGF, const Expr *E)
CallArgList - Type for representing both the value and type of arguments in a call.
void add(RValue rvalue, QualType type)
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
llvm::Value * EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E, llvm::Triple::ArchType Arch)
llvm::Value * EmitFromMemory(llvm::Value *Value, QualType Ty)
EmitFromMemory - Change a scalar value from its memory representation to its value representation.
llvm::Value * EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID, const CallExpr *E)
void FinishFunction(SourceLocation EndLoc=SourceLocation())
FinishFunction - Complete IR generation of the current function.
llvm::Value * EmitLifetimeStart(llvm::TypeSize Size, llvm::Value *Addr)
std::pair< RValue, llvm::Value * > EmitAtomicCompareExchange(LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, llvm::AtomicOrdering Success=llvm::AtomicOrdering::SequentiallyConsistent, llvm::AtomicOrdering Failure=llvm::AtomicOrdering::SequentiallyConsistent, bool IsWeak=false, AggValueSlot Slot=AggValueSlot::ignored())
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
llvm::Value * EmitSVEPredicateCast(llvm::Value *Pred, llvm::ScalableVectorType *VTy)
llvm::CallInst * EmitTrapCall(llvm::Intrinsic::ID IntrID)
Emit a call to trap or debugtrap and attach function attribute "trap-func-name" if specified.
SanitizerSet SanOpts
Sanitizers enabled for this function.
RValue EmitBuiltinIsAligned(const CallExpr *E)
Emit IR for __builtin_is_aligned.
LValue EmitAggExprToLValue(const Expr *E)
EmitAggExprToLValue - Emit the computation of the specified expression of aggregate type into a tempo...
void EmitNonNullArgCheck(RValue RV, QualType ArgType, SourceLocation ArgLoc, AbstractCallee AC, unsigned ParmNum)
Create a check for a function parameter that may potentially be declared as non-null.
llvm::Value * EmitHexagonBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
void GetAArch64SVEProcessedOperands(unsigned BuiltinID, const CallExpr *E, SmallVectorImpl< llvm::Value * > &Ops, SVETypeFlags TypeFlags)
llvm::Value * EmitAMDGPUBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
void EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr)
void pushLifetimeExtendedDestroy(CleanupKind kind, Address addr, QualType type, Destroyer *destroyer, bool useEHCleanupForArray)
llvm::Value * EmitCheckedArgForBuiltin(const Expr *E, BuiltinCheckKind Kind)
Emits an argument for a call to a builtin.
llvm::Value * EmitSVEGatherLoad(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
CleanupKind getARCCleanupKind()
Retrieves the default cleanup kind for an ARC cleanup.
llvm::Value * EmitRISCVCpuSupports(const CallExpr *E)
llvm::Value * EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitHLSLBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitAArch64SVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
RValue emitBuiltinOSLogFormat(const CallExpr &E)
Emit IR for __builtin_os_log_format.
LValue EmitLValue(const Expr *E, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitLValue - Emit code to compute a designator that specifies the location of the expression.
llvm::Value * EmitVAStartEnd(llvm::Value *ArgValue, bool IsStart)
Emits a call to an LLVM variable-argument intrinsic, either llvm.va_start or llvm....
llvm::Value * EmitSVEMaskedStore(const CallExpr *, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
llvm::Value * EmitSVEReinterpret(llvm::Value *Val, llvm::Type *Ty)
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
llvm::Value * EmitSEHExceptionInfo()
RValue EmitBuiltinAlignTo(const CallExpr *E, bool AlignUp)
Emit IR for __builtin_align_up/__builtin_align_down.
const LangOptions & getLangOpts() const
void ProcessOrderScopeAMDGCN(llvm::Value *Order, llvm::Value *Scope, llvm::AtomicOrdering &AO, llvm::SyncScope::ID &SSID)
llvm::Constant * EmitCheckTypeDescriptor(QualType T)
Emit a description of a type in a format suitable for passing to a runtime sanitizer handler.
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
void EmitTrapCheck(llvm::Value *Checked, SanitizerHandler CheckHandlerID)
Create a basic block that will call the trap intrinsic, and emit a conditional branch to it,...
void EmitUnreachable(SourceLocation Loc)
Emit a reached-unreachable diagnostic if Loc is valid and runtime checking is enabled.
llvm::Value * EmitSVETupleCreate(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType, ArrayRef< llvm::Value * > Ops)
RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee, ReturnValueSlot ReturnValue, const CallArgList &Args, llvm::CallBase **callOrInvoke, bool IsMustTail, SourceLocation Loc, bool IsVirtualFunctionPointerThunk=false)
EmitCall - Generate a call of the given function, expecting the given result type,...
Address makeNaturalAddressForPointer(llvm::Value *Ptr, QualType T, CharUnits Alignment=CharUnits::Zero(), bool ForPointeeType=false, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
Construct an address with the natural alignment of T.
ComplexPairTy EmitComplexExpr(const Expr *E, bool IgnoreReal=false, bool IgnoreImag=false)
EmitComplexExpr - Emit the computation of the specified expression of complex type,...
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
TypeCheckKind
Situations in which we might emit a check for the suitability of a pointer or glvalue.
@ TCK_Store
Checking the destination of a store. Must be suitably sized and aligned.
@ TCK_Load
Checking the operand of a load. Must be suitably sized and aligned.
llvm::Value * EmitSMELdrStr(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::Value * FormSVEBuiltinResult(llvm::Value *Call)
FormSVEBuiltinResult - Returns the struct of scalable vectors as a wider vector.
llvm::Value * EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx, const llvm::ElementCount &Count)
llvm::Type * ConvertTypeForMem(QualType T)
llvm::Value * EmitSVEMaskedLoad(const CallExpr *, llvm::Type *ReturnTy, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID, bool IsZExtReturn)
bool AlwaysEmitXRayCustomEvents() const
AlwaysEmitXRayCustomEvents - Return true if we must unconditionally emit XRay custom event handling c...
llvm::Value * EmitSVEDupX(llvm::Value *Scalar)
RawAddress CreateMemTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
@ Default
! No language constraints on evaluation order.
const TargetInfo & getTarget() const
llvm::Value * vectorWrapScalar16(llvm::Value *Op)
llvm::Function * LookupNeonLLVMIntrinsic(unsigned IntrinsicID, unsigned Modifier, llvm::Type *ArgTy, const CallExpr *E)
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
llvm::Value * EmitSEHExceptionCode()
Address EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitPointerWithAlignment - Given an expression with a pointer type, emit the value and compute our be...
llvm::Value * EmitTargetBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
EmitTargetBuiltinExpr - Emit the given builtin call.
llvm::Value * EmitCountedByFieldExpr(const Expr *Base, const FieldDecl *FAMDecl, const FieldDecl *CountDecl)
Build an expression accessing the "counted_by" field.
RValue EmitAnyExprToTemp(const Expr *E)
EmitAnyExprToTemp - Similarly to EmitAnyExpr(), however, the result will always be accessible even if...
RValue EmitCoroutineIntrinsic(const CallExpr *E, unsigned int IID)
llvm::Value * EmitAArch64SMEBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
RValue EmitAMDGPUDevicePrintfCallExpr(const CallExpr *E)
Address EmitArrayToPointerDecay(const Expr *Array, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
void EmitCheck(ArrayRef< std::pair< llvm::Value *, SanitizerMask > > Checked, SanitizerHandler Check, ArrayRef< llvm::Constant * > StaticArgs, ArrayRef< llvm::Value * > DynamicArgs)
Create a basic block that will either trap or call a handler function in the UBSan runtime with the p...
RValue EmitBuiltinNewDeleteCall(const FunctionProtoType *Type, const CallExpr *TheCallExpr, bool IsDelete)
llvm::Value * EmitRISCVCpuInit()
llvm::Value * EmitSVETupleSetOrGet(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType, ArrayRef< llvm::Value * > Ops)
RValue EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
llvm::Value * EmitSVEPrefetchLoad(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
llvm::Value * EmitAArch64CompareBuiltinExpr(llvm::Value *Op, llvm::Type *Ty, const llvm::CmpInst::Predicate Fp, const llvm::CmpInst::Predicate Ip, const llvm::Twine &Name="")
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
AggValueSlot CreateAggTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateAggTemp - Create a temporary memory object for the given aggregate type.
llvm::ScalableVectorType * getSVEType(const SVETypeFlags &TypeFlags)
RValue emitRotate(const CallExpr *E, bool IsRotateRight)
llvm::Constant * EmitCheckSourceLocation(SourceLocation Loc)
Emit a description of a source location in a format suitable for passing to a runtime sanitizer handl...
void ErrorUnsupported(const Stmt *S, const char *Type)
ErrorUnsupported - Print out an error that codegen doesn't support the specified stmt yet.
CGDebugInfo * getDebugInfo()
const FieldDecl * FindFlexibleArrayMemberFieldAndOffset(ASTContext &Ctx, const RecordDecl *RD, const FieldDecl *FAMDecl, uint64_t &Offset)
Address EmitVAListRef(const Expr *E)
llvm::Value * EmitNeonShiftVector(llvm::Value *V, llvm::Type *Ty, bool negateForRightShift)
llvm::Value * EmitSVEMovl(const SVETypeFlags &TypeFlags, llvm::ArrayRef< llvm::Value * > Ops, unsigned BuiltinID)
void emitAlignmentAssumption(llvm::Value *PtrValue, QualType Ty, SourceLocation Loc, SourceLocation AssumptionLoc, llvm::Value *Alignment, llvm::Value *OffsetValue=nullptr)
const TargetCodeGenInfo & getTargetHooks() const
llvm::Value * EmitARMBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
llvm::Type * getEltType(const SVETypeFlags &TypeFlags)
void EmitAggExpr(const Expr *E, AggValueSlot AS)
EmitAggExpr - Emit the computation of the specified expression of aggregate type.
bool ShouldXRayInstrumentFunction() const
ShouldXRayInstrument - Return true if the current function should be instrumented with XRay nop sleds...
llvm::Value * EmitSVEPMull(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
llvm::Value * EmitToMemory(llvm::Value *Value, QualType Ty)
EmitToMemory - Change a scalar value from its value representation to its in-memory representation.
RValue EmitOpenMPDevicePrintfCallExpr(const CallExpr *E)
bool IsInPreservedAIRegion
True if CodeGen currently emits code inside presereved access index region.
llvm::Value * EmitARCRetain(QualType type, llvm::Value *value)
bool AlwaysEmitXRayTypedEvents() const
AlwaysEmitXRayTypedEvents - Return true if clang must unconditionally emit XRay typed event handling ...
void SetSqrtFPAccuracy(llvm::Value *Val)
Set the minimum required accuracy of the given sqrt operation based on CodeGenOpts.
llvm::Value * EmitSVEScatterStore(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::CallInst * EmitNounwindRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
ASTContext & getContext() const
llvm::Function * generateBuiltinOSLogHelperFunction(const analyze_os_log::OSLogBufferLayout &Layout, CharUnits BufferAlignment)
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
CGCallee EmitCallee(const Expr *E)
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
llvm::Value * EmitScalarOrConstFoldImmArg(unsigned ICEArguments, unsigned Idx, const CallExpr *E)
void checkTargetFeatures(const CallExpr *E, const FunctionDecl *TargetDecl)
llvm::Value * BuildVector(ArrayRef< llvm::Value * > Ops)
llvm::Value * EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitARMCDEBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
llvm::CallInst * EmitRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
llvm::Type * ConvertType(QualType T)
llvm::CallBase * EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee, ArrayRef< llvm::Value * > args, const Twine &name="")
llvm::Value * EmitSystemZBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitSMEReadWrite(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, LValue LV, QualType Type, SanitizerSet SkippedChecks=SanitizerSet(), llvm::Value *ArraySize=nullptr)
llvm::Value * EmitSMELd1St1(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::Value * EmitPPCBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
static Destroyer destroyARCStrongPrecise
void EmitARCIntrinsicUse(ArrayRef< llvm::Value * > values)
RValue EmitNVPTXDevicePrintfCallExpr(const CallExpr *E)
llvm::Value * EmitSVEStructLoad(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Address EmitMSVAListRef(const Expr *E)
Emit a "reference" to a __builtin_ms_va_list; this is always the value of the expression,...
llvm::Value * EmitCheckedInBoundsGEP(llvm::Type *ElemTy, llvm::Value *Ptr, ArrayRef< llvm::Value * > IdxList, bool SignedIndices, bool IsSubtraction, SourceLocation Loc, const Twine &Name="")
Same as IRBuilder::CreateInBoundsGEP, but additionally emits a check to detect undefined behavior whe...
llvm::Value * EmitNeonRShiftImm(llvm::Value *Vec, llvm::Value *Amt, llvm::Type *Ty, bool usgn, const char *name)
SmallVector< llvm::Type *, 2 > getSVEOverloadTypes(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType, ArrayRef< llvm::Value * > Ops)
static bool hasAggregateEvaluationKind(QualType T)
const FieldDecl * FindCountedByField(const FieldDecl *FD)
Find the FieldDecl specified in a FAM's "counted_by" attribute.
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
llvm::Value * EmitARMMVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
llvm::Value * EmitSVEStructStore(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
llvm::Value * EmitSEHAbnormalTermination()
llvm::Value * EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E)
Address ReturnValue
ReturnValue - The temporary alloca to hold the return value.
llvm::Value * EmitSVEAllTruePred(const SVETypeFlags &TypeFlags)
RValue GetUndefRValue(QualType Ty)
GetUndefRValue - Get an appropriate 'undef' rvalue for the given type.
llvm::Type * SVEBuiltinMemEltTy(const SVETypeFlags &TypeFlags)
SVEBuiltinMemEltTy - Returns the memory element type for this memory access builtin.
llvm::LLVMContext & getLLVMContext()
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
void AddAMDGPUFenceAddressSpaceMMRA(llvm::Instruction *Inst, const CallExpr *E)
llvm::Value * EmitSMEZero(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::Value * EmitRISCVBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
llvm::Value * EmitCommonNeonBuiltinExpr(unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic, const char *NameHint, unsigned Modifier, const CallExpr *E, SmallVectorImpl< llvm::Value * > &Ops, Address PtrOp0, Address PtrOp1, llvm::Triple::ArchType Arch)
llvm::Value * EmitNeonCall(llvm::Function *F, SmallVectorImpl< llvm::Value * > &O, const char *name, unsigned shift=0, bool rightshift=false)
llvm::Value * EmitAnnotationCall(llvm::Function *AnnotationFn, llvm::Value *AnnotatedVal, StringRef AnnotationStr, SourceLocation Location, const AnnotateAttr *Attr)
Emit an annotation call (intrinsic).
llvm::ScalableVectorType * getSVEPredType(const SVETypeFlags &TypeFlags)
llvm::Value * EmitSVEGatherPrefetch(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
llvm::Value * EmitBPFBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
This class organizes the cross-function state that is used while generating LLVM code.
CGHLSLRuntime & getHLSLRuntime()
Return a reference to the configured HLSL runtime.
llvm::Module & getModule() const
llvm::FunctionCallee CreateRuntimeFunction(llvm::FunctionType *Ty, StringRef Name, llvm::AttributeList ExtraAttrs=llvm::AttributeList(), bool Local=false, bool AssumeConvergent=false)
Create or return a runtime function declaration with the specified type and name.
llvm::Constant * getRawFunctionPointer(GlobalDecl GD, llvm::Type *Ty=nullptr)
Return a function pointer for a reference to the given function.
llvm::Constant * getBuiltinLibFunction(const FunctionDecl *FD, unsigned BuiltinID)
Given a builtin id for a function like "__builtin_fabsf", return a Function* for "fabsf".
DiagnosticsEngine & getDiags() const
void ErrorUnsupported(const Stmt *S, const char *Type)
Print out an error that codegen doesn't support the specified stmt yet.
const LangOptions & getLangOpts() const
CGCUDARuntime & getCUDARuntime()
Return a reference to the configured CUDA runtime.
CodeGenTypes & getTypes()
CGOpenCLRuntime & getOpenCLRuntime()
Return a reference to the configured OpenCL runtime.
const TargetInfo & getTarget() const
const llvm::DataLayout & getDataLayout() const
void Error(SourceLocation loc, StringRef error)
Emit a general error that something can't be done.
CGCXXABI & getCXXABI() const
llvm::Constant * GetFunctionStart(const ValueDecl *Decl)
const llvm::Triple & getTriple() const
void DecorateInstructionWithTBAA(llvm::Instruction *Inst, TBAAAccessInfo TBAAInfo)
DecorateInstructionWithTBAA - Decorate the instruction with a TBAA tag.
llvm::Constant * CreateRuntimeVariable(llvm::Type *Ty, StringRef Name)
Create a new runtime global variable with the specified type and name.
TBAAAccessInfo getTBAAAccessInfo(QualType AccessType)
getTBAAAccessInfo - Get TBAA information that describes an access to an object of the given type.
ASTContext & getContext() const
const CodeGenOptions & getCodeGenOpts() const
StringRef getMangledName(GlobalDecl GD)
CharUnits getNaturalPointeeTypeAlignment(QualType T, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
llvm::LLVMContext & getLLVMContext()
CGObjCRuntime & getObjCRuntime()
Return a reference to the configured Objective-C runtime.
void SetLLVMFunctionAttributes(GlobalDecl GD, const CGFunctionInfo &Info, llvm::Function *F, bool IsThunk)
Set the LLVM function attributes (sext, zext, etc).
void SetLLVMFunctionAttributesForDefinition(const Decl *D, llvm::Function *F)
Set the LLVM function attributes which only apply to a function definition.
llvm::Function * getIntrinsic(unsigned IID, ArrayRef< llvm::Type * > Tys=std::nullopt)
ConstantAddress GetAddrOfConstantCString(const std::string &Str, const char *GlobalName=nullptr)
Returns a pointer to a character array containing the literal and a terminating '\0' character.
llvm::Type * ConvertType(QualType T)
ConvertType - Convert type T into a llvm::Type.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
const CGFunctionInfo & arrangeBuiltinFunctionCall(QualType resultType, const CallArgList &args)
llvm::Constant * emitAbstract(const Expr *E, QualType T)
Emit the result of the given expression as an abstract constant, asserting that it succeeded.
Information for lazily generating a cleanup.
FunctionArgList - Type for representing both the decl and type of parameters to a function.
LValue - This represents an lvalue references.
llvm::Value * getRawBitFieldPointer(CodeGenFunction &CGF) const
llvm::Value * getPointer(CodeGenFunction &CGF) const
Address getAddress() const
void setNontemporal(bool Value)
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
static RValue getIgnored()
static RValue get(llvm::Value *V)
static RValue getAggregate(Address addr, bool isVolatile=false)
Convert an Address to an RValue.
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
An abstract representation of an aligned address.
llvm::Value * getPointer() const
static RawAddress invalid()
ReturnValueSlot - Contains the address where the return value of a function can be stored,...
virtual llvm::Value * encodeReturnAddress(CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const
Performs the code-generation required to convert the address of an instruction into a return address ...
virtual llvm::Value * decodeReturnAddress(CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const
Performs the code-generation required to convert a return address as stored by the system into the ac...
const T & getABIInfo() const
virtual int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const
Determines the DWARF register number for the stack pointer, for exception-handling purposes.
virtual llvm::Value * testFPKind(llvm::Value *V, unsigned BuiltinID, CGBuilderTy &Builder, CodeGenModule &CGM) const
Performs a target specific test of a floating point value for things like IsNaN, Infinity,...
Complex values, per C99 6.2.5p11.
Represents a concrete matrix type with constant number of rows and columns.
RecordDecl * getOuterLexicalRecordContext()
Retrieve the outermost lexically enclosing record context.
bool isImplicit() const
isImplicit - Indicates whether the declaration was implicitly generated by the implementation.
FunctionDecl * getAsFunction() LLVM_READONLY
Returns the function itself, or the templated function if this is a function template.
DeclContext * getDeclContext()
static bool isFlexibleArrayMemberLike(ASTContext &Context, const Decl *D, QualType Ty, LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel, bool IgnoreTemplateOrMacroSubstitution)
Whether it resembles a flexible array member.
Concrete class used by the front-end to report problems and issues.
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
This represents one expression.
bool EvaluateAsInt(EvalResult &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer,...
Expr * IgnoreParenCasts() LLVM_READONLY
Skip past any parentheses and casts which might surround this expression until reaching a fixed point...
llvm::APSInt EvaluateKnownConstInt(const ASTContext &Ctx, SmallVectorImpl< PartialDiagnosticAt > *Diag=nullptr) const
EvaluateKnownConstInt - Call EvaluateAsRValue and return the folded integer.
Expr * IgnoreParenImpCasts() LLVM_READONLY
Skip past any parentheses and implicit casts which might surround this expression until reaching a fi...
bool EvaluateAsFloat(llvm::APFloat &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsFloat - Return true if this is a constant which we can fold and convert to a floating point...
Expr * IgnoreParens() LLVM_READONLY
Skip past any parentheses which might surround this expression until reaching a fixed point.
@ NPC_ValueDependentIsNotNull
Specifies that a value-dependent expression should be considered to never be a null pointer constant.
ExprObjectKind getObjectKind() const
getObjectKind - The object kind that this expression produces.
bool EvaluateAsRValue(EvalResult &Result, const ASTContext &Ctx, bool InConstantContext=false) const
EvaluateAsRValue - Return true if this is a constant which we can fold to an rvalue using any crazy t...
bool HasSideEffects(const ASTContext &Ctx, bool IncludePossibleEffects=true) const
HasSideEffects - This routine returns true for all those expressions which have any effect other than...
std::optional< std::string > tryEvaluateString(ASTContext &Ctx) const
If the current Expr can be evaluated to a pointer to a null-terminated constant string,...
Expr * IgnoreImpCasts() LLVM_READONLY
Skip past any implicit casts which might surround this expression until reaching a fixed point.
NullPointerConstantKind isNullPointerConstant(ASTContext &Ctx, NullPointerConstantValueDependence NPC) const
isNullPointerConstant - C99 6.3.2.3p3 - Test if this reduces down to a Null pointer constant.
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
std::optional< llvm::APSInt > getIntegerConstantExpr(const ASTContext &Ctx, SourceLocation *Loc=nullptr) const
isIntegerConstantExpr - Return the value if this expression is a valid integer constant expression.
bool tryEvaluateObjectSize(uint64_t &Result, ASTContext &Ctx, unsigned Type) const
If the current Expr is a pointer, this will try to statically determine the number of bytes available...
const ValueDecl * getAsBuiltinConstantDeclRef(const ASTContext &Context) const
If this expression is an unambiguous reference to a single declaration, in the style of __builtin_fun...
Represents difference between two FPOptions values.
Represents a member of a struct/union/class.
Represents a function declaration or definition.
const ParmVarDecl * getParamDecl(unsigned i) const
unsigned getBuiltinID(bool ConsiderWrapperFunctions=false) const
Returns a value indicating whether this function corresponds to a builtin function.
Represents a prototype with parameter type info, e.g.
GlobalDecl - represents a global declaration.
const Decl * getDecl() const
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
StrictFlexArraysLevelKind
StringRef getName() const
Get the name of identifier for this declaration as a StringRef.
std::string getNameAsString() const
Get a human-readable name for the declaration, even if it is one of the special kinds of names (C++ c...
Flags to identify the types for overloaded Neon builtins.
EltType getEltType() const
PointerType - C99 6.7.5.1 - Pointer Declarators.
QualType getPointeeType() const
A (possibly-)qualified type.
bool isVolatileQualified() const
Determine whether this type is volatile-qualified.
bool isWebAssemblyFuncrefType() const
Returns true if it is a WebAssembly Funcref Type.
LangAS getAddressSpace() const
Return the address space of this type.
bool isWebAssemblyExternrefType() const
Returns true if it is a WebAssembly Externref Type.
The collection of all-type qualifiers we support.
Represents a struct/union/class.
field_range fields() const
Flags to identify the types for overloaded SVE builtins.
bool isZExtReturn() const
bool isReverseUSDOT() const
bool isOverloadNone() const
MemEltType getMemEltType() const
bool isGatherLoad() const
bool isOverloadCvt() const
EltType getEltType() const
bool isOverloadDefault() const
bool isOverloadWhileRW() const
bool isReverseMergeAnyAccOp() const
bool isReductionQV() const
bool isInsertOp1SVALL() const
bool isAppendSVALL() const
bool isReverseMergeAnyBinOp() const
bool isStructStore() const
bool isTupleCreate() const
bool isGatherPrefetch() const
bool hasSplatOperand() const
MergeType getMergeType() const
bool isByteIndexed() const
bool isStructLoad() const
bool isOverloadWhileOrMultiVecCvt() const
unsigned getSplatOperand() const
bool isScatterStore() const
bool isReverseCompare() const
Scope - A scope is a transient data structure that is used while parsing the program.
Encodes a location in the source.
SourceLocation getBeginLoc() const LLVM_READONLY
Exposes information about the current target.
TargetOptions & getTargetOpts() const
Retrieve the target options.
virtual bool hasLegalHalfType() const
Determine whether _Float16 is supported on this target.
const llvm::Triple & getTriple() const
Returns the target triple of the primary target.
bool isLittleEndian() const
unsigned getMaxOpenCLWorkGroupSize() const
virtual bool checkArithmeticFenceSupported() const
Controls if __arithmetic_fence is supported in the targeted backend.
unsigned getSuitableAlign() const
Return the alignment that is the largest alignment ever used for any scalar/SIMD data type on the tar...
virtual std::string_view getClobbers() const =0
Returns a string of target-specific clobbers, in LLVM format.
llvm::CodeObjectVersionKind CodeObjectVersion
Code object version for AMDGPU.
The base class of the type hierarchy.
CXXRecordDecl * getAsCXXRecordDecl() const
Retrieves the CXXRecordDecl that this type refers to, either because the type is a RecordType or beca...
bool isBlockPointerType() const
bool isBooleanType() const
bool isSignedIntegerType() const
Return true if this is an integer type that is signed, according to C99 6.2.5p4 [char,...
bool isComplexType() const
isComplexType() does not include complex integers (a GCC extension).
bool isCountAttributedType() const
bool isPointerType() const
bool isIntegerType() const
isIntegerType() does not include complex integers (a GCC extension).
const T * castAs() const
Member-template castAs<specific type>.
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
bool isIntegralOrEnumerationType() const
Determine whether this type is an integral or enumeration type.
bool hasUnsignedIntegerRepresentation() const
Determine whether this type has an unsigned integer representation of some sort, e....
bool isBitIntType() const
bool hasSignedIntegerRepresentation() const
Determine whether this type has an signed integer representation of some sort, e.g....
bool isObjCObjectPointerType() const
bool hasFloatingRepresentation() const
Determine whether this type has a floating-point representation of some sort, e.g....
bool isFloatingType() const
bool isUnsignedIntegerType() const
Return true if this is an integer type that is unsigned, according to C99 6.2.5p6 [which returns true...
const T * getAs() const
Member-template getAs<specific type>'.
bool isRecordType() const
bool isSizelessVectorType() const
Returns true for all scalable vector types.
QualType getSizelessVectorEltType(const ASTContext &Ctx) const
Returns the representative type for the element of a sizeless vector builtin type.
RecordDecl * getAsRecordDecl() const
Retrieves the RecordDecl this type refers to.
Represent the declaration of a variable (in which case it is an lvalue) a function (in which case it ...
Represents a GCC generic vector type.
unsigned getNumElements() const
SmallVector< OSLogBufferItem, 4 > Items
unsigned char getNumArgsByte() const
unsigned char getSummaryByte() const
Defines the clang::TargetInfo interface.
@ Type
The l-value was considered opaque, so the alignment was determined from a type.
llvm::Constant * initializationPatternFor(CodeGenModule &, llvm::Type *)
TypeEvaluationKind
The kind of evaluation to perform on values of a particular type.
@ EHCleanup
Denotes a cleanup that should run when a scope is exited using exceptional control flow (a throw stat...
constexpr XRayInstrMask Typed
constexpr XRayInstrMask Custom
bool computeOSLogBufferLayout(clang::ASTContext &Ctx, const clang::CallExpr *E, OSLogBufferLayout &layout)
const void * Store
Store - This opaque type encapsulates an immutable mapping from locations to values.
bool Dup(InterpState &S, CodePtr OpPC)
bool Ret(InterpState &S, CodePtr &PC, APValue &Result)
bool Zero(InterpState &S, CodePtr OpPC)
bool Mul(InterpState &S, CodePtr OpPC)
bool Neg(InterpState &S, CodePtr OpPC)
bool Load(InterpState &S, CodePtr OpPC)
bool Cast(InterpState &S, CodePtr OpPC)
The JSON file list parser is used to communicate input to InstallAPI.
@ Vector
'vector' clause, allowed on 'loop', Combined, and 'routine' directives.
@ DType
'dtype' clause, an alias for 'device_type', stored separately for diagnostic purposes.
@ OK_BitField
A bitfield object is a bitfield on a C or C++ record.
bool operator<(DeclarationName LHS, DeclarationName RHS)
Ordering on two declaration names.
@ Asm
Assembly: we accept this only so that we can preprocess it.
@ Result
The result type of a method or function.
LangAS
Defines the address space values used by the address space qualifier of QualType.
const FunctionProtoType * T
@ Success
Template argument deduction was successful.
@ Other
Other implicit parameter.
Diagnostic wrappers for TextAPI types for error reporting.
llvm::PointerType * VoidPtrTy
llvm::IntegerType * Int64Ty
llvm::PointerType * ConstGlobalsPtrTy
void* in the address space for constant globals
CharUnits getIntAlign() const
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
llvm::Type * HalfTy
half, bfloat, float, double
llvm::IntegerType * SizeTy
llvm::IntegerType * Int32Ty
llvm::IntegerType * IntPtrTy
llvm::IntegerType * IntTy
int
llvm::IntegerType * Int16Ty
llvm::PointerType * Int8PtrTy
llvm::PointerType * UnqualPtrTy
llvm::PointerType * AllocaInt8PtrTy
LangAS getASTAllocaAddressSpace() const
EvalResult is a struct with detailed info about an evaluated expression.
APValue Val
Val - This is the value the expression can be folded to.
void clear(SanitizerMask K=SanitizerKind::All)
Disable the sanitizers specified in K.
void set(SanitizerMask K, bool Value)
Enable or disable a certain (single) sanitizer.
bool has(SanitizerMask K) const
Check if a certain (single) sanitizer is enabled.
bool has(XRayInstrMask K) const
#define scalbln(__x, __y)
#define copysign(__x, __y)
#define remquo(__x, __y, __z)
#define nextafter(__x, __y)
#define nexttoward(__x, __y)
#define remainder(__x, __y)
#define fma(__x, __y, __z)