34#include "llvm/ADT/APFloat.h"
35#include "llvm/ADT/APInt.h"
36#include "llvm/ADT/FloatingPointMode.h"
37#include "llvm/ADT/SmallPtrSet.h"
38#include "llvm/ADT/StringExtras.h"
39#include "llvm/Analysis/ValueTracking.h"
40#include "llvm/IR/DataLayout.h"
41#include "llvm/IR/InlineAsm.h"
42#include "llvm/IR/Intrinsics.h"
43#include "llvm/IR/IntrinsicsAArch64.h"
44#include "llvm/IR/IntrinsicsAMDGPU.h"
45#include "llvm/IR/IntrinsicsARM.h"
46#include "llvm/IR/IntrinsicsBPF.h"
47#include "llvm/IR/IntrinsicsDirectX.h"
48#include "llvm/IR/IntrinsicsHexagon.h"
49#include "llvm/IR/IntrinsicsNVPTX.h"
50#include "llvm/IR/IntrinsicsPowerPC.h"
51#include "llvm/IR/IntrinsicsR600.h"
52#include "llvm/IR/IntrinsicsRISCV.h"
53#include "llvm/IR/IntrinsicsS390.h"
54#include "llvm/IR/IntrinsicsVE.h"
55#include "llvm/IR/IntrinsicsWebAssembly.h"
56#include "llvm/IR/IntrinsicsX86.h"
57#include "llvm/IR/MDBuilder.h"
58#include "llvm/IR/MatrixBuilder.h"
59#include "llvm/Support/ConvertUTF.h"
60#include "llvm/Support/MathExtras.h"
61#include "llvm/Support/ScopedPrinter.h"
62#include "llvm/TargetParser/AArch64TargetParser.h"
63#include "llvm/TargetParser/X86TargetParser.h"
68using namespace CodeGen;
72 Align AlignmentInBytes) {
74 switch (CGF.
getLangOpts().getTrivialAutoVarInit()) {
75 case LangOptions::TrivialAutoVarInitKind::Uninitialized:
78 case LangOptions::TrivialAutoVarInitKind::Zero:
79 Byte = CGF.
Builder.getInt8(0x00);
81 case LangOptions::TrivialAutoVarInitKind::Pattern: {
83 Byte = llvm::dyn_cast<llvm::ConstantInt>(
91 I->addAnnotationMetadata(
"auto-init");
106 static SmallDenseMap<unsigned, StringRef, 64> F128Builtins{
107 {Builtin::BI__builtin___fprintf_chk,
"__fprintf_chkieee128"},
108 {Builtin::BI__builtin___printf_chk,
"__printf_chkieee128"},
109 {Builtin::BI__builtin___snprintf_chk,
"__snprintf_chkieee128"},
110 {Builtin::BI__builtin___sprintf_chk,
"__sprintf_chkieee128"},
111 {Builtin::BI__builtin___vfprintf_chk,
"__vfprintf_chkieee128"},
112 {Builtin::BI__builtin___vprintf_chk,
"__vprintf_chkieee128"},
113 {Builtin::BI__builtin___vsnprintf_chk,
"__vsnprintf_chkieee128"},
114 {Builtin::BI__builtin___vsprintf_chk,
"__vsprintf_chkieee128"},
115 {Builtin::BI__builtin_fprintf,
"__fprintfieee128"},
116 {Builtin::BI__builtin_printf,
"__printfieee128"},
117 {Builtin::BI__builtin_snprintf,
"__snprintfieee128"},
118 {Builtin::BI__builtin_sprintf,
"__sprintfieee128"},
119 {Builtin::BI__builtin_vfprintf,
"__vfprintfieee128"},
120 {Builtin::BI__builtin_vprintf,
"__vprintfieee128"},
121 {Builtin::BI__builtin_vsnprintf,
"__vsnprintfieee128"},
122 {Builtin::BI__builtin_vsprintf,
"__vsprintfieee128"},
123 {Builtin::BI__builtin_fscanf,
"__fscanfieee128"},
124 {Builtin::BI__builtin_scanf,
"__scanfieee128"},
125 {Builtin::BI__builtin_sscanf,
"__sscanfieee128"},
126 {Builtin::BI__builtin_vfscanf,
"__vfscanfieee128"},
127 {Builtin::BI__builtin_vscanf,
"__vscanfieee128"},
128 {Builtin::BI__builtin_vsscanf,
"__vsscanfieee128"},
129 {Builtin::BI__builtin_nexttowardf128,
"__nexttowardieee128"},
135 static SmallDenseMap<unsigned, StringRef, 4> AIXLongDouble64Builtins{
136 {Builtin::BI__builtin_frexpl,
"frexp"},
137 {Builtin::BI__builtin_ldexpl,
"ldexp"},
138 {Builtin::BI__builtin_modfl,
"modf"},
144 if (FD->
hasAttr<AsmLabelAttr>())
150 &
getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad() &&
151 F128Builtins.contains(BuiltinID))
152 Name = F128Builtins[BuiltinID];
155 &llvm::APFloat::IEEEdouble() &&
156 AIXLongDouble64Builtins.contains(BuiltinID))
157 Name = AIXLongDouble64Builtins[BuiltinID];
162 llvm::FunctionType *Ty =
165 return GetOrCreateLLVMFunction(Name, Ty, D,
false);
171 QualType T, llvm::IntegerType *IntType) {
174 if (
V->getType()->isPointerTy())
175 return CGF.
Builder.CreatePtrToInt(
V, IntType);
177 assert(
V->getType() == IntType);
182 QualType T, llvm::Type *ResultType) {
185 if (ResultType->isPointerTy())
186 return CGF.
Builder.CreateIntToPtr(
V, ResultType);
188 assert(
V->getType() == ResultType);
199 if (Align % Bytes != 0) {
212 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
222 llvm::IntegerType *IntType = llvm::IntegerType::get(
226 llvm::Type *ValueType = Val->getType();
254 llvm::AtomicRMWInst::BinOp Kind,
263 llvm::AtomicRMWInst::BinOp Kind,
265 Instruction::BinaryOps Op,
266 bool Invert =
false) {
275 llvm::IntegerType *IntType = llvm::IntegerType::get(
279 llvm::Type *ValueType = Val->getType();
283 Kind, DestAddr, Val, llvm::AtomicOrdering::SequentiallyConsistent);
288 llvm::ConstantInt::getAllOnesValue(IntType));
312 llvm::IntegerType *IntType = llvm::IntegerType::get(
316 llvm::Type *ValueType = Cmp->getType();
321 DestAddr, Cmp, New, llvm::AtomicOrdering::SequentiallyConsistent,
322 llvm::AtomicOrdering::SequentiallyConsistent);
325 return CGF.
Builder.CreateZExt(CGF.
Builder.CreateExtractValue(Pair, 1),
348 AtomicOrdering SuccessOrdering = AtomicOrdering::SequentiallyConsistent) {
363 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release ?
364 AtomicOrdering::Monotonic :
372 DestAddr, Comparand, Exchange, SuccessOrdering, FailureOrdering);
373 Result->setVolatile(
true);
391 AtomicOrdering SuccessOrdering) {
398 assert(DestPtr->getType()->isPointerTy());
399 assert(!ExchangeHigh->getType()->isPointerTy());
400 assert(!ExchangeLow->getType()->isPointerTy());
403 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release
404 ? AtomicOrdering::Monotonic
409 llvm::Type *Int128Ty = llvm::IntegerType::get(CGF.
getLLVMContext(), 128);
410 Address DestAddr(DestPtr, Int128Ty,
415 ExchangeHigh = CGF.
Builder.CreateZExt(ExchangeHigh, Int128Ty);
416 ExchangeLow = CGF.
Builder.CreateZExt(ExchangeLow, Int128Ty);
418 CGF.
Builder.CreateShl(ExchangeHigh, llvm::ConstantInt::get(Int128Ty, 64));
419 llvm::Value *Exchange = CGF.
Builder.CreateOr(ExchangeHigh, ExchangeLow);
425 SuccessOrdering, FailureOrdering);
431 CXI->setVolatile(
true);
443 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
449 AtomicRMWInst::Add, DestAddr, ConstantInt::get(IntTy, 1), Ordering);
450 return CGF.
Builder.CreateAdd(
Result, ConstantInt::get(IntTy, 1));
455 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
461 AtomicRMWInst::Sub, DestAddr, ConstantInt::get(IntTy, 1), Ordering);
462 return CGF.
Builder.CreateSub(
Result, ConstantInt::get(IntTy, 1));
473 Load->setVolatile(
true);
483 llvm::StoreInst *Store =
485 Store->setVolatile(
true);
493 const CallExpr *E,
unsigned IntrinsicID,
494 unsigned ConstrainedIntrinsicID) {
497 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
498 if (CGF.
Builder.getIsFPConstrained()) {
500 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0 });
503 return CGF.
Builder.CreateCall(F, Src0);
510 const CallExpr *E,
unsigned IntrinsicID,
511 unsigned ConstrainedIntrinsicID) {
515 if (CGF.
Builder.getIsFPConstrained()) {
516 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
518 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0, Src1 });
521 return CGF.
Builder.CreateCall(F, { Src0, Src1 });
528 llvm::Intrinsic::ID ConstrainedIntrinsicID) {
532 if (CGF.
Builder.getIsFPConstrained()) {
533 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
535 {Src0->getType(), Src1->getType()});
536 return CGF.
Builder.CreateConstrainedFPCall(F, {Src0, Src1});
541 return CGF.
Builder.CreateCall(F, {Src0, Src1});
547 const CallExpr *E,
unsigned IntrinsicID,
548 unsigned ConstrainedIntrinsicID) {
553 if (CGF.
Builder.getIsFPConstrained()) {
554 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
556 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0, Src1, Src2 });
559 return CGF.
Builder.CreateCall(F, { Src0, Src1, Src2 });
566 unsigned IntrinsicID,
567 unsigned ConstrainedIntrinsicID,
571 if (CGF.
Builder.getIsFPConstrained())
576 if (CGF.
Builder.getIsFPConstrained())
577 return CGF.
Builder.CreateConstrainedFPCall(F, Args);
579 return CGF.
Builder.CreateCall(F, Args);
585 unsigned IntrinsicID,
586 llvm::StringRef Name =
"") {
590 return CGF.
Builder.CreateCall(F, Src0, Name);
596 unsigned IntrinsicID) {
601 return CGF.
Builder.CreateCall(F, { Src0, Src1 });
607 unsigned IntrinsicID) {
613 return CGF.
Builder.CreateCall(F, { Src0, Src1, Src2 });
619 unsigned IntrinsicID) {
624 return CGF.
Builder.CreateCall(F, {Src0, Src1});
630 unsigned IntrinsicID,
631 unsigned ConstrainedIntrinsicID) {
635 if (CGF.
Builder.getIsFPConstrained()) {
636 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
638 {ResultType, Src0->getType()});
639 return CGF.
Builder.CreateConstrainedFPCall(F, {Src0});
643 return CGF.
Builder.CreateCall(F, Src0);
648 llvm::Intrinsic::ID IntrinsicID) {
656 llvm::Value *
Call = CGF.
Builder.CreateCall(F, Src0);
658 llvm::Value *Exp = CGF.
Builder.CreateExtractValue(
Call, 1);
669 Call->setDoesNotAccessMemory();
678 llvm::Type *Ty =
V->getType();
679 int Width = Ty->getPrimitiveSizeInBits();
680 llvm::Type *IntTy = llvm::IntegerType::get(
C, Width);
682 if (Ty->isPPC_FP128Ty()) {
692 Value *ShiftCst = llvm::ConstantInt::get(IntTy, Width);
697 IntTy = llvm::IntegerType::get(
C, Width);
700 Value *Zero = llvm::Constant::getNullValue(IntTy);
701 return CGF.
Builder.CreateICmpSLT(
V, Zero);
705 const CallExpr *E, llvm::Constant *calleeValue) {
720 const llvm::Intrinsic::ID IntrinsicID,
721 llvm::Value *
X, llvm::Value *Y,
722 llvm::Value *&Carry) {
724 assert(
X->getType() == Y->getType() &&
725 "Arguments must be the same type. (Did you forget to make sure both "
726 "arguments have the same integer width?)");
729 llvm::Value *Tmp = CGF.
Builder.CreateCall(Callee, {
X, Y});
730 Carry = CGF.
Builder.CreateExtractValue(Tmp, 1);
731 return CGF.
Builder.CreateExtractValue(Tmp, 0);
735 unsigned IntrinsicID,
738 llvm::MDNode *RNode = MDHelper.createRange(APInt(32, low), APInt(32, high));
740 llvm::Instruction *
Call = CGF.
Builder.CreateCall(F);
741 Call->setMetadata(llvm::LLVMContext::MD_range, RNode);
742 Call->setMetadata(llvm::LLVMContext::MD_noundef,
748 struct WidthAndSignedness {
754static WidthAndSignedness
768static struct WidthAndSignedness
770 assert(Types.size() > 0 &&
"Empty list of types.");
774 for (
const auto &
Type : Types) {
783 for (
const auto &
Type : Types) {
785 if (Width < MinWidth) {
794 Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend;
804 return From == To || (From == 0 && To == 1) || (From == 3 && To == 2);
809 return ConstantInt::get(ResType, (
Type & 2) ? 0 : -1,
true);
813CodeGenFunction::evaluateOrEmitBuiltinObjectSize(
const Expr *E,
unsigned Type,
814 llvm::IntegerType *ResType,
815 llvm::Value *EmittedE,
819 return emitBuiltinObjectSize(E,
Type, ResType, EmittedE, IsDynamic);
820 return ConstantInt::get(ResType, ObjectSize,
true);
827 unsigned FieldNo = 0;
831 if (
const auto *Field = dyn_cast<FieldDecl>(D);
832 Field && (Name.empty() ||
Field->getNameAsString() == Name) &&
834 Ctx, Field,
Field->getType(), StrictFlexArraysLevel,
841 if (
const auto *
Record = dyn_cast<RecordDecl>(D))
849 if (!IsUnion && isa<FieldDecl>(D))
860 if (
const auto *FD = dyn_cast<FieldDecl>(D);
861 FD && FD->hasAttr<CountedByAttr>()) {
865 if (
const auto *Rec = dyn_cast<RecordDecl>(D))
873CodeGenFunction::emitFlexibleArrayMemberSize(
const Expr *E,
unsigned Type,
874 llvm::IntegerType *ResType) {
903 const Expr *Idx =
nullptr;
905 if (
const auto *UO = dyn_cast<UnaryOperator>(
Base);
906 UO && UO->getOpcode() == UO_AddrOf) {
908 if (
const auto *ASE = dyn_cast<ArraySubscriptExpr>(SubExpr)) {
909 Base = ASE->getBase()->IgnoreParenImpCasts();
912 if (
const auto *IL = dyn_cast<IntegerLiteral>(Idx)) {
913 int64_t Val = IL->getValue().getSExtValue();
930 if (
const auto *ME = dyn_cast<MemberExpr>(
Base)) {
932 const ValueDecl *VD = ME->getMemberDecl();
935 }
else if (
const auto *DRE = dyn_cast<DeclRefExpr>(
Base)) {
937 QualType Ty = DRE->getDecl()->getType();
978 if (!FAMDecl || !FAMDecl->
hasAttr<CountedByAttr>())
994 CountedByInst =
Builder.CreateIntCast(CountedByInst, ResType, IsSigned);
997 Value *IdxInst =
nullptr;
1005 IdxInst =
Builder.CreateIntCast(IdxInst, ResType, IdxSigned);
1010 Builder.CreateSub(CountedByInst, IdxInst,
"", !IsSigned, IsSigned);
1016 llvm::Constant *ElemSize =
1017 llvm::ConstantInt::get(ResType,
Size.getQuantity(), IsSigned);
1019 Builder.CreateMul(CountedByInst, ElemSize,
"", !IsSigned, IsSigned);
1020 FAMSize =
Builder.CreateIntCast(FAMSize, ResType, IsSigned);
1021 Value *Res = FAMSize;
1023 if (isa<DeclRefExpr>(
Base)) {
1028 llvm::Constant *FAMOffset = ConstantInt::get(ResType, Offset, IsSigned);
1029 Value *OffsetAndFAMSize =
1030 Builder.CreateAdd(FAMOffset, Res,
"", !IsSigned, IsSigned);
1033 llvm::Constant *SizeofStruct =
1039 ?
Builder.CreateBinaryIntrinsic(llvm::Intrinsic::smax,
1040 OffsetAndFAMSize, SizeofStruct)
1041 :
Builder.CreateBinaryIntrinsic(llvm::Intrinsic::umax,
1042 OffsetAndFAMSize, SizeofStruct);
1052 return Builder.CreateSelect(Cmp, Res, ConstantInt::get(ResType, 0, IsSigned));
1065CodeGenFunction::emitBuiltinObjectSize(
const Expr *E,
unsigned Type,
1066 llvm::IntegerType *ResType,
1067 llvm::Value *EmittedE,
bool IsDynamic) {
1071 auto *Param = dyn_cast<ParmVarDecl>(D->getDecl());
1072 auto *PS = D->getDecl()->getAttr<PassObjectSizeAttr>();
1073 if (Param !=
nullptr && PS !=
nullptr &&
1075 auto Iter = SizeArguments.find(Param);
1076 assert(
Iter != SizeArguments.end());
1079 auto DIter = LocalDeclMap.find(D);
1080 assert(DIter != LocalDeclMap.end());
1090 if (
Value *
V = emitFlexibleArrayMemberSize(E,
Type, ResType))
1101 assert(Ptr->
getType()->isPointerTy() &&
1102 "Non-pointer passed to __builtin_object_size?");
1118 enum ActionKind : uint8_t { TestOnly, Complement, Reset,
Set };
1119 enum InterlockingKind : uint8_t {
1128 InterlockingKind Interlocking;
1131 static BitTest decodeBitTestBuiltin(
unsigned BuiltinID);
1135BitTest BitTest::decodeBitTestBuiltin(
unsigned BuiltinID) {
1136 switch (BuiltinID) {
1138 case Builtin::BI_bittest:
1139 return {TestOnly, Unlocked,
false};
1140 case Builtin::BI_bittestandcomplement:
1141 return {Complement, Unlocked,
false};
1142 case Builtin::BI_bittestandreset:
1143 return {Reset, Unlocked,
false};
1144 case Builtin::BI_bittestandset:
1145 return {
Set, Unlocked,
false};
1146 case Builtin::BI_interlockedbittestandreset:
1147 return {Reset, Sequential,
false};
1148 case Builtin::BI_interlockedbittestandset:
1149 return {
Set, Sequential,
false};
1152 case Builtin::BI_bittest64:
1153 return {TestOnly, Unlocked,
true};
1154 case Builtin::BI_bittestandcomplement64:
1155 return {Complement, Unlocked,
true};
1156 case Builtin::BI_bittestandreset64:
1157 return {Reset, Unlocked,
true};
1158 case Builtin::BI_bittestandset64:
1159 return {
Set, Unlocked,
true};
1160 case Builtin::BI_interlockedbittestandreset64:
1161 return {Reset, Sequential,
true};
1162 case Builtin::BI_interlockedbittestandset64:
1163 return {
Set, Sequential,
true};
1166 case Builtin::BI_interlockedbittestandset_acq:
1167 return {
Set, Acquire,
false};
1168 case Builtin::BI_interlockedbittestandset_rel:
1169 return {
Set, Release,
false};
1170 case Builtin::BI_interlockedbittestandset_nf:
1171 return {
Set, NoFence,
false};
1172 case Builtin::BI_interlockedbittestandreset_acq:
1173 return {Reset, Acquire,
false};
1174 case Builtin::BI_interlockedbittestandreset_rel:
1175 return {Reset, Release,
false};
1176 case Builtin::BI_interlockedbittestandreset_nf:
1177 return {Reset, NoFence,
false};
1179 llvm_unreachable(
"expected only bittest intrinsics");
1184 case BitTest::TestOnly:
return '\0';
1185 case BitTest::Complement:
return 'c';
1186 case BitTest::Reset:
return 'r';
1187 case BitTest::Set:
return 's';
1189 llvm_unreachable(
"invalid action");
1197 char SizeSuffix = BT.Is64Bit ?
'q' :
'l';
1201 raw_svector_ostream AsmOS(
Asm);
1202 if (BT.Interlocking != BitTest::Unlocked)
1207 AsmOS << SizeSuffix <<
" $2, ($1)";
1210 std::string Constraints =
"={@ccc},r,r,~{cc},~{memory}";
1212 if (!MachineClobbers.empty()) {
1214 Constraints += MachineClobbers;
1216 llvm::IntegerType *IntType = llvm::IntegerType::get(
1219 llvm::FunctionType *FTy =
1220 llvm::FunctionType::get(CGF.
Int8Ty, {CGF.UnqualPtrTy, IntType},
false);
1222 llvm::InlineAsm *IA =
1223 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
1224 return CGF.
Builder.CreateCall(IA, {BitBase, BitPos});
1227static llvm::AtomicOrdering
1230 case BitTest::Unlocked:
return llvm::AtomicOrdering::NotAtomic;
1231 case BitTest::Sequential:
return llvm::AtomicOrdering::SequentiallyConsistent;
1232 case BitTest::Acquire:
return llvm::AtomicOrdering::Acquire;
1233 case BitTest::Release:
return llvm::AtomicOrdering::Release;
1234 case BitTest::NoFence:
return llvm::AtomicOrdering::Monotonic;
1236 llvm_unreachable(
"invalid interlocking");
1249 BitTest BT = BitTest::decodeBitTestBuiltin(BuiltinID);
1261 BitPos, llvm::ConstantInt::get(BitPos->
getType(), 3),
"bittest.byteidx");
1264 ByteIndex,
"bittest.byteaddr"),
1268 llvm::ConstantInt::get(CGF.
Int8Ty, 0x7));
1271 Value *Mask =
nullptr;
1272 if (BT.Action != BitTest::TestOnly) {
1273 Mask = CGF.
Builder.CreateShl(llvm::ConstantInt::get(CGF.
Int8Ty, 1), PosLow,
1280 Value *OldByte =
nullptr;
1281 if (Ordering != llvm::AtomicOrdering::NotAtomic) {
1284 llvm::AtomicRMWInst::BinOp RMWOp = llvm::AtomicRMWInst::Or;
1285 if (BT.Action == BitTest::Reset) {
1286 Mask = CGF.
Builder.CreateNot(Mask);
1287 RMWOp = llvm::AtomicRMWInst::And;
1293 Value *NewByte =
nullptr;
1294 switch (BT.Action) {
1295 case BitTest::TestOnly:
1298 case BitTest::Complement:
1299 NewByte = CGF.
Builder.CreateXor(OldByte, Mask);
1301 case BitTest::Reset:
1302 NewByte = CGF.
Builder.CreateAnd(OldByte, CGF.
Builder.CreateNot(Mask));
1305 NewByte = CGF.
Builder.CreateOr(OldByte, Mask);
1314 Value *ShiftedByte = CGF.
Builder.CreateLShr(OldByte, PosLow,
"bittest.shr");
1316 ShiftedByte, llvm::ConstantInt::get(CGF.
Int8Ty, 1),
"bittest.res");
1325 raw_svector_ostream AsmOS(
Asm);
1326 llvm::IntegerType *RetType = CGF.
Int32Ty;
1328 switch (BuiltinID) {
1329 case clang::PPC::BI__builtin_ppc_ldarx:
1333 case clang::PPC::BI__builtin_ppc_lwarx:
1337 case clang::PPC::BI__builtin_ppc_lharx:
1341 case clang::PPC::BI__builtin_ppc_lbarx:
1346 llvm_unreachable(
"Expected only PowerPC load reserve intrinsics");
1349 AsmOS <<
"$0, ${1:y}";
1351 std::string Constraints =
"=r,*Z,~{memory}";
1353 if (!MachineClobbers.empty()) {
1355 Constraints += MachineClobbers;
1359 llvm::FunctionType *FTy = llvm::FunctionType::get(RetType, {PtrType},
false);
1361 llvm::InlineAsm *IA =
1362 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
1363 llvm::CallInst *CI = CGF.
Builder.CreateCall(IA, {Addr});
1365 0, Attribute::get(CGF.
getLLVMContext(), Attribute::ElementType, RetType));
1370enum class MSVCSetJmpKind {
1382 llvm::Value *Arg1 =
nullptr;
1383 llvm::Type *Arg1Ty =
nullptr;
1385 bool IsVarArg =
false;
1386 if (SJKind == MSVCSetJmpKind::_setjmp3) {
1389 Arg1 = llvm::ConstantInt::get(CGF.
IntTy, 0);
1392 Name = SJKind == MSVCSetJmpKind::_setjmp ?
"_setjmp" :
"_setjmpex";
1395 Arg1 = CGF.
Builder.CreateCall(
1398 Arg1 = CGF.
Builder.CreateCall(
1400 llvm::ConstantInt::get(CGF.
Int32Ty, 0));
1404 llvm::Type *ArgTypes[2] = {CGF.
Int8PtrTy, Arg1Ty};
1405 llvm::AttributeList ReturnsTwiceAttr = llvm::AttributeList::get(
1407 llvm::Attribute::ReturnsTwice);
1409 llvm::FunctionType::get(CGF.
IntTy, ArgTypes, IsVarArg), Name,
1410 ReturnsTwiceAttr,
true);
1412 llvm::Value *Buf = CGF.
Builder.CreateBitOrPointerCast(
1414 llvm::Value *Args[] = {Buf, Arg1};
1416 CB->setAttributes(ReturnsTwiceAttr);
1464static std::optional<CodeGenFunction::MSVCIntrin>
1467 switch (BuiltinID) {
1469 return std::nullopt;
1470 case clang::ARM::BI_BitScanForward:
1471 case clang::ARM::BI_BitScanForward64:
1472 return MSVCIntrin::_BitScanForward;
1473 case clang::ARM::BI_BitScanReverse:
1474 case clang::ARM::BI_BitScanReverse64:
1475 return MSVCIntrin::_BitScanReverse;
1476 case clang::ARM::BI_InterlockedAnd64:
1477 return MSVCIntrin::_InterlockedAnd;
1478 case clang::ARM::BI_InterlockedExchange64:
1479 return MSVCIntrin::_InterlockedExchange;
1480 case clang::ARM::BI_InterlockedExchangeAdd64:
1481 return MSVCIntrin::_InterlockedExchangeAdd;
1482 case clang::ARM::BI_InterlockedExchangeSub64:
1483 return MSVCIntrin::_InterlockedExchangeSub;
1484 case clang::ARM::BI_InterlockedOr64:
1485 return MSVCIntrin::_InterlockedOr;
1486 case clang::ARM::BI_InterlockedXor64:
1487 return MSVCIntrin::_InterlockedXor;
1488 case clang::ARM::BI_InterlockedDecrement64:
1489 return MSVCIntrin::_InterlockedDecrement;
1490 case clang::ARM::BI_InterlockedIncrement64:
1491 return MSVCIntrin::_InterlockedIncrement;
1492 case clang::ARM::BI_InterlockedExchangeAdd8_acq:
1493 case clang::ARM::BI_InterlockedExchangeAdd16_acq:
1494 case clang::ARM::BI_InterlockedExchangeAdd_acq:
1495 case clang::ARM::BI_InterlockedExchangeAdd64_acq:
1496 return MSVCIntrin::_InterlockedExchangeAdd_acq;
1497 case clang::ARM::BI_InterlockedExchangeAdd8_rel:
1498 case clang::ARM::BI_InterlockedExchangeAdd16_rel:
1499 case clang::ARM::BI_InterlockedExchangeAdd_rel:
1500 case clang::ARM::BI_InterlockedExchangeAdd64_rel:
1501 return MSVCIntrin::_InterlockedExchangeAdd_rel;
1502 case clang::ARM::BI_InterlockedExchangeAdd8_nf:
1503 case clang::ARM::BI_InterlockedExchangeAdd16_nf:
1504 case clang::ARM::BI_InterlockedExchangeAdd_nf:
1505 case clang::ARM::BI_InterlockedExchangeAdd64_nf:
1506 return MSVCIntrin::_InterlockedExchangeAdd_nf;
1507 case clang::ARM::BI_InterlockedExchange8_acq:
1508 case clang::ARM::BI_InterlockedExchange16_acq:
1509 case clang::ARM::BI_InterlockedExchange_acq:
1510 case clang::ARM::BI_InterlockedExchange64_acq:
1511 return MSVCIntrin::_InterlockedExchange_acq;
1512 case clang::ARM::BI_InterlockedExchange8_rel:
1513 case clang::ARM::BI_InterlockedExchange16_rel:
1514 case clang::ARM::BI_InterlockedExchange_rel:
1515 case clang::ARM::BI_InterlockedExchange64_rel:
1516 return MSVCIntrin::_InterlockedExchange_rel;
1517 case clang::ARM::BI_InterlockedExchange8_nf:
1518 case clang::ARM::BI_InterlockedExchange16_nf:
1519 case clang::ARM::BI_InterlockedExchange_nf:
1520 case clang::ARM::BI_InterlockedExchange64_nf:
1521 return MSVCIntrin::_InterlockedExchange_nf;
1522 case clang::ARM::BI_InterlockedCompareExchange8_acq:
1523 case clang::ARM::BI_InterlockedCompareExchange16_acq:
1524 case clang::ARM::BI_InterlockedCompareExchange_acq:
1525 case clang::ARM::BI_InterlockedCompareExchange64_acq:
1526 return MSVCIntrin::_InterlockedCompareExchange_acq;
1527 case clang::ARM::BI_InterlockedCompareExchange8_rel:
1528 case clang::ARM::BI_InterlockedCompareExchange16_rel:
1529 case clang::ARM::BI_InterlockedCompareExchange_rel:
1530 case clang::ARM::BI_InterlockedCompareExchange64_rel:
1531 return MSVCIntrin::_InterlockedCompareExchange_rel;
1532 case clang::ARM::BI_InterlockedCompareExchange8_nf:
1533 case clang::ARM::BI_InterlockedCompareExchange16_nf:
1534 case clang::ARM::BI_InterlockedCompareExchange_nf:
1535 case clang::ARM::BI_InterlockedCompareExchange64_nf:
1536 return MSVCIntrin::_InterlockedCompareExchange_nf;
1537 case clang::ARM::BI_InterlockedOr8_acq:
1538 case clang::ARM::BI_InterlockedOr16_acq:
1539 case clang::ARM::BI_InterlockedOr_acq:
1540 case clang::ARM::BI_InterlockedOr64_acq:
1541 return MSVCIntrin::_InterlockedOr_acq;
1542 case clang::ARM::BI_InterlockedOr8_rel:
1543 case clang::ARM::BI_InterlockedOr16_rel:
1544 case clang::ARM::BI_InterlockedOr_rel:
1545 case clang::ARM::BI_InterlockedOr64_rel:
1546 return MSVCIntrin::_InterlockedOr_rel;
1547 case clang::ARM::BI_InterlockedOr8_nf:
1548 case clang::ARM::BI_InterlockedOr16_nf:
1549 case clang::ARM::BI_InterlockedOr_nf:
1550 case clang::ARM::BI_InterlockedOr64_nf:
1551 return MSVCIntrin::_InterlockedOr_nf;
1552 case clang::ARM::BI_InterlockedXor8_acq:
1553 case clang::ARM::BI_InterlockedXor16_acq:
1554 case clang::ARM::BI_InterlockedXor_acq:
1555 case clang::ARM::BI_InterlockedXor64_acq:
1556 return MSVCIntrin::_InterlockedXor_acq;
1557 case clang::ARM::BI_InterlockedXor8_rel:
1558 case clang::ARM::BI_InterlockedXor16_rel:
1559 case clang::ARM::BI_InterlockedXor_rel:
1560 case clang::ARM::BI_InterlockedXor64_rel:
1561 return MSVCIntrin::_InterlockedXor_rel;
1562 case clang::ARM::BI_InterlockedXor8_nf:
1563 case clang::ARM::BI_InterlockedXor16_nf:
1564 case clang::ARM::BI_InterlockedXor_nf:
1565 case clang::ARM::BI_InterlockedXor64_nf:
1566 return MSVCIntrin::_InterlockedXor_nf;
1567 case clang::ARM::BI_InterlockedAnd8_acq:
1568 case clang::ARM::BI_InterlockedAnd16_acq:
1569 case clang::ARM::BI_InterlockedAnd_acq:
1570 case clang::ARM::BI_InterlockedAnd64_acq:
1571 return MSVCIntrin::_InterlockedAnd_acq;
1572 case clang::ARM::BI_InterlockedAnd8_rel:
1573 case clang::ARM::BI_InterlockedAnd16_rel:
1574 case clang::ARM::BI_InterlockedAnd_rel:
1575 case clang::ARM::BI_InterlockedAnd64_rel:
1576 return MSVCIntrin::_InterlockedAnd_rel;
1577 case clang::ARM::BI_InterlockedAnd8_nf:
1578 case clang::ARM::BI_InterlockedAnd16_nf:
1579 case clang::ARM::BI_InterlockedAnd_nf:
1580 case clang::ARM::BI_InterlockedAnd64_nf:
1581 return MSVCIntrin::_InterlockedAnd_nf;
1582 case clang::ARM::BI_InterlockedIncrement16_acq:
1583 case clang::ARM::BI_InterlockedIncrement_acq:
1584 case clang::ARM::BI_InterlockedIncrement64_acq:
1585 return MSVCIntrin::_InterlockedIncrement_acq;
1586 case clang::ARM::BI_InterlockedIncrement16_rel:
1587 case clang::ARM::BI_InterlockedIncrement_rel:
1588 case clang::ARM::BI_InterlockedIncrement64_rel:
1589 return MSVCIntrin::_InterlockedIncrement_rel;
1590 case clang::ARM::BI_InterlockedIncrement16_nf:
1591 case clang::ARM::BI_InterlockedIncrement_nf:
1592 case clang::ARM::BI_InterlockedIncrement64_nf:
1593 return MSVCIntrin::_InterlockedIncrement_nf;
1594 case clang::ARM::BI_InterlockedDecrement16_acq:
1595 case clang::ARM::BI_InterlockedDecrement_acq:
1596 case clang::ARM::BI_InterlockedDecrement64_acq:
1597 return MSVCIntrin::_InterlockedDecrement_acq;
1598 case clang::ARM::BI_InterlockedDecrement16_rel:
1599 case clang::ARM::BI_InterlockedDecrement_rel:
1600 case clang::ARM::BI_InterlockedDecrement64_rel:
1601 return MSVCIntrin::_InterlockedDecrement_rel;
1602 case clang::ARM::BI_InterlockedDecrement16_nf:
1603 case clang::ARM::BI_InterlockedDecrement_nf:
1604 case clang::ARM::BI_InterlockedDecrement64_nf:
1605 return MSVCIntrin::_InterlockedDecrement_nf;
1607 llvm_unreachable(
"must return from switch");
1610static std::optional<CodeGenFunction::MSVCIntrin>
1613 switch (BuiltinID) {
1615 return std::nullopt;
1616 case clang::AArch64::BI_BitScanForward:
1617 case clang::AArch64::BI_BitScanForward64:
1618 return MSVCIntrin::_BitScanForward;
1619 case clang::AArch64::BI_BitScanReverse:
1620 case clang::AArch64::BI_BitScanReverse64:
1621 return MSVCIntrin::_BitScanReverse;
1622 case clang::AArch64::BI_InterlockedAnd64:
1623 return MSVCIntrin::_InterlockedAnd;
1624 case clang::AArch64::BI_InterlockedExchange64:
1625 return MSVCIntrin::_InterlockedExchange;
1626 case clang::AArch64::BI_InterlockedExchangeAdd64:
1627 return MSVCIntrin::_InterlockedExchangeAdd;
1628 case clang::AArch64::BI_InterlockedExchangeSub64:
1629 return MSVCIntrin::_InterlockedExchangeSub;
1630 case clang::AArch64::BI_InterlockedOr64:
1631 return MSVCIntrin::_InterlockedOr;
1632 case clang::AArch64::BI_InterlockedXor64:
1633 return MSVCIntrin::_InterlockedXor;
1634 case clang::AArch64::BI_InterlockedDecrement64:
1635 return MSVCIntrin::_InterlockedDecrement;
1636 case clang::AArch64::BI_InterlockedIncrement64:
1637 return MSVCIntrin::_InterlockedIncrement;
1638 case clang::AArch64::BI_InterlockedExchangeAdd8_acq:
1639 case clang::AArch64::BI_InterlockedExchangeAdd16_acq:
1640 case clang::AArch64::BI_InterlockedExchangeAdd_acq:
1641 case clang::AArch64::BI_InterlockedExchangeAdd64_acq:
1642 return MSVCIntrin::_InterlockedExchangeAdd_acq;
1643 case clang::AArch64::BI_InterlockedExchangeAdd8_rel:
1644 case clang::AArch64::BI_InterlockedExchangeAdd16_rel:
1645 case clang::AArch64::BI_InterlockedExchangeAdd_rel:
1646 case clang::AArch64::BI_InterlockedExchangeAdd64_rel:
1647 return MSVCIntrin::_InterlockedExchangeAdd_rel;
1648 case clang::AArch64::BI_InterlockedExchangeAdd8_nf:
1649 case clang::AArch64::BI_InterlockedExchangeAdd16_nf:
1650 case clang::AArch64::BI_InterlockedExchangeAdd_nf:
1651 case clang::AArch64::BI_InterlockedExchangeAdd64_nf:
1652 return MSVCIntrin::_InterlockedExchangeAdd_nf;
1653 case clang::AArch64::BI_InterlockedExchange8_acq:
1654 case clang::AArch64::BI_InterlockedExchange16_acq:
1655 case clang::AArch64::BI_InterlockedExchange_acq:
1656 case clang::AArch64::BI_InterlockedExchange64_acq:
1657 return MSVCIntrin::_InterlockedExchange_acq;
1658 case clang::AArch64::BI_InterlockedExchange8_rel:
1659 case clang::AArch64::BI_InterlockedExchange16_rel:
1660 case clang::AArch64::BI_InterlockedExchange_rel:
1661 case clang::AArch64::BI_InterlockedExchange64_rel:
1662 return MSVCIntrin::_InterlockedExchange_rel;
1663 case clang::AArch64::BI_InterlockedExchange8_nf:
1664 case clang::AArch64::BI_InterlockedExchange16_nf:
1665 case clang::AArch64::BI_InterlockedExchange_nf:
1666 case clang::AArch64::BI_InterlockedExchange64_nf:
1667 return MSVCIntrin::_InterlockedExchange_nf;
1668 case clang::AArch64::BI_InterlockedCompareExchange8_acq:
1669 case clang::AArch64::BI_InterlockedCompareExchange16_acq:
1670 case clang::AArch64::BI_InterlockedCompareExchange_acq:
1671 case clang::AArch64::BI_InterlockedCompareExchange64_acq:
1672 return MSVCIntrin::_InterlockedCompareExchange_acq;
1673 case clang::AArch64::BI_InterlockedCompareExchange8_rel:
1674 case clang::AArch64::BI_InterlockedCompareExchange16_rel:
1675 case clang::AArch64::BI_InterlockedCompareExchange_rel:
1676 case clang::AArch64::BI_InterlockedCompareExchange64_rel:
1677 return MSVCIntrin::_InterlockedCompareExchange_rel;
1678 case clang::AArch64::BI_InterlockedCompareExchange8_nf:
1679 case clang::AArch64::BI_InterlockedCompareExchange16_nf:
1680 case clang::AArch64::BI_InterlockedCompareExchange_nf:
1681 case clang::AArch64::BI_InterlockedCompareExchange64_nf:
1682 return MSVCIntrin::_InterlockedCompareExchange_nf;
1683 case clang::AArch64::BI_InterlockedCompareExchange128:
1684 return MSVCIntrin::_InterlockedCompareExchange128;
1685 case clang::AArch64::BI_InterlockedCompareExchange128_acq:
1686 return MSVCIntrin::_InterlockedCompareExchange128_acq;
1687 case clang::AArch64::BI_InterlockedCompareExchange128_nf:
1688 return MSVCIntrin::_InterlockedCompareExchange128_nf;
1689 case clang::AArch64::BI_InterlockedCompareExchange128_rel:
1690 return MSVCIntrin::_InterlockedCompareExchange128_rel;
1691 case clang::AArch64::BI_InterlockedOr8_acq:
1692 case clang::AArch64::BI_InterlockedOr16_acq:
1693 case clang::AArch64::BI_InterlockedOr_acq:
1694 case clang::AArch64::BI_InterlockedOr64_acq:
1695 return MSVCIntrin::_InterlockedOr_acq;
1696 case clang::AArch64::BI_InterlockedOr8_rel:
1697 case clang::AArch64::BI_InterlockedOr16_rel:
1698 case clang::AArch64::BI_InterlockedOr_rel:
1699 case clang::AArch64::BI_InterlockedOr64_rel:
1700 return MSVCIntrin::_InterlockedOr_rel;
1701 case clang::AArch64::BI_InterlockedOr8_nf:
1702 case clang::AArch64::BI_InterlockedOr16_nf:
1703 case clang::AArch64::BI_InterlockedOr_nf:
1704 case clang::AArch64::BI_InterlockedOr64_nf:
1705 return MSVCIntrin::_InterlockedOr_nf;
1706 case clang::AArch64::BI_InterlockedXor8_acq:
1707 case clang::AArch64::BI_InterlockedXor16_acq:
1708 case clang::AArch64::BI_InterlockedXor_acq:
1709 case clang::AArch64::BI_InterlockedXor64_acq:
1710 return MSVCIntrin::_InterlockedXor_acq;
1711 case clang::AArch64::BI_InterlockedXor8_rel:
1712 case clang::AArch64::BI_InterlockedXor16_rel:
1713 case clang::AArch64::BI_InterlockedXor_rel:
1714 case clang::AArch64::BI_InterlockedXor64_rel:
1715 return MSVCIntrin::_InterlockedXor_rel;
1716 case clang::AArch64::BI_InterlockedXor8_nf:
1717 case clang::AArch64::BI_InterlockedXor16_nf:
1718 case clang::AArch64::BI_InterlockedXor_nf:
1719 case clang::AArch64::BI_InterlockedXor64_nf:
1720 return MSVCIntrin::_InterlockedXor_nf;
1721 case clang::AArch64::BI_InterlockedAnd8_acq:
1722 case clang::AArch64::BI_InterlockedAnd16_acq:
1723 case clang::AArch64::BI_InterlockedAnd_acq:
1724 case clang::AArch64::BI_InterlockedAnd64_acq:
1725 return MSVCIntrin::_InterlockedAnd_acq;
1726 case clang::AArch64::BI_InterlockedAnd8_rel:
1727 case clang::AArch64::BI_InterlockedAnd16_rel:
1728 case clang::AArch64::BI_InterlockedAnd_rel:
1729 case clang::AArch64::BI_InterlockedAnd64_rel:
1730 return MSVCIntrin::_InterlockedAnd_rel;
1731 case clang::AArch64::BI_InterlockedAnd8_nf:
1732 case clang::AArch64::BI_InterlockedAnd16_nf:
1733 case clang::AArch64::BI_InterlockedAnd_nf:
1734 case clang::AArch64::BI_InterlockedAnd64_nf:
1735 return MSVCIntrin::_InterlockedAnd_nf;
1736 case clang::AArch64::BI_InterlockedIncrement16_acq:
1737 case clang::AArch64::BI_InterlockedIncrement_acq:
1738 case clang::AArch64::BI_InterlockedIncrement64_acq:
1739 return MSVCIntrin::_InterlockedIncrement_acq;
1740 case clang::AArch64::BI_InterlockedIncrement16_rel:
1741 case clang::AArch64::BI_InterlockedIncrement_rel:
1742 case clang::AArch64::BI_InterlockedIncrement64_rel:
1743 return MSVCIntrin::_InterlockedIncrement_rel;
1744 case clang::AArch64::BI_InterlockedIncrement16_nf:
1745 case clang::AArch64::BI_InterlockedIncrement_nf:
1746 case clang::AArch64::BI_InterlockedIncrement64_nf:
1747 return MSVCIntrin::_InterlockedIncrement_nf;
1748 case clang::AArch64::BI_InterlockedDecrement16_acq:
1749 case clang::AArch64::BI_InterlockedDecrement_acq:
1750 case clang::AArch64::BI_InterlockedDecrement64_acq:
1751 return MSVCIntrin::_InterlockedDecrement_acq;
1752 case clang::AArch64::BI_InterlockedDecrement16_rel:
1753 case clang::AArch64::BI_InterlockedDecrement_rel:
1754 case clang::AArch64::BI_InterlockedDecrement64_rel:
1755 return MSVCIntrin::_InterlockedDecrement_rel;
1756 case clang::AArch64::BI_InterlockedDecrement16_nf:
1757 case clang::AArch64::BI_InterlockedDecrement_nf:
1758 case clang::AArch64::BI_InterlockedDecrement64_nf:
1759 return MSVCIntrin::_InterlockedDecrement_nf;
1761 llvm_unreachable(
"must return from switch");
1764static std::optional<CodeGenFunction::MSVCIntrin>
1767 switch (BuiltinID) {
1769 return std::nullopt;
1770 case clang::X86::BI_BitScanForward:
1771 case clang::X86::BI_BitScanForward64:
1772 return MSVCIntrin::_BitScanForward;
1773 case clang::X86::BI_BitScanReverse:
1774 case clang::X86::BI_BitScanReverse64:
1775 return MSVCIntrin::_BitScanReverse;
1776 case clang::X86::BI_InterlockedAnd64:
1777 return MSVCIntrin::_InterlockedAnd;
1778 case clang::X86::BI_InterlockedCompareExchange128:
1779 return MSVCIntrin::_InterlockedCompareExchange128;
1780 case clang::X86::BI_InterlockedExchange64:
1781 return MSVCIntrin::_InterlockedExchange;
1782 case clang::X86::BI_InterlockedExchangeAdd64:
1783 return MSVCIntrin::_InterlockedExchangeAdd;
1784 case clang::X86::BI_InterlockedExchangeSub64:
1785 return MSVCIntrin::_InterlockedExchangeSub;
1786 case clang::X86::BI_InterlockedOr64:
1787 return MSVCIntrin::_InterlockedOr;
1788 case clang::X86::BI_InterlockedXor64:
1789 return MSVCIntrin::_InterlockedXor;
1790 case clang::X86::BI_InterlockedDecrement64:
1791 return MSVCIntrin::_InterlockedDecrement;
1792 case clang::X86::BI_InterlockedIncrement64:
1793 return MSVCIntrin::_InterlockedIncrement;
1795 llvm_unreachable(
"must return from switch");
1801 switch (BuiltinID) {
1802 case MSVCIntrin::_BitScanForward:
1803 case MSVCIntrin::_BitScanReverse: {
1807 llvm::Type *ArgType = ArgValue->
getType();
1808 llvm::Type *IndexType = IndexAddress.getElementType();
1811 Value *ArgZero = llvm::Constant::getNullValue(ArgType);
1812 Value *ResZero = llvm::Constant::getNullValue(ResultType);
1813 Value *ResOne = llvm::ConstantInt::get(ResultType, 1);
1818 PHINode *
Result =
Builder.CreatePHI(ResultType, 2,
"bitscan_result");
1821 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, ArgZero);
1823 Builder.CreateCondBr(IsZero, End, NotZero);
1826 Builder.SetInsertPoint(NotZero);
1828 if (BuiltinID == MSVCIntrin::_BitScanForward) {
1831 ZeroCount =
Builder.CreateIntCast(ZeroCount, IndexType,
false);
1834 unsigned ArgWidth = cast<llvm::IntegerType>(ArgType)->getBitWidth();
1835 Value *ArgTypeLastIndex = llvm::ConstantInt::get(IndexType, ArgWidth - 1);
1839 ZeroCount =
Builder.CreateIntCast(ZeroCount, IndexType,
false);
1840 Value *Index =
Builder.CreateNSWSub(ArgTypeLastIndex, ZeroCount);
1844 Result->addIncoming(ResOne, NotZero);
1849 case MSVCIntrin::_InterlockedAnd:
1851 case MSVCIntrin::_InterlockedExchange:
1853 case MSVCIntrin::_InterlockedExchangeAdd:
1855 case MSVCIntrin::_InterlockedExchangeSub:
1857 case MSVCIntrin::_InterlockedOr:
1859 case MSVCIntrin::_InterlockedXor:
1861 case MSVCIntrin::_InterlockedExchangeAdd_acq:
1863 AtomicOrdering::Acquire);
1864 case MSVCIntrin::_InterlockedExchangeAdd_rel:
1866 AtomicOrdering::Release);
1867 case MSVCIntrin::_InterlockedExchangeAdd_nf:
1869 AtomicOrdering::Monotonic);
1870 case MSVCIntrin::_InterlockedExchange_acq:
1872 AtomicOrdering::Acquire);
1873 case MSVCIntrin::_InterlockedExchange_rel:
1875 AtomicOrdering::Release);
1876 case MSVCIntrin::_InterlockedExchange_nf:
1878 AtomicOrdering::Monotonic);
1879 case MSVCIntrin::_InterlockedCompareExchange_acq:
1881 case MSVCIntrin::_InterlockedCompareExchange_rel:
1883 case MSVCIntrin::_InterlockedCompareExchange_nf:
1885 case MSVCIntrin::_InterlockedCompareExchange128:
1887 *
this, E, AtomicOrdering::SequentiallyConsistent);
1888 case MSVCIntrin::_InterlockedCompareExchange128_acq:
1890 case MSVCIntrin::_InterlockedCompareExchange128_rel:
1892 case MSVCIntrin::_InterlockedCompareExchange128_nf:
1894 case MSVCIntrin::_InterlockedOr_acq:
1896 AtomicOrdering::Acquire);
1897 case MSVCIntrin::_InterlockedOr_rel:
1899 AtomicOrdering::Release);
1900 case MSVCIntrin::_InterlockedOr_nf:
1902 AtomicOrdering::Monotonic);
1903 case MSVCIntrin::_InterlockedXor_acq:
1905 AtomicOrdering::Acquire);
1906 case MSVCIntrin::_InterlockedXor_rel:
1908 AtomicOrdering::Release);
1909 case MSVCIntrin::_InterlockedXor_nf:
1911 AtomicOrdering::Monotonic);
1912 case MSVCIntrin::_InterlockedAnd_acq:
1914 AtomicOrdering::Acquire);
1915 case MSVCIntrin::_InterlockedAnd_rel:
1917 AtomicOrdering::Release);
1918 case MSVCIntrin::_InterlockedAnd_nf:
1920 AtomicOrdering::Monotonic);
1921 case MSVCIntrin::_InterlockedIncrement_acq:
1923 case MSVCIntrin::_InterlockedIncrement_rel:
1925 case MSVCIntrin::_InterlockedIncrement_nf:
1927 case MSVCIntrin::_InterlockedDecrement_acq:
1929 case MSVCIntrin::_InterlockedDecrement_rel:
1931 case MSVCIntrin::_InterlockedDecrement_nf:
1934 case MSVCIntrin::_InterlockedDecrement:
1936 case MSVCIntrin::_InterlockedIncrement:
1939 case MSVCIntrin::__fastfail: {
1944 StringRef
Asm, Constraints;
1949 case llvm::Triple::x86:
1950 case llvm::Triple::x86_64:
1952 Constraints =
"{cx}";
1954 case llvm::Triple::thumb:
1956 Constraints =
"{r0}";
1958 case llvm::Triple::aarch64:
1959 Asm =
"brk #0xF003";
1960 Constraints =
"{w0}";
1962 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy, {
Int32Ty},
false);
1963 llvm::InlineAsm *IA =
1964 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
1965 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
1967 llvm::Attribute::NoReturn);
1969 CI->setAttributes(NoReturnAttr);
1973 llvm_unreachable(
"Incorrect MSVC intrinsic!");
1979 CallObjCArcUse(llvm::Value *
object) :
object(
object) {}
1989 BuiltinCheckKind Kind) {
1991 &&
"Unsupported builtin check kind");
1997 SanitizerScope SanScope(
this);
1999 ArgValue, llvm::Constant::getNullValue(ArgValue->
getType()));
2000 EmitCheck(std::make_pair(Cond, SanitizerKind::Builtin),
2001 SanitizerHandler::InvalidBuiltin,
2003 llvm::ConstantInt::get(
Builder.getInt8Ty(), Kind)},
2009 return CGF.
Builder.CreateBinaryIntrinsic(
2010 Intrinsic::abs, ArgValue,
2011 ConstantInt::get(CGF.
Builder.getInt1Ty(), HasNSW));
2015 bool SanitizeOverflow) {
2019 if (
const auto *VCI = dyn_cast<llvm::ConstantInt>(ArgValue)) {
2020 if (!VCI->isMinSignedValue())
2021 return EmitAbs(CGF, ArgValue,
true);
2024 CodeGenFunction::SanitizerScope SanScope(&CGF);
2026 Constant *Zero = Constant::getNullValue(ArgValue->
getType());
2027 Value *ResultAndOverflow = CGF.
Builder.CreateBinaryIntrinsic(
2028 Intrinsic::ssub_with_overflow, Zero, ArgValue);
2031 CGF.
Builder.CreateExtractValue(ResultAndOverflow, 1));
2034 if (SanitizeOverflow) {
2035 CGF.
EmitCheck({{NotOverflow, SanitizerKind::SignedIntegerOverflow}},
2036 SanitizerHandler::NegateOverflow,
2041 CGF.
EmitTrapCheck(NotOverflow, SanitizerHandler::SubOverflow);
2043 Value *CmpResult = CGF.
Builder.CreateICmpSLT(ArgValue, Zero,
"abscond");
2044 return CGF.
Builder.CreateSelect(CmpResult,
Result, ArgValue,
"abs");
2049 QualType UnsignedTy =
C.getIntTypeForBitwidth(Size * 8,
false);
2050 return C.getCanonicalType(UnsignedTy);
2060 raw_svector_ostream OS(Name);
2061 OS <<
"__os_log_helper";
2065 for (
const auto &Item : Layout.
Items)
2066 OS <<
"_" <<
int(Item.getSizeByte()) <<
"_"
2067 <<
int(Item.getDescriptorByte());
2070 if (llvm::Function *F =
CGM.
getModule().getFunction(Name))
2080 for (
unsigned int I = 0, E = Layout.
Items.size(); I < E; ++I) {
2081 char Size = Layout.
Items[I].getSizeByte();
2088 &Ctx.
Idents.
get(std::string(
"arg") + llvm::to_string(I)), ArgTy,
2090 ArgTys.emplace_back(ArgTy);
2101 llvm::Function *Fn = llvm::Function::Create(
2102 FuncTy, llvm::GlobalValue::LinkOnceODRLinkage, Name, &
CGM.
getModule());
2103 Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);
2106 Fn->setDoesNotThrow();
2110 Fn->addFnAttr(llvm::Attribute::NoInline);
2128 for (
const auto &Item : Layout.
Items) {
2130 Builder.getInt8(Item.getDescriptorByte()),
2133 Builder.getInt8(Item.getSizeByte()),
2137 if (!
Size.getQuantity())
2155 "__builtin_os_log_format takes at least 2 arguments");
2166 for (
const auto &Item : Layout.
Items) {
2167 int Size = Item.getSizeByte();
2171 llvm::Value *ArgVal;
2175 for (
unsigned I = 0, E = Item.getMaskType().size(); I < E; ++I)
2176 Val |= ((
uint64_t)Item.getMaskType()[I]) << I * 8;
2177 ArgVal = llvm::Constant::getIntegerValue(
Int64Ty, llvm::APInt(64, Val));
2178 }
else if (
const Expr *TheExpr = Item.getExpr()) {
2184 auto LifetimeExtendObject = [&](
const Expr *E) {
2192 if (isa<CallExpr>(E) || isa<ObjCMessageExpr>(E))
2197 if (TheExpr->getType()->isObjCRetainableType() &&
2198 getLangOpts().ObjCAutoRefCount && LifetimeExtendObject(TheExpr)) {
2200 "Only scalar can be a ObjC retainable type");
2201 if (!isa<Constant>(ArgVal)) {
2215 pushCleanupAfterFullExpr<CallObjCArcUse>(Cleanup, ArgVal);
2219 ArgVal =
Builder.getInt32(Item.getConstValue().getQuantity());
2222 unsigned ArgValSize =
2226 ArgVal =
Builder.CreateBitOrPointerCast(ArgVal,
IntTy);
2242 unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info,
2243 WidthAndSignedness ResultInfo) {
2244 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2245 Op1Info.Width == Op2Info.Width && Op2Info.Width == ResultInfo.Width &&
2246 !Op1Info.Signed && !Op2Info.Signed && ResultInfo.Signed;
2251 const clang::Expr *Op2, WidthAndSignedness Op2Info,
2253 WidthAndSignedness ResultInfo) {
2255 Builtin::BI__builtin_mul_overflow, Op1Info, Op2Info, ResultInfo) &&
2256 "Cannot specialize this multiply");
2261 llvm::Value *HasOverflow;
2263 CGF, llvm::Intrinsic::umul_with_overflow, V1, V2, HasOverflow);
2268 auto IntMax = llvm::APInt::getSignedMaxValue(ResultInfo.Width);
2269 llvm::Value *IntMaxValue = llvm::ConstantInt::get(
Result->getType(), IntMax);
2271 llvm::Value *IntMaxOverflow = CGF.
Builder.CreateICmpUGT(
Result, IntMaxValue);
2272 HasOverflow = CGF.
Builder.CreateOr(HasOverflow, IntMaxOverflow);
2284 WidthAndSignedness Op1Info,
2285 WidthAndSignedness Op2Info,
2286 WidthAndSignedness ResultInfo) {
2287 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2288 std::max(Op1Info.Width, Op2Info.Width) >= ResultInfo.Width &&
2289 Op1Info.Signed != Op2Info.Signed;
2296 WidthAndSignedness Op1Info,
const clang::Expr *Op2,
2297 WidthAndSignedness Op2Info,
2299 WidthAndSignedness ResultInfo) {
2301 Op2Info, ResultInfo) &&
2302 "Not a mixed-sign multipliction we can specialize");
2305 const clang::Expr *SignedOp = Op1Info.Signed ? Op1 : Op2;
2306 const clang::Expr *UnsignedOp = Op1Info.Signed ? Op2 : Op1;
2309 unsigned SignedOpWidth = Op1Info.Signed ? Op1Info.Width : Op2Info.Width;
2310 unsigned UnsignedOpWidth = Op1Info.Signed ? Op2Info.Width : Op1Info.Width;
2313 if (SignedOpWidth < UnsignedOpWidth)
2315 if (UnsignedOpWidth < SignedOpWidth)
2318 llvm::Type *OpTy =
Signed->getType();
2319 llvm::Value *Zero = llvm::Constant::getNullValue(OpTy);
2322 unsigned OpWidth = std::max(Op1Info.Width, Op2Info.Width);
2325 llvm::Value *IsNegative = CGF.
Builder.CreateICmpSLT(
Signed, Zero);
2326 llvm::Value *AbsOfNegative = CGF.
Builder.CreateSub(Zero,
Signed);
2327 llvm::Value *AbsSigned =
2328 CGF.
Builder.CreateSelect(IsNegative, AbsOfNegative,
Signed);
2331 llvm::Value *UnsignedOverflow;
2332 llvm::Value *UnsignedResult =
2336 llvm::Value *Overflow, *
Result;
2337 if (ResultInfo.Signed) {
2341 llvm::APInt::getSignedMaxValue(ResultInfo.Width).zext(OpWidth);
2342 llvm::Value *MaxResult =
2343 CGF.
Builder.CreateAdd(llvm::ConstantInt::get(OpTy, IntMax),
2344 CGF.
Builder.CreateZExt(IsNegative, OpTy));
2345 llvm::Value *SignedOverflow =
2346 CGF.
Builder.CreateICmpUGT(UnsignedResult, MaxResult);
2347 Overflow = CGF.
Builder.CreateOr(UnsignedOverflow, SignedOverflow);
2350 llvm::Value *NegativeResult = CGF.
Builder.CreateNeg(UnsignedResult);
2351 llvm::Value *SignedResult =
2352 CGF.
Builder.CreateSelect(IsNegative, NegativeResult, UnsignedResult);
2356 llvm::Value *Underflow = CGF.
Builder.CreateAnd(
2357 IsNegative, CGF.
Builder.CreateIsNotNull(UnsignedResult));
2358 Overflow = CGF.
Builder.CreateOr(UnsignedOverflow, Underflow);
2359 if (ResultInfo.Width < OpWidth) {
2361 llvm::APInt::getMaxValue(ResultInfo.Width).zext(OpWidth);
2362 llvm::Value *TruncOverflow = CGF.
Builder.CreateICmpUGT(
2363 UnsignedResult, llvm::ConstantInt::get(OpTy, IntMax));
2364 Overflow = CGF.
Builder.CreateOr(Overflow, TruncOverflow);
2369 IsNegative, CGF.
Builder.CreateNeg(UnsignedResult), UnsignedResult);
2373 assert(Overflow &&
Result &&
"Missing overflow or result");
2384 llvm::SmallPtrSetImpl<const Decl *> &Seen) {
2393 if (!Seen.insert(
Record).second)
2396 assert(
Record->hasDefinition() &&
2397 "Incomplete types should already be diagnosed");
2399 if (
Record->isDynamicClass())
2424 llvm::Type *Ty = Src->getType();
2425 ShiftAmt =
Builder.CreateIntCast(ShiftAmt, Ty,
false);
2428 unsigned IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2435 switch (BuiltinID) {
2436#define MUTATE_LDBL(func) \
2437 case Builtin::BI__builtin_##func##l: \
2438 return Builtin::BI__builtin_##func##f128;
2507 if (CGF.
Builder.getIsFPConstrained() &&
2508 CGF.
Builder.getDefaultConstrainedExcept() != fp::ebIgnore) {
2520 auto UBF = CGF->
CGM.
getModule().getOrInsertFunction(Name, FnTy);
2523 for (
auto &&FormalTy : FnTy->params())
2524 Args.push_back(llvm::PoisonValue::get(FormalTy));
2537 !
Result.hasSideEffects()) {
2541 if (
Result.Val.isFloat())
2550 if (
getTarget().getTriple().isPPC64() &&
2551 &
getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad())
2558 const unsigned BuiltinIDIfNoAsmLabel =
2559 FD->
hasAttr<AsmLabelAttr>() ? 0 : BuiltinID;
2561 std::optional<bool> ErrnoOverriden;
2567 if (OP.hasMathErrnoOverride())
2568 ErrnoOverriden = OP.getMathErrnoOverride();
2577 bool ErrnoOverridenToFalseWithOpt =
2578 ErrnoOverriden.has_value() && !ErrnoOverriden.value() && !OptNone &&
2596 switch (BuiltinID) {
2597 case Builtin::BI__builtin_fma:
2598 case Builtin::BI__builtin_fmaf:
2599 case Builtin::BI__builtin_fmal:
2600 case Builtin::BIfma:
2601 case Builtin::BIfmaf:
2602 case Builtin::BIfmal: {
2604 if (Trip.isGNUEnvironment() || Trip.isOSMSVCRT())
2612 bool ConstWithoutErrnoAndExceptions =
2614 bool ConstWithoutExceptions =
2632 bool ConstWithoutErrnoOrExceptions =
2633 ConstWithoutErrnoAndExceptions || ConstWithoutExceptions;
2634 bool GenerateIntrinsics =
2635 (ConstAlways && !OptNone) ||
2637 !(ErrnoOverriden.has_value() && ErrnoOverriden.value()) && !OptNone);
2638 if (!GenerateIntrinsics) {
2639 GenerateIntrinsics =
2640 ConstWithoutErrnoOrExceptions && !ConstWithoutErrnoAndExceptions;
2641 if (!GenerateIntrinsics)
2642 GenerateIntrinsics =
2643 ConstWithoutErrnoOrExceptions &&
2645 !(ErrnoOverriden.has_value() && ErrnoOverriden.value()) && !OptNone);
2646 if (!GenerateIntrinsics)
2647 GenerateIntrinsics =
2648 ConstWithoutErrnoOrExceptions && ErrnoOverridenToFalseWithOpt;
2650 if (GenerateIntrinsics) {
2651 switch (BuiltinIDIfNoAsmLabel) {
2652 case Builtin::BIceil:
2653 case Builtin::BIceilf:
2654 case Builtin::BIceill:
2655 case Builtin::BI__builtin_ceil:
2656 case Builtin::BI__builtin_ceilf:
2657 case Builtin::BI__builtin_ceilf16:
2658 case Builtin::BI__builtin_ceill:
2659 case Builtin::BI__builtin_ceilf128:
2662 Intrinsic::experimental_constrained_ceil));
2664 case Builtin::BIcopysign:
2665 case Builtin::BIcopysignf:
2666 case Builtin::BIcopysignl:
2667 case Builtin::BI__builtin_copysign:
2668 case Builtin::BI__builtin_copysignf:
2669 case Builtin::BI__builtin_copysignf16:
2670 case Builtin::BI__builtin_copysignl:
2671 case Builtin::BI__builtin_copysignf128:
2674 case Builtin::BIcos:
2675 case Builtin::BIcosf:
2676 case Builtin::BIcosl:
2677 case Builtin::BI__builtin_cos:
2678 case Builtin::BI__builtin_cosf:
2679 case Builtin::BI__builtin_cosf16:
2680 case Builtin::BI__builtin_cosl:
2681 case Builtin::BI__builtin_cosf128:
2684 Intrinsic::experimental_constrained_cos));
2686 case Builtin::BIexp:
2687 case Builtin::BIexpf:
2688 case Builtin::BIexpl:
2689 case Builtin::BI__builtin_exp:
2690 case Builtin::BI__builtin_expf:
2691 case Builtin::BI__builtin_expf16:
2692 case Builtin::BI__builtin_expl:
2693 case Builtin::BI__builtin_expf128:
2696 Intrinsic::experimental_constrained_exp));
2698 case Builtin::BIexp2:
2699 case Builtin::BIexp2f:
2700 case Builtin::BIexp2l:
2701 case Builtin::BI__builtin_exp2:
2702 case Builtin::BI__builtin_exp2f:
2703 case Builtin::BI__builtin_exp2f16:
2704 case Builtin::BI__builtin_exp2l:
2705 case Builtin::BI__builtin_exp2f128:
2708 Intrinsic::experimental_constrained_exp2));
2709 case Builtin::BI__builtin_exp10:
2710 case Builtin::BI__builtin_exp10f:
2711 case Builtin::BI__builtin_exp10f16:
2712 case Builtin::BI__builtin_exp10l:
2713 case Builtin::BI__builtin_exp10f128: {
2715 if (
Builder.getIsFPConstrained())
2719 case Builtin::BIfabs:
2720 case Builtin::BIfabsf:
2721 case Builtin::BIfabsl:
2722 case Builtin::BI__builtin_fabs:
2723 case Builtin::BI__builtin_fabsf:
2724 case Builtin::BI__builtin_fabsf16:
2725 case Builtin::BI__builtin_fabsl:
2726 case Builtin::BI__builtin_fabsf128:
2729 case Builtin::BIfloor:
2730 case Builtin::BIfloorf:
2731 case Builtin::BIfloorl:
2732 case Builtin::BI__builtin_floor:
2733 case Builtin::BI__builtin_floorf:
2734 case Builtin::BI__builtin_floorf16:
2735 case Builtin::BI__builtin_floorl:
2736 case Builtin::BI__builtin_floorf128:
2739 Intrinsic::experimental_constrained_floor));
2741 case Builtin::BIfma:
2742 case Builtin::BIfmaf:
2743 case Builtin::BIfmal:
2744 case Builtin::BI__builtin_fma:
2745 case Builtin::BI__builtin_fmaf:
2746 case Builtin::BI__builtin_fmaf16:
2747 case Builtin::BI__builtin_fmal:
2748 case Builtin::BI__builtin_fmaf128:
2751 Intrinsic::experimental_constrained_fma));
2753 case Builtin::BIfmax:
2754 case Builtin::BIfmaxf:
2755 case Builtin::BIfmaxl:
2756 case Builtin::BI__builtin_fmax:
2757 case Builtin::BI__builtin_fmaxf:
2758 case Builtin::BI__builtin_fmaxf16:
2759 case Builtin::BI__builtin_fmaxl:
2760 case Builtin::BI__builtin_fmaxf128:
2763 Intrinsic::experimental_constrained_maxnum));
2765 case Builtin::BIfmin:
2766 case Builtin::BIfminf:
2767 case Builtin::BIfminl:
2768 case Builtin::BI__builtin_fmin:
2769 case Builtin::BI__builtin_fminf:
2770 case Builtin::BI__builtin_fminf16:
2771 case Builtin::BI__builtin_fminl:
2772 case Builtin::BI__builtin_fminf128:
2775 Intrinsic::experimental_constrained_minnum));
2779 case Builtin::BIfmod:
2780 case Builtin::BIfmodf:
2781 case Builtin::BIfmodl:
2782 case Builtin::BI__builtin_fmod:
2783 case Builtin::BI__builtin_fmodf:
2784 case Builtin::BI__builtin_fmodf16:
2785 case Builtin::BI__builtin_fmodl:
2786 case Builtin::BI__builtin_fmodf128: {
2787 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
2793 case Builtin::BIlog:
2794 case Builtin::BIlogf:
2795 case Builtin::BIlogl:
2796 case Builtin::BI__builtin_log:
2797 case Builtin::BI__builtin_logf:
2798 case Builtin::BI__builtin_logf16:
2799 case Builtin::BI__builtin_logl:
2800 case Builtin::BI__builtin_logf128:
2803 Intrinsic::experimental_constrained_log));
2805 case Builtin::BIlog10:
2806 case Builtin::BIlog10f:
2807 case Builtin::BIlog10l:
2808 case Builtin::BI__builtin_log10:
2809 case Builtin::BI__builtin_log10f:
2810 case Builtin::BI__builtin_log10f16:
2811 case Builtin::BI__builtin_log10l:
2812 case Builtin::BI__builtin_log10f128:
2815 Intrinsic::experimental_constrained_log10));
2817 case Builtin::BIlog2:
2818 case Builtin::BIlog2f:
2819 case Builtin::BIlog2l:
2820 case Builtin::BI__builtin_log2:
2821 case Builtin::BI__builtin_log2f:
2822 case Builtin::BI__builtin_log2f16:
2823 case Builtin::BI__builtin_log2l:
2824 case Builtin::BI__builtin_log2f128:
2827 Intrinsic::experimental_constrained_log2));
2829 case Builtin::BInearbyint:
2830 case Builtin::BInearbyintf:
2831 case Builtin::BInearbyintl:
2832 case Builtin::BI__builtin_nearbyint:
2833 case Builtin::BI__builtin_nearbyintf:
2834 case Builtin::BI__builtin_nearbyintl:
2835 case Builtin::BI__builtin_nearbyintf128:
2837 Intrinsic::nearbyint,
2838 Intrinsic::experimental_constrained_nearbyint));
2840 case Builtin::BIpow:
2841 case Builtin::BIpowf:
2842 case Builtin::BIpowl:
2843 case Builtin::BI__builtin_pow:
2844 case Builtin::BI__builtin_powf:
2845 case Builtin::BI__builtin_powf16:
2846 case Builtin::BI__builtin_powl:
2847 case Builtin::BI__builtin_powf128:
2850 Intrinsic::experimental_constrained_pow));
2852 case Builtin::BIrint:
2853 case Builtin::BIrintf:
2854 case Builtin::BIrintl:
2855 case Builtin::BI__builtin_rint:
2856 case Builtin::BI__builtin_rintf:
2857 case Builtin::BI__builtin_rintf16:
2858 case Builtin::BI__builtin_rintl:
2859 case Builtin::BI__builtin_rintf128:
2862 Intrinsic::experimental_constrained_rint));
2864 case Builtin::BIround:
2865 case Builtin::BIroundf:
2866 case Builtin::BIroundl:
2867 case Builtin::BI__builtin_round:
2868 case Builtin::BI__builtin_roundf:
2869 case Builtin::BI__builtin_roundf16:
2870 case Builtin::BI__builtin_roundl:
2871 case Builtin::BI__builtin_roundf128:
2874 Intrinsic::experimental_constrained_round));
2876 case Builtin::BIroundeven:
2877 case Builtin::BIroundevenf:
2878 case Builtin::BIroundevenl:
2879 case Builtin::BI__builtin_roundeven:
2880 case Builtin::BI__builtin_roundevenf:
2881 case Builtin::BI__builtin_roundevenf16:
2882 case Builtin::BI__builtin_roundevenl:
2883 case Builtin::BI__builtin_roundevenf128:
2885 Intrinsic::roundeven,
2886 Intrinsic::experimental_constrained_roundeven));
2888 case Builtin::BIsin:
2889 case Builtin::BIsinf:
2890 case Builtin::BIsinl:
2891 case Builtin::BI__builtin_sin:
2892 case Builtin::BI__builtin_sinf:
2893 case Builtin::BI__builtin_sinf16:
2894 case Builtin::BI__builtin_sinl:
2895 case Builtin::BI__builtin_sinf128:
2898 Intrinsic::experimental_constrained_sin));
2900 case Builtin::BIsqrt:
2901 case Builtin::BIsqrtf:
2902 case Builtin::BIsqrtl:
2903 case Builtin::BI__builtin_sqrt:
2904 case Builtin::BI__builtin_sqrtf:
2905 case Builtin::BI__builtin_sqrtf16:
2906 case Builtin::BI__builtin_sqrtl:
2907 case Builtin::BI__builtin_sqrtf128:
2908 case Builtin::BI__builtin_elementwise_sqrt: {
2910 *
this, E, Intrinsic::sqrt, Intrinsic::experimental_constrained_sqrt);
2914 case Builtin::BItrunc:
2915 case Builtin::BItruncf:
2916 case Builtin::BItruncl:
2917 case Builtin::BI__builtin_trunc:
2918 case Builtin::BI__builtin_truncf:
2919 case Builtin::BI__builtin_truncf16:
2920 case Builtin::BI__builtin_truncl:
2921 case Builtin::BI__builtin_truncf128:
2924 Intrinsic::experimental_constrained_trunc));
2926 case Builtin::BIlround:
2927 case Builtin::BIlroundf:
2928 case Builtin::BIlroundl:
2929 case Builtin::BI__builtin_lround:
2930 case Builtin::BI__builtin_lroundf:
2931 case Builtin::BI__builtin_lroundl:
2932 case Builtin::BI__builtin_lroundf128:
2934 *
this, E, Intrinsic::lround,
2935 Intrinsic::experimental_constrained_lround));
2937 case Builtin::BIllround:
2938 case Builtin::BIllroundf:
2939 case Builtin::BIllroundl:
2940 case Builtin::BI__builtin_llround:
2941 case Builtin::BI__builtin_llroundf:
2942 case Builtin::BI__builtin_llroundl:
2943 case Builtin::BI__builtin_llroundf128:
2945 *
this, E, Intrinsic::llround,
2946 Intrinsic::experimental_constrained_llround));
2948 case Builtin::BIlrint:
2949 case Builtin::BIlrintf:
2950 case Builtin::BIlrintl:
2951 case Builtin::BI__builtin_lrint:
2952 case Builtin::BI__builtin_lrintf:
2953 case Builtin::BI__builtin_lrintl:
2954 case Builtin::BI__builtin_lrintf128:
2956 *
this, E, Intrinsic::lrint,
2957 Intrinsic::experimental_constrained_lrint));
2959 case Builtin::BIllrint:
2960 case Builtin::BIllrintf:
2961 case Builtin::BIllrintl:
2962 case Builtin::BI__builtin_llrint:
2963 case Builtin::BI__builtin_llrintf:
2964 case Builtin::BI__builtin_llrintl:
2965 case Builtin::BI__builtin_llrintf128:
2967 *
this, E, Intrinsic::llrint,
2968 Intrinsic::experimental_constrained_llrint));
2969 case Builtin::BI__builtin_ldexp:
2970 case Builtin::BI__builtin_ldexpf:
2971 case Builtin::BI__builtin_ldexpl:
2972 case Builtin::BI__builtin_ldexpf16:
2973 case Builtin::BI__builtin_ldexpf128: {
2975 *
this, E, Intrinsic::ldexp,
2976 Intrinsic::experimental_constrained_ldexp));
2986 Value *Val = A.getPointer();
2992 SkippedChecks.
set(SanitizerKind::All);
2993 SkippedChecks.
clear(SanitizerKind::Alignment);
2996 if (
auto *CE = dyn_cast<ImplicitCastExpr>(Arg))
2997 if (CE->getCastKind() == CK_BitCast)
2998 Arg = CE->getSubExpr();
2999 EmitTypeCheck(Kind, Loc, Val, Arg->getType(), A.getAlignment(),
3004 switch (BuiltinIDIfNoAsmLabel) {
3006 case Builtin::BI__builtin___CFStringMakeConstantString:
3007 case Builtin::BI__builtin___NSStringMakeConstantString:
3009 case Builtin::BI__builtin_stdarg_start:
3010 case Builtin::BI__builtin_va_start:
3011 case Builtin::BI__va_start:
3012 case Builtin::BI__builtin_va_end:
3016 BuiltinID != Builtin::BI__builtin_va_end);
3018 case Builtin::BI__builtin_va_copy: {
3024 case Builtin::BIabs:
3025 case Builtin::BIlabs:
3026 case Builtin::BIllabs:
3027 case Builtin::BI__builtin_abs:
3028 case Builtin::BI__builtin_labs:
3029 case Builtin::BI__builtin_llabs: {
3030 bool SanitizeOverflow =
SanOpts.
has(SanitizerKind::SignedIntegerOverflow);
3033 switch (
getLangOpts().getSignedOverflowBehavior()) {
3038 if (!SanitizeOverflow) {
3050 case Builtin::BI__builtin_complex: {
3055 case Builtin::BI__builtin_conj:
3056 case Builtin::BI__builtin_conjf:
3057 case Builtin::BI__builtin_conjl:
3058 case Builtin::BIconj:
3059 case Builtin::BIconjf:
3060 case Builtin::BIconjl: {
3062 Value *Real = ComplexVal.first;
3063 Value *Imag = ComplexVal.second;
3064 Imag =
Builder.CreateFNeg(Imag,
"neg");
3067 case Builtin::BI__builtin_creal:
3068 case Builtin::BI__builtin_crealf:
3069 case Builtin::BI__builtin_creall:
3070 case Builtin::BIcreal:
3071 case Builtin::BIcrealf:
3072 case Builtin::BIcreall: {
3077 case Builtin::BI__builtin_preserve_access_index: {
3098 case Builtin::BI__builtin_cimag:
3099 case Builtin::BI__builtin_cimagf:
3100 case Builtin::BI__builtin_cimagl:
3101 case Builtin::BIcimag:
3102 case Builtin::BIcimagf:
3103 case Builtin::BIcimagl: {
3108 case Builtin::BI__builtin_clrsb:
3109 case Builtin::BI__builtin_clrsbl:
3110 case Builtin::BI__builtin_clrsbll: {
3114 llvm::Type *ArgType = ArgValue->
getType();
3118 Value *
Zero = llvm::Constant::getNullValue(ArgType);
3119 Value *IsNeg =
Builder.CreateICmpSLT(ArgValue, Zero,
"isneg");
3121 Value *Tmp =
Builder.CreateSelect(IsNeg, Inverse, ArgValue);
3128 case Builtin::BI__builtin_ctzs:
3129 case Builtin::BI__builtin_ctz:
3130 case Builtin::BI__builtin_ctzl:
3131 case Builtin::BI__builtin_ctzll: {
3134 llvm::Type *ArgType = ArgValue->
getType();
3140 if (
Result->getType() != ResultType)
3145 case Builtin::BI__builtin_clzs:
3146 case Builtin::BI__builtin_clz:
3147 case Builtin::BI__builtin_clzl:
3148 case Builtin::BI__builtin_clzll: {
3151 llvm::Type *ArgType = ArgValue->
getType();
3157 if (
Result->getType() != ResultType)
3162 case Builtin::BI__builtin_ffs:
3163 case Builtin::BI__builtin_ffsl:
3164 case Builtin::BI__builtin_ffsll: {
3168 llvm::Type *ArgType = ArgValue->
getType();
3173 Builder.CreateAdd(
Builder.CreateCall(F, {ArgValue, Builder.getTrue()}),
3174 llvm::ConstantInt::get(ArgType, 1));
3175 Value *
Zero = llvm::Constant::getNullValue(ArgType);
3176 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, Zero,
"iszero");
3178 if (
Result->getType() != ResultType)
3183 case Builtin::BI__builtin_parity:
3184 case Builtin::BI__builtin_parityl:
3185 case Builtin::BI__builtin_parityll: {
3189 llvm::Type *ArgType = ArgValue->
getType();
3195 if (
Result->getType() != ResultType)
3200 case Builtin::BI__lzcnt16:
3201 case Builtin::BI__lzcnt:
3202 case Builtin::BI__lzcnt64: {
3205 llvm::Type *ArgType = ArgValue->
getType();
3210 if (
Result->getType() != ResultType)
3215 case Builtin::BI__popcnt16:
3216 case Builtin::BI__popcnt:
3217 case Builtin::BI__popcnt64:
3218 case Builtin::BI__builtin_popcount:
3219 case Builtin::BI__builtin_popcountl:
3220 case Builtin::BI__builtin_popcountll:
3221 case Builtin::BI__builtin_popcountg: {
3224 llvm::Type *ArgType = ArgValue->
getType();
3229 if (
Result->getType() != ResultType)
3234 case Builtin::BI__builtin_unpredictable: {
3240 case Builtin::BI__builtin_expect: {
3242 llvm::Type *ArgType = ArgValue->
getType();
3253 Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue},
"expval");
3256 case Builtin::BI__builtin_expect_with_probability: {
3258 llvm::Type *ArgType = ArgValue->
getType();
3261 llvm::APFloat Probability(0.0);
3264 assert(EvalSucceed &&
"probability should be able to evaluate as float");
3266 bool LoseInfo =
false;
3267 Probability.convert(llvm::APFloat::IEEEdouble(),
3268 llvm::RoundingMode::Dynamic, &LoseInfo);
3270 Constant *Confidence = ConstantFP::get(Ty, Probability);
3280 FnExpect, {ArgValue, ExpectedValue, Confidence},
"expval");
3283 case Builtin::BI__builtin_assume_aligned: {
3286 Value *OffsetValue =
3290 ConstantInt *AlignmentCI = cast<ConstantInt>(AlignmentValue);
3291 if (AlignmentCI->getValue().ugt(llvm::Value::MaximumAlignment))
3292 AlignmentCI = ConstantInt::get(AlignmentCI->getIntegerType(),
3293 llvm::Value::MaximumAlignment);
3297 AlignmentCI, OffsetValue);
3300 case Builtin::BI__assume:
3301 case Builtin::BI__builtin_assume: {
3307 Builder.CreateCall(FnAssume, ArgValue);
3310 case Builtin::BI__builtin_assume_separate_storage: {
3317 Value *Values[] = {Value0, Value1};
3318 OperandBundleDefT<Value *> OBD(
"separate_storage", Values);
3322 case Builtin::BI__arithmetic_fence: {
3325 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3326 llvm::FastMathFlags FMF =
Builder.getFastMathFlags();
3327 bool isArithmeticFenceEnabled =
3328 FMF.allowReassoc() &&
3332 if (isArithmeticFenceEnabled) {
3335 Value *Real =
Builder.CreateArithmeticFence(ComplexVal.first,
3337 Value *Imag =
Builder.CreateArithmeticFence(ComplexVal.second,
3342 Value *Real = ComplexVal.first;
3343 Value *Imag = ComplexVal.second;
3347 if (isArithmeticFenceEnabled)
3352 case Builtin::BI__builtin_bswap16:
3353 case Builtin::BI__builtin_bswap32:
3354 case Builtin::BI__builtin_bswap64:
3355 case Builtin::BI_byteswap_ushort:
3356 case Builtin::BI_byteswap_ulong:
3357 case Builtin::BI_byteswap_uint64: {
3360 case Builtin::BI__builtin_bitreverse8:
3361 case Builtin::BI__builtin_bitreverse16:
3362 case Builtin::BI__builtin_bitreverse32:
3363 case Builtin::BI__builtin_bitreverse64: {
3366 case Builtin::BI__builtin_rotateleft8:
3367 case Builtin::BI__builtin_rotateleft16:
3368 case Builtin::BI__builtin_rotateleft32:
3369 case Builtin::BI__builtin_rotateleft64:
3370 case Builtin::BI_rotl8:
3371 case Builtin::BI_rotl16:
3372 case Builtin::BI_rotl:
3373 case Builtin::BI_lrotl:
3374 case Builtin::BI_rotl64:
3377 case Builtin::BI__builtin_rotateright8:
3378 case Builtin::BI__builtin_rotateright16:
3379 case Builtin::BI__builtin_rotateright32:
3380 case Builtin::BI__builtin_rotateright64:
3381 case Builtin::BI_rotr8:
3382 case Builtin::BI_rotr16:
3383 case Builtin::BI_rotr:
3384 case Builtin::BI_lrotr:
3385 case Builtin::BI_rotr64:
3388 case Builtin::BI__builtin_constant_p: {
3399 return RValue::get(ConstantInt::get(ResultType, 0));
3404 return RValue::get(ConstantInt::get(ResultType, 0));
3416 if (
Result->getType() != ResultType)
3420 case Builtin::BI__builtin_dynamic_object_size:
3421 case Builtin::BI__builtin_object_size: {
3428 bool IsDynamic = BuiltinID == Builtin::BI__builtin_dynamic_object_size;
3430 nullptr, IsDynamic));
3432 case Builtin::BI__builtin_prefetch: {
3436 llvm::ConstantInt::get(
Int32Ty, 0);
3438 llvm::ConstantInt::get(
Int32Ty, 3);
3444 case Builtin::BI__builtin_readcyclecounter: {
3448 case Builtin::BI__builtin_readsteadycounter: {
3452 case Builtin::BI__builtin___clear_cache: {
3458 case Builtin::BI__builtin_trap:
3461 case Builtin::BI__debugbreak:
3464 case Builtin::BI__builtin_unreachable: {
3473 case Builtin::BI__builtin_powi:
3474 case Builtin::BI__builtin_powif:
3475 case Builtin::BI__builtin_powil: {
3479 if (
Builder.getIsFPConstrained()) {
3482 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3489 { Src0->getType(), Src1->getType() });
3492 case Builtin::BI__builtin_frexpl: {
3496 if (&
getTarget().getLongDoubleFormat() == &llvm::APFloat::PPCDoubleDouble())
3500 case Builtin::BI__builtin_frexp:
3501 case Builtin::BI__builtin_frexpf:
3502 case Builtin::BI__builtin_frexpf128:
3503 case Builtin::BI__builtin_frexpf16:
3505 case Builtin::BI__builtin_isgreater:
3506 case Builtin::BI__builtin_isgreaterequal:
3507 case Builtin::BI__builtin_isless:
3508 case Builtin::BI__builtin_islessequal:
3509 case Builtin::BI__builtin_islessgreater:
3510 case Builtin::BI__builtin_isunordered: {
3513 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3517 switch (BuiltinID) {
3518 default: llvm_unreachable(
"Unknown ordered comparison");
3519 case Builtin::BI__builtin_isgreater:
3520 LHS =
Builder.CreateFCmpOGT(LHS, RHS,
"cmp");
3522 case Builtin::BI__builtin_isgreaterequal:
3523 LHS =
Builder.CreateFCmpOGE(LHS, RHS,
"cmp");
3525 case Builtin::BI__builtin_isless:
3526 LHS =
Builder.CreateFCmpOLT(LHS, RHS,
"cmp");
3528 case Builtin::BI__builtin_islessequal:
3529 LHS =
Builder.CreateFCmpOLE(LHS, RHS,
"cmp");
3531 case Builtin::BI__builtin_islessgreater:
3532 LHS =
Builder.CreateFCmpONE(LHS, RHS,
"cmp");
3534 case Builtin::BI__builtin_isunordered:
3535 LHS =
Builder.CreateFCmpUNO(LHS, RHS,
"cmp");
3542 case Builtin::BI__builtin_isnan: {
3543 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3552 case Builtin::BI__builtin_issignaling: {
3553 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3560 case Builtin::BI__builtin_isinf: {
3561 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3570 case Builtin::BIfinite:
3571 case Builtin::BI__finite:
3572 case Builtin::BIfinitef:
3573 case Builtin::BI__finitef:
3574 case Builtin::BIfinitel:
3575 case Builtin::BI__finitel:
3576 case Builtin::BI__builtin_isfinite: {
3577 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3586 case Builtin::BI__builtin_isnormal: {
3587 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3594 case Builtin::BI__builtin_issubnormal: {
3595 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3598 Builder.CreateZExt(
Builder.createIsFPClass(
V, FPClassTest::fcSubnormal),
3602 case Builtin::BI__builtin_iszero: {
3603 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3610 case Builtin::BI__builtin_isfpclass: {
3615 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3621 case Builtin::BI__builtin_nondeterministic_value: {
3630 case Builtin::BI__builtin_elementwise_abs: {
3635 QT = VecTy->getElementType();
3639 Builder.getFalse(),
nullptr,
"elt.abs");
3646 case Builtin::BI__builtin_elementwise_ceil:
3649 case Builtin::BI__builtin_elementwise_exp:
3652 case Builtin::BI__builtin_elementwise_exp2:
3655 case Builtin::BI__builtin_elementwise_log:
3658 case Builtin::BI__builtin_elementwise_log2:
3661 case Builtin::BI__builtin_elementwise_log10:
3664 case Builtin::BI__builtin_elementwise_pow: {
3667 case Builtin::BI__builtin_elementwise_bitreverse:
3670 case Builtin::BI__builtin_elementwise_cos:
3673 case Builtin::BI__builtin_elementwise_floor:
3676 case Builtin::BI__builtin_elementwise_roundeven:
3679 case Builtin::BI__builtin_elementwise_round:
3682 case Builtin::BI__builtin_elementwise_rint:
3685 case Builtin::BI__builtin_elementwise_nearbyint:
3688 case Builtin::BI__builtin_elementwise_sin:
3692 case Builtin::BI__builtin_elementwise_trunc:
3695 case Builtin::BI__builtin_elementwise_canonicalize:
3697 emitUnaryBuiltin(*
this, E, llvm::Intrinsic::canonicalize,
"elt.canonicalize"));
3698 case Builtin::BI__builtin_elementwise_copysign:
3700 case Builtin::BI__builtin_elementwise_fma:
3702 case Builtin::BI__builtin_elementwise_add_sat:
3703 case Builtin::BI__builtin_elementwise_sub_sat: {
3707 assert(Op0->
getType()->isIntOrIntVectorTy() &&
"integer type expected");
3710 Ty = VecTy->getElementType();
3713 if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_add_sat)
3714 Opc = IsSigned ? llvm::Intrinsic::sadd_sat : llvm::Intrinsic::uadd_sat;
3716 Opc = IsSigned ? llvm::Intrinsic::ssub_sat : llvm::Intrinsic::usub_sat;
3717 Result =
Builder.CreateBinaryIntrinsic(Opc, Op0, Op1,
nullptr,
"elt.sat");
3721 case Builtin::BI__builtin_elementwise_max: {
3725 if (Op0->
getType()->isIntOrIntVectorTy()) {
3728 Ty = VecTy->getElementType();
3730 ? llvm::Intrinsic::smax
3731 : llvm::Intrinsic::umax,
3732 Op0, Op1,
nullptr,
"elt.max");
3737 case Builtin::BI__builtin_elementwise_min: {
3741 if (Op0->
getType()->isIntOrIntVectorTy()) {
3744 Ty = VecTy->getElementType();
3746 ? llvm::Intrinsic::smin
3747 : llvm::Intrinsic::umin,
3748 Op0, Op1,
nullptr,
"elt.min");
3754 case Builtin::BI__builtin_reduce_max: {
3755 auto GetIntrinsicID = [](
QualType QT) {
3757 QT = VecTy->getElementType();
3759 return llvm::Intrinsic::vector_reduce_smax;
3761 return llvm::Intrinsic::vector_reduce_umax;
3763 return llvm::Intrinsic::vector_reduce_fmax;
3766 *
this, E, GetIntrinsicID(E->
getArg(0)->
getType()),
"rdx.min"));
3769 case Builtin::BI__builtin_reduce_min: {
3770 auto GetIntrinsicID = [](
QualType QT) {
3772 QT = VecTy->getElementType();
3774 return llvm::Intrinsic::vector_reduce_smin;
3776 return llvm::Intrinsic::vector_reduce_umin;
3778 return llvm::Intrinsic::vector_reduce_fmin;
3782 *
this, E, GetIntrinsicID(E->
getArg(0)->
getType()),
"rdx.min"));
3785 case Builtin::BI__builtin_reduce_add:
3787 *
this, E, llvm::Intrinsic::vector_reduce_add,
"rdx.add"));
3788 case Builtin::BI__builtin_reduce_mul:
3790 *
this, E, llvm::Intrinsic::vector_reduce_mul,
"rdx.mul"));
3791 case Builtin::BI__builtin_reduce_xor:
3793 *
this, E, llvm::Intrinsic::vector_reduce_xor,
"rdx.xor"));
3794 case Builtin::BI__builtin_reduce_or:
3796 *
this, E, llvm::Intrinsic::vector_reduce_or,
"rdx.or"));
3797 case Builtin::BI__builtin_reduce_and:
3799 *
this, E, llvm::Intrinsic::vector_reduce_and,
"rdx.and"));
3801 case Builtin::BI__builtin_matrix_transpose: {
3805 Value *
Result = MB.CreateMatrixTranspose(MatValue, MatrixTy->getNumRows(),
3806 MatrixTy->getNumColumns());
3810 case Builtin::BI__builtin_matrix_column_major_load: {
3816 assert(PtrTy &&
"arg0 must be of pointer type");
3825 ResultTy->getNumRows(), ResultTy->getNumColumns(),
3830 case Builtin::BI__builtin_matrix_column_major_store: {
3838 assert(PtrTy &&
"arg1 must be of pointer type");
3845 Stride, IsVolatile, MatrixTy->getNumRows(), MatrixTy->getNumColumns());
3849 case Builtin::BI__builtin_isinf_sign: {
3851 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3856 AbsArg, ConstantFP::getInfinity(Arg->
getType()),
"isinf");
3862 Value *NegativeOne = ConstantInt::get(
IntTy, -1);
3863 Value *SignResult =
Builder.CreateSelect(IsNeg, NegativeOne, One);
3868 case Builtin::BI__builtin_flt_rounds: {
3873 if (
Result->getType() != ResultType)
3879 case Builtin::BI__builtin_set_flt_rounds: {
3887 case Builtin::BI__builtin_fpclassify: {
3888 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3899 "fpclassify_result");
3903 Value *IsZero =
Builder.CreateFCmpOEQ(
V, Constant::getNullValue(Ty),
3907 Builder.CreateCondBr(IsZero, End, NotZero);
3911 Builder.SetInsertPoint(NotZero);
3915 Builder.CreateCondBr(IsNan, End, NotNan);
3916 Result->addIncoming(NanLiteral, NotZero);
3919 Builder.SetInsertPoint(NotNan);
3922 Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(
V->getType()),
3926 Builder.CreateCondBr(IsInf, End, NotInf);
3927 Result->addIncoming(InfLiteral, NotNan);
3930 Builder.SetInsertPoint(NotInf);
3931 APFloat Smallest = APFloat::getSmallestNormalized(
3934 Builder.CreateFCmpUGE(VAbs, ConstantFP::get(
V->getContext(), Smallest),
3936 Value *NormalResult =
3940 Result->addIncoming(NormalResult, NotInf);
3953 case Builtin::BIalloca:
3954 case Builtin::BI_alloca:
3955 case Builtin::BI__builtin_alloca_uninitialized:
3956 case Builtin::BI__builtin_alloca: {
3960 const Align SuitableAlignmentInBytes =
3964 AllocaInst *AI =
Builder.CreateAlloca(
Builder.getInt8Ty(), Size);
3965 AI->setAlignment(SuitableAlignmentInBytes);
3966 if (BuiltinID != Builtin::BI__builtin_alloca_uninitialized)
3978 case Builtin::BI__builtin_alloca_with_align_uninitialized:
3979 case Builtin::BI__builtin_alloca_with_align: {
3982 auto *AlignmentInBitsCI = cast<ConstantInt>(AlignmentInBitsValue);
3983 unsigned AlignmentInBits = AlignmentInBitsCI->getZExtValue();
3984 const Align AlignmentInBytes =
3986 AllocaInst *AI =
Builder.CreateAlloca(
Builder.getInt8Ty(), Size);
3987 AI->setAlignment(AlignmentInBytes);
3988 if (BuiltinID != Builtin::BI__builtin_alloca_with_align_uninitialized)
4000 case Builtin::BIbzero:
4001 case Builtin::BI__builtin_bzero: {
4010 case Builtin::BIbcopy:
4011 case Builtin::BI__builtin_bcopy: {
4023 case Builtin::BImemcpy:
4024 case Builtin::BI__builtin_memcpy:
4025 case Builtin::BImempcpy:
4026 case Builtin::BI__builtin_mempcpy: {
4033 if (BuiltinID == Builtin::BImempcpy ||
4034 BuiltinID == Builtin::BI__builtin_mempcpy)
4041 case Builtin::BI__builtin_memcpy_inline: {
4052 case Builtin::BI__builtin_char_memchr:
4053 BuiltinID = Builtin::BI__builtin_memchr;
4056 case Builtin::BI__builtin___memcpy_chk: {
4063 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
4064 if (
Size.ugt(DstSize))
4068 Value *SizeVal = llvm::ConstantInt::get(
Builder.getContext(), Size);
4073 case Builtin::BI__builtin_objc_memmove_collectable: {
4078 DestAddr, SrcAddr, SizeVal);
4082 case Builtin::BI__builtin___memmove_chk: {
4089 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
4090 if (
Size.ugt(DstSize))
4094 Value *SizeVal = llvm::ConstantInt::get(
Builder.getContext(), Size);
4099 case Builtin::BImemmove:
4100 case Builtin::BI__builtin_memmove: {
4109 case Builtin::BImemset:
4110 case Builtin::BI__builtin_memset: {
4120 case Builtin::BI__builtin_memset_inline: {
4131 case Builtin::BI__builtin___memset_chk: {
4138 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
4139 if (
Size.ugt(DstSize))
4144 Value *SizeVal = llvm::ConstantInt::get(
Builder.getContext(), Size);
4148 case Builtin::BI__builtin_wmemchr: {
4151 if (!
getTarget().getTriple().isOSMSVCRT())
4159 BasicBlock *Entry =
Builder.GetInsertBlock();
4164 Builder.CreateCondBr(SizeEq0, Exit, CmpEq);
4168 StrPhi->addIncoming(Str, Entry);
4170 SizePhi->addIncoming(Size, Entry);
4174 Value *FoundChr =
Builder.CreateConstInBoundsGEP1_32(WCharTy, StrPhi, 0);
4176 Builder.CreateCondBr(StrEqChr, Exit, Next);
4179 Value *NextStr =
Builder.CreateConstInBoundsGEP1_32(WCharTy, StrPhi, 1);
4181 Value *NextSizeEq0 =
4182 Builder.CreateICmpEQ(NextSize, ConstantInt::get(
SizeTy, 0));
4183 Builder.CreateCondBr(NextSizeEq0, Exit, CmpEq);
4184 StrPhi->addIncoming(NextStr, Next);
4185 SizePhi->addIncoming(NextSize, Next);
4189 Ret->addIncoming(llvm::Constant::getNullValue(Str->
getType()), Entry);
4190 Ret->addIncoming(llvm::Constant::getNullValue(Str->
getType()), Next);
4191 Ret->addIncoming(FoundChr, CmpEq);
4194 case Builtin::BI__builtin_wmemcmp: {
4197 if (!
getTarget().getTriple().isOSMSVCRT())
4206 BasicBlock *Entry =
Builder.GetInsertBlock();
4212 Builder.CreateCondBr(SizeEq0, Exit, CmpGT);
4216 DstPhi->addIncoming(Dst, Entry);
4218 SrcPhi->addIncoming(Src, Entry);
4220 SizePhi->addIncoming(Size, Entry);
4226 Builder.CreateCondBr(DstGtSrc, Exit, CmpLT);
4230 Builder.CreateCondBr(DstLtSrc, Exit, Next);
4233 Value *NextDst =
Builder.CreateConstInBoundsGEP1_32(WCharTy, DstPhi, 1);
4234 Value *NextSrc =
Builder.CreateConstInBoundsGEP1_32(WCharTy, SrcPhi, 1);
4236 Value *NextSizeEq0 =
4237 Builder.CreateICmpEQ(NextSize, ConstantInt::get(
SizeTy, 0));
4238 Builder.CreateCondBr(NextSizeEq0, Exit, CmpGT);
4239 DstPhi->addIncoming(NextDst, Next);
4240 SrcPhi->addIncoming(NextSrc, Next);
4241 SizePhi->addIncoming(NextSize, Next);
4245 Ret->addIncoming(ConstantInt::get(
IntTy, 0), Entry);
4246 Ret->addIncoming(ConstantInt::get(
IntTy, 1), CmpGT);
4247 Ret->addIncoming(ConstantInt::get(
IntTy, -1), CmpLT);
4248 Ret->addIncoming(ConstantInt::get(
IntTy, 0), Next);
4251 case Builtin::BI__builtin_dwarf_cfa: {
4264 llvm::ConstantInt::get(
Int32Ty, Offset)));
4266 case Builtin::BI__builtin_return_address: {
4272 case Builtin::BI_ReturnAddress: {
4276 case Builtin::BI__builtin_frame_address: {
4282 case Builtin::BI__builtin_extract_return_addr: {
4287 case Builtin::BI__builtin_frob_return_addr: {
4292 case Builtin::BI__builtin_dwarf_sp_column: {
4293 llvm::IntegerType *Ty
4302 case Builtin::BI__builtin_init_dwarf_reg_size_table: {
4308 case Builtin::BI__builtin_eh_return: {
4312 llvm::IntegerType *
IntTy = cast<llvm::IntegerType>(Int->
getType());
4313 assert((
IntTy->getBitWidth() == 32 ||
IntTy->getBitWidth() == 64) &&
4314 "LLVM's __builtin_eh_return only supports 32- and 64-bit variants");
4317 : Intrinsic::eh_return_i64);
4318 Builder.CreateCall(F, {Int, Ptr});
4326 case Builtin::BI__builtin_unwind_init: {
4331 case Builtin::BI__builtin_extend_pointer: {
4356 case Builtin::BI__builtin_setjmp: {
4363 ConstantInt::get(
Int32Ty, 0));
4377 case Builtin::BI__builtin_longjmp: {
4391 case Builtin::BI__builtin_launder: {
4400 case Builtin::BI__sync_fetch_and_add:
4401 case Builtin::BI__sync_fetch_and_sub:
4402 case Builtin::BI__sync_fetch_and_or:
4403 case Builtin::BI__sync_fetch_and_and:
4404 case Builtin::BI__sync_fetch_and_xor:
4405 case Builtin::BI__sync_fetch_and_nand:
4406 case Builtin::BI__sync_add_and_fetch:
4407 case Builtin::BI__sync_sub_and_fetch:
4408 case Builtin::BI__sync_and_and_fetch:
4409 case Builtin::BI__sync_or_and_fetch:
4410 case Builtin::BI__sync_xor_and_fetch:
4411 case Builtin::BI__sync_nand_and_fetch:
4412 case Builtin::BI__sync_val_compare_and_swap:
4413 case Builtin::BI__sync_bool_compare_and_swap:
4414 case Builtin::BI__sync_lock_test_and_set:
4415 case Builtin::BI__sync_lock_release:
4416 case Builtin::BI__sync_swap:
4417 llvm_unreachable(
"Shouldn't make it through sema");
4418 case Builtin::BI__sync_fetch_and_add_1:
4419 case Builtin::BI__sync_fetch_and_add_2:
4420 case Builtin::BI__sync_fetch_and_add_4:
4421 case Builtin::BI__sync_fetch_and_add_8:
4422 case Builtin::BI__sync_fetch_and_add_16:
4424 case Builtin::BI__sync_fetch_and_sub_1:
4425 case Builtin::BI__sync_fetch_and_sub_2:
4426 case Builtin::BI__sync_fetch_and_sub_4:
4427 case Builtin::BI__sync_fetch_and_sub_8:
4428 case Builtin::BI__sync_fetch_and_sub_16:
4430 case Builtin::BI__sync_fetch_and_or_1:
4431 case Builtin::BI__sync_fetch_and_or_2:
4432 case Builtin::BI__sync_fetch_and_or_4:
4433 case Builtin::BI__sync_fetch_and_or_8:
4434 case Builtin::BI__sync_fetch_and_or_16:
4436 case Builtin::BI__sync_fetch_and_and_1:
4437 case Builtin::BI__sync_fetch_and_and_2:
4438 case Builtin::BI__sync_fetch_and_and_4:
4439 case Builtin::BI__sync_fetch_and_and_8:
4440 case Builtin::BI__sync_fetch_and_and_16:
4442 case Builtin::BI__sync_fetch_and_xor_1:
4443 case Builtin::BI__sync_fetch_and_xor_2:
4444 case Builtin::BI__sync_fetch_and_xor_4:
4445 case Builtin::BI__sync_fetch_and_xor_8:
4446 case Builtin::BI__sync_fetch_and_xor_16:
4448 case Builtin::BI__sync_fetch_and_nand_1:
4449 case Builtin::BI__sync_fetch_and_nand_2:
4450 case Builtin::BI__sync_fetch_and_nand_4:
4451 case Builtin::BI__sync_fetch_and_nand_8:
4452 case Builtin::BI__sync_fetch_and_nand_16:
4456 case Builtin::BI__sync_fetch_and_min:
4458 case Builtin::BI__sync_fetch_and_max:
4460 case Builtin::BI__sync_fetch_and_umin:
4462 case Builtin::BI__sync_fetch_and_umax:
4465 case Builtin::BI__sync_add_and_fetch_1:
4466 case Builtin::BI__sync_add_and_fetch_2:
4467 case Builtin::BI__sync_add_and_fetch_4:
4468 case Builtin::BI__sync_add_and_fetch_8:
4469 case Builtin::BI__sync_add_and_fetch_16:
4471 llvm::Instruction::Add);
4472 case Builtin::BI__sync_sub_and_fetch_1:
4473 case Builtin::BI__sync_sub_and_fetch_2:
4474 case Builtin::BI__sync_sub_and_fetch_4:
4475 case Builtin::BI__sync_sub_and_fetch_8:
4476 case Builtin::BI__sync_sub_and_fetch_16:
4478 llvm::Instruction::Sub);
4479 case Builtin::BI__sync_and_and_fetch_1:
4480 case Builtin::BI__sync_and_and_fetch_2:
4481 case Builtin::BI__sync_and_and_fetch_4:
4482 case Builtin::BI__sync_and_and_fetch_8:
4483 case Builtin::BI__sync_and_and_fetch_16:
4485 llvm::Instruction::And);
4486 case Builtin::BI__sync_or_and_fetch_1:
4487 case Builtin::BI__sync_or_and_fetch_2:
4488 case Builtin::BI__sync_or_and_fetch_4:
4489 case Builtin::BI__sync_or_and_fetch_8:
4490 case Builtin::BI__sync_or_and_fetch_16:
4492 llvm::Instruction::Or);
4493 case Builtin::BI__sync_xor_and_fetch_1:
4494 case Builtin::BI__sync_xor_and_fetch_2:
4495 case Builtin::BI__sync_xor_and_fetch_4:
4496 case Builtin::BI__sync_xor_and_fetch_8:
4497 case Builtin::BI__sync_xor_and_fetch_16:
4499 llvm::Instruction::Xor);
4500 case Builtin::BI__sync_nand_and_fetch_1:
4501 case Builtin::BI__sync_nand_and_fetch_2:
4502 case Builtin::BI__sync_nand_and_fetch_4:
4503 case Builtin::BI__sync_nand_and_fetch_8:
4504 case Builtin::BI__sync_nand_and_fetch_16:
4506 llvm::Instruction::And,
true);
4508 case Builtin::BI__sync_val_compare_and_swap_1:
4509 case Builtin::BI__sync_val_compare_and_swap_2:
4510 case Builtin::BI__sync_val_compare_and_swap_4:
4511 case Builtin::BI__sync_val_compare_and_swap_8:
4512 case Builtin::BI__sync_val_compare_and_swap_16:
4515 case Builtin::BI__sync_bool_compare_and_swap_1:
4516 case Builtin::BI__sync_bool_compare_and_swap_2:
4517 case Builtin::BI__sync_bool_compare_and_swap_4:
4518 case Builtin::BI__sync_bool_compare_and_swap_8:
4519 case Builtin::BI__sync_bool_compare_and_swap_16:
4522 case Builtin::BI__sync_swap_1:
4523 case Builtin::BI__sync_swap_2:
4524 case Builtin::BI__sync_swap_4:
4525 case Builtin::BI__sync_swap_8:
4526 case Builtin::BI__sync_swap_16:
4529 case Builtin::BI__sync_lock_test_and_set_1:
4530 case Builtin::BI__sync_lock_test_and_set_2:
4531 case Builtin::BI__sync_lock_test_and_set_4:
4532 case Builtin::BI__sync_lock_test_and_set_8:
4533 case Builtin::BI__sync_lock_test_and_set_16:
4536 case Builtin::BI__sync_lock_release_1:
4537 case Builtin::BI__sync_lock_release_2:
4538 case Builtin::BI__sync_lock_release_4:
4539 case Builtin::BI__sync_lock_release_8:
4540 case Builtin::BI__sync_lock_release_16: {
4546 llvm::StoreInst *
Store =
4548 Store->setAtomic(llvm::AtomicOrdering::Release);
4552 case Builtin::BI__sync_synchronize: {
4560 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent);
4564 case Builtin::BI__builtin_nontemporal_load:
4566 case Builtin::BI__builtin_nontemporal_store:
4568 case Builtin::BI__c11_atomic_is_lock_free:
4569 case Builtin::BI__atomic_is_lock_free: {
4573 const char *LibCallName =
"__atomic_is_lock_free";
4577 if (BuiltinID == Builtin::BI__atomic_is_lock_free)
4591 case Builtin::BI__atomic_test_and_set: {
4603 if (isa<llvm::ConstantInt>(Order)) {
4604 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
4605 AtomicRMWInst *
Result =
nullptr;
4610 llvm::AtomicOrdering::Monotonic);
4615 llvm::AtomicOrdering::Acquire);
4619 llvm::AtomicOrdering::Release);
4624 llvm::AtomicOrdering::AcquireRelease);
4628 llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
4629 llvm::AtomicOrdering::SequentiallyConsistent);
4632 Result->setVolatile(Volatile);
4638 llvm::BasicBlock *BBs[5] = {
4645 llvm::AtomicOrdering Orders[5] = {
4646 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Acquire,
4647 llvm::AtomicOrdering::Release, llvm::AtomicOrdering::AcquireRelease,
4648 llvm::AtomicOrdering::SequentiallyConsistent};
4650 Order =
Builder.CreateIntCast(Order,
Builder.getInt32Ty(),
false);
4651 llvm::SwitchInst *SI =
Builder.CreateSwitch(Order, BBs[0]);
4653 Builder.SetInsertPoint(ContBB);
4656 for (
unsigned i = 0; i < 5; ++i) {
4657 Builder.SetInsertPoint(BBs[i]);
4659 Ptr, NewVal, Orders[i]);
4660 RMW->setVolatile(Volatile);
4661 Result->addIncoming(RMW, BBs[i]);
4665 SI->addCase(
Builder.getInt32(0), BBs[0]);
4666 SI->addCase(
Builder.getInt32(1), BBs[1]);
4667 SI->addCase(
Builder.getInt32(2), BBs[1]);
4668 SI->addCase(
Builder.getInt32(3), BBs[2]);
4669 SI->addCase(
Builder.getInt32(4), BBs[3]);
4670 SI->addCase(
Builder.getInt32(5), BBs[4]);
4672 Builder.SetInsertPoint(ContBB);
4676 case Builtin::BI__atomic_clear: {
4685 if (isa<llvm::ConstantInt>(Order)) {
4686 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
4691 Store->setOrdering(llvm::AtomicOrdering::Monotonic);
4694 Store->setOrdering(llvm::AtomicOrdering::Release);
4697 Store->setOrdering(llvm::AtomicOrdering::SequentiallyConsistent);
4705 llvm::BasicBlock *BBs[3] = {
4710 llvm::AtomicOrdering Orders[3] = {
4711 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Release,
4712 llvm::AtomicOrdering::SequentiallyConsistent};
4714 Order =
Builder.CreateIntCast(Order,
Builder.getInt32Ty(),
false);
4715 llvm::SwitchInst *SI =
Builder.CreateSwitch(Order, BBs[0]);
4717 for (
unsigned i = 0; i < 3; ++i) {
4718 Builder.SetInsertPoint(BBs[i]);
4720 Store->setOrdering(Orders[i]);
4724 SI->addCase(
Builder.getInt32(0), BBs[0]);
4725 SI->addCase(
Builder.getInt32(3), BBs[1]);
4726 SI->addCase(
Builder.getInt32(5), BBs[2]);
4728 Builder.SetInsertPoint(ContBB);
4732 case Builtin::BI__atomic_thread_fence:
4733 case Builtin::BI__atomic_signal_fence:
4734 case Builtin::BI__c11_atomic_thread_fence:
4735 case Builtin::BI__c11_atomic_signal_fence: {
4736 llvm::SyncScope::ID SSID;
4737 if (BuiltinID == Builtin::BI__atomic_signal_fence ||
4738 BuiltinID == Builtin::BI__c11_atomic_signal_fence)
4739 SSID = llvm::SyncScope::SingleThread;
4741 SSID = llvm::SyncScope::System;
4743 if (isa<llvm::ConstantInt>(Order)) {
4744 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
4751 Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
4754 Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
4757 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
4760 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
4766 llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB;
4773 Order =
Builder.CreateIntCast(Order,
Builder.getInt32Ty(),
false);
4774 llvm::SwitchInst *SI =
Builder.CreateSwitch(Order, ContBB);
4776 Builder.SetInsertPoint(AcquireBB);
4777 Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
4779 SI->addCase(
Builder.getInt32(1), AcquireBB);
4780 SI->addCase(
Builder.getInt32(2), AcquireBB);
4782 Builder.SetInsertPoint(ReleaseBB);
4783 Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
4785 SI->addCase(
Builder.getInt32(3), ReleaseBB);
4787 Builder.SetInsertPoint(AcqRelBB);
4788 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
4790 SI->addCase(
Builder.getInt32(4), AcqRelBB);
4792 Builder.SetInsertPoint(SeqCstBB);
4793 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
4795 SI->addCase(
Builder.getInt32(5), SeqCstBB);
4797 Builder.SetInsertPoint(ContBB);
4801 case Builtin::BI__builtin_signbit:
4802 case Builtin::BI__builtin_signbitf:
4803 case Builtin::BI__builtin_signbitl: {
4808 case Builtin::BI__warn_memset_zero_len:
4810 case Builtin::BI__annotation: {
4815 assert(Str->getCharByteWidth() == 2);
4816 StringRef WideBytes = Str->getBytes();
4817 std::string StrUtf8;
4818 if (!convertUTF16ToUTF8String(
4819 ArrayRef(WideBytes.data(), WideBytes.size()), StrUtf8)) {
4823 Strings.push_back(llvm::MDString::get(
getLLVMContext(), StrUtf8));
4833 case Builtin::BI__builtin_annotation: {
4842 StringRef Str = cast<StringLiteral>(AnnotationStrExpr)->getString();
4846 case Builtin::BI__builtin_addcb:
4847 case Builtin::BI__builtin_addcs:
4848 case Builtin::BI__builtin_addc:
4849 case Builtin::BI__builtin_addcl:
4850 case Builtin::BI__builtin_addcll:
4851 case Builtin::BI__builtin_subcb:
4852 case Builtin::BI__builtin_subcs:
4853 case Builtin::BI__builtin_subc:
4854 case Builtin::BI__builtin_subcl:
4855 case Builtin::BI__builtin_subcll: {
4881 llvm::Intrinsic::ID IntrinsicId;
4882 switch (BuiltinID) {
4883 default: llvm_unreachable(
"Unknown multiprecision builtin id.");
4884 case Builtin::BI__builtin_addcb:
4885 case Builtin::BI__builtin_addcs:
4886 case Builtin::BI__builtin_addc:
4887 case Builtin::BI__builtin_addcl:
4888 case Builtin::BI__builtin_addcll:
4889 IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
4891 case Builtin::BI__builtin_subcb:
4892 case Builtin::BI__builtin_subcs:
4893 case Builtin::BI__builtin_subc:
4894 case Builtin::BI__builtin_subcl:
4895 case Builtin::BI__builtin_subcll:
4896 IntrinsicId = llvm::Intrinsic::usub_with_overflow;
4901 llvm::Value *Carry1;
4904 llvm::Value *Carry2;
4906 Sum1, Carryin, Carry2);
4907 llvm::Value *CarryOut =
Builder.CreateZExt(
Builder.CreateOr(Carry1, Carry2),
4913 case Builtin::BI__builtin_add_overflow:
4914 case Builtin::BI__builtin_sub_overflow:
4915 case Builtin::BI__builtin_mul_overflow: {
4923 WidthAndSignedness LeftInfo =
4925 WidthAndSignedness RightInfo =
4927 WidthAndSignedness ResultInfo =
4934 RightInfo, ResultArg, ResultQTy,
4940 *
this, LeftArg, LeftInfo, RightArg, RightInfo, ResultArg, ResultQTy,
4943 WidthAndSignedness EncompassingInfo =
4946 llvm::Type *EncompassingLLVMTy =
4951 llvm::Intrinsic::ID IntrinsicId;
4952 switch (BuiltinID) {
4954 llvm_unreachable(
"Unknown overflow builtin id.");
4955 case Builtin::BI__builtin_add_overflow:
4956 IntrinsicId = EncompassingInfo.Signed
4957 ? llvm::Intrinsic::sadd_with_overflow
4958 : llvm::Intrinsic::uadd_with_overflow;
4960 case Builtin::BI__builtin_sub_overflow:
4961 IntrinsicId = EncompassingInfo.Signed
4962 ? llvm::Intrinsic::ssub_with_overflow
4963 : llvm::Intrinsic::usub_with_overflow;
4965 case Builtin::BI__builtin_mul_overflow:
4966 IntrinsicId = EncompassingInfo.Signed
4967 ? llvm::Intrinsic::smul_with_overflow
4968 : llvm::Intrinsic::umul_with_overflow;
4977 Left =
Builder.CreateIntCast(Left, EncompassingLLVMTy, LeftInfo.Signed);
4978 Right =
Builder.CreateIntCast(Right, EncompassingLLVMTy, RightInfo.Signed);
4981 llvm::Value *Overflow, *
Result;
4984 if (EncompassingInfo.Width > ResultInfo.Width) {
4987 llvm::Value *ResultTrunc =
Builder.CreateTrunc(
Result, ResultLLVMTy);
4991 llvm::Value *ResultTruncExt =
Builder.CreateIntCast(
4992 ResultTrunc, EncompassingLLVMTy, ResultInfo.Signed);
4993 llvm::Value *TruncationOverflow =
4996 Overflow =
Builder.CreateOr(Overflow, TruncationOverflow);
5008 case Builtin::BI__builtin_uadd_overflow:
5009 case Builtin::BI__builtin_uaddl_overflow:
5010 case Builtin::BI__builtin_uaddll_overflow:
5011 case Builtin::BI__builtin_usub_overflow:
5012 case Builtin::BI__builtin_usubl_overflow:
5013 case Builtin::BI__builtin_usubll_overflow:
5014 case Builtin::BI__builtin_umul_overflow:
5015 case Builtin::BI__builtin_umull_overflow:
5016 case Builtin::BI__builtin_umulll_overflow:
5017 case Builtin::BI__builtin_sadd_overflow:
5018 case Builtin::BI__builtin_saddl_overflow:
5019 case Builtin::BI__builtin_saddll_overflow:
5020 case Builtin::BI__builtin_ssub_overflow:
5021 case Builtin::BI__builtin_ssubl_overflow:
5022 case Builtin::BI__builtin_ssubll_overflow:
5023 case Builtin::BI__builtin_smul_overflow:
5024 case Builtin::BI__builtin_smull_overflow:
5025 case Builtin::BI__builtin_smulll_overflow: {
5035 llvm::Intrinsic::ID IntrinsicId;
5036 switch (BuiltinID) {
5037 default: llvm_unreachable(
"Unknown overflow builtin id.");
5038 case Builtin::BI__builtin_uadd_overflow:
5039 case Builtin::BI__builtin_uaddl_overflow:
5040 case Builtin::BI__builtin_uaddll_overflow:
5041 IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
5043 case Builtin::BI__builtin_usub_overflow:
5044 case Builtin::BI__builtin_usubl_overflow:
5045 case Builtin::BI__builtin_usubll_overflow:
5046 IntrinsicId = llvm::Intrinsic::usub_with_overflow;
5048 case Builtin::BI__builtin_umul_overflow:
5049 case Builtin::BI__builtin_umull_overflow:
5050 case Builtin::BI__builtin_umulll_overflow:
5051 IntrinsicId = llvm::Intrinsic::umul_with_overflow;
5053 case Builtin::BI__builtin_sadd_overflow:
5054 case Builtin::BI__builtin_saddl_overflow:
5055 case Builtin::BI__builtin_saddll_overflow:
5056 IntrinsicId = llvm::Intrinsic::sadd_with_overflow;
5058 case Builtin::BI__builtin_ssub_overflow:
5059 case Builtin::BI__builtin_ssubl_overflow:
5060 case Builtin::BI__builtin_ssubll_overflow:
5061 IntrinsicId = llvm::Intrinsic::ssub_with_overflow;
5063 case Builtin::BI__builtin_smul_overflow:
5064 case Builtin::BI__builtin_smull_overflow:
5065 case Builtin::BI__builtin_smulll_overflow:
5066 IntrinsicId = llvm::Intrinsic::smul_with_overflow;
5077 case Builtin::BIaddressof:
5078 case Builtin::BI__addressof:
5079 case Builtin::BI__builtin_addressof:
5081 case Builtin::BI__builtin_function_start:
5084 case Builtin::BI__builtin_operator_new:
5087 case Builtin::BI__builtin_operator_delete:
5092 case Builtin::BI__builtin_is_aligned:
5094 case Builtin::BI__builtin_align_up:
5096 case Builtin::BI__builtin_align_down:
5099 case Builtin::BI__noop:
5102 case Builtin::BI__builtin_call_with_static_chain: {
5109 case Builtin::BI_InterlockedExchange8:
5110 case Builtin::BI_InterlockedExchange16:
5111 case Builtin::BI_InterlockedExchange:
5112 case Builtin::BI_InterlockedExchangePointer:
5115 case Builtin::BI_InterlockedCompareExchangePointer:
5116 case Builtin::BI_InterlockedCompareExchangePointer_nf: {
5118 llvm::IntegerType *IntType = IntegerType::get(
5124 RTy = Exchange->getType();
5125 Exchange =
Builder.CreatePtrToInt(Exchange, IntType);
5127 llvm::Value *Comparand =
5131 BuiltinID == Builtin::BI_InterlockedCompareExchangePointer_nf ?
5132 AtomicOrdering::Monotonic : AtomicOrdering::SequentiallyConsistent;
5135 Ordering, Ordering);
5136 Result->setVolatile(
true);
5142 case Builtin::BI_InterlockedCompareExchange8:
5143 case Builtin::BI_InterlockedCompareExchange16:
5144 case Builtin::BI_InterlockedCompareExchange:
5145 case Builtin::BI_InterlockedCompareExchange64:
5147 case Builtin::BI_InterlockedIncrement16:
5148 case Builtin::BI_InterlockedIncrement:
5151 case Builtin::BI_InterlockedDecrement16:
5152 case Builtin::BI_InterlockedDecrement:
5155 case Builtin::BI_InterlockedAnd8:
5156 case Builtin::BI_InterlockedAnd16:
5157 case Builtin::BI_InterlockedAnd:
5159 case Builtin::BI_InterlockedExchangeAdd8:
5160 case Builtin::BI_InterlockedExchangeAdd16:
5161 case Builtin::BI_InterlockedExchangeAdd:
5164 case Builtin::BI_InterlockedExchangeSub8:
5165 case Builtin::BI_InterlockedExchangeSub16:
5166 case Builtin::BI_InterlockedExchangeSub:
5169 case Builtin::BI_InterlockedOr8:
5170 case Builtin::BI_InterlockedOr16:
5171 case Builtin::BI_InterlockedOr:
5173 case Builtin::BI_InterlockedXor8:
5174 case Builtin::BI_InterlockedXor16:
5175 case Builtin::BI_InterlockedXor:
5178 case Builtin::BI_bittest64:
5179 case Builtin::BI_bittest:
5180 case Builtin::BI_bittestandcomplement64:
5181 case Builtin::BI_bittestandcomplement:
5182 case Builtin::BI_bittestandreset64:
5183 case Builtin::BI_bittestandreset:
5184 case Builtin::BI_bittestandset64:
5185 case Builtin::BI_bittestandset:
5186 case Builtin::BI_interlockedbittestandreset:
5187 case Builtin::BI_interlockedbittestandreset64:
5188 case Builtin::BI_interlockedbittestandset64:
5189 case Builtin::BI_interlockedbittestandset:
5190 case Builtin::BI_interlockedbittestandset_acq:
5191 case Builtin::BI_interlockedbittestandset_rel:
5192 case Builtin::BI_interlockedbittestandset_nf:
5193 case Builtin::BI_interlockedbittestandreset_acq:
5194 case Builtin::BI_interlockedbittestandreset_rel:
5195 case Builtin::BI_interlockedbittestandreset_nf:
5200 case Builtin::BI__iso_volatile_load8:
5201 case Builtin::BI__iso_volatile_load16:
5202 case Builtin::BI__iso_volatile_load32:
5203 case Builtin::BI__iso_volatile_load64:
5205 case Builtin::BI__iso_volatile_store8:
5206 case Builtin::BI__iso_volatile_store16:
5207 case Builtin::BI__iso_volatile_store32:
5208 case Builtin::BI__iso_volatile_store64:
5211 case Builtin::BI__builtin_ptrauth_auth:
5212 case Builtin::BI__builtin_ptrauth_auth_and_resign:
5213 case Builtin::BI__builtin_ptrauth_blend_discriminator:
5214 case Builtin::BI__builtin_ptrauth_sign_generic_data:
5215 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
5216 case Builtin::BI__builtin_ptrauth_strip: {
5223 llvm::Type *OrigValueType = Args[0]->getType();
5224 if (OrigValueType->isPointerTy())
5227 switch (BuiltinID) {
5228 case Builtin::BI__builtin_ptrauth_auth_and_resign:
5229 if (Args[4]->getType()->isPointerTy())
5233 case Builtin::BI__builtin_ptrauth_auth:
5234 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
5235 if (Args[2]->getType()->isPointerTy())
5239 case Builtin::BI__builtin_ptrauth_sign_generic_data:
5240 if (Args[1]->getType()->isPointerTy())
5244 case Builtin::BI__builtin_ptrauth_blend_discriminator:
5245 case Builtin::BI__builtin_ptrauth_strip:
5250 auto IntrinsicID = [&]() ->
unsigned {
5251 switch (BuiltinID) {
5252 case Builtin::BI__builtin_ptrauth_auth:
5253 return llvm::Intrinsic::ptrauth_auth;
5254 case Builtin::BI__builtin_ptrauth_auth_and_resign:
5255 return llvm::Intrinsic::ptrauth_resign;
5256 case Builtin::BI__builtin_ptrauth_blend_discriminator:
5257 return llvm::Intrinsic::ptrauth_blend;
5258 case Builtin::BI__builtin_ptrauth_sign_generic_data:
5259 return llvm::Intrinsic::ptrauth_sign_generic;
5260 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
5261 return llvm::Intrinsic::ptrauth_sign;
5262 case Builtin::BI__builtin_ptrauth_strip:
5263 return llvm::Intrinsic::ptrauth_strip;
5265 llvm_unreachable(
"bad ptrauth intrinsic");
5270 if (BuiltinID != Builtin::BI__builtin_ptrauth_sign_generic_data &&
5271 BuiltinID != Builtin::BI__builtin_ptrauth_blend_discriminator &&
5272 OrigValueType->isPointerTy()) {
5278 case Builtin::BI__exception_code:
5279 case Builtin::BI_exception_code:
5281 case Builtin::BI__exception_info:
5282 case Builtin::BI_exception_info:
5284 case Builtin::BI__abnormal_termination:
5285 case Builtin::BI_abnormal_termination:
5287 case Builtin::BI_setjmpex:
5292 case Builtin::BI_setjmp:
5295 if (
getTarget().getTriple().getArch() == llvm::Triple::x86)
5297 else if (
getTarget().getTriple().getArch() == llvm::Triple::aarch64)
5304 case Builtin::BImove:
5305 case Builtin::BImove_if_noexcept:
5306 case Builtin::BIforward:
5307 case Builtin::BIforward_like:
5308 case Builtin::BIas_const:
5310 case Builtin::BI__GetExceptionInfo: {
5311 if (llvm::GlobalVariable *GV =
5317 case Builtin::BI__fastfail:
5320 case Builtin::BI__builtin_coro_id:
5322 case Builtin::BI__builtin_coro_promise:
5324 case Builtin::BI__builtin_coro_resume:
5327 case Builtin::BI__builtin_coro_frame:
5329 case Builtin::BI__builtin_coro_noop:
5331 case Builtin::BI__builtin_coro_free:
5333 case Builtin::BI__builtin_coro_destroy:
5336 case Builtin::BI__builtin_coro_done:
5338 case Builtin::BI__builtin_coro_alloc:
5340 case Builtin::BI__builtin_coro_begin:
5342 case Builtin::BI__builtin_coro_end:
5344 case Builtin::BI__builtin_coro_suspend:
5346 case Builtin::BI__builtin_coro_size:
5348 case Builtin::BI__builtin_coro_align:
5352 case Builtin::BIread_pipe:
5353 case Builtin::BIwrite_pipe: {
5357 Value *PacketSize = OpenCLRT.getPipeElemSize(E->
getArg(0));
5358 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->
getArg(0));
5361 unsigned GenericAS =
5363 llvm::Type *I8PTy = llvm::PointerType::get(
getLLVMContext(), GenericAS);
5367 const char *Name = (BuiltinID == Builtin::BIread_pipe) ?
"__read_pipe_2"
5372 llvm::FunctionType *FTy = llvm::FunctionType::get(
5377 {Arg0, BCast, PacketSize, PacketAlign}));
5380 "Illegal number of parameters to pipe function");
5381 const char *Name = (BuiltinID == Builtin::BIread_pipe) ?
"__read_pipe_4"
5388 llvm::FunctionType *FTy = llvm::FunctionType::get(
5397 {Arg0, Arg1, Arg2, BCast, PacketSize, PacketAlign}));
5402 case Builtin::BIreserve_read_pipe:
5403 case Builtin::BIreserve_write_pipe:
5404 case Builtin::BIwork_group_reserve_read_pipe:
5405 case Builtin::BIwork_group_reserve_write_pipe:
5406 case Builtin::BIsub_group_reserve_read_pipe:
5407 case Builtin::BIsub_group_reserve_write_pipe: {
5410 if (BuiltinID == Builtin::BIreserve_read_pipe)
5411 Name =
"__reserve_read_pipe";
5412 else if (BuiltinID == Builtin::BIreserve_write_pipe)
5413 Name =
"__reserve_write_pipe";
5414 else if (BuiltinID == Builtin::BIwork_group_reserve_read_pipe)
5415 Name =
"__work_group_reserve_read_pipe";
5416 else if (BuiltinID == Builtin::BIwork_group_reserve_write_pipe)
5417 Name =
"__work_group_reserve_write_pipe";
5418 else if (BuiltinID == Builtin::BIsub_group_reserve_read_pipe)
5419 Name =
"__sub_group_reserve_read_pipe";
5421 Name =
"__sub_group_reserve_write_pipe";
5427 Value *PacketSize = OpenCLRT.getPipeElemSize(E->
getArg(0));
5428 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->
getArg(0));
5432 llvm::FunctionType *FTy = llvm::FunctionType::get(
5439 {Arg0, Arg1, PacketSize, PacketAlign}));
5443 case Builtin::BIcommit_read_pipe:
5444 case Builtin::BIcommit_write_pipe:
5445 case Builtin::BIwork_group_commit_read_pipe:
5446 case Builtin::BIwork_group_commit_write_pipe:
5447 case Builtin::BIsub_group_commit_read_pipe:
5448 case Builtin::BIsub_group_commit_write_pipe: {
5450 if (BuiltinID == Builtin::BIcommit_read_pipe)
5451 Name =
"__commit_read_pipe";
5452 else if (BuiltinID == Builtin::BIcommit_write_pipe)
5453 Name =
"__commit_write_pipe";
5454 else if (BuiltinID == Builtin::BIwork_group_commit_read_pipe)
5455 Name =
"__work_group_commit_read_pipe";
5456 else if (BuiltinID == Builtin::BIwork_group_commit_write_pipe)
5457 Name =
"__work_group_commit_write_pipe";
5458 else if (BuiltinID == Builtin::BIsub_group_commit_read_pipe)
5459 Name =
"__sub_group_commit_read_pipe";
5461 Name =
"__sub_group_commit_write_pipe";
5466 Value *PacketSize = OpenCLRT.getPipeElemSize(E->
getArg(0));
5467 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->
getArg(0));
5471 llvm::FunctionType *FTy =
5476 {Arg0, Arg1, PacketSize, PacketAlign}));
5479 case Builtin::BIget_pipe_num_packets:
5480 case Builtin::BIget_pipe_max_packets: {
5481 const char *BaseName;
5483 if (BuiltinID == Builtin::BIget_pipe_num_packets)
5484 BaseName =
"__get_pipe_num_packets";
5486 BaseName =
"__get_pipe_max_packets";
5487 std::string Name = std::string(BaseName) +
5488 std::string(PipeTy->isReadOnly() ?
"_ro" :
"_wo");
5493 Value *PacketSize = OpenCLRT.getPipeElemSize(E->
getArg(0));
5494 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->
getArg(0));
5496 llvm::FunctionType *FTy = llvm::FunctionType::get(
5500 {Arg0, PacketSize, PacketAlign}));
5504 case Builtin::BIto_global:
5505 case Builtin::BIto_local:
5506 case Builtin::BIto_private: {
5508 auto NewArgT = llvm::PointerType::get(
5511 auto NewRetT = llvm::PointerType::get(
5515 auto FTy = llvm::FunctionType::get(NewRetT, {NewArgT},
false);
5516 llvm::Value *NewArg;
5517 if (Arg0->
getType()->getPointerAddressSpace() !=
5518 NewArgT->getPointerAddressSpace())
5521 NewArg =
Builder.CreateBitOrPointerCast(Arg0, NewArgT);
5537 case Builtin::BIenqueue_kernel: {
5542 llvm::Type *GenericVoidPtrTy =
Builder.getPtrTy(
5554 Name =
"__enqueue_kernel_basic";
5555 llvm::Type *ArgTys[] = {QueueTy,
Int32Ty, RangeTy, GenericVoidPtrTy,
5557 llvm::FunctionType *FTy = llvm::FunctionType::get(
5563 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5564 llvm::Value *
Block =
5565 Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5567 AttrBuilder B(
Builder.getContext());
5569 llvm::AttributeList ByValAttrSet =
5570 llvm::AttributeList::get(
CGM.
getModule().getContext(), 3U, B);
5574 {Queue, Flags, Range, Kernel, Block});
5575 RTCall->setAttributes(ByValAttrSet);
5578 assert(NumArgs >= 5 &&
"Invalid enqueue_kernel signature");
5582 auto CreateArrayForSizeVar = [=](
unsigned First)
5583 -> std::tuple<llvm::Value *, llvm::Value *, llvm::Value *> {
5584 llvm::APInt ArraySize(32, NumArgs -
First);
5586 getContext().getSizeType(), ArraySize,
nullptr,
5590 llvm::Value *TmpPtr = Tmp.getPointer();
5593 llvm::Value *ElemPtr;
5596 auto *
Zero = llvm::ConstantInt::get(
IntTy, 0);
5597 for (
unsigned I =
First; I < NumArgs; ++I) {
5598 auto *Index = llvm::ConstantInt::get(
IntTy, I -
First);
5608 return std::tie(ElemPtr, TmpSize, TmpPtr);
5614 Name =
"__enqueue_kernel_varargs";
5618 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5619 auto *
Block =
Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5620 llvm::Value *ElemPtr, *TmpSize, *TmpPtr;
5621 std::tie(ElemPtr, TmpSize, TmpPtr) = CreateArrayForSizeVar(4);
5625 llvm::Value *
const Args[] = {Queue, Flags,
5629 llvm::Type *
const ArgTys[] = {
5630 QueueTy,
IntTy, RangeTy, GenericVoidPtrTy,
5631 GenericVoidPtrTy,
IntTy, ElemPtr->getType()};
5633 llvm::FunctionType *FTy = llvm::FunctionType::get(
Int32Ty, ArgTys,
false);
5642 llvm::PointerType *PtrTy = llvm::PointerType::get(
5646 llvm::Value *NumEvents =
5652 llvm::Value *EventWaitList =
nullptr;
5655 EventWaitList = llvm::ConstantPointerNull::get(PtrTy);
5661 EventWaitList =
Builder.CreatePointerCast(EventWaitList, PtrTy);
5663 llvm::Value *EventRet =
nullptr;
5666 EventRet = llvm::ConstantPointerNull::get(PtrTy);
5675 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5676 llvm::Value *
Block =
5677 Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5679 std::vector<llvm::Type *> ArgTys = {
5681 PtrTy, PtrTy, GenericVoidPtrTy, GenericVoidPtrTy};
5683 std::vector<llvm::Value *> Args = {Queue, Flags,
Range,
5684 NumEvents, EventWaitList, EventRet,
5689 Name =
"__enqueue_kernel_basic_events";
5690 llvm::FunctionType *FTy = llvm::FunctionType::get(
5698 Args.push_back(ConstantInt::get(
Int32Ty, NumArgs - 7));
5700 Name =
"__enqueue_kernel_events_varargs";
5702 llvm::Value *ElemPtr, *TmpSize, *TmpPtr;
5703 std::tie(ElemPtr, TmpSize, TmpPtr) = CreateArrayForSizeVar(7);
5704 Args.push_back(ElemPtr);
5705 ArgTys.push_back(ElemPtr->getType());
5707 llvm::FunctionType *FTy = llvm::FunctionType::get(
5720 case Builtin::BIget_kernel_work_group_size: {
5721 llvm::Type *GenericVoidPtrTy =
Builder.getPtrTy(
5726 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5727 Value *Arg =
Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5730 llvm::FunctionType::get(
IntTy, {GenericVoidPtrTy, GenericVoidPtrTy},
5732 "__get_kernel_work_group_size_impl"),
5735 case Builtin::BIget_kernel_preferred_work_group_size_multiple: {
5736 llvm::Type *GenericVoidPtrTy =
Builder.getPtrTy(
5741 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5742 Value *Arg =
Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5745 llvm::FunctionType::get(
IntTy, {GenericVoidPtrTy, GenericVoidPtrTy},
5747 "__get_kernel_preferred_work_group_size_multiple_impl"),
5750 case Builtin::BIget_kernel_max_sub_group_size_for_ndrange:
5751 case Builtin::BIget_kernel_sub_group_count_for_ndrange: {
5752 llvm::Type *GenericVoidPtrTy =
Builder.getPtrTy(
5759 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5762 BuiltinID == Builtin::BIget_kernel_max_sub_group_size_for_ndrange
5763 ?
"__get_kernel_max_sub_group_size_for_ndrange_impl"
5764 :
"__get_kernel_sub_group_count_for_ndrange_impl";
5767 llvm::FunctionType::get(
5768 IntTy, {NDRange->getType(), GenericVoidPtrTy, GenericVoidPtrTy},
5771 {NDRange, Kernel, Block}));
5774 case Builtin::BI__builtin_store_half:
5775 case Builtin::BI__builtin_store_halff: {
5782 case Builtin::BI__builtin_load_half: {
5787 case Builtin::BI__builtin_load_halff: {
5792 case Builtin::BI__builtin_printf:
5793 case Builtin::BIprintf:
5794 if (
getTarget().getTriple().isNVPTX() ||
5805 case Builtin::BI__builtin_canonicalize:
5806 case Builtin::BI__builtin_canonicalizef:
5807 case Builtin::BI__builtin_canonicalizef16:
5808 case Builtin::BI__builtin_canonicalizel:
5811 case Builtin::BI__builtin_thread_pointer: {
5812 if (!
getContext().getTargetInfo().isTLSSupported())
5817 case Builtin::BI__builtin_os_log_format:
5820 case Builtin::BI__xray_customevent: {
5833 auto FTy = F->getFunctionType();
5834 auto Arg0 = E->
getArg(0);
5836 auto Arg0Ty = Arg0->
getType();
5837 auto PTy0 = FTy->getParamType(0);
5838 if (PTy0 != Arg0Val->getType()) {
5839 if (Arg0Ty->isArrayType())
5842 Arg0Val =
Builder.CreatePointerCast(Arg0Val, PTy0);
5845 auto PTy1 = FTy->getParamType(1);
5847 Arg1 =
Builder.CreateTruncOrBitCast(Arg1, PTy1);
5851 case Builtin::BI__xray_typedevent: {
5867 auto FTy = F->getFunctionType();
5869 auto PTy0 = FTy->getParamType(0);
5871 Arg0 =
Builder.CreateTruncOrBitCast(Arg0, PTy0);
5872 auto Arg1 = E->
getArg(1);
5874 auto Arg1Ty = Arg1->
getType();
5875 auto PTy1 = FTy->getParamType(1);
5876 if (PTy1 != Arg1Val->getType()) {
5877 if (Arg1Ty->isArrayType())
5880 Arg1Val =
Builder.CreatePointerCast(Arg1Val, PTy1);
5883 auto PTy2 = FTy->getParamType(2);
5885 Arg2 =
Builder.CreateTruncOrBitCast(Arg2, PTy2);
5889 case Builtin::BI__builtin_ms_va_start:
5890 case Builtin::BI__builtin_ms_va_end:
5893 BuiltinID == Builtin::BI__builtin_ms_va_start));
5895 case Builtin::BI__builtin_ms_va_copy: {
5912 case Builtin::BI__builtin_get_device_side_mangled_name: {
5916 llvm::Constant *Zeros[] = {llvm::ConstantInt::get(
SizeTy, 0),
5917 llvm::ConstantInt::get(
SizeTy, 0)};
5918 auto *Ptr = llvm::ConstantExpr::getGetElementPtr(Str.getElementType(),
5919 Str.getPointer(), Zeros);
5945 LargestVectorWidth = std::max(LargestVectorWidth, VectorWidth);
5949 Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic;
5951 llvm::Triple::getArchTypePrefix(
getTarget().getTriple().getArch());
5952 if (!Prefix.empty()) {
5953 IntrinsicID = Intrinsic::getIntrinsicForClangBuiltin(Prefix.data(), Name);
5957 if (IntrinsicID == Intrinsic::not_intrinsic)
5958 IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(Prefix.data(), Name);
5961 if (IntrinsicID != Intrinsic::not_intrinsic) {
5966 unsigned ICEArguments = 0;
5972 llvm::FunctionType *FTy = F->getFunctionType();
5974 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; ++i) {
5978 llvm::Type *PTy = FTy->getParamType(i);
5979 if (PTy != ArgValue->
getType()) {
5981 if (
auto *PtrTy = dyn_cast<llvm::PointerType>(PTy)) {
5982 if (PtrTy->getAddressSpace() !=
5983 ArgValue->
getType()->getPointerAddressSpace()) {
5986 PtrTy->getAddressSpace()));
5992 if (PTy->isX86_AMXTy())
5993 ArgValue =
Builder.CreateIntrinsic(Intrinsic::x86_cast_vector_to_tile,
5994 {ArgValue->
getType()}, {ArgValue});
5996 ArgValue =
Builder.CreateBitCast(ArgValue, PTy);
5999 Args.push_back(ArgValue);
6005 llvm::Type *RetTy =
VoidTy;
6009 if (RetTy !=
V->getType()) {
6011 if (
auto *PtrTy = dyn_cast<llvm::PointerType>(RetTy)) {
6012 if (PtrTy->getAddressSpace() !=
V->getType()->getPointerAddressSpace()) {
6015 PtrTy->getAddressSpace()));
6021 if (
V->getType()->isX86_AMXTy())
6022 V =
Builder.CreateIntrinsic(Intrinsic::x86_cast_tile_to_vector, {RetTy},
6028 if (RetTy->isVoidTy())
6048 if (
V->getType()->isVoidTy())
6055 llvm_unreachable(
"No current target builtin returns complex");
6057 llvm_unreachable(
"Bad evaluation kind in EmitBuiltinExpr");
6074 unsigned BuiltinID,
const CallExpr *E,
6076 llvm::Triple::ArchType Arch) {
6088 case llvm::Triple::arm:
6089 case llvm::Triple::armeb:
6090 case llvm::Triple::thumb:
6091 case llvm::Triple::thumbeb:
6093 case llvm::Triple::aarch64:
6094 case llvm::Triple::aarch64_32:
6095 case llvm::Triple::aarch64_be:
6097 case llvm::Triple::bpfeb:
6098 case llvm::Triple::bpfel:
6100 case llvm::Triple::x86:
6101 case llvm::Triple::x86_64:
6103 case llvm::Triple::ppc:
6104 case llvm::Triple::ppcle:
6105 case llvm::Triple::ppc64:
6106 case llvm::Triple::ppc64le:
6108 case llvm::Triple::r600:
6109 case llvm::Triple::amdgcn:
6111 case llvm::Triple::systemz:
6113 case llvm::Triple::nvptx:
6114 case llvm::Triple::nvptx64:
6116 case llvm::Triple::wasm32:
6117 case llvm::Triple::wasm64:
6119 case llvm::Triple::hexagon:
6121 case llvm::Triple::riscv32:
6122 case llvm::Triple::riscv64:
6133 assert(
getContext().getAuxTargetInfo() &&
"Missing aux target info");
6145 bool HasLegalHalfType =
true,
6147 bool AllowBFloatArgsAndRet =
true) {
6148 int IsQuad = TypeFlags.
isQuad();
6152 return llvm::FixedVectorType::get(CGF->
Int8Ty, V1Ty ? 1 : (8 << IsQuad));
6155 return llvm::FixedVectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
6157 if (AllowBFloatArgsAndRet)
6158 return llvm::FixedVectorType::get(CGF->
BFloatTy, V1Ty ? 1 : (4 << IsQuad));
6160 return llvm::FixedVectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
6162 if (HasLegalHalfType)
6163 return llvm::FixedVectorType::get(CGF->
HalfTy, V1Ty ? 1 : (4 << IsQuad));
6165 return llvm::FixedVectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
6167 return llvm::FixedVectorType::get(CGF->
Int32Ty, V1Ty ? 1 : (2 << IsQuad));
6170 return llvm::FixedVectorType::get(CGF->
Int64Ty, V1Ty ? 1 : (1 << IsQuad));
6175 return llvm::FixedVectorType::get(CGF->
Int8Ty, 16);
6177 return llvm::FixedVectorType::get(CGF->
FloatTy, V1Ty ? 1 : (2 << IsQuad));
6179 return llvm::FixedVectorType::get(CGF->
DoubleTy, V1Ty ? 1 : (1 << IsQuad));
6181 llvm_unreachable(
"Unknown vector element type!");
6186 int IsQuad = IntTypeFlags.
isQuad();
6189 return llvm::FixedVectorType::get(CGF->
HalfTy, (4 << IsQuad));
6191 return llvm::FixedVectorType::get(CGF->
FloatTy, (2 << IsQuad));
6193 return llvm::FixedVectorType::get(CGF->
DoubleTy, (1 << IsQuad));
6195 llvm_unreachable(
"Type can't be converted to floating-point!");
6200 const ElementCount &Count) {
6201 Value *SV = llvm::ConstantVector::getSplat(Count,
C);
6202 return Builder.CreateShuffleVector(
V,
V, SV,
"lane");
6206 ElementCount EC = cast<llvm::VectorType>(
V->getType())->getElementCount();
6212 unsigned shift,
bool rightshift) {
6214 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
6215 ai != ae; ++ai, ++j) {
6216 if (F->isConstrainedFPIntrinsic())
6217 if (ai->getType()->isMetadataTy())
6219 if (shift > 0 && shift == j)
6222 Ops[j] =
Builder.CreateBitCast(Ops[j], ai->getType(), name);
6225 if (F->isConstrainedFPIntrinsic())
6226 return Builder.CreateConstrainedFPCall(F, Ops, name);
6228 return Builder.CreateCall(F, Ops, name);
6233 int SV = cast<ConstantInt>(
V)->getSExtValue();
6234 return ConstantInt::get(Ty, neg ? -SV : SV);
6239 llvm::Type *Ty,
bool usgn,
6241 llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
6243 int ShiftAmt = cast<ConstantInt>(Shift)->getSExtValue();
6244 int EltSize = VTy->getScalarSizeInBits();
6246 Vec =
Builder.CreateBitCast(Vec, Ty);
6250 if (ShiftAmt == EltSize) {
6253 return llvm::ConstantAggregateZero::get(VTy);
6258 Shift = ConstantInt::get(VTy->getElementType(), ShiftAmt);
6264 return Builder.CreateLShr(Vec, Shift, name);
6266 return Builder.CreateAShr(Vec, Shift, name);
6292struct ARMVectorIntrinsicInfo {
6293 const char *NameHint;
6295 unsigned LLVMIntrinsic;
6296 unsigned AltLLVMIntrinsic;
6299 bool operator<(
unsigned RHSBuiltinID)
const {
6300 return BuiltinID < RHSBuiltinID;
6302 bool operator<(
const ARMVectorIntrinsicInfo &TE)
const {
6303 return BuiltinID < TE.BuiltinID;
6308#define NEONMAP0(NameBase) \
6309 { #NameBase, NEON::BI__builtin_neon_ ## NameBase, 0, 0, 0 }
6311#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
6312 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
6313 Intrinsic::LLVMIntrinsic, 0, TypeModifier }
6315#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier) \
6316 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
6317 Intrinsic::LLVMIntrinsic, Intrinsic::AltLLVMIntrinsic, \
6321 NEONMAP1(__a32_vcvt_bf16_f32, arm_neon_vcvtfp2bf, 0),
6328 NEONMAP1(vabs_v, arm_neon_vabs, 0),
6329 NEONMAP1(vabsq_v, arm_neon_vabs, 0),
6333 NEONMAP1(vaesdq_u8, arm_neon_aesd, 0),
6334 NEONMAP1(vaeseq_u8, arm_neon_aese, 0),
6335 NEONMAP1(vaesimcq_u8, arm_neon_aesimc, 0),
6336 NEONMAP1(vaesmcq_u8, arm_neon_aesmc, 0),
6337 NEONMAP1(vbfdot_f32, arm_neon_bfdot, 0),
6338 NEONMAP1(vbfdotq_f32, arm_neon_bfdot, 0),
6339 NEONMAP1(vbfmlalbq_f32, arm_neon_bfmlalb, 0),
6340 NEONMAP1(vbfmlaltq_f32, arm_neon_bfmlalt, 0),
6341 NEONMAP1(vbfmmlaq_f32, arm_neon_bfmmla, 0),
6354 NEONMAP1(vcage_v, arm_neon_vacge, 0),
6355 NEONMAP1(vcageq_v, arm_neon_vacge, 0),
6356 NEONMAP1(vcagt_v, arm_neon_vacgt, 0),
6357 NEONMAP1(vcagtq_v, arm_neon_vacgt, 0),
6358 NEONMAP1(vcale_v, arm_neon_vacge, 0),
6359 NEONMAP1(vcaleq_v, arm_neon_vacge, 0),
6360 NEONMAP1(vcalt_v, arm_neon_vacgt, 0),
6361 NEONMAP1(vcaltq_v, arm_neon_vacgt, 0),
6378 NEONMAP1(vcvt_f16_f32, arm_neon_vcvtfp2hf, 0),
6381 NEONMAP1(vcvt_f32_f16, arm_neon_vcvthf2fp, 0),
6383 NEONMAP1(vcvt_n_f16_s16, arm_neon_vcvtfxs2fp, 0),
6384 NEONMAP1(vcvt_n_f16_u16, arm_neon_vcvtfxu2fp, 0),
6385 NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
6386 NEONMAP1(vcvt_n_s16_f16, arm_neon_vcvtfp2fxs, 0),
6387 NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0),
6388 NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0),
6389 NEONMAP1(vcvt_n_u16_f16, arm_neon_vcvtfp2fxu, 0),
6390 NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0),
6391 NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0),
6398 NEONMAP1(vcvta_s16_f16, arm_neon_vcvtas, 0),
6399 NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0),
6400 NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0),
6401 NEONMAP1(vcvta_u16_f16, arm_neon_vcvtau, 0),
6402 NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0),
6403 NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0),
6404 NEONMAP1(vcvtaq_s16_f16, arm_neon_vcvtas, 0),
6405 NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0),
6406 NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0),
6407 NEONMAP1(vcvtaq_u16_f16, arm_neon_vcvtau, 0),
6408 NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0),
6409 NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0),
6410 NEONMAP1(vcvth_bf16_f32, arm_neon_vcvtbfp2bf, 0),
6411 NEONMAP1(vcvtm_s16_f16, arm_neon_vcvtms, 0),
6412 NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0),
6413 NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0),
6414 NEONMAP1(vcvtm_u16_f16, arm_neon_vcvtmu, 0),
6415 NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0),
6416 NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0),
6417 NEONMAP1(vcvtmq_s16_f16, arm_neon_vcvtms, 0),
6418 NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0),
6419 NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0),
6420 NEONMAP1(vcvtmq_u16_f16, arm_neon_vcvtmu, 0),
6421 NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0),
6422 NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0),
6423 NEONMAP1(vcvtn_s16_f16, arm_neon_vcvtns, 0),
6424 NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0),
6425 NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0),
6426 NEONMAP1(vcvtn_u16_f16, arm_neon_vcvtnu, 0),
6427 NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0),
6428 NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0),
6429 NEONMAP1(vcvtnq_s16_f16, arm_neon_vcvtns, 0),
6430 NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0),
6431 NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0),
6432 NEONMAP1(vcvtnq_u16_f16, arm_neon_vcvtnu, 0),
6433 NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0),
6434 NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0),
6435 NEONMAP1(vcvtp_s16_f16, arm_neon_vcvtps, 0),
6436 NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0),
6437 NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0),
6438 NEONMAP1(vcvtp_u16_f16, arm_neon_vcvtpu, 0),
6439 NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0),
6440 NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0),
6441 NEONMAP1(vcvtpq_s16_f16, arm_neon_vcvtps, 0),
6442 NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0),
6443 NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0),
6444 NEONMAP1(vcvtpq_u16_f16, arm_neon_vcvtpu, 0),
6445 NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0),
6446 NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0),
6450 NEONMAP1(vcvtq_n_f16_s16, arm_neon_vcvtfxs2fp, 0),
6451 NEONMAP1(vcvtq_n_f16_u16, arm_neon_vcvtfxu2fp, 0),
6452 NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
6453 NEONMAP1(vcvtq_n_s16_f16, arm_neon_vcvtfp2fxs, 0),
6454 NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0),
6455 NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0),
6456 NEONMAP1(vcvtq_n_u16_f16, arm_neon_vcvtfp2fxu, 0),
6457 NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0),
6458 NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0),
6465 NEONMAP1(vdot_s32, arm_neon_sdot, 0),
6466 NEONMAP1(vdot_u32, arm_neon_udot, 0),
6467 NEONMAP1(vdotq_s32, arm_neon_sdot, 0),
6468 NEONMAP1(vdotq_u32, arm_neon_udot, 0),
6478 NEONMAP1(vld1_v, arm_neon_vld1, 0),
6479 NEONMAP1(vld1_x2_v, arm_neon_vld1x2, 0),
6480 NEONMAP1(vld1_x3_v, arm_neon_vld1x3, 0),
6481 NEONMAP1(vld1_x4_v, arm_neon_vld1x4, 0),
6483 NEONMAP1(vld1q_v, arm_neon_vld1, 0),
6484 NEONMAP1(vld1q_x2_v, arm_neon_vld1x2, 0),
6485 NEONMAP1(vld1q_x3_v, arm_neon_vld1x3, 0),
6486 NEONMAP1(vld1q_x4_v, arm_neon_vld1x4, 0),
6487 NEONMAP1(vld2_dup_v, arm_neon_vld2dup, 0),
6488 NEONMAP1(vld2_lane_v, arm_neon_vld2lane, 0),
6489 NEONMAP1(vld2_v, arm_neon_vld2, 0),
6490 NEONMAP1(vld2q_dup_v, arm_neon_vld2dup, 0),
6491 NEONMAP1(vld2q_lane_v, arm_neon_vld2lane, 0),
6492 NEONMAP1(vld2q_v, arm_neon_vld2, 0),
6493 NEONMAP1(vld3_dup_v, arm_neon_vld3dup, 0),
6494 NEONMAP1(vld3_lane_v, arm_neon_vld3lane, 0),
6495 NEONMAP1(vld3_v, arm_neon_vld3, 0),
6496 NEONMAP1(vld3q_dup_v, arm_neon_vld3dup, 0),
6497 NEONMAP1(vld3q_lane_v, arm_neon_vld3lane, 0),
6498 NEONMAP1(vld3q_v, arm_neon_vld3, 0),
6499 NEONMAP1(vld4_dup_v, arm_neon_vld4dup, 0),
6500 NEONMAP1(vld4_lane_v, arm_neon_vld4lane, 0),
6501 NEONMAP1(vld4_v, arm_neon_vld4, 0),
6502 NEONMAP1(vld4q_dup_v, arm_neon_vld4dup, 0),
6503 NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0),
6504 NEONMAP1(vld4q_v, arm_neon_vld4, 0),
6513 NEONMAP1(vmmlaq_s32, arm_neon_smmla, 0),
6514 NEONMAP1(vmmlaq_u32, arm_neon_ummla, 0),
6532 NEONMAP2(vqdmlal_v, arm_neon_vqdmull, sadd_sat, 0),
6533 NEONMAP2(vqdmlsl_v, arm_neon_vqdmull, ssub_sat, 0),
6557 NEONMAP1(vqshlu_n_v, arm_neon_vqshiftsu, 0),
6558 NEONMAP1(vqshluq_n_v, arm_neon_vqshiftsu, 0),
6562 NEONMAP2(vrecpe_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
6563 NEONMAP2(vrecpeq_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
6586 NEONMAP2(vrsqrte_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
6587 NEONMAP2(vrsqrteq_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
6591 NEONMAP1(vsha1su0q_u32, arm_neon_sha1su0, 0),
6592 NEONMAP1(vsha1su1q_u32, arm_neon_sha1su1, 0),
6593 NEONMAP1(vsha256h2q_u32, arm_neon_sha256h2, 0),
6594 NEONMAP1(vsha256hq_u32, arm_neon_sha256h, 0),
6595 NEONMAP1(vsha256su0q_u32, arm_neon_sha256su0, 0),
6596 NEONMAP1(vsha256su1q_u32, arm_neon_sha256su1, 0),
6605 NEONMAP1(vst1_v, arm_neon_vst1, 0),
6606 NEONMAP1(vst1_x2_v, arm_neon_vst1x2, 0),
6607 NEONMAP1(vst1_x3_v, arm_neon_vst1x3, 0),
6608 NEONMAP1(vst1_x4_v, arm_neon_vst1x4, 0),
6609 NEONMAP1(vst1q_v, arm_neon_vst1, 0),
6610 NEONMAP1(vst1q_x2_v, arm_neon_vst1x2, 0),
6611 NEONMAP1(vst1q_x3_v, arm_neon_vst1x3, 0),
6612 NEONMAP1(vst1q_x4_v, arm_neon_vst1x4, 0),
6613 NEONMAP1(vst2_lane_v, arm_neon_vst2lane, 0),
6614 NEONMAP1(vst2_v, arm_neon_vst2, 0),
6615 NEONMAP1(vst2q_lane_v, arm_neon_vst2lane, 0),
6616 NEONMAP1(vst2q_v, arm_neon_vst2, 0),
6617 NEONMAP1(vst3_lane_v, arm_neon_vst3lane, 0),
6618 NEONMAP1(vst3_v, arm_neon_vst3, 0),
6619 NEONMAP1(vst3q_lane_v, arm_neon_vst3lane, 0),
6620 NEONMAP1(vst3q_v, arm_neon_vst3, 0),
6621 NEONMAP1(vst4_lane_v, arm_neon_vst4lane, 0),
6622 NEONMAP1(vst4_v, arm_neon_vst4, 0),
6623 NEONMAP1(vst4q_lane_v, arm_neon_vst4lane, 0),
6624 NEONMAP1(vst4q_v, arm_neon_vst4, 0),
6630 NEONMAP1(vusdot_s32, arm_neon_usdot, 0),
6631 NEONMAP1(vusdotq_s32, arm_neon_usdot, 0),
6632 NEONMAP1(vusmmlaq_s32, arm_neon_usmmla, 0),
6640 NEONMAP1(__a64_vcvtq_low_bf16_f32, aarch64_neon_bfcvtn, 0),
6645 NEONMAP1(vabs_v, aarch64_neon_abs, 0),
6646 NEONMAP1(vabsq_v, aarch64_neon_abs, 0),
6651 NEONMAP1(vaesdq_u8, aarch64_crypto_aesd, 0),
6652 NEONMAP1(vaeseq_u8, aarch64_crypto_aese, 0),
6653 NEONMAP1(vaesimcq_u8, aarch64_crypto_aesimc, 0),
6654 NEONMAP1(vaesmcq_u8, aarch64_crypto_aesmc, 0),
6663 NEONMAP1(vbfdot_f32, aarch64_neon_bfdot, 0),
6664 NEONMAP1(vbfdotq_f32, aarch64_neon_bfdot, 0),
6665 NEONMAP1(vbfmlalbq_f32, aarch64_neon_bfmlalb, 0),
6666 NEONMAP1(vbfmlaltq_f32, aarch64_neon_bfmlalt, 0),
6667 NEONMAP1(vbfmmlaq_f32, aarch64_neon_bfmmla, 0),
6678 NEONMAP1(vcage_v, aarch64_neon_facge, 0),
6679 NEONMAP1(vcageq_v, aarch64_neon_facge, 0),
6680 NEONMAP1(vcagt_v, aarch64_neon_facgt, 0),
6681 NEONMAP1(vcagtq_v, aarch64_neon_facgt, 0),
6682 NEONMAP1(vcale_v, aarch64_neon_facge, 0),
6683 NEONMAP1(vcaleq_v, aarch64_neon_facge, 0),
6684 NEONMAP1(vcalt_v, aarch64_neon_facgt, 0),
6685 NEONMAP1(vcaltq_v, aarch64_neon_facgt, 0),
6722 NEONMAP1(vcvt_f16_f32, aarch64_neon_vcvtfp2hf, 0),
6725 NEONMAP1(vcvt_f32_f16, aarch64_neon_vcvthf2fp, 0),
6727 NEONMAP1(vcvt_n_f16_s16, aarch64_neon_vcvtfxs2fp, 0),
6728 NEONMAP1(vcvt_n_f16_u16, aarch64_neon_vcvtfxu2fp, 0),
6729 NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
6730 NEONMAP2(vcvt_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
6731 NEONMAP1(vcvt_n_s16_f16, aarch64_neon_vcvtfp2fxs, 0),
6732 NEONMAP1(vcvt_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
6733 NEONMAP1(vcvt_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
6734 NEONMAP1(vcvt_n_u16_f16, aarch64_neon_vcvtfp2fxu, 0),
6735 NEONMAP1(vcvt_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
6736 NEONMAP1(vcvt_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
6740 NEONMAP1(vcvtq_high_bf16_f32, aarch64_neon_bfcvtn2, 0),
6741 NEONMAP1(vcvtq_n_f16_s16, aarch64_neon_vcvtfxs2fp, 0),
6742 NEONMAP1(vcvtq_n_f16_u16, aarch64_neon_vcvtfxu2fp, 0),
6743 NEONMAP2(vcvtq_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
6744 NEONMAP2(vcvtq_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
6745 NEONMAP1(vcvtq_n_s16_f16, aarch64_neon_vcvtfp2fxs, 0),
6746 NEONMAP1(vcvtq_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
6747 NEONMAP1(vcvtq_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
6748 NEONMAP1(vcvtq_n_u16_f16, aarch64_neon_vcvtfp2fxu, 0),
6749 NEONMAP1(vcvtq_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
6750 NEONMAP1(vcvtq_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
6752 NEONMAP1(vdot_s32, aarch64_neon_sdot, 0),
6753 NEONMAP1(vdot_u32, aarch64_neon_udot, 0),
6754 NEONMAP1(vdotq_s32, aarch64_neon_sdot, 0),
6755 NEONMAP1(vdotq_u32, aarch64_neon_udot, 0),
6768 NEONMAP1(vfmlal_high_f16, aarch64_neon_fmlal2, 0),
6769 NEONMAP1(vfmlal_low_f16, aarch64_neon_fmlal, 0),
6770 NEONMAP1(vfmlalq_high_f16, aarch64_neon_fmlal2, 0),
6771 NEONMAP1(vfmlalq_low_f16, aarch64_neon_fmlal, 0),
6772 NEONMAP1(vfmlsl_high_f16, aarch64_neon_fmlsl2, 0),
6773 NEONMAP1(vfmlsl_low_f16, aarch64_neon_fmlsl, 0),
6774 NEONMAP1(vfmlslq_high_f16, aarch64_neon_fmlsl2, 0),
6775 NEONMAP1(vfmlslq_low_f16, aarch64_neon_fmlsl, 0),
6780 NEONMAP1(vld1_x2_v, aarch64_neon_ld1x2, 0),
6781 NEONMAP1(vld1_x3_v, aarch64_neon_ld1x3, 0),
6782 NEONMAP1(vld1_x4_v, aarch64_neon_ld1x4, 0),
6783 NEONMAP1(vld1q_x2_v, aarch64_neon_ld1x2, 0),
6784 NEONMAP1(vld1q_x3_v, aarch64_neon_ld1x3, 0),
6785 NEONMAP1(vld1q_x4_v, aarch64_neon_ld1x4, 0),
6786 NEONMAP1(vmmlaq_s32, aarch64_neon_smmla, 0),
6787 NEONMAP1(vmmlaq_u32, aarch64_neon_ummla, 0),
6800 NEONMAP2(vqdmlal_v, aarch64_neon_sqdmull, aarch64_neon_sqadd, 0),
6801 NEONMAP2(vqdmlsl_v, aarch64_neon_sqdmull, aarch64_neon_sqsub, 0),
6802 NEONMAP1(vqdmulh_lane_v, aarch64_neon_sqdmulh_lane, 0),
6803 NEONMAP1(vqdmulh_laneq_v, aarch64_neon_sqdmulh_laneq, 0),
6805 NEONMAP1(vqdmulhq_lane_v, aarch64_neon_sqdmulh_lane, 0),
6806 NEONMAP1(vqdmulhq_laneq_v, aarch64_neon_sqdmulh_laneq, 0),
6821 NEONMAP1(vqrdmulh_lane_v, aarch64_neon_sqrdmulh_lane, 0),
6822 NEONMAP1(vqrdmulh_laneq_v, aarch64_neon_sqrdmulh_laneq, 0),
6824 NEONMAP1(vqrdmulhq_lane_v, aarch64_neon_sqrdmulh_lane, 0),
6825 NEONMAP1(vqrdmulhq_laneq_v, aarch64_neon_sqrdmulh_laneq, 0),
6833 NEONMAP1(vqshlu_n_v, aarch64_neon_sqshlu, 0),
6834 NEONMAP1(vqshluq_n_v, aarch64_neon_sqshlu, 0),
6838 NEONMAP1(vrax1q_u64, aarch64_crypto_rax1, 0),
6839 NEONMAP2(vrecpe_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
6840 NEONMAP2(vrecpeq_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
6867 NEONMAP2(vrsqrte_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
6868 NEONMAP2(vrsqrteq_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
6872 NEONMAP1(vsha1su0q_u32, aarch64_crypto_sha1su0, 0),
6873 NEONMAP1(vsha1su1q_u32, aarch64_crypto_sha1su1, 0),
6874 NEONMAP1(vsha256h2q_u32, aarch64_crypto_sha256h2, 0),
6875 NEONMAP1(vsha256hq_u32, aarch64_crypto_sha256h, 0),
6876 NEONMAP1(vsha256su0q_u32, aarch64_crypto_sha256su0, 0),
6877 NEONMAP1(vsha256su1q_u32, aarch64_crypto_sha256su1, 0),
6878 NEONMAP1(vsha512h2q_u64, aarch64_crypto_sha512h2, 0),
6879 NEONMAP1(vsha512hq_u64, aarch64_crypto_sha512h, 0),
6880 NEONMAP1(vsha512su0q_u64, aarch64_crypto_sha512su0, 0),
6881 NEONMAP1(vsha512su1q_u64, aarch64_crypto_sha512su1, 0),
6890 NEONMAP1(vsm3partw1q_u32, aarch64_crypto_sm3partw1, 0),
6891 NEONMAP1(vsm3partw2q_u32, aarch64_crypto_sm3partw2, 0),
6892 NEONMAP1(vsm3ss1q_u32, aarch64_crypto_sm3ss1, 0),
6893 NEONMAP1(vsm3tt1aq_u32, aarch64_crypto_sm3tt1a, 0),
6894 NEONMAP1(vsm3tt1bq_u32, aarch64_crypto_sm3tt1b, 0),
6895 NEONMAP1(vsm3tt2aq_u32, aarch64_crypto_sm3tt2a, 0),
6896 NEONMAP1(vsm3tt2bq_u32, aarch64_crypto_sm3tt2b, 0),
6897 NEONMAP1(vsm4ekeyq_u32, aarch64_crypto_sm4ekey, 0),
6898 NEONMAP1(vsm4eq_u32, aarch64_crypto_sm4e, 0),
6899 NEONMAP1(vst1_x2_v, aarch64_neon_st1x2, 0),
6900 NEONMAP1(vst1_x3_v, aarch64_neon_st1x3, 0),
6901 NEONMAP1(vst1_x4_v, aarch64_neon_st1x4, 0),
6902 NEONMAP1(vst1q_x2_v, aarch64_neon_st1x2, 0),
6903 NEONMAP1(vst1q_x3_v, aarch64_neon_st1x3, 0),
6904 NEONMAP1(vst1q_x4_v, aarch64_neon_st1x4, 0),
6908 NEONMAP1(vusdot_s32, aarch64_neon_usdot, 0),
6909 NEONMAP1(vusdotq_s32, aarch64_neon_usdot, 0),
6910 NEONMAP1(vusmmlaq_s32, aarch64_neon_usmmla, 0),
6911 NEONMAP1(vxarq_u64, aarch64_crypto_xar, 0),
6949 NEONMAP1(vcvth_bf16_f32, aarch64_neon_bfcvt, 0),
6968 NEONMAP1(vcvtxd_f32_f64, aarch64_sisd_fcvtxn, 0),
6989 NEONMAP1(vmull_p64, aarch64_neon_pmull64, 0),
7017 NEONMAP1(vqdmulls_s32, aarch64_neon_sqdmulls_scalar, 0),
7098 NEONMAP1(vsha1cq_u32, aarch64_crypto_sha1c, 0),
7099 NEONMAP1(vsha1h_u32, aarch64_crypto_sha1h, 0),
7100 NEONMAP1(vsha1mq_u32, aarch64_crypto_sha1m, 0),
7101 NEONMAP1(vsha1pq_u32, aarch64_crypto_sha1p, 0),
7155 { NEON::BI__builtin_neon_splat_lane_bf16, NEON::BI__builtin_neon_splat_lane_v, },
7156 { NEON::BI__builtin_neon_splat_laneq_bf16, NEON::BI__builtin_neon_splat_laneq_v, },
7157 { NEON::BI__builtin_neon_splatq_lane_bf16, NEON::BI__builtin_neon_splatq_lane_v, },
7158 { NEON::BI__builtin_neon_splatq_laneq_bf16, NEON::BI__builtin_neon_splatq_laneq_v, },
7159 { NEON::BI__builtin_neon_vabd_f16, NEON::BI__builtin_neon_vabd_v, },
7160 { NEON::BI__builtin_neon_vabdq_f16, NEON::BI__builtin_neon_vabdq_v, },
7161 { NEON::BI__builtin_neon_vabs_f16, NEON::BI__builtin_neon_vabs_v, },
7162 { NEON::BI__builtin_neon_vabsq_f16, NEON::BI__builtin_neon_vabsq_v, },
7163 { NEON::BI__builtin_neon_vbsl_f16, NEON::BI__builtin_neon_vbsl_v, },
7164 { NEON::BI__builtin_neon_vbslq_f16, NEON::BI__builtin_neon_vbslq_v, },
7165 { NEON::BI__builtin_neon_vcage_f16, NEON::BI__builtin_neon_vcage_v, },
7166 { NEON::BI__builtin_neon_vcageq_f16, NEON::BI__builtin_neon_vcageq_v, },
7167 { NEON::BI__builtin_neon_vcagt_f16, NEON::BI__builtin_neon_vcagt_v, },
7168 { NEON::BI__builtin_neon_vcagtq_f16, NEON::BI__builtin_neon_vcagtq_v, },
7169 { NEON::BI__builtin_neon_vcale_f16, NEON::BI__builtin_neon_vcale_v, },
7170 { NEON::BI__builtin_neon_vcaleq_f16, NEON::BI__builtin_neon_vcaleq_v, },
7171 { NEON::BI__builtin_neon_vcalt_f16, NEON::BI__builtin_neon_vcalt_v, },
7172 { NEON::BI__builtin_neon_vcaltq_f16, NEON::BI__builtin_neon_vcaltq_v, },
7173 { NEON::BI__builtin_neon_vceqz_f16, NEON::BI__builtin_neon_vceqz_v, },
7174 { NEON::BI__builtin_neon_vceqzq_f16, NEON::BI__builtin_neon_vceqzq_v, },
7175 { NEON::BI__builtin_neon_vcgez_f16, NEON::BI__builtin_neon_vcgez_v, },
7176 { NEON::BI__builtin_neon_vcgezq_f16, NEON::BI__builtin_neon_vcgezq_v, },
7177 { NEON::BI__builtin_neon_vcgtz_f16, NEON::BI__builtin_neon_vcgtz_v, },
7178 { NEON::BI__builtin_neon_vcgtzq_f16, NEON::BI__builtin_neon_vcgtzq_v, },
7179 { NEON::BI__builtin_neon_vclez_f16, NEON::BI__builtin_neon_vclez_v, },
7180 { NEON::BI__builtin_neon_vclezq_f16, NEON::BI__builtin_neon_vclezq_v, },
7181 { NEON::BI__builtin_neon_vcltz_f16, NEON::BI__builtin_neon_vcltz_v, },
7182 { NEON::BI__builtin_neon_vcltzq_f16, NEON::BI__builtin_neon_vcltzq_v, },
7183 { NEON::BI__builtin_neon_vext_f16, NEON::BI__builtin_neon_vext_v, },
7184 { NEON::BI__builtin_neon_vextq_f16, NEON::BI__builtin_neon_vextq_v, },
7185 { NEON::BI__builtin_neon_vfma_f16, NEON::BI__builtin_neon_vfma_v, },
7186 { NEON::BI__builtin_neon_vfma_lane_f16, NEON::BI__builtin_neon_vfma_lane_v, },
7187 { NEON::BI__builtin_neon_vfma_laneq_f16, NEON::BI__builtin_neon_vfma_laneq_v, },
7188 { NEON::BI__builtin_neon_vfmaq_f16, NEON::BI__builtin_neon_vfmaq_v, },
7189 { NEON::BI__builtin_neon_vfmaq_lane_f16, NEON::BI__builtin_neon_vfmaq_lane_v, },
7190 { NEON::BI__builtin_neon_vfmaq_laneq_f16, NEON::BI__builtin_neon_vfmaq_laneq_v, },
7191 { NEON::BI__builtin_neon_vld1_bf16_x2, NEON::BI__builtin_neon_vld1_x2_v },
7192 { NEON::BI__builtin_neon_vld1_bf16_x3, NEON::BI__builtin_neon_vld1_x3_v },
7193 { NEON::BI__builtin_neon_vld1_bf16_x4, NEON::BI__builtin_neon_vld1_x4_v },
7194 { NEON::BI__builtin_neon_vld1_bf16, NEON::BI__builtin_neon_vld1_v },
7195 { NEON::BI__builtin_neon_vld1_dup_bf16, NEON::BI__builtin_neon_vld1_dup_v },
7196 { NEON::BI__builtin_neon_vld1_lane_bf16, NEON::BI__builtin_neon_vld1_lane_v },
7197 { NEON::BI__builtin_neon_vld1q_bf16_x2, NEON::BI__builtin_neon_vld1q_x2_v },
7198 { NEON::BI__builtin_neon_vld1q_bf16_x3, NEON::BI__builtin_neon_vld1q_x3_v },
7199 { NEON::BI__builtin_neon_vld1q_bf16_x4, NEON::BI__builtin_neon_vld1q_x4_v },
7200 { NEON::BI__builtin_neon_vld1q_bf16, NEON::BI__builtin_neon_vld1q_v },
7201 { NEON::BI__builtin_neon_vld1q_dup_bf16, NEON::BI__builtin_neon_vld1q_dup_v },
7202 { NEON::BI__builtin_neon_vld1q_lane_bf16, NEON::BI__builtin_neon_vld1q_lane_v },
7203 { NEON::BI__builtin_neon_vld2_bf16, NEON::BI__builtin_neon_vld2_v },
7204 { NEON::BI__builtin_neon_vld2_dup_bf16, NEON::BI__builtin_neon_vld2_dup_v },
7205 { NEON::BI__builtin_neon_vld2_lane_bf16, NEON::BI__builtin_neon_vld2_lane_v },
7206 { NEON::BI__builtin_neon_vld2q_bf16, NEON::BI__builtin_neon_vld2q_v },
7207 { NEON::BI__builtin_neon_vld2q_dup_bf16, NEON::BI__builtin_neon_vld2q_dup_v },
7208 { NEON::BI__builtin_neon_vld2q_lane_bf16, NEON::BI__builtin_neon_vld2q_lane_v },
7209 { NEON::BI__builtin_neon_vld3_bf16, NEON::BI__builtin_neon_vld3_v },
7210 { NEON::BI__builtin_neon_vld3_dup_bf16, NEON::BI__builtin_neon_vld3_dup_v },
7211 { NEON::BI__builtin_neon_vld3_lane_bf16, NEON::BI__builtin_neon_vld3_lane_v },
7212 { NEON::BI__builtin_neon_vld3q_bf16, NEON::BI__builtin_neon_vld3q_v },
7213 { NEON::BI__builtin_neon_vld3q_dup_bf16, NEON::BI__builtin_neon_vld3q_dup_v },
7214 { NEON::BI__builtin_neon_vld3q_lane_bf16, NEON::BI__builtin_neon_vld3q_lane_v },
7215 { NEON::BI__builtin_neon_vld4_bf16, NEON::BI__builtin_neon_vld4_v },
7216 { NEON::BI__builtin_neon_vld4_dup_bf16, NEON::BI__builtin_neon_vld4_dup_v },
7217 { NEON::BI__builtin_neon_vld4_lane_bf16, NEON::BI__builtin_neon_vld4_lane_v },
7218 { NEON::BI__builtin_neon_vld4q_bf16, NEON::BI__builtin_neon_vld4q_v },
7219 { NEON::BI__builtin_neon_vld4q_dup_bf16, NEON::BI__builtin_neon_vld4q_dup_v },
7220 { NEON::BI__builtin_neon_vld4q_lane_bf16, NEON::BI__builtin_neon_vld4q_lane_v },
7221 { NEON::BI__builtin_neon_vmax_f16, NEON::BI__builtin_neon_vmax_v, },
7222 { NEON::BI__builtin_neon_vmaxnm_f16, NEON::BI__builtin_neon_vmaxnm_v, },
7223 { NEON::BI__builtin_neon_vmaxnmq_f16, NEON::BI__builtin_neon_vmaxnmq_v, },
7224 { NEON::BI__builtin_neon_vmaxq_f16, NEON::BI__builtin_neon_vmaxq_v, },
7225 { NEON::BI__builtin_neon_vmin_f16, NEON::BI__builtin_neon_vmin_v, },
7226 { NEON::BI__builtin_neon_vminnm_f16, NEON::BI__builtin_neon_vminnm_v, },
7227 { NEON::BI__builtin_neon_vminnmq_f16, NEON::BI__builtin_neon_vminnmq_v, },
7228 { NEON::BI__builtin_neon_vminq_f16, NEON::BI__builtin_neon_vminq_v, },
7229 { NEON::BI__builtin_neon_vmulx_f16, NEON::BI__builtin_neon_vmulx_v, },
7230 { NEON::BI__builtin_neon_vmulxq_f16, NEON::BI__builtin_neon_vmulxq_v, },
7231 { NEON::BI__builtin_neon_vpadd_f16, NEON::BI__builtin_neon_vpadd_v, },
7232 { NEON::BI__builtin_neon_vpaddq_f16, NEON::BI__builtin_neon_vpaddq_v, },
7233 { NEON::BI__builtin_neon_vpmax_f16, NEON::BI__builtin_neon_vpmax_v, },
7234 { NEON::BI__builtin_neon_vpmaxnm_f16, NEON::BI__builtin_neon_vpmaxnm_v, },
7235 { NEON::BI__builtin_neon_vpmaxnmq_f16, NEON::BI__builtin_neon_vpmaxnmq_v, },
7236 { NEON::BI__builtin_neon_vpmaxq_f16, NEON::BI__builtin_neon_vpmaxq_v, },
7237 { NEON::BI__builtin_neon_vpmin_f16, NEON::BI__builtin_neon_vpmin_v, },
7238 { NEON::BI__builtin_neon_vpminnm_f16, NEON::BI__builtin_neon_vpminnm_v, },
7239 { NEON::BI__builtin_neon_vpminnmq_f16, NEON::BI__builtin_neon_vpminnmq_v, },
7240 { NEON::BI__builtin_neon_vpminq_f16, NEON::BI__builtin_neon_vpminq_v, },
7241 { NEON::BI__builtin_neon_vrecpe_f16, NEON::BI__builtin_neon_vrecpe_v, },
7242 { NEON::BI__builtin_neon_vrecpeq_f16, NEON::BI__builtin_neon_vrecpeq_v, },
7243 { NEON::BI__builtin_neon_vrecps_f16, NEON::BI__builtin_neon_vrecps_v, },
7244 { NEON::BI__builtin_neon_vrecpsq_f16, NEON::BI__builtin_neon_vrecpsq_v, },
7245 { NEON::BI__builtin_neon_vrnd_f16, NEON::BI__builtin_neon_vrnd_v, },
7246 { NEON::BI__builtin_neon_vrnda_f16, NEON::BI__builtin_neon_vrnda_v, },
7247 { NEON::BI__builtin_neon_vrndaq_f16, NEON::BI__builtin_neon_vrndaq_v, },
7248 { NEON::BI__builtin_neon_vrndi_f16, NEON::BI__builtin_neon_vrndi_v, },
7249 { NEON::BI__builtin_neon_vrndiq_f16, NEON::BI__builtin_neon_vrndiq_v, },
7250 { NEON::BI__builtin_neon_vrndm_f16, NEON::BI__builtin_neon_vrndm_v, },
7251 { NEON::BI__builtin_neon_vrndmq_f16, NEON::BI__builtin_neon_vrndmq_v, },
7252 { NEON::BI__builtin_neon_vrndn_f16, NEON::BI__builtin_neon_vrndn_v, },
7253 { NEON::BI__builtin_neon_vrndnq_f16, NEON::BI__builtin_neon_vrndnq_v, },
7254 { NEON::BI__builtin_neon_vrndp_f16, NEON::BI__builtin_neon_vrndp_v, },
7255 { NEON::BI__builtin_neon_vrndpq_f16, NEON::BI__builtin_neon_vrndpq_v, },
7256 { NEON::BI__builtin_neon_vrndq_f16, NEON::BI__builtin_neon_vrndq_v, },
7257 { NEON::BI__builtin_neon_vrndx_f16, NEON::BI__builtin_neon_vrndx_v, },
7258 { NEON::BI__builtin_neon_vrndxq_f16, NEON::BI__builtin_neon_vrndxq_v, },
7259 { NEON::BI__builtin_neon_vrsqrte_f16, NEON::BI__builtin_neon_vrsqrte_v, },
7260 { NEON::BI__builtin_neon_vrsqrteq_f16, NEON::BI__builtin_neon_vrsqrteq_v, },
7261 { NEON::BI__builtin_neon_vrsqrts_f16, NEON::BI__builtin_neon_vrsqrts_v, },
7262 { NEON::BI__builtin_neon_vrsqrtsq_f16, NEON::BI__builtin_neon_vrsqrtsq_v, },
7263 { NEON::BI__builtin_neon_vsqrt_f16, NEON::BI__builtin_neon_vsqrt_v, },
7264 { NEON::BI__builtin_neon_vsqrtq_f16, NEON::BI__builtin_neon_vsqrtq_v, },
7265 { NEON::BI__builtin_neon_vst1_bf16_x2, NEON::BI__builtin_neon_vst1_x2_v },
7266 { NEON::BI__builtin_neon_vst1_bf16_x3, NEON::BI__builtin_neon_vst1_x3_v },
7267 { NEON::BI__builtin_neon_vst1_bf16_x4, NEON::BI__builtin_neon_vst1_x4_v },
7268 { NEON::BI__builtin_neon_vst1_bf16, NEON::BI__builtin_neon_vst1_v },
7269 { NEON::BI__builtin_neon_vst1_lane_bf16, NEON::BI__builtin_neon_vst1_lane_v },
7270 { NEON::BI__builtin_neon_vst1q_bf16_x2, NEON::BI__builtin_neon_vst1q_x2_v },
7271 { NEON::BI__builtin_neon_vst1q_bf16_x3, NEON::BI__builtin_neon_vst1q_x3_v },
7272 { NEON::BI__builtin_neon_vst1q_bf16_x4, NEON::BI__builtin_neon_vst1q_x4_v },
7273 { NEON::BI__builtin_neon_vst1q_bf16, NEON::BI__builtin_neon_vst1q_v },
7274 { NEON::BI__builtin_neon_vst1q_lane_bf16, NEON::BI__builtin_neon_vst1q_lane_v },
7275 { NEON::BI__builtin_neon_vst2_bf16, NEON::BI__builtin_neon_vst2_v },
7276 { NEON::BI__builtin_neon_vst2_lane_bf16, NEON::BI__builtin_neon_vst2_lane_v },
7277 { NEON::BI__builtin_neon_vst2q_bf16, NEON::BI__builtin_neon_vst2q_v },
7278 { NEON::BI__builtin_neon_vst2q_lane_bf16, NEON::BI__builtin_neon_vst2q_lane_v },
7279 { NEON::BI__builtin_neon_vst3_bf16, NEON::BI__builtin_neon_vst3_v },
7280 { NEON::BI__builtin_neon_vst3_lane_bf16, NEON::BI__builtin_neon_vst3_lane_v },
7281 { NEON::BI__builtin_neon_vst3q_bf16, NEON::BI__builtin_neon_vst3q_v },
7282 { NEON::BI__builtin_neon_vst3q_lane_bf16, NEON::BI__builtin_neon_vst3q_lane_v },
7283 { NEON::BI__builtin_neon_vst4_bf16, NEON::BI__builtin_neon_vst4_v },
7284 { NEON::BI__builtin_neon_vst4_lane_bf16, NEON::BI__builtin_neon_vst4_lane_v },
7285 { NEON::BI__builtin_neon_vst4q_bf16, NEON::BI__builtin_neon_vst4q_v },
7286 { NEON::BI__builtin_neon_vst4q_lane_bf16, NEON::BI__builtin_neon_vst4q_lane_v },
7287 { NEON::BI__builtin_neon_vtrn_f16, NEON::BI__builtin_neon_vtrn_v, },
7288 { NEON::BI__builtin_neon_vtrnq_f16, NEON::BI__builtin_neon_vtrnq_v, },
7289 { NEON::BI__builtin_neon_vuzp_f16, NEON::BI__builtin_neon_vuzp_v, },
7290 { NEON::BI__builtin_neon_vuzpq_f16, NEON::BI__builtin_neon_vuzpq_v, },
7291 { NEON::BI__builtin_neon_vzip_f16, NEON::BI__builtin_neon_vzip_v, },
7292 { NEON::BI__builtin_neon_vzipq_f16, NEON::BI__builtin_neon_vzipq_v, },
7296 { NEON::BI__builtin_neon_vldap1_lane_u64, NEON::BI__builtin_neon_vldap1_lane_s64 },
7297 { NEON::BI__builtin_neon_vldap1_lane_f64, NEON::BI__builtin_neon_vldap1_lane_s64 },
7298 { NEON::BI__builtin_neon_vldap1_lane_p64, NEON::BI__builtin_neon_vldap1_lane_s64 },
7299 { NEON::BI__builtin_neon_vldap1q_lane_u64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
7300 { NEON::BI__builtin_neon_vldap1q_lane_f64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
7301 { NEON::BI__builtin_neon_vldap1q_lane_p64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
7302 { NEON::BI__builtin_neon_vstl1_lane_u64, NEON::BI__builtin_neon_vstl1_lane_s64 },
7303 { NEON::BI__builtin_neon_vstl1_lane_f64, NEON::BI__builtin_neon_vstl1_lane_s64 },
7304 { NEON::BI__builtin_neon_vstl1_lane_p64, NEON::BI__builtin_neon_vstl1_lane_s64 },
7305 { NEON::BI__builtin_neon_vstl1q_lane_u64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
7306 { NEON::BI__builtin_neon_vstl1q_lane_f64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
7307 { NEON::BI__builtin_neon_vstl1q_lane_p64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
7314#define SVEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
7316 #NameBase, SVE::BI__builtin_sve_##NameBase, Intrinsic::LLVMIntrinsic, 0, \
7320#define SVEMAP2(NameBase, TypeModifier) \
7321 { #NameBase, SVE::BI__builtin_sve_##NameBase, 0, 0, TypeModifier }
7323#define GET_SVE_LLVM_INTRINSIC_MAP
7324#include "clang/Basic/arm_sve_builtin_cg.inc"
7325#include "clang/Basic/BuiltinsAArch64NeonSVEBridge_cg.def"
7326#undef GET_SVE_LLVM_INTRINSIC_MAP
7332#define SMEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
7334 #NameBase, SME::BI__builtin_sme_##NameBase, Intrinsic::LLVMIntrinsic, 0, \
7338#define SMEMAP2(NameBase, TypeModifier) \
7339 { #NameBase, SME::BI__builtin_sme_##NameBase, 0, 0, TypeModifier }
7341#define GET_SME_LLVM_INTRINSIC_MAP
7342#include "clang/Basic/arm_sme_builtin_cg.inc"
7343#undef GET_SME_LLVM_INTRINSIC_MAP
7356static const ARMVectorIntrinsicInfo *
7358 unsigned BuiltinID,
bool &MapProvenSorted) {
7361 if (!MapProvenSorted) {
7362 assert(llvm::is_sorted(IntrinsicMap));
7363 MapProvenSorted =
true;
7367 const ARMVectorIntrinsicInfo *Builtin =
7368 llvm::lower_bound(IntrinsicMap, BuiltinID);
7370 if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID)
7378 llvm::Type *ArgType,
7391 Ty = llvm::FixedVectorType::get(
7392 Ty, VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1);
7399 int Elts = VectorSize ? VectorSize / ArgType->getPrimitiveSizeInBits() : 1;
7400 ArgType = llvm::FixedVectorType::get(ArgType, Elts);
7404 Tys.push_back(ArgType);
7407 Tys.push_back(ArgType);
7418 unsigned BuiltinID = SISDInfo.BuiltinID;
7419 unsigned int Int = SISDInfo.LLVMIntrinsic;
7420 unsigned Modifier = SISDInfo.TypeModifier;
7421 const char *
s = SISDInfo.NameHint;
7423 switch (BuiltinID) {
7424 case NEON::BI__builtin_neon_vcled_s64:
7425 case NEON::BI__builtin_neon_vcled_u64:
7426 case NEON::BI__builtin_neon_vcles_f32:
7427 case NEON::BI__builtin_neon_vcled_f64:
7428 case NEON::BI__builtin_neon_vcltd_s64:
7429 case NEON::BI__builtin_neon_vcltd_u64:
7430 case NEON::BI__builtin_neon_vclts_f32:
7431 case NEON::BI__builtin_neon_vcltd_f64:
7432 case NEON::BI__builtin_neon_vcales_f32:
7433 case NEON::BI__builtin_neon_vcaled_f64:
7434 case NEON::BI__builtin_neon_vcalts_f32:
7435 case NEON::BI__builtin_neon_vcaltd_f64:
7439 std::swap(Ops[0], Ops[1]);
7443 assert(Int &&
"Generic code assumes a valid intrinsic");
7451 ConstantInt *C0 = ConstantInt::get(CGF.
SizeTy, 0);
7452 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
7453 ai != ae; ++ai, ++j) {
7454 llvm::Type *ArgTy = ai->getType();
7455 if (Ops[j]->getType()->getPrimitiveSizeInBits() ==
7456 ArgTy->getPrimitiveSizeInBits())
7459 assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy());
7462 Ops[j] = CGF.
Builder.CreateTruncOrBitCast(
7463 Ops[j], cast<llvm::VectorType>(ArgTy)->getElementType());
7465 CGF.
Builder.CreateInsertElement(PoisonValue::get(ArgTy), Ops[j], C0);
7470 if (ResultType->getPrimitiveSizeInBits().getFixedValue() <
7471 Result->getType()->getPrimitiveSizeInBits().getFixedValue())
7478 unsigned BuiltinID,
unsigned LLVMIntrinsic,
unsigned AltLLVMIntrinsic,
7479 const char *NameHint,
unsigned Modifier,
const CallExpr *E,
7481 llvm::Triple::ArchType Arch) {
7484 std::optional<llvm::APSInt> NeonTypeConst =
7491 bool Usgn =
Type.isUnsigned();
7492 bool Quad =
Type.isQuad();
7494 const bool AllowBFloatArgsAndRet =
7497 llvm::FixedVectorType *VTy =
7498 GetNeonType(
this,
Type, HasLegalHalfType,
false, AllowBFloatArgsAndRet);
7499 llvm::Type *Ty = VTy;
7503 auto getAlignmentValue32 = [&](
Address addr) ->
Value* {
7504 return Builder.getInt32(addr.getAlignment().getQuantity());
7507 unsigned Int = LLVMIntrinsic;
7509 Int = AltLLVMIntrinsic;
7511 switch (BuiltinID) {
7513 case NEON::BI__builtin_neon_splat_lane_v:
7514 case NEON::BI__builtin_neon_splat_laneq_v:
7515 case NEON::BI__builtin_neon_splatq_lane_v:
7516 case NEON::BI__builtin_neon_splatq_laneq_v: {
7517 auto NumElements = VTy->getElementCount();
7518 if (BuiltinID == NEON::BI__builtin_neon_splatq_lane_v)
7519 NumElements = NumElements * 2;
7520 if (BuiltinID == NEON::BI__builtin_neon_splat_laneq_v)
7521 NumElements = NumElements.divideCoefficientBy(2);
7523 Ops[0] =
Builder.CreateBitCast(Ops[0], VTy);
7524 return EmitNeonSplat(Ops[0], cast<ConstantInt>(Ops[1]), NumElements);
7526 case NEON::BI__builtin_neon_vpadd_v:
7527 case NEON::BI__builtin_neon_vpaddq_v:
7529 if (VTy->getElementType()->isFloatingPointTy() &&
7530 Int == Intrinsic::aarch64_neon_addp)
7531 Int = Intrinsic::aarch64_neon_faddp;
7533 case NEON::BI__builtin_neon_vabs_v:
7534 case NEON::BI__builtin_neon_vabsq_v:
7535 if (VTy->getElementType()->isFloatingPointTy())
7538 case NEON::BI__builtin_neon_vadd_v:
7539 case NEON::BI__builtin_neon_vaddq_v: {
7540 llvm::Type *VTy = llvm::FixedVectorType::get(
Int8Ty, Quad ? 16 : 8);
7541 Ops[0] =
Builder.CreateBitCast(Ops[0], VTy);
7542 Ops[1] =
Builder.CreateBitCast(Ops[1], VTy);
7543 Ops[0] =
Builder.CreateXor(Ops[0], Ops[1]);
7544 return Builder.CreateBitCast(Ops[0], Ty);
7546 case NEON::BI__builtin_neon_vaddhn_v: {
7547 llvm::FixedVectorType *SrcTy =
7548 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
7551 Ops[0] =
Builder.CreateBitCast(Ops[0], SrcTy);
7552 Ops[1] =
Builder.CreateBitCast(Ops[1], SrcTy);
7553 Ops[0] =
Builder.CreateAdd(Ops[0], Ops[1],
"vaddhn");
7556 Constant *ShiftAmt =
7557 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
7558 Ops[0] =
Builder.CreateLShr(Ops[0], ShiftAmt,
"vaddhn");
7561 return Builder.CreateTrunc(Ops[0], VTy,
"vaddhn");
7563 case NEON::BI__builtin_neon_vcale_v:
7564 case NEON::BI__builtin_neon_vcaleq_v:
7565 case NEON::BI__builtin_neon_vcalt_v:
7566 case NEON::BI__builtin_neon_vcaltq_v:
7567 std::swap(Ops[0], Ops[1]);
7569 case NEON::BI__builtin_neon_vcage_v:
7570 case NEON::BI__builtin_neon_vcageq_v:
7571 case NEON::BI__builtin_neon_vcagt_v:
7572 case NEON::BI__builtin_neon_vcagtq_v: {
7574 switch (VTy->getScalarSizeInBits()) {
7575 default: llvm_unreachable(
"unexpected type");
7586 auto *VecFlt = llvm::FixedVectorType::get(Ty, VTy->getNumElements());
7587 llvm::Type *Tys[] = { VTy, VecFlt };
7591 case NEON::BI__builtin_neon_vceqz_v:
7592 case NEON::BI__builtin_neon_vceqzq_v:
7594 ICmpInst::ICMP_EQ,
"vceqz");
7595 case NEON::BI__builtin_neon_vcgez_v:
7596 case NEON::BI__builtin_neon_vcgezq_v:
7598 ICmpInst::ICMP_SGE,
"vcgez");
7599 case NEON::BI__builtin_neon_vclez_v:
7600 case NEON::BI__builtin_neon_vclezq_v:
7602 ICmpInst::ICMP_SLE,
"vclez");
7603 case NEON::BI__builtin_neon_vcgtz_v:
7604 case NEON::BI__builtin_neon_vcgtzq_v:
7606 ICmpInst::ICMP_SGT,
"vcgtz");
7607 case NEON::BI__builtin_neon_vcltz_v:
7608 case NEON::BI__builtin_neon_vcltzq_v:
7610 ICmpInst::ICMP_SLT,
"vcltz");
7611 case NEON::BI__builtin_neon_vclz_v:
7612 case NEON::BI__builtin_neon_vclzq_v:
7617 case NEON::BI__builtin_neon_vcvt_f32_v:
7618 case NEON::BI__builtin_neon_vcvtq_f32_v:
7619 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
7622 return Usgn ?
Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
7623 :
Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
7624 case NEON::BI__builtin_neon_vcvt_f16_s16:
7625 case NEON::BI__builtin_neon_vcvt_f16_u16:
7626 case NEON::BI__builtin_neon_vcvtq_f16_s16:
7627 case NEON::BI__builtin_neon_vcvtq_f16_u16:
7628 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
7631 return Usgn ?
Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
7632 :
Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
7633 case NEON::BI__builtin_neon_vcvt_n_f16_s16:
7634 case NEON::BI__builtin_neon_vcvt_n_f16_u16:
7635 case NEON::BI__builtin_neon_vcvtq_n_f16_s16:
7636 case NEON::BI__builtin_neon_vcvtq_n_f16_u16: {
7641 case NEON::BI__builtin_neon_vcvt_n_f32_v:
7642 case NEON::BI__builtin_neon_vcvt_n_f64_v:
7643 case NEON::BI__builtin_neon_vcvtq_n_f32_v:
7644 case NEON::BI__builtin_neon_vcvtq_n_f64_v: {
7646 Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
7650 case NEON::BI__builtin_neon_vcvt_n_s16_f16:
7651 case NEON::BI__builtin_neon_vcvt_n_s32_v:
7652 case NEON::BI__builtin_neon_vcvt_n_u16_f16:
7653 case NEON::BI__builtin_neon_vcvt_n_u32_v:
7654 case NEON::BI__builtin_neon_vcvt_n_s64_v:
7655 case NEON::BI__builtin_neon_vcvt_n_u64_v:
7656 case NEON::BI__builtin_neon_vcvtq_n_s16_f16:
7657 case NEON::BI__builtin_neon_vcvtq_n_s32_v:
7658 case NEON::BI__builtin_neon_vcvtq_n_u16_f16:
7659 case NEON::BI__builtin_neon_vcvtq_n_u32_v:
7660 case NEON::BI__builtin_neon_vcvtq_n_s64_v:
7661 case NEON::BI__builtin_neon_vcvtq_n_u64_v: {
7666 case NEON::BI__builtin_neon_vcvt_s32_v:
7667 case NEON::BI__builtin_neon_vcvt_u32_v:
7668 case NEON::BI__builtin_neon_vcvt_s64_v:
7669 case NEON::BI__builtin_neon_vcvt_u64_v:
7670 case NEON::BI__builtin_neon_vcvt_s16_f16:
7671 case NEON::BI__builtin_neon_vcvt_u16_f16:
7672 case NEON::BI__builtin_neon_vcvtq_s32_v:
7673 case NEON::BI__builtin_neon_vcvtq_u32_v:
7674 case NEON::BI__builtin_neon_vcvtq_s64_v:
7675 case NEON::BI__builtin_neon_vcvtq_u64_v:
7676 case NEON::BI__builtin_neon_vcvtq_s16_f16:
7677 case NEON::BI__builtin_neon_vcvtq_u16_f16: {
7679 return Usgn ?
Builder.CreateFPToUI(Ops[0], Ty,
"vcvt")
7680 :
Builder.CreateFPToSI(Ops[0], Ty,
"vcvt");
7682 case NEON::BI__builtin_neon_vcvta_s16_f16:
7683 case NEON::BI__builtin_neon_vcvta_s32_v:
7684 case NEON::BI__builtin_neon_vcvta_s64_v:
7685 case NEON::BI__builtin_neon_vcvta_u16_f16:
7686 case NEON::BI__builtin_neon_vcvta_u32_v:
7687 case NEON::BI__builtin_neon_vcvta_u64_v:
7688 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
7689 case NEON::BI__builtin_neon_vcvtaq_s32_v:
7690 case NEON::BI__builtin_neon_vcvtaq_s64_v:
7691 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
7692 case NEON::BI__builtin_neon_vcvtaq_u32_v:
7693 case NEON::BI__builtin_neon_vcvtaq_u64_v:
7694 case NEON::BI__builtin_neon_vcvtn_s16_f16:
7695 case NEON::BI__builtin_neon_vcvtn_s32_v:
7696 case NEON::BI__builtin_neon_vcvtn_s64_v:
7697 case NEON::BI__builtin_neon_vcvtn_u16_f16:
7698 case NEON::BI__builtin_neon_vcvtn_u32_v:
7699 case NEON::BI__builtin_neon_vcvtn_u64_v:
7700 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
7701 case NEON::BI__builtin_neon_vcvtnq_s32_v:
7702 case NEON::BI__builtin_neon_vcvtnq_s64_v:
7703 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
7704 case NEON::BI__builtin_neon_vcvtnq_u32_v:
7705 case NEON::BI__builtin_neon_vcvtnq_u64_v:
7706 case NEON::BI__builtin_neon_vcvtp_s16_f16:
7707 case NEON::BI__builtin_neon_vcvtp_s32_v:
7708 case NEON::BI__builtin_neon_vcvtp_s64_v:
7709 case NEON::BI__builtin_neon_vcvtp_u16_f16:
7710 case NEON::BI__builtin_neon_vcvtp_u32_v:
7711 case NEON::BI__builtin_neon_vcvtp_u64_v:
7712 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
7713 case NEON::BI__builtin_neon_vcvtpq_s32_v:
7714 case NEON::BI__builtin_neon_vcvtpq_s64_v:
7715 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
7716 case NEON::BI__builtin_neon_vcvtpq_u32_v:
7717 case NEON::BI__builtin_neon_vcvtpq_u64_v:
7718 case NEON::BI__builtin_neon_vcvtm_s16_f16:
7719 case NEON::BI__builtin_neon_vcvtm_s32_v:
7720 case NEON::BI__builtin_neon_vcvtm_s64_v:
7721 case NEON::BI__builtin_neon_vcvtm_u16_f16:
7722 case NEON::BI__builtin_neon_vcvtm_u32_v:
7723 case NEON::BI__builtin_neon_vcvtm_u64_v:
7724 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
7725 case NEON::BI__builtin_neon_vcvtmq_s32_v:
7726 case NEON::BI__builtin_neon_vcvtmq_s64_v:
7727 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
7728 case NEON::BI__builtin_neon_vcvtmq_u32_v:
7729 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
7733 case NEON::BI__builtin_neon_vcvtx_f32_v: {
7734 llvm::Type *Tys[2] = { VTy->getTruncatedElementVectorType(VTy), Ty};
7738 case NEON::BI__builtin_neon_vext_v:
7739 case NEON::BI__builtin_neon_vextq_v: {
7740 int CV = cast<ConstantInt>(Ops[2])->getSExtValue();
7742 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
7743 Indices.push_back(i+CV);
7745 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
7746 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
7747 return Builder.CreateShuffleVector(Ops[0], Ops[1], Indices,
"vext");
7749 case NEON::BI__builtin_neon_vfma_v:
7750 case NEON::BI__builtin_neon_vfmaq_v: {
7751 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
7752 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
7753 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
7757 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
7758 {Ops[1], Ops[2], Ops[0]});
7760 case NEON::BI__builtin_neon_vld1_v:
7761 case NEON::BI__builtin_neon_vld1q_v: {
7763 Ops.push_back(getAlignmentValue32(PtrOp0));
7766 case NEON::BI__builtin_neon_vld1_x2_v:
7767 case NEON::BI__builtin_neon_vld1q_x2_v:
7768 case NEON::BI__builtin_neon_vld1_x3_v:
7769 case NEON::BI__builtin_neon_vld1q_x3_v:
7770 case NEON::BI__builtin_neon_vld1_x4_v:
7771 case NEON::BI__builtin_neon_vld1q_x4_v: {
7774 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld1xN");
7777 case NEON::BI__builtin_neon_vld2_v:
7778 case NEON::BI__builtin_neon_vld2q_v:
7779 case NEON::BI__builtin_neon_vld3_v:
7780 case NEON::BI__builtin_neon_vld3q_v:
7781 case NEON::BI__builtin_neon_vld4_v:
7782 case NEON::BI__builtin_neon_vld4q_v:
7783 case NEON::BI__builtin_neon_vld2_dup_v:
7784 case NEON::BI__builtin_neon_vld2q_dup_v:
7785 case NEON::BI__builtin_neon_vld3_dup_v:
7786 case NEON::BI__builtin_neon_vld3q_dup_v:
7787 case NEON::BI__builtin_neon_vld4_dup_v:
7788 case NEON::BI__builtin_neon_vld4q_dup_v: {
7791 Value *Align = getAlignmentValue32(PtrOp1);
7792 Ops[1] =
Builder.CreateCall(F, {Ops[1], Align}, NameHint);
7795 case NEON::BI__builtin_neon_vld1_dup_v:
7796 case NEON::BI__builtin_neon_vld1q_dup_v: {
7797 Value *
V = PoisonValue::get(Ty);
7800 llvm::Constant *CI = ConstantInt::get(
SizeTy, 0);
7801 Ops[0] =
Builder.CreateInsertElement(
V, Ld, CI);
7804 case NEON::BI__builtin_neon_vld2_lane_v:
7805 case NEON::BI__builtin_neon_vld2q_lane_v:
7806 case NEON::BI__builtin_neon_vld3_lane_v:
7807 case NEON::BI__builtin_neon_vld3q_lane_v:
7808 case NEON::BI__builtin_neon_vld4_lane_v:
7809 case NEON::BI__builtin_neon_vld4q_lane_v: {
7812 for (
unsigned I = 2; I < Ops.size() - 1; ++I)
7813 Ops[I] =
Builder.CreateBitCast(Ops[I], Ty);
7814 Ops.push_back(getAlignmentValue32(PtrOp1));
7818 case NEON::BI__builtin_neon_vmovl_v: {
7819 llvm::FixedVectorType *DTy =
7820 llvm::FixedVectorType::getTruncatedElementVectorType(VTy);
7821 Ops[0] =
Builder.CreateBitCast(Ops[0], DTy);
7823 return Builder.CreateZExt(Ops[0], Ty,
"vmovl");
7824 return Builder.CreateSExt(Ops[0], Ty,
"vmovl");
7826 case NEON::BI__builtin_neon_vmovn_v: {
7827 llvm::FixedVectorType *QTy =
7828 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
7829 Ops[0] =
Builder.CreateBitCast(Ops[0], QTy);
7830 return Builder.CreateTrunc(Ops[0], Ty,
"vmovn");
7832 case NEON::BI__builtin_neon_vmull_v:
7838 Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls;
7839 Int =
Type.isPoly() ? (
unsigned)Intrinsic::arm_neon_vmullp : Int;
7841 case NEON::BI__builtin_neon_vpadal_v:
7842 case NEON::BI__builtin_neon_vpadalq_v: {
7844 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
7848 llvm::FixedVectorType::get(EltTy, VTy->getNumElements() * 2);
7849 llvm::Type *Tys[2] = { Ty, NarrowTy };
7852 case NEON::BI__builtin_neon_vpaddl_v:
7853 case NEON::BI__builtin_neon_vpaddlq_v: {
7855 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
7856 llvm::Type *EltTy = llvm::IntegerType::get(
getLLVMContext(), EltBits / 2);
7858 llvm::FixedVectorType::get(EltTy, VTy->getNumElements() * 2);
7859 llvm::Type *Tys[2] = { Ty, NarrowTy };
7862 case NEON::BI__builtin_neon_vqdmlal_v:
7863 case NEON::BI__builtin_neon_vqdmlsl_v: {
7870 case NEON::BI__builtin_neon_vqdmulhq_lane_v:
7871 case NEON::BI__builtin_neon_vqdmulh_lane_v:
7872 case NEON::BI__builtin_neon_vqrdmulhq_lane_v:
7873 case NEON::BI__builtin_neon_vqrdmulh_lane_v: {
7874 auto *RTy = cast<llvm::FixedVectorType>(Ty);
7875 if (BuiltinID == NEON::BI__builtin_neon_vqdmulhq_lane_v ||
7876 BuiltinID == NEON::BI__builtin_neon_vqrdmulhq_lane_v)
7877 RTy = llvm::FixedVectorType::get(RTy->getElementType(),
7878 RTy->getNumElements() * 2);
7879 llvm::Type *Tys[2] = {
7884 case NEON::BI__builtin_neon_vqdmulhq_laneq_v:
7885 case NEON::BI__builtin_neon_vqdmulh_laneq_v:
7886 case NEON::BI__builtin_neon_vqrdmulhq_laneq_v:
7887 case NEON::BI__builtin_neon_vqrdmulh_laneq_v: {
7888 llvm::Type *Tys[2] = {
7893 case NEON::BI__builtin_neon_vqshl_n_v:
7894 case NEON::BI__builtin_neon_vqshlq_n_v:
7897 case NEON::BI__builtin_neon_vqshlu_n_v:
7898 case NEON::BI__builtin_neon_vqshluq_n_v:
7901 case NEON::BI__builtin_neon_vrecpe_v:
7902 case NEON::BI__builtin_neon_vrecpeq_v:
7903 case NEON::BI__builtin_neon_vrsqrte_v:
7904 case NEON::BI__builtin_neon_vrsqrteq_v:
7905 Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic;
7907 case NEON::BI__builtin_neon_vrndi_v:
7908 case NEON::BI__builtin_neon_vrndiq_v:
7909 Int =
Builder.getIsFPConstrained()
7910 ? Intrinsic::experimental_constrained_nearbyint
7911 : Intrinsic::nearbyint;
7913 case NEON::BI__builtin_neon_vrshr_n_v:
7914 case NEON::BI__builtin_neon_vrshrq_n_v:
7917 case NEON::BI__builtin_neon_vsha512hq_u64:
7918 case NEON::BI__builtin_neon_vsha512h2q_u64:
7919 case NEON::BI__builtin_neon_vsha512su0q_u64:
7920 case NEON::BI__builtin_neon_vsha512su1q_u64: {
7924 case NEON::BI__builtin_neon_vshl_n_v:
7925 case NEON::BI__builtin_neon_vshlq_n_v:
7927 return Builder.CreateShl(
Builder.CreateBitCast(Ops[0],Ty), Ops[1],
7929 case NEON::BI__builtin_neon_vshll_n_v: {
7930 llvm::FixedVectorType *SrcTy =
7931 llvm::FixedVectorType::getTruncatedElementVectorType(VTy);
7932 Ops[0] =
Builder.CreateBitCast(Ops[0], SrcTy);
7934 Ops[0] =
Builder.CreateZExt(Ops[0], VTy);
7936 Ops[0] =
Builder.CreateSExt(Ops[0], VTy);
7938 return Builder.CreateShl(Ops[0], Ops[1],
"vshll_n");
7940 case NEON::BI__builtin_neon_vshrn_n_v: {
7941 llvm::FixedVectorType *SrcTy =
7942 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
7943 Ops[0] =
Builder.CreateBitCast(Ops[0], SrcTy);
7946 Ops[0] =
Builder.CreateLShr(Ops[0], Ops[1]);
7948 Ops[0] =
Builder.CreateAShr(Ops[0], Ops[1]);
7949 return Builder.CreateTrunc(Ops[0], Ty,
"vshrn_n");
7951 case NEON::BI__builtin_neon_vshr_n_v:
7952 case NEON::BI__builtin_neon_vshrq_n_v:
7954 case NEON::BI__builtin_neon_vst1_v:
7955 case NEON::BI__builtin_neon_vst1q_v:
7956 case NEON::BI__builtin_neon_vst2_v:
7957 case NEON::BI__builtin_neon_vst2q_v:
7958 case NEON::BI__builtin_neon_vst3_v:
7959 case NEON::BI__builtin_neon_vst3q_v:
7960 case NEON::BI__builtin_neon_vst4_v:
7961 case NEON::BI__builtin_neon_vst4q_v:
7962 case NEON::BI__builtin_neon_vst2_lane_v:
7963 case NEON::BI__builtin_neon_vst2q_lane_v:
7964 case NEON::BI__builtin_neon_vst3_lane_v:
7965 case NEON::BI__builtin_neon_vst3q_lane_v:
7966 case NEON::BI__builtin_neon_vst4_lane_v:
7967 case NEON::BI__builtin_neon_vst4q_lane_v: {
7969 Ops.push_back(getAlignmentValue32(PtrOp0));
7972 case NEON::BI__builtin_neon_vsm3partw1q_u32:
7973 case NEON::BI__builtin_neon_vsm3partw2q_u32:
7974 case NEON::BI__builtin_neon_vsm3ss1q_u32:
7975 case NEON::BI__builtin_neon_vsm4ekeyq_u32:
7976 case NEON::BI__builtin_neon_vsm4eq_u32: {
7980 case NEON::BI__builtin_neon_vsm3tt1aq_u32:
7981 case NEON::BI__builtin_neon_vsm3tt1bq_u32:
7982 case NEON::BI__builtin_neon_vsm3tt2aq_u32:
7983 case NEON::BI__builtin_neon_vsm3tt2bq_u32: {
7988 case NEON::BI__builtin_neon_vst1_x2_v:
7989 case NEON::BI__builtin_neon_vst1q_x2_v:
7990 case NEON::BI__builtin_neon_vst1_x3_v:
7991 case NEON::BI__builtin_neon_vst1q_x3_v:
7992 case NEON::BI__builtin_neon_vst1_x4_v:
7993 case NEON::BI__builtin_neon_vst1q_x4_v: {
7996 if (Arch == llvm::Triple::aarch64 || Arch == llvm::Triple::aarch64_be ||
7997 Arch == llvm::Triple::aarch64_32) {
7999 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
8005 case NEON::BI__builtin_neon_vsubhn_v: {
8006 llvm::FixedVectorType *SrcTy =
8007 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
8010 Ops[0] =
Builder.CreateBitCast(Ops[0], SrcTy);
8011 Ops[1] =
Builder.CreateBitCast(Ops[1], SrcTy);
8012 Ops[0] =
Builder.CreateSub(Ops[0], Ops[1],
"vsubhn");
8015 Constant *ShiftAmt =
8016 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
8017 Ops[0] =
Builder.CreateLShr(Ops[0], ShiftAmt,
"vsubhn");
8020 return Builder.CreateTrunc(Ops[0], VTy,
"vsubhn");
8022 case NEON::BI__builtin_neon_vtrn_v:
8023 case NEON::BI__builtin_neon_vtrnq_v: {
8024 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
8025 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
8026 Value *SV =
nullptr;
8028 for (
unsigned vi = 0; vi != 2; ++vi) {
8030 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
8031 Indices.push_back(i+vi);
8032 Indices.push_back(i+e+vi);
8034 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8035 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vtrn");
8040 case NEON::BI__builtin_neon_vtst_v:
8041 case NEON::BI__builtin_neon_vtstq_v: {
8042 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
8043 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
8044 Ops[0] =
Builder.CreateAnd(Ops[0], Ops[1]);
8045 Ops[0] =
Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
8046 ConstantAggregateZero::get(Ty));
8047 return Builder.CreateSExt(Ops[0], Ty,
"vtst");
8049 case NEON::BI__builtin_neon_vuzp_v:
8050 case NEON::BI__builtin_neon_vuzpq_v: {
8051 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
8052 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
8053 Value *SV =
nullptr;
8055 for (
unsigned vi = 0; vi != 2; ++vi) {
8057 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
8058 Indices.push_back(2*i+vi);
8060 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8061 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vuzp");
8066 case NEON::BI__builtin_neon_vxarq_u64: {
8071 case NEON::BI__builtin_neon_vzip_v:
8072 case NEON::BI__builtin_neon_vzipq_v: {
8073 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
8074 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
8075 Value *SV =
nullptr;
8077 for (
unsigned vi = 0; vi != 2; ++vi) {
8079 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
8080 Indices.push_back((i + vi*e) >> 1);
8081 Indices.push_back(((i + vi*e) >> 1)+e);
8083 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8084 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vzip");
8089 case NEON::BI__builtin_neon_vdot_s32:
8090 case NEON::BI__builtin_neon_vdot_u32:
8091 case NEON::BI__builtin_neon_vdotq_s32:
8092 case NEON::BI__builtin_neon_vdotq_u32: {
8094 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8095 llvm::Type *Tys[2] = { Ty, InputTy };
8098 case NEON::BI__builtin_neon_vfmlal_low_f16:
8099 case NEON::BI__builtin_neon_vfmlalq_low_f16: {
8101 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8102 llvm::Type *Tys[2] = { Ty, InputTy };
8105 case NEON::BI__builtin_neon_vfmlsl_low_f16:
8106 case NEON::BI__builtin_neon_vfmlslq_low_f16: {
8108 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8109 llvm::Type *Tys[2] = { Ty, InputTy };
8112 case NEON::BI__builtin_neon_vfmlal_high_f16:
8113 case NEON::BI__builtin_neon_vfmlalq_high_f16: {
8115 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8116 llvm::Type *Tys[2] = { Ty, InputTy };
8119 case NEON::BI__builtin_neon_vfmlsl_high_f16:
8120 case NEON::BI__builtin_neon_vfmlslq_high_f16: {
8122 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8123 llvm::Type *Tys[2] = { Ty, InputTy };
8126 case NEON::BI__builtin_neon_vmmlaq_s32:
8127 case NEON::BI__builtin_neon_vmmlaq_u32: {
8129 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8130 llvm::Type *Tys[2] = { Ty, InputTy };
8133 case NEON::BI__builtin_neon_vusmmlaq_s32: {
8135 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8136 llvm::Type *Tys[2] = { Ty, InputTy };
8139 case NEON::BI__builtin_neon_vusdot_s32:
8140 case NEON::BI__builtin_neon_vusdotq_s32: {
8142 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8143 llvm::Type *Tys[2] = { Ty, InputTy };
8146 case NEON::BI__builtin_neon_vbfdot_f32:
8147 case NEON::BI__builtin_neon_vbfdotq_f32: {
8148 llvm::Type *InputTy =
8149 llvm::FixedVectorType::get(
BFloatTy, Ty->getPrimitiveSizeInBits() / 16);
8150 llvm::Type *Tys[2] = { Ty, InputTy };
8153 case NEON::BI__builtin_neon___a32_vcvt_bf16_f32: {
8154 llvm::Type *Tys[1] = { Ty };
8161 assert(Int &&
"Expected valid intrinsic number");
8174 Value *Op, llvm::Type *Ty,
const CmpInst::Predicate Fp,
8175 const CmpInst::Predicate Ip,
const Twine &Name) {
8176 llvm::Type *OTy = Op->
getType();
8182 if (BitCastInst *BI = dyn_cast<BitCastInst>(Op))
8183 OTy = BI->getOperand(0)->getType();
8185 Op =
Builder.CreateBitCast(Op, OTy);
8186 if (OTy->getScalarType()->isFloatingPointTy()) {
8187 if (Fp == CmpInst::FCMP_OEQ)
8188 Op =
Builder.CreateFCmp(Fp, Op, Constant::getNullValue(OTy));
8190 Op =
Builder.CreateFCmpS(Fp, Op, Constant::getNullValue(OTy));
8192 Op =
Builder.CreateICmp(Ip, Op, Constant::getNullValue(OTy));
8194 return Builder.CreateSExt(Op, Ty, Name);
8199 llvm::Type *ResTy,
unsigned IntID,
8203 TblOps.push_back(ExtOp);
8207 auto *TblTy = cast<llvm::FixedVectorType>(Ops[0]->getType());
8208 for (
unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) {
8209 Indices.push_back(2*i);
8210 Indices.push_back(2*i+1);
8213 int PairPos = 0, End = Ops.size() - 1;
8214 while (PairPos < End) {
8215 TblOps.push_back(CGF.
Builder.CreateShuffleVector(Ops[PairPos],
8216 Ops[PairPos+1], Indices,
8223 if (PairPos == End) {
8224 Value *ZeroTbl = ConstantAggregateZero::get(TblTy);
8225 TblOps.push_back(CGF.
Builder.CreateShuffleVector(Ops[PairPos],
8226 ZeroTbl, Indices, Name));
8230 TblOps.push_back(IndexOp);
8236Value *CodeGenFunction::GetValueForARMHint(
unsigned BuiltinID) {
8238 switch (BuiltinID) {
8241 case clang::ARM::BI__builtin_arm_nop:
8244 case clang::ARM::BI__builtin_arm_yield:
8245 case clang::ARM::BI__yield:
8248 case clang::ARM::BI__builtin_arm_wfe:
8249 case clang::ARM::BI__wfe:
8252 case clang::ARM::BI__builtin_arm_wfi:
8253 case clang::ARM::BI__wfi:
8256 case clang::ARM::BI__builtin_arm_sev:
8257 case clang::ARM::BI__sev:
8260 case clang::ARM::BI__builtin_arm_sevl:
8261 case clang::ARM::BI__sevl:
8279 llvm::Type *RegisterType,
8280 llvm::Type *ValueType,
bool isExecHi) {
8285 llvm::Value *
Call = Builder.CreateCall(F, {Builder.getInt1(
true)});
8288 Value *Rt2 = Builder.CreateLShr(
Call, 32);
8289 Rt2 = Builder.CreateTrunc(Rt2, CGF.
Int32Ty);
8301 llvm::Type *RegisterType,
8302 llvm::Type *ValueType,
8304 StringRef SysReg =
"") {
8306 assert((RegisterType->isIntegerTy(32) || RegisterType->isIntegerTy(64) ||
8307 RegisterType->isIntegerTy(128)) &&
8308 "Unsupported size for register.");
8314 if (SysReg.empty()) {
8316 SysReg = cast<clang::StringLiteral>(SysRegStrExpr)->getString();
8319 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) };
8320 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
8321 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
8323 llvm::Type *Types[] = { RegisterType };
8325 bool MixedTypes = RegisterType->isIntegerTy(64) && ValueType->isIntegerTy(32);
8326 assert(!(RegisterType->isIntegerTy(32) && ValueType->isIntegerTy(64))
8327 &&
"Can't fit 64-bit value in 32-bit register");
8329 if (AccessKind !=
Write) {
8332 AccessKind ==
VolatileRead ? llvm::Intrinsic::read_volatile_register
8333 : llvm::Intrinsic::read_register,
8335 llvm::Value *
Call = Builder.CreateCall(F, Metadata);
8339 return Builder.CreateTrunc(
Call, ValueType);
8341 if (ValueType->isPointerTy())
8343 return Builder.CreateIntToPtr(
Call, ValueType);
8348 llvm::Function *F = CGM.
getIntrinsic(llvm::Intrinsic::write_register, Types);
8352 ArgValue = Builder.CreateZExt(ArgValue, RegisterType);
8353 return Builder.CreateCall(F, { Metadata, ArgValue });
8356 if (ValueType->isPointerTy()) {
8358 ArgValue = Builder.CreatePtrToInt(ArgValue, RegisterType);
8359 return Builder.CreateCall(F, { Metadata, ArgValue });
8362 return Builder.CreateCall(F, { Metadata, ArgValue });
8368 switch (BuiltinID) {
8370 case NEON::BI__builtin_neon_vget_lane_i8:
8371 case NEON::BI__builtin_neon_vget_lane_i16:
8372 case NEON::BI__builtin_neon_vget_lane_bf16:
8373 case NEON::BI__builtin_neon_vget_lane_i32:
8374 case NEON::BI__builtin_neon_vget_lane_i64:
8375 case NEON::BI__builtin_neon_vget_lane_f32:
8376 case NEON::BI__builtin_neon_vgetq_lane_i8:
8377 case NEON::BI__builtin_neon_vgetq_lane_i16:
8378 case NEON::BI__builtin_neon_vgetq_lane_bf16:
8379 case NEON::BI__builtin_neon_vgetq_lane_i32:
8380 case NEON::BI__builtin_neon_vgetq_lane_i64:
8381 case NEON::BI__builtin_neon_vgetq_lane_f32:
8382 case NEON::BI__builtin_neon_vduph_lane_bf16:
8383 case NEON::BI__builtin_neon_vduph_laneq_bf16:
8384 case NEON::BI__builtin_neon_vset_lane_i8:
8385 case NEON::BI__builtin_neon_vset_lane_i16:
8386 case NEON::BI__builtin_neon_vset_lane_bf16:
8387 case NEON::BI__builtin_neon_vset_lane_i32:
8388 case NEON::BI__builtin_neon_vset_lane_i64:
8389 case NEON::BI__builtin_neon_vset_lane_f32:
8390 case NEON::BI__builtin_neon_vsetq_lane_i8:
8391 case NEON::BI__builtin_neon_vsetq_lane_i16:
8392 case NEON::BI__builtin_neon_vsetq_lane_bf16:
8393 case NEON::BI__builtin_neon_vsetq_lane_i32:
8394 case NEON::BI__builtin_neon_vsetq_lane_i64:
8395 case NEON::BI__builtin_neon_vsetq_lane_f32:
8396 case NEON::BI__builtin_neon_vsha1h_u32:
8397 case NEON::BI__builtin_neon_vsha1cq_u32:
8398 case NEON::BI__builtin_neon_vsha1pq_u32:
8399 case NEON::BI__builtin_neon_vsha1mq_u32:
8400 case NEON::BI__builtin_neon_vcvth_bf16_f32:
8401 case clang::ARM::BI_MoveToCoprocessor:
8402 case clang::ARM::BI_MoveToCoprocessor2:
8411 llvm::Triple::ArchType Arch) {
8412 if (
auto Hint = GetValueForARMHint(BuiltinID))
8415 if (BuiltinID == clang::ARM::BI__emit) {
8417 llvm::FunctionType *FTy =
8418 llvm::FunctionType::get(
VoidTy,
false);
8422 llvm_unreachable(
"Sema will ensure that the parameter is constant");
8425 uint64_t ZExtValue =
Value.zextOrTrunc(IsThumb ? 16 : 32).getZExtValue();
8427 llvm::InlineAsm *Emit =
8428 IsThumb ? InlineAsm::get(FTy,
".inst.n 0x" + utohexstr(ZExtValue),
"",
8430 : InlineAsm::get(FTy,
".inst 0x" + utohexstr(ZExtValue),
"",
8433 return Builder.CreateCall(Emit);
8436 if (BuiltinID == clang::ARM::BI__builtin_arm_dbg) {
8441 if (BuiltinID == clang::ARM::BI__builtin_arm_prefetch) {
8453 if (BuiltinID == clang::ARM::BI__builtin_arm_rbit) {
8456 CGM.
getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
8459 if (BuiltinID == clang::ARM::BI__builtin_arm_clz ||
8460 BuiltinID == clang::ARM::BI__builtin_arm_clz64) {
8464 if (BuiltinID == clang::ARM::BI__builtin_arm_clz64)
8470 if (BuiltinID == clang::ARM::BI__builtin_arm_cls) {
8474 if (BuiltinID == clang::ARM::BI__builtin_arm_cls64) {
8480 if (BuiltinID == clang::ARM::BI__clear_cache) {
8481 assert(E->
getNumArgs() == 2 &&
"__clear_cache takes 2 arguments");
8484 for (
unsigned i = 0; i < 2; i++)
8487 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
8488 StringRef Name = FD->
getName();
8492 if (BuiltinID == clang::ARM::BI__builtin_arm_mcrr ||
8493 BuiltinID == clang::ARM::BI__builtin_arm_mcrr2) {
8496 switch (BuiltinID) {
8497 default: llvm_unreachable(
"unexpected builtin");
8498 case clang::ARM::BI__builtin_arm_mcrr:
8501 case clang::ARM::BI__builtin_arm_mcrr2:
8523 return Builder.CreateCall(F, {Coproc, Opc1, Rt, Rt2, CRm});
8526 if (BuiltinID == clang::ARM::BI__builtin_arm_mrrc ||
8527 BuiltinID == clang::ARM::BI__builtin_arm_mrrc2) {
8530 switch (BuiltinID) {
8531 default: llvm_unreachable(
"unexpected builtin");
8532 case clang::ARM::BI__builtin_arm_mrrc:
8535 case clang::ARM::BI__builtin_arm_mrrc2:
8543 Value *RtAndRt2 =
Builder.CreateCall(F, {Coproc, Opc1, CRm});
8553 Value *ShiftCast = llvm::ConstantInt::get(
Int64Ty, 32);
8554 RtAndRt2 =
Builder.CreateShl(Rt, ShiftCast,
"shl",
true);
8555 RtAndRt2 =
Builder.CreateOr(RtAndRt2, Rt1);
8560 if (BuiltinID == clang::ARM::BI__builtin_arm_ldrexd ||
8561 ((BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
8562 BuiltinID == clang::ARM::BI__builtin_arm_ldaex) &&
8564 BuiltinID == clang::ARM::BI__ldrexd) {
8567 switch (BuiltinID) {
8568 default: llvm_unreachable(
"unexpected builtin");
8569 case clang::ARM::BI__builtin_arm_ldaex:
8572 case clang::ARM::BI__builtin_arm_ldrexd:
8573 case clang::ARM::BI__builtin_arm_ldrex:
8574 case clang::ARM::BI__ldrexd:
8588 Val =
Builder.CreateShl(Val0, ShiftCst,
"shl",
true );
8589 Val =
Builder.CreateOr(Val, Val1);
8593 if (BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
8594 BuiltinID == clang::ARM::BI__builtin_arm_ldaex) {
8603 BuiltinID == clang::ARM::BI__builtin_arm_ldaex ? Intrinsic::arm_ldaex
8604 : Intrinsic::arm_ldrex,
8606 CallInst *Val =
Builder.CreateCall(F, LoadAddr,
"ldrex");
8610 if (RealResTy->isPointerTy())
8611 return Builder.CreateIntToPtr(Val, RealResTy);
8613 llvm::Type *IntResTy = llvm::IntegerType::get(
8615 return Builder.CreateBitCast(
Builder.CreateTruncOrBitCast(Val, IntResTy),
8620 if (BuiltinID == clang::ARM::BI__builtin_arm_strexd ||
8621 ((BuiltinID == clang::ARM::BI__builtin_arm_stlex ||
8622 BuiltinID == clang::ARM::BI__builtin_arm_strex) &&
8625 BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlexd
8626 : Intrinsic::arm_strexd);
8639 return Builder.CreateCall(F, {Arg0, Arg1, StPtr},
"strexd");
8642 if (BuiltinID == clang::ARM::BI__builtin_arm_strex ||
8643 BuiltinID == clang::ARM::BI__builtin_arm_stlex) {
8648 llvm::Type *StoreTy =
8651 if (StoreVal->
getType()->isPointerTy())
8654 llvm::Type *
IntTy = llvm::IntegerType::get(
8662 BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlex
8663 : Intrinsic::arm_strex,
8666 CallInst *CI =
Builder.CreateCall(F, {StoreVal, StoreAddr},
"strex");
8668 1, Attribute::get(
getLLVMContext(), Attribute::ElementType, StoreTy));
8672 if (BuiltinID == clang::ARM::BI__builtin_arm_clrex) {
8678 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
8679 switch (BuiltinID) {
8680 case clang::ARM::BI__builtin_arm_crc32b:
8681 CRCIntrinsicID = Intrinsic::arm_crc32b;
break;
8682 case clang::ARM::BI__builtin_arm_crc32cb:
8683 CRCIntrinsicID = Intrinsic::arm_crc32cb;
break;
8684 case clang::ARM::BI__builtin_arm_crc32h:
8685 CRCIntrinsicID = Intrinsic::arm_crc32h;
break;
8686 case clang::ARM::BI__builtin_arm_crc32ch:
8687 CRCIntrinsicID = Intrinsic::arm_crc32ch;
break;
8688 case clang::ARM::BI__builtin_arm_crc32w:
8689 case clang::ARM::BI__builtin_arm_crc32d:
8690 CRCIntrinsicID = Intrinsic::arm_crc32w;
break;
8691 case clang::ARM::BI__builtin_arm_crc32cw:
8692 case clang::ARM::BI__builtin_arm_crc32cd:
8693 CRCIntrinsicID = Intrinsic::arm_crc32cw;
break;
8696 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
8702 if (BuiltinID == clang::ARM::BI__builtin_arm_crc32d ||
8703 BuiltinID == clang::ARM::BI__builtin_arm_crc32cd) {
8711 return Builder.CreateCall(F, {Res, Arg1b});
8716 return Builder.CreateCall(F, {Arg0, Arg1});
8720 if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
8721 BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
8722 BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
8723 BuiltinID == clang::ARM::BI__builtin_arm_wsr ||
8724 BuiltinID == clang::ARM::BI__builtin_arm_wsr64 ||
8725 BuiltinID == clang::ARM::BI__builtin_arm_wsrp) {
8728 if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
8729 BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
8730 BuiltinID == clang::ARM::BI__builtin_arm_rsrp)
8733 bool IsPointerBuiltin = BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
8734 BuiltinID == clang::ARM::BI__builtin_arm_wsrp;
8736 bool Is64Bit = BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
8737 BuiltinID == clang::ARM::BI__builtin_arm_wsr64;
8739 llvm::Type *ValueType;
8740 llvm::Type *RegisterType;
8741 if (IsPointerBuiltin) {
8744 }
else if (Is64Bit) {
8745 ValueType = RegisterType =
Int64Ty;
8747 ValueType = RegisterType =
Int32Ty;
8754 if (BuiltinID == ARM::BI__builtin_sponentry) {
8773 return P.first == BuiltinID;
8776 BuiltinID = It->second;
8780 unsigned ICEArguments = 0;
8785 auto getAlignmentValue32 = [&](
Address addr) ->
Value* {
8786 return Builder.getInt32(addr.getAlignment().getQuantity());
8793 unsigned NumArgs = E->
getNumArgs() - (HasExtraArg ? 1 : 0);
8794 for (
unsigned i = 0, e = NumArgs; i != e; i++) {
8796 switch (BuiltinID) {
8797 case NEON::BI__builtin_neon_vld1_v:
8798 case NEON::BI__builtin_neon_vld1q_v:
8799 case NEON::BI__builtin_neon_vld1q_lane_v:
8800 case NEON::BI__builtin_neon_vld1_lane_v:
8801 case NEON::BI__builtin_neon_vld1_dup_v:
8802 case NEON::BI__builtin_neon_vld1q_dup_v:
8803 case NEON::BI__builtin_neon_vst1_v:
8804 case NEON::BI__builtin_neon_vst1q_v:
8805 case NEON::BI__builtin_neon_vst1q_lane_v:
8806 case NEON::BI__builtin_neon_vst1_lane_v:
8807 case NEON::BI__builtin_neon_vst2_v:
8808 case NEON::BI__builtin_neon_vst2q_v:
8809 case NEON::BI__builtin_neon_vst2_lane_v:
8810 case NEON::BI__builtin_neon_vst2q_lane_v:
8811 case NEON::BI__builtin_neon_vst3_v:
8812 case NEON::BI__builtin_neon_vst3q_v:
8813 case NEON::BI__builtin_neon_vst3_lane_v:
8814 case NEON::BI__builtin_neon_vst3q_lane_v:
8815 case NEON::BI__builtin_neon_vst4_v:
8816 case NEON::BI__builtin_neon_vst4q_v:
8817 case NEON::BI__builtin_neon_vst4_lane_v:
8818 case NEON::BI__builtin_neon_vst4q_lane_v:
8827 switch (BuiltinID) {
8828 case NEON::BI__builtin_neon_vld2_v:
8829 case NEON::BI__builtin_neon_vld2q_v:
8830 case NEON::BI__builtin_neon_vld3_v:
8831 case NEON::BI__builtin_neon_vld3q_v:
8832 case NEON::BI__builtin_neon_vld4_v:
8833 case NEON::BI__builtin_neon_vld4q_v:
8834 case NEON::BI__builtin_neon_vld2_lane_v:
8835 case NEON::BI__builtin_neon_vld2q_lane_v:
8836 case NEON::BI__builtin_neon_vld3_lane_v:
8837 case NEON::BI__builtin_neon_vld3q_lane_v:
8838 case NEON::BI__builtin_neon_vld4_lane_v:
8839 case NEON::BI__builtin_neon_vld4q_lane_v:
8840 case NEON::BI__builtin_neon_vld2_dup_v:
8841 case NEON::BI__builtin_neon_vld2q_dup_v:
8842 case NEON::BI__builtin_neon_vld3_dup_v:
8843 case NEON::BI__builtin_neon_vld3q_dup_v:
8844 case NEON::BI__builtin_neon_vld4_dup_v:
8845 case NEON::BI__builtin_neon_vld4q_dup_v:
8857 switch (BuiltinID) {
8860 case NEON::BI__builtin_neon_vget_lane_i8:
8861 case NEON::BI__builtin_neon_vget_lane_i16:
8862 case NEON::BI__builtin_neon_vget_lane_i32:
8863 case NEON::BI__builtin_neon_vget_lane_i64:
8864 case NEON::BI__builtin_neon_vget_lane_bf16:
8865 case NEON::BI__builtin_neon_vget_lane_f32:
8866 case NEON::BI__builtin_neon_vgetq_lane_i8:
8867 case NEON::BI__builtin_neon_vgetq_lane_i16:
8868 case NEON::BI__builtin_neon_vgetq_lane_i32:
8869 case NEON::BI__builtin_neon_vgetq_lane_i64:
8870 case NEON::BI__builtin_neon_vgetq_lane_bf16:
8871 case NEON::BI__builtin_neon_vgetq_lane_f32:
8872 case NEON::BI__builtin_neon_vduph_lane_bf16:
8873 case NEON::BI__builtin_neon_vduph_laneq_bf16:
8874 return Builder.CreateExtractElement(Ops[0], Ops[1],
"vget_lane");
8876 case NEON::BI__builtin_neon_vrndns_f32: {
8878 llvm::Type *Tys[] = {Arg->
getType()};
8880 return Builder.CreateCall(F, {Arg},
"vrndn"); }
8882 case NEON::BI__builtin_neon_vset_lane_i8:
8883 case NEON::BI__builtin_neon_vset_lane_i16:
8884 case NEON::BI__builtin_neon_vset_lane_i32:
8885 case NEON::BI__builtin_neon_vset_lane_i64:
8886 case NEON::BI__builtin_neon_vset_lane_bf16:
8887 case NEON::BI__builtin_neon_vset_lane_f32:
8888 case NEON::BI__builtin_neon_vsetq_lane_i8:
8889 case NEON::BI__builtin_neon_vsetq_lane_i16:
8890 case NEON::BI__builtin_neon_vsetq_lane_i32:
8891 case NEON::BI__builtin_neon_vsetq_lane_i64:
8892 case NEON::BI__builtin_neon_vsetq_lane_bf16:
8893 case NEON::BI__builtin_neon_vsetq_lane_f32:
8894 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
8896 case NEON::BI__builtin_neon_vsha1h_u32:
8899 case NEON::BI__builtin_neon_vsha1cq_u32:
8902 case NEON::BI__builtin_neon_vsha1pq_u32:
8905 case NEON::BI__builtin_neon_vsha1mq_u32:
8909 case NEON::BI__builtin_neon_vcvth_bf16_f32: {
8916 case clang::ARM::BI_MoveToCoprocessor:
8917 case clang::ARM::BI_MoveToCoprocessor2: {
8919 ? Intrinsic::arm_mcr
8920 : Intrinsic::arm_mcr2);
8921 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0],
8922 Ops[3], Ops[4], Ops[5]});
8927 assert(HasExtraArg);
8929 std::optional<llvm::APSInt>
Result =
8934 if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f ||
8935 BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_d) {
8938 if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f)
8944 bool usgn =
Result->getZExtValue() == 1;
8945 unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr;
8949 return Builder.CreateCall(F, Ops,
"vcvtr");
8954 bool usgn =
Type.isUnsigned();
8955 bool rightShift =
false;
8957 llvm::FixedVectorType *VTy =
8960 llvm::Type *Ty = VTy;
8971 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
8972 Builtin->NameHint, Builtin->TypeModifier, E, Ops, PtrOp0, PtrOp1, Arch);
8975 switch (BuiltinID) {
8976 default:
return nullptr;
8977 case NEON::BI__builtin_neon_vld1q_lane_v:
8980 if (VTy->getElementType()->isIntegerTy(64)) {
8982 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
8983 int Lane = cast<ConstantInt>(Ops[2])->getZExtValue();
8984 Value *SV = llvm::ConstantVector::get(ConstantInt::get(
Int32Ty, 1-Lane));
8985 Ops[1] =
Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
8987 Ty = llvm::FixedVectorType::get(VTy->getElementType(), 1);
8990 Value *Align = getAlignmentValue32(PtrOp0);
8993 int Indices[] = {1 - Lane, Lane};
8994 return Builder.CreateShuffleVector(Ops[1], Ld, Indices,
"vld1q_lane");
8997 case NEON::BI__builtin_neon_vld1_lane_v: {
8998 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
9001 return Builder.CreateInsertElement(Ops[1], Ld, Ops[2],
"vld1_lane");
9003 case NEON::BI__builtin_neon_vqrshrn_n_v:
9005 usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns;
9008 case NEON::BI__builtin_neon_vqrshrun_n_v:
9010 Ops,
"vqrshrun_n", 1,
true);
9011 case NEON::BI__builtin_neon_vqshrn_n_v:
9012 Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns;
9015 case NEON::BI__builtin_neon_vqshrun_n_v:
9017 Ops,
"vqshrun_n", 1,
true);
9018 case NEON::BI__builtin_neon_vrecpe_v:
9019 case NEON::BI__builtin_neon_vrecpeq_v:
9022 case NEON::BI__builtin_neon_vrshrn_n_v:
9024 Ops,
"vrshrn_n", 1,
true);
9025 case NEON::BI__builtin_neon_vrsra_n_v:
9026 case NEON::BI__builtin_neon_vrsraq_n_v:
9027 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
9028 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
9030 Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
9032 return Builder.CreateAdd(Ops[0], Ops[1],
"vrsra_n");
9033 case NEON::BI__builtin_neon_vsri_n_v:
9034 case NEON::BI__builtin_neon_vsriq_n_v:
9037 case NEON::BI__builtin_neon_vsli_n_v:
9038 case NEON::BI__builtin_neon_vsliq_n_v:
9042 case NEON::BI__builtin_neon_vsra_n_v:
9043 case NEON::BI__builtin_neon_vsraq_n_v:
9044 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
9046 return Builder.CreateAdd(Ops[0], Ops[1]);
9047 case NEON::BI__builtin_neon_vst1q_lane_v:
9050 if (VTy->getElementType()->isIntegerTy(64)) {
9051 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
9052 Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2]));
9053 Ops[1] =
Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
9054 Ops[2] = getAlignmentValue32(PtrOp0);
9055 llvm::Type *Tys[] = {
Int8PtrTy, Ops[1]->getType()};
9060 case NEON::BI__builtin_neon_vst1_lane_v: {
9061 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
9062 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2]);
9066 case NEON::BI__builtin_neon_vtbl1_v:
9069 case NEON::BI__builtin_neon_vtbl2_v:
9072 case NEON::BI__builtin_neon_vtbl3_v:
9075 case NEON::BI__builtin_neon_vtbl4_v:
9078 case NEON::BI__builtin_neon_vtbx1_v:
9081 case NEON::BI__builtin_neon_vtbx2_v:
9084 case NEON::BI__builtin_neon_vtbx3_v:
9087 case NEON::BI__builtin_neon_vtbx4_v:
9093template<
typename Integer>
9102 return Unsigned ? Builder.CreateZExt(
V, T) : Builder.CreateSExt(
V, T);
9112 unsigned LaneBits = cast<llvm::VectorType>(
V->getType())
9114 ->getPrimitiveSizeInBits();
9115 if (Shift == LaneBits) {
9120 return llvm::Constant::getNullValue(
V->getType());
9124 return Unsigned ? Builder.CreateLShr(
V, Shift) : Builder.CreateAShr(
V, Shift);
9131 unsigned Elements = 128 /
V->getType()->getPrimitiveSizeInBits();
9132 return Builder.CreateVectorSplat(Elements,
V);
9138 llvm::Type *DestType) {
9151 V->getType()->getScalarSizeInBits() != DestType->getScalarSizeInBits()) {
9152 return Builder.CreateCall(
9154 {DestType, V->getType()}),
9157 return Builder.CreateBitCast(
V, DestType);
9165 unsigned InputElements =
9166 cast<llvm::FixedVectorType>(
V->getType())->getNumElements();
9167 for (
unsigned i = 0; i < InputElements; i += 2)
9168 Indices.push_back(i + Odd);
9169 return Builder.CreateShuffleVector(
V, Indices);
9175 assert(V0->getType() == V1->getType() &&
"Can't zip different vector types");
9177 unsigned InputElements =
9178 cast<llvm::FixedVectorType>(V0->getType())->getNumElements();
9179 for (
unsigned i = 0; i < InputElements; i++) {
9180 Indices.push_back(i);
9181 Indices.push_back(i + InputElements);
9183 return Builder.CreateShuffleVector(V0, V1, Indices);
9186template<
unsigned HighBit,
unsigned OtherBits>
9190 llvm::Type *T = cast<llvm::VectorType>(VT)->getElementType();
9191 unsigned LaneBits = T->getPrimitiveSizeInBits();
9192 uint32_t
Value = HighBit << (LaneBits - 1);
9194 Value |= (1UL << (LaneBits - 1)) - 1;
9195 llvm::Value *Lane = llvm::ConstantInt::get(T,
Value);
9201 unsigned ReverseWidth) {
9205 unsigned LaneSize =
V->getType()->getScalarSizeInBits();
9206 unsigned Elements = 128 / LaneSize;
9207 unsigned Mask = ReverseWidth / LaneSize - 1;
9208 for (
unsigned i = 0; i < Elements; i++)
9209 Indices.push_back(i ^ Mask);
9210 return Builder.CreateShuffleVector(
V, Indices);
9216 llvm::Triple::ArchType Arch) {
9217 enum class CustomCodeGen { VLD24, VST24 } CustomCodeGenType;
9218 Intrinsic::ID IRIntr;
9219 unsigned NumVectors;
9222 switch (BuiltinID) {
9223 #include "clang/Basic/arm_mve_builtin_cg.inc"
9234 switch (CustomCodeGenType) {
9236 case CustomCodeGen::VLD24: {
9240 auto MvecCType = E->
getType();
9242 assert(MvecLType->isStructTy() &&
9243 "Return type for vld[24]q should be a struct");
9244 assert(MvecLType->getStructNumElements() == 1 &&
9245 "Return-type struct for vld[24]q should have one element");
9246 auto MvecLTypeInner = MvecLType->getStructElementType(0);
9247 assert(MvecLTypeInner->isArrayTy() &&
9248 "Return-type struct for vld[24]q should contain an array");
9249 assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
9250 "Array member of return-type struct vld[24]q has wrong length");
9251 auto VecLType = MvecLTypeInner->getArrayElementType();
9253 Tys.push_back(VecLType);
9255 auto Addr = E->
getArg(0);
9261 Value *MvecOut = PoisonValue::get(MvecLType);
9262 for (
unsigned i = 0; i < NumVectors; ++i) {
9263 Value *Vec =
Builder.CreateExtractValue(LoadResult, i);
9264 MvecOut =
Builder.CreateInsertValue(MvecOut, Vec, {0, i});
9273 case CustomCodeGen::VST24: {
9277 auto Addr = E->
getArg(0);
9283 assert(MvecLType->isStructTy() &&
"Data type for vst2q should be a struct");
9284 assert(MvecLType->getStructNumElements() == 1 &&
9285 "Data-type struct for vst2q should have one element");
9286 auto MvecLTypeInner = MvecLType->getStructElementType(0);
9287 assert(MvecLTypeInner->isArrayTy() &&
9288 "Data-type struct for vst2q should contain an array");
9289 assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
9290 "Array member of return-type struct vld[24]q has wrong length");
9291 auto VecLType = MvecLTypeInner->getArrayElementType();
9293 Tys.push_back(VecLType);
9298 for (
unsigned i = 0; i < NumVectors; i++)
9299 Ops.push_back(
Builder.CreateExtractValue(Mvec, {0, i}));
9302 Value *ToReturn =
nullptr;
9303 for (
unsigned i = 0; i < NumVectors; i++) {
9304 Ops.push_back(llvm::ConstantInt::get(
Int32Ty, i));
9305 ToReturn =
Builder.CreateCall(F, Ops);
9311 llvm_unreachable(
"unknown custom codegen type.");
9317 llvm::Triple::ArchType Arch) {
9318 switch (BuiltinID) {
9321#include "clang/Basic/arm_cde_builtin_cg.inc"
9328 llvm::Triple::ArchType Arch) {
9329 unsigned int Int = 0;
9330 const char *
s =
nullptr;
9332 switch (BuiltinID) {
9335 case NEON::BI__builtin_neon_vtbl1_v:
9336 case NEON::BI__builtin_neon_vqtbl1_v:
9337 case NEON::BI__builtin_neon_vqtbl1q_v:
9338 case NEON::BI__builtin_neon_vtbl2_v:
9339 case NEON::BI__builtin_neon_vqtbl2_v:
9340 case NEON::BI__builtin_neon_vqtbl2q_v:
9341 case NEON::BI__builtin_neon_vtbl3_v:
9342 case NEON::BI__builtin_neon_vqtbl3_v:
9343 case NEON::BI__builtin_neon_vqtbl3q_v:
9344 case NEON::BI__builtin_neon_vtbl4_v:
9345 case NEON::BI__builtin_neon_vqtbl4_v:
9346 case NEON::BI__builtin_neon_vqtbl4q_v:
9348 case NEON::BI__builtin_neon_vtbx1_v:
9349 case NEON::BI__builtin_neon_vqtbx1_v:
9350 case NEON::BI__builtin_neon_vqtbx1q_v:
9351 case NEON::BI__builtin_neon_vtbx2_v:
9352 case NEON::BI__builtin_neon_vqtbx2_v:
9353 case NEON::BI__builtin_neon_vqtbx2q_v:
9354 case NEON::BI__builtin_neon_vtbx3_v:
9355 case NEON::BI__builtin_neon_vqtbx3_v:
9356 case NEON::BI__builtin_neon_vqtbx3q_v:
9357 case NEON::BI__builtin_neon_vtbx4_v:
9358 case NEON::BI__builtin_neon_vqtbx4_v:
9359 case NEON::BI__builtin_neon_vqtbx4q_v:
9367 std::optional<llvm::APSInt>
Result =
9382 switch (BuiltinID) {
9383 case NEON::BI__builtin_neon_vtbl1_v: {
9385 Ty, Intrinsic::aarch64_neon_tbl1,
"vtbl1");
9387 case NEON::BI__builtin_neon_vtbl2_v: {
9389 Ty, Intrinsic::aarch64_neon_tbl1,
"vtbl1");
9391 case NEON::BI__builtin_neon_vtbl3_v: {
9393 Ty, Intrinsic::aarch64_neon_tbl2,
"vtbl2");
9395 case NEON::BI__builtin_neon_vtbl4_v: {
9397 Ty, Intrinsic::aarch64_neon_tbl2,
"vtbl2");
9399 case NEON::BI__builtin_neon_vtbx1_v: {
9402 Intrinsic::aarch64_neon_tbl1,
"vtbl1");
9404 llvm::Constant *EightV = ConstantInt::get(Ty, 8);
9405 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV);
9406 CmpRes = Builder.CreateSExt(CmpRes, Ty);
9408 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
9409 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
9410 return Builder.CreateOr(EltsFromInput, EltsFromTbl,
"vtbx");
9412 case NEON::BI__builtin_neon_vtbx2_v: {
9414 Ty, Intrinsic::aarch64_neon_tbx1,
"vtbx1");
9416 case NEON::BI__builtin_neon_vtbx3_v: {
9419 Intrinsic::aarch64_neon_tbl2,
"vtbl2");
9421 llvm::Constant *TwentyFourV = ConstantInt::get(Ty, 24);
9422 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4],
9424 CmpRes = Builder.CreateSExt(CmpRes, Ty);
9426 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
9427 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
9428 return Builder.CreateOr(EltsFromInput, EltsFromTbl,
"vtbx");
9430 case NEON::BI__builtin_neon_vtbx4_v: {
9432 Ty, Intrinsic::aarch64_neon_tbx2,
"vtbx2");
9434 case NEON::BI__builtin_neon_vqtbl1_v:
9435 case NEON::BI__builtin_neon_vqtbl1q_v:
9436 Int = Intrinsic::aarch64_neon_tbl1;
s =
"vtbl1";
break;
9437 case NEON::BI__builtin_neon_vqtbl2_v:
9438 case NEON::BI__builtin_neon_vqtbl2q_v: {
9439 Int = Intrinsic::aarch64_neon_tbl2;
s =
"vtbl2";
break;
9440 case NEON::BI__builtin_neon_vqtbl3_v:
9441 case NEON::BI__builtin_neon_vqtbl3q_v:
9442 Int = Intrinsic::aarch64_neon_tbl3;
s =
"vtbl3";
break;
9443 case NEON::BI__builtin_neon_vqtbl4_v:
9444 case NEON::BI__builtin_neon_vqtbl4q_v:
9445 Int = Intrinsic::aarch64_neon_tbl4;
s =
"vtbl4";
break;
9446 case NEON::BI__builtin_neon_vqtbx1_v:
9447 case NEON::BI__builtin_neon_vqtbx1q_v:
9448 Int = Intrinsic::aarch64_neon_tbx1;
s =
"vtbx1";
break;
9449 case NEON::BI__builtin_neon_vqtbx2_v:
9450 case NEON::BI__builtin_neon_vqtbx2q_v:
9451 Int = Intrinsic::aarch64_neon_tbx2;
s =
"vtbx2";
break;
9452 case NEON::BI__builtin_neon_vqtbx3_v:
9453 case NEON::BI__builtin_neon_vqtbx3q_v:
9454 Int = Intrinsic::aarch64_neon_tbx3;
s =
"vtbx3";
break;
9455 case NEON::BI__builtin_neon_vqtbx4_v:
9456 case NEON::BI__builtin_neon_vqtbx4q_v:
9457 Int = Intrinsic::aarch64_neon_tbx4;
s =
"vtbx4";
break;
9469 auto *VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
9471 Value *
V = PoisonValue::get(VTy);
9472 llvm::Constant *CI = ConstantInt::get(
SizeTy, 0);
9473 Op =
Builder.CreateInsertElement(
V, Op, CI);
9482 case SVETypeFlags::MemEltTyDefault:
9484 case SVETypeFlags::MemEltTyInt8:
9486 case SVETypeFlags::MemEltTyInt16:
9488 case SVETypeFlags::MemEltTyInt32:
9490 case SVETypeFlags::MemEltTyInt64:
9493 llvm_unreachable(
"Unknown MemEltType");
9499 llvm_unreachable(
"Invalid SVETypeFlag!");
9501 case SVETypeFlags::EltTyInt8:
9503 case SVETypeFlags::EltTyInt16:
9505 case SVETypeFlags::EltTyInt32:
9507 case SVETypeFlags::EltTyInt64:
9509 case SVETypeFlags::EltTyInt128:
9512 case SVETypeFlags::EltTyFloat16:
9514 case SVETypeFlags::EltTyFloat32:
9516 case SVETypeFlags::EltTyFloat64:
9519 case SVETypeFlags::EltTyBFloat16:
9522 case SVETypeFlags::EltTyBool8:
9523 case SVETypeFlags::EltTyBool16:
9524 case SVETypeFlags::EltTyBool32:
9525 case SVETypeFlags::EltTyBool64:
9532llvm::ScalableVectorType *
9535 default: llvm_unreachable(
"Unhandled SVETypeFlag!");
9537 case SVETypeFlags::EltTyInt8:
9538 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 16);
9539 case SVETypeFlags::EltTyInt16:
9540 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 8);
9541 case SVETypeFlags::EltTyInt32:
9542 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 4);
9543 case SVETypeFlags::EltTyInt64:
9544 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 2);
9546 case SVETypeFlags::EltTyBFloat16:
9547 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 8);
9548 case SVETypeFlags::EltTyFloat16:
9549 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 8);
9550 case SVETypeFlags::EltTyFloat32:
9551 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 4);
9552 case SVETypeFlags::EltTyFloat64:
9553 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 2);
9555 case SVETypeFlags::EltTyBool8:
9556 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 16);
9557 case SVETypeFlags::EltTyBool16:
9558 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 8);
9559 case SVETypeFlags::EltTyBool32:
9560 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 4);
9561 case SVETypeFlags::EltTyBool64:
9562 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 2);
9567llvm::ScalableVectorType *
9571 llvm_unreachable(
"Invalid SVETypeFlag!");
9573 case SVETypeFlags::EltTyInt8:
9574 return llvm::ScalableVectorType::get(
Builder.getInt8Ty(), 16);
9575 case SVETypeFlags::EltTyInt16:
9576 return llvm::ScalableVectorType::get(
Builder.getInt16Ty(), 8);
9577 case SVETypeFlags::EltTyInt32:
9578 return llvm::ScalableVectorType::get(
Builder.getInt32Ty(), 4);
9579 case SVETypeFlags::EltTyInt64:
9580 return llvm::ScalableVectorType::get(
Builder.getInt64Ty(), 2);
9582 case SVETypeFlags::EltTyFloat16:
9583 return llvm::ScalableVectorType::get(
Builder.getHalfTy(), 8);
9584 case SVETypeFlags::EltTyBFloat16:
9585 return llvm::ScalableVectorType::get(
Builder.getBFloatTy(), 8);
9586 case SVETypeFlags::EltTyFloat32:
9587 return llvm::ScalableVectorType::get(
Builder.getFloatTy(), 4);
9588 case SVETypeFlags::EltTyFloat64:
9589 return llvm::ScalableVectorType::get(
Builder.getDoubleTy(), 2);
9591 case SVETypeFlags::EltTyBool8:
9592 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 16);
9593 case SVETypeFlags::EltTyBool16:
9594 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 8);
9595 case SVETypeFlags::EltTyBool32:
9596 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 4);
9597 case SVETypeFlags::EltTyBool64:
9598 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 2);
9613 return llvm::ScalableVectorType::get(EltTy, NumElts);
9619 llvm::ScalableVectorType *VTy) {
9621 if (isa<TargetExtType>(Pred->
getType()) &&
9622 cast<TargetExtType>(Pred->
getType())->getName() ==
"aarch64.svcount")
9625 auto *RTy = llvm::VectorType::get(IntegerType::get(
getLLVMContext(), 1), VTy);
9630 llvm::Type *IntrinsicTy;
9631 switch (VTy->getMinNumElements()) {
9633 llvm_unreachable(
"unsupported element count!");
9638 IntID = Intrinsic::aarch64_sve_convert_from_svbool;
9642 IntID = Intrinsic::aarch64_sve_convert_to_svbool;
9643 IntrinsicTy = Pred->
getType();
9649 assert(
C->getType() == RTy &&
"Unexpected return type!");
9657 auto *OverloadedTy =
9661 if (Ops[1]->getType()->isVectorTy())
9681 Ops[0], cast<llvm::ScalableVectorType>(F->getArg(0)->getType()));
9686 if (Ops.size() == 2) {
9687 assert(Ops[1]->getType()->isVectorTy() &&
"Scalar base requires an offset");
9688 Ops.push_back(ConstantInt::get(
Int64Ty, 0));
9693 if (!TypeFlags.
isByteIndexed() && Ops[1]->getType()->isVectorTy()) {
9694 unsigned BytesPerElt =
9695 OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
9696 Ops[2] =
Builder.CreateShl(Ops[2], Log2_32(BytesPerElt));
9711 auto *OverloadedTy =
9716 Ops.insert(Ops.begin(), Ops.pop_back_val());
9719 if (Ops[2]->getType()->isVectorTy())
9734 if (Ops.size() == 3) {
9735 assert(Ops[1]->getType()->isVectorTy() &&
"Scalar base requires an offset");
9736 Ops.push_back(ConstantInt::get(
Int64Ty, 0));
9741 Ops[0] =
Builder.CreateTrunc(Ops[0], OverloadedTy);
9751 Ops[1], cast<llvm::ScalableVectorType>(F->getArg(1)->getType()));
9755 if (!TypeFlags.
isByteIndexed() && Ops[2]->getType()->isVectorTy()) {
9756 unsigned BytesPerElt =
9757 OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
9758 Ops[3] =
Builder.CreateShl(Ops[3], Log2_32(BytesPerElt));
9761 return Builder.CreateCall(F, Ops);
9769 auto *OverloadedTy = dyn_cast<llvm::ScalableVectorType>(Ops[1]->getType());
9771 OverloadedTy = cast<llvm::ScalableVectorType>(Ops[2]->getType());
9777 if (Ops[1]->getType()->isVectorTy()) {
9778 if (Ops.size() == 3) {
9780 Ops.push_back(ConstantInt::get(
Int64Ty, 0));
9783 std::swap(Ops[2], Ops[3]);
9787 unsigned BytesPerElt = MemEltTy->getPrimitiveSizeInBits() / 8;
9788 if (BytesPerElt > 1)
9789 Ops[2] =
Builder.CreateShl(Ops[2], Log2_32(BytesPerElt));
9794 return Builder.CreateCall(F, Ops);
9800 llvm::ScalableVectorType *VTy =
getSVEType(TypeFlags);
9804 case Intrinsic::aarch64_sve_ld2_sret:
9805 case Intrinsic::aarch64_sve_ld1_pn_x2:
9806 case Intrinsic::aarch64_sve_ldnt1_pn_x2:
9807 case Intrinsic::aarch64_sve_ld2q_sret:
9810 case Intrinsic::aarch64_sve_ld3_sret:
9811 case Intrinsic::aarch64_sve_ld3q_sret:
9814 case Intrinsic::aarch64_sve_ld4_sret:
9815 case Intrinsic::aarch64_sve_ld1_pn_x4:
9816 case Intrinsic::aarch64_sve_ldnt1_pn_x4:
9817 case Intrinsic::aarch64_sve_ld4q_sret:
9821 llvm_unreachable(
"unknown intrinsic!");
9823 auto RetTy = llvm::VectorType::get(VTy->getElementType(),
9824 VTy->getElementCount() * N);
9827 Value *BasePtr = Ops[1];
9835 unsigned MinElts = VTy->getMinNumElements();
9836 Value *
Ret = llvm::PoisonValue::get(RetTy);
9837 for (
unsigned I = 0; I < N; I++) {
9840 Ret =
Builder.CreateInsertVector(RetTy, Ret, SRet, Idx);
9848 llvm::ScalableVectorType *VTy =
getSVEType(TypeFlags);
9852 case Intrinsic::aarch64_sve_st2:
9853 case Intrinsic::aarch64_sve_st1_pn_x2:
9854 case Intrinsic::aarch64_sve_stnt1_pn_x2:
9855 case Intrinsic::aarch64_sve_st2q:
9858 case Intrinsic::aarch64_sve_st3:
9859 case Intrinsic::aarch64_sve_st3q:
9862 case Intrinsic::aarch64_sve_st4:
9863 case Intrinsic::aarch64_sve_st1_pn_x4:
9864 case Intrinsic::aarch64_sve_stnt1_pn_x4:
9865 case Intrinsic::aarch64_sve_st4q:
9869 llvm_unreachable(
"unknown intrinsic!");
9873 Value *BasePtr = Ops[1];
9876 if (Ops.size() > (2 + N))
9882 for (
unsigned I = Ops.size() - N; I < Ops.size(); ++I)
9883 Operands.push_back(Ops[I]);
9884 Operands.append({Predicate, BasePtr});
9887 return Builder.CreateCall(F, Operands);
9895 unsigned BuiltinID) {
9907 llvm::ScalableVectorType *Ty =
getSVEType(TypeFlags);
9913 llvm::Type *OverloadedTy =
getSVEType(TypeFlags);
9920 unsigned BuiltinID) {
9923 auto *MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
9926 Value *BasePtr = Ops[1];
9932 Value *PrfOp = Ops.back();
9935 return Builder.CreateCall(F, {Predicate, BasePtr, PrfOp});
9939 llvm::Type *ReturnTy,
9941 unsigned IntrinsicID,
9942 bool IsZExtReturn) {
9949 auto VectorTy = cast<llvm::ScalableVectorType>(ReturnTy);
9950 llvm::ScalableVectorType *MemoryTy =
nullptr;
9951 llvm::ScalableVectorType *PredTy =
nullptr;
9952 bool IsQuadLoad =
false;
9953 switch (IntrinsicID) {
9954 case Intrinsic::aarch64_sve_ld1uwq:
9955 case Intrinsic::aarch64_sve_ld1udq:
9956 MemoryTy = llvm::ScalableVectorType::get(MemEltTy, 1);
9957 PredTy = llvm::ScalableVectorType::get(
9962 MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
9968 Value *BasePtr = Ops[1];
9976 cast<llvm::Instruction>(
Builder.CreateCall(F, {Predicate, BasePtr}));
9983 return IsZExtReturn ?
Builder.CreateZExt(Load, VectorTy)
9984 :
Builder.CreateSExt(Load, VectorTy);
9989 unsigned IntrinsicID) {
9996 auto VectorTy = cast<llvm::ScalableVectorType>(Ops.back()->getType());
9997 auto MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
9999 auto PredTy = MemoryTy;
10000 auto AddrMemoryTy = MemoryTy;
10001 bool IsQuadStore =
false;
10003 switch (IntrinsicID) {
10004 case Intrinsic::aarch64_sve_st1wq:
10005 case Intrinsic::aarch64_sve_st1dq:
10006 AddrMemoryTy = llvm::ScalableVectorType::get(MemEltTy, 1);
10008 llvm::ScalableVectorType::get(IntegerType::get(
getLLVMContext(), 1), 1);
10009 IsQuadStore =
true;
10015 Value *BasePtr = Ops[1];
10018 if (Ops.size() == 4)
10023 IsQuadStore ? Ops.back() :
Builder.CreateTrunc(Ops.back(), MemoryTy);
10028 cast<llvm::Instruction>(
Builder.CreateCall(F, {Val, Predicate, BasePtr}));
10041 NewOps.push_back(Ops[2]);
10043 llvm::Value *BasePtr = Ops[3];
10047 if (Ops.size() == 5) {
10050 llvm::Value *StreamingVectorLengthCall =
10051 Builder.CreateCall(StreamingVectorLength);
10052 llvm::Value *Mulvl =
10053 Builder.CreateMul(StreamingVectorLengthCall, Ops[4],
"mulvl");
10057 NewOps.push_back(BasePtr);
10058 NewOps.push_back(Ops[0]);
10059 NewOps.push_back(Ops[1]);
10061 return Builder.CreateCall(F, NewOps);
10073 return Builder.CreateCall(F, Ops);
10080 if (Ops.size() == 0)
10081 Ops.push_back(llvm::ConstantInt::get(
Int32Ty, 255));
10083 return Builder.CreateCall(F, Ops);
10089 if (Ops.size() == 2)
10090 Ops.push_back(
Builder.getInt32(0));
10094 return Builder.CreateCall(F, Ops);
10100 return Builder.CreateVectorSplat(
10101 cast<llvm::VectorType>(Ty)->getElementCount(), Scalar);
10115 return Builder.CreateBitCast(Val, Ty);
10120 auto *SplatZero = Constant::getNullValue(Ty);
10121 Ops.insert(Ops.begin(), SplatZero);
10126 auto *SplatUndef = UndefValue::get(Ty);
10127 Ops.insert(Ops.begin(), SplatUndef);
10132 llvm::Type *ResultType,
10137 llvm::Type *DefaultType =
getSVEType(TypeFlags);
10140 return {DefaultType, Ops[1]->getType()};
10146 return {Ops[0]->getType(), Ops.back()->getType()};
10148 if (TypeFlags.
isReductionQV() && !ResultType->isScalableTy() &&
10149 ResultType->isVectorTy())
10150 return {ResultType, Ops[1]->getType()};
10153 return {DefaultType};
10160 "Expects TypleFlag isTupleSet or TypeFlags.isTupleSet()");
10162 unsigned I = cast<ConstantInt>(Ops[1])->getSExtValue();
10163 auto *SingleVecTy = dyn_cast<llvm::ScalableVectorType>(
10164 TypeFlags.
isTupleSet() ? Ops[2]->getType() : Ty);
10166 I * SingleVecTy->getMinNumElements());
10169 return Builder.CreateInsertVector(Ty, Ops[0], Ops[2], Idx);
10170 return Builder.CreateExtractVector(Ty, Ops[0], Idx);
10176 assert(TypeFlags.
isTupleCreate() &&
"Expects TypleFlag isTupleCreate");
10178 auto *SrcTy = dyn_cast<llvm::ScalableVectorType>(Ops[0]->getType());
10179 unsigned MinElts = SrcTy->getMinNumElements();
10180 Value *
Call = llvm::PoisonValue::get(Ty);
10181 for (
unsigned I = 0; I < Ops.size(); I++) {
10192 auto *StructTy = dyn_cast<StructType>(
Call->getType());
10196 auto *VTy = dyn_cast<ScalableVectorType>(StructTy->getTypeAtIndex(0
U));
10199 unsigned N = StructTy->getNumElements();
10202 bool IsPredTy = VTy->getElementType()->isIntegerTy(1);
10203 unsigned MinElts = IsPredTy ? 16 : VTy->getMinNumElements();
10205 ScalableVectorType *WideVTy =
10206 ScalableVectorType::get(VTy->getElementType(), MinElts * N);
10207 Value *
Ret = llvm::PoisonValue::get(WideVTy);
10208 for (
unsigned I = 0; I < N; ++I) {
10210 assert(SRet->
getType() == VTy &&
"Unexpected type for result value");
10215 SRet, ScalableVectorType::get(
Builder.getInt1Ty(), 16));
10217 Ret =
Builder.CreateInsertVector(WideVTy, Ret, SRet, Idx);
10228 unsigned ICEArguments = 0;
10237 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; i++) {
10238 bool IsICE = ICEArguments & (1 << i);
10244 std::optional<llvm::APSInt>
Result =
10246 assert(
Result &&
"Expected argument to be a constant");
10256 if (IsTupleGetOrSet || !isa<ScalableVectorType>(Arg->getType())) {
10257 Ops.push_back(Arg);
10261 auto *VTy = cast<ScalableVectorType>(Arg->getType());
10262 unsigned MinElts = VTy->getMinNumElements();
10263 bool IsPred = VTy->getElementType()->isIntegerTy(1);
10264 unsigned N = (MinElts * VTy->getScalarSizeInBits()) / (IsPred ? 16 : 128);
10267 Ops.push_back(Arg);
10271 for (
unsigned I = 0; I < N; ++I) {
10274 ScalableVectorType::get(VTy->getElementType(), MinElts / N);
10275 Ops.push_back(
Builder.CreateExtractVector(NewVTy, Arg, Idx));
10283 if (BuiltinID >= SVE::BI__builtin_sve_reinterpret_s8_s8 &&
10284 BuiltinID <= SVE::BI__builtin_sve_reinterpret_f64_f64_x4) {
10299 else if (TypeFlags.
isStore())
10317 else if (TypeFlags.
isUndef())
10318 return UndefValue::get(Ty);
10319 else if (Builtin->LLVMIntrinsic != 0) {
10320 if (TypeFlags.
getMergeType() == SVETypeFlags::MergeZeroExp)
10323 if (TypeFlags.
getMergeType() == SVETypeFlags::MergeAnyExp)
10329 Ops.push_back(
Builder.getInt32( 31));
10331 Ops.insert(&Ops[1],
Builder.getInt32( 31));
10334 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
10335 if (
auto PredTy = dyn_cast<llvm::VectorType>(Ops[i]->getType()))
10336 if (PredTy->getElementType()->isIntegerTy(1))
10346 std::swap(Ops[1], Ops[2]);
10348 std::swap(Ops[1], Ops[2]);
10351 std::swap(Ops[1], Ops[2]);
10354 std::swap(Ops[1], Ops[3]);
10357 if (TypeFlags.
getMergeType() == SVETypeFlags::MergeZero) {
10358 llvm::Type *OpndTy = Ops[1]->getType();
10359 auto *SplatZero = Constant::getNullValue(OpndTy);
10360 Ops[1] =
Builder.CreateSelect(Ops[0], Ops[1], SplatZero);
10368 if (
auto PredTy = dyn_cast<llvm::VectorType>(
Call->getType()))
10369 if (PredTy->getScalarType()->isIntegerTy(1))
10375 switch (BuiltinID) {
10379 case SVE::BI__builtin_sve_svreinterpret_b: {
10383 CGM.
getIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool, SVCountTy);
10384 return Builder.CreateCall(CastFromSVCountF, Ops[0]);
10386 case SVE::BI__builtin_sve_svreinterpret_c: {
10390 CGM.
getIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool, SVCountTy);
10391 return Builder.CreateCall(CastToSVCountF, Ops[0]);
10394 case SVE::BI__builtin_sve_svpsel_lane_b8:
10395 case SVE::BI__builtin_sve_svpsel_lane_b16:
10396 case SVE::BI__builtin_sve_svpsel_lane_b32:
10397 case SVE::BI__builtin_sve_svpsel_lane_b64:
10398 case SVE::BI__builtin_sve_svpsel_lane_c8:
10399 case SVE::BI__builtin_sve_svpsel_lane_c16:
10400 case SVE::BI__builtin_sve_svpsel_lane_c32:
10401 case SVE::BI__builtin_sve_svpsel_lane_c64: {
10402 bool IsSVCount = isa<TargetExtType>(Ops[0]->getType());
10403 assert(((!IsSVCount || cast<TargetExtType>(Ops[0]->getType())->
getName() ==
10404 "aarch64.svcount")) &&
10405 "Unexpected TargetExtType");
10409 CGM.
getIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool, SVCountTy);
10411 CGM.
getIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool, SVCountTy);
10415 llvm::Value *Ops0 =
10416 IsSVCount ?
Builder.CreateCall(CastFromSVCountF, Ops[0]) : Ops[0];
10418 llvm::Value *PSel =
Builder.CreateCall(F, {Ops0, Ops1, Ops[2]});
10419 return IsSVCount ?
Builder.CreateCall(CastToSVCountF, PSel) : PSel;
10421 case SVE::BI__builtin_sve_svmov_b_z: {
10424 llvm::Type* OverloadedTy =
getSVEType(TypeFlags);
10426 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[1]});
10429 case SVE::BI__builtin_sve_svnot_b_z: {
10432 llvm::Type* OverloadedTy =
getSVEType(TypeFlags);
10434 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[0]});
10437 case SVE::BI__builtin_sve_svmovlb_u16:
10438 case SVE::BI__builtin_sve_svmovlb_u32:
10439 case SVE::BI__builtin_sve_svmovlb_u64:
10440 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllb);
10442 case SVE::BI__builtin_sve_svmovlb_s16:
10443 case SVE::BI__builtin_sve_svmovlb_s32:
10444 case SVE::BI__builtin_sve_svmovlb_s64:
10445 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllb);
10447 case SVE::BI__builtin_sve_svmovlt_u16:
10448 case SVE::BI__builtin_sve_svmovlt_u32:
10449 case SVE::BI__builtin_sve_svmovlt_u64:
10450 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllt);
10452 case SVE::BI__builtin_sve_svmovlt_s16:
10453 case SVE::BI__builtin_sve_svmovlt_s32:
10454 case SVE::BI__builtin_sve_svmovlt_s64:
10455 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllt);
10457 case SVE::BI__builtin_sve_svpmullt_u16:
10458 case SVE::BI__builtin_sve_svpmullt_u64:
10459 case SVE::BI__builtin_sve_svpmullt_n_u16:
10460 case SVE::BI__builtin_sve_svpmullt_n_u64:
10461 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullt_pair);
10463 case SVE::BI__builtin_sve_svpmullb_u16:
10464 case SVE::BI__builtin_sve_svpmullb_u64:
10465 case SVE::BI__builtin_sve_svpmullb_n_u16:
10466 case SVE::BI__builtin_sve_svpmullb_n_u64:
10467 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullb_pair);
10469 case SVE::BI__builtin_sve_svdup_n_b8:
10470 case SVE::BI__builtin_sve_svdup_n_b16:
10471 case SVE::BI__builtin_sve_svdup_n_b32:
10472 case SVE::BI__builtin_sve_svdup_n_b64: {
10474 Builder.CreateICmpNE(Ops[0], Constant::getNullValue(Ops[0]->getType()));
10475 llvm::ScalableVectorType *OverloadedTy =
getSVEType(TypeFlags);
10480 case SVE::BI__builtin_sve_svdupq_n_b8:
10481 case SVE::BI__builtin_sve_svdupq_n_b16:
10482 case SVE::BI__builtin_sve_svdupq_n_b32:
10483 case SVE::BI__builtin_sve_svdupq_n_b64:
10484 case SVE::BI__builtin_sve_svdupq_n_u8:
10485 case SVE::BI__builtin_sve_svdupq_n_s8:
10486 case SVE::BI__builtin_sve_svdupq_n_u64:
10487 case SVE::BI__builtin_sve_svdupq_n_f64:
10488 case SVE::BI__builtin_sve_svdupq_n_s64:
10489 case SVE::BI__builtin_sve_svdupq_n_u16:
10490 case SVE::BI__builtin_sve_svdupq_n_f16:
10491 case SVE::BI__builtin_sve_svdupq_n_bf16:
10492 case SVE::BI__builtin_sve_svdupq_n_s16:
10493 case SVE::BI__builtin_sve_svdupq_n_u32:
10494 case SVE::BI__builtin_sve_svdupq_n_f32:
10495 case SVE::BI__builtin_sve_svdupq_n_s32: {
10498 unsigned NumOpnds = Ops.size();
10501 cast<llvm::VectorType>(Ty)->getElementType()->isIntegerTy(1);
10506 llvm::Type *EltTy = Ops[0]->getType();
10511 for (
unsigned I = 0; I < NumOpnds; ++I)
10512 VecOps.push_back(
Builder.CreateZExt(Ops[I], EltTy));
10517 OverloadedTy, PoisonValue::get(OverloadedTy), Vec,
Builder.getInt64(0));
10532 : Intrinsic::aarch64_sve_cmpne_wide,
10539 case SVE::BI__builtin_sve_svpfalse_b:
10540 return ConstantInt::getFalse(Ty);
10542 case SVE::BI__builtin_sve_svpfalse_c: {
10543 auto SVBoolTy = ScalableVectorType::get(
Builder.getInt1Ty(), 16);
10546 return Builder.CreateCall(CastToSVCountF, ConstantInt::getFalse(SVBoolTy));
10549 case SVE::BI__builtin_sve_svlen_bf16:
10550 case SVE::BI__builtin_sve_svlen_f16:
10551 case SVE::BI__builtin_sve_svlen_f32:
10552 case SVE::BI__builtin_sve_svlen_f64:
10553 case SVE::BI__builtin_sve_svlen_s8:
10554 case SVE::BI__builtin_sve_svlen_s16:
10555 case SVE::BI__builtin_sve_svlen_s32:
10556 case SVE::BI__builtin_sve_svlen_s64:
10557 case SVE::BI__builtin_sve_svlen_u8:
10558 case SVE::BI__builtin_sve_svlen_u16:
10559 case SVE::BI__builtin_sve_svlen_u32:
10560 case SVE::BI__builtin_sve_svlen_u64: {
10562 auto VTy = cast<llvm::VectorType>(
getSVEType(TF));
10564 llvm::ConstantInt::get(Ty, VTy->getElementCount().getKnownMinValue());
10570 case SVE::BI__builtin_sve_svtbl2_u8:
10571 case SVE::BI__builtin_sve_svtbl2_s8:
10572 case SVE::BI__builtin_sve_svtbl2_u16:
10573 case SVE::BI__builtin_sve_svtbl2_s16:
10574 case SVE::BI__builtin_sve_svtbl2_u32:
10575 case SVE::BI__builtin_sve_svtbl2_s32:
10576 case SVE::BI__builtin_sve_svtbl2_u64:
10577 case SVE::BI__builtin_sve_svtbl2_s64:
10578 case SVE::BI__builtin_sve_svtbl2_f16:
10579 case SVE::BI__builtin_sve_svtbl2_bf16:
10580 case SVE::BI__builtin_sve_svtbl2_f32:
10581 case SVE::BI__builtin_sve_svtbl2_f64: {
10583 auto VTy = cast<llvm::ScalableVectorType>(
getSVEType(TF));
10585 return Builder.CreateCall(F, Ops);
10588 case SVE::BI__builtin_sve_svset_neonq_s8:
10589 case SVE::BI__builtin_sve_svset_neonq_s16:
10590 case SVE::BI__builtin_sve_svset_neonq_s32:
10591 case SVE::BI__builtin_sve_svset_neonq_s64:
10592 case SVE::BI__builtin_sve_svset_neonq_u8:
10593 case SVE::BI__builtin_sve_svset_neonq_u16:
10594 case SVE::BI__builtin_sve_svset_neonq_u32:
10595 case SVE::BI__builtin_sve_svset_neonq_u64:
10596 case SVE::BI__builtin_sve_svset_neonq_f16:
10597 case SVE::BI__builtin_sve_svset_neonq_f32:
10598 case SVE::BI__builtin_sve_svset_neonq_f64:
10599 case SVE::BI__builtin_sve_svset_neonq_bf16: {
10600 return Builder.CreateInsertVector(Ty, Ops[0], Ops[1],
Builder.getInt64(0));
10603 case SVE::BI__builtin_sve_svget_neonq_s8:
10604 case SVE::BI__builtin_sve_svget_neonq_s16:
10605 case SVE::BI__builtin_sve_svget_neonq_s32:
10606 case SVE::BI__builtin_sve_svget_neonq_s64:
10607 case SVE::BI__builtin_sve_svget_neonq_u8:
10608 case SVE::BI__builtin_sve_svget_neonq_u16:
10609 case SVE::BI__builtin_sve_svget_neonq_u32:
10610 case SVE::BI__builtin_sve_svget_neonq_u64:
10611 case SVE::BI__builtin_sve_svget_neonq_f16:
10612 case SVE::BI__builtin_sve_svget_neonq_f32:
10613 case SVE::BI__builtin_sve_svget_neonq_f64:
10614 case SVE::BI__builtin_sve_svget_neonq_bf16: {
10615 return Builder.CreateExtractVector(Ty, Ops[0],
Builder.getInt64(0));
10618 case SVE::BI__builtin_sve_svdup_neonq_s8:
10619 case SVE::BI__builtin_sve_svdup_neonq_s16:
10620 case SVE::BI__builtin_sve_svdup_neonq_s32:
10621 case SVE::BI__builtin_sve_svdup_neonq_s64:
10622 case SVE::BI__builtin_sve_svdup_neonq_u8:
10623 case SVE::BI__builtin_sve_svdup_neonq_u16:
10624 case SVE::BI__builtin_sve_svdup_neonq_u32:
10625 case SVE::BI__builtin_sve_svdup_neonq_u64:
10626 case SVE::BI__builtin_sve_svdup_neonq_f16:
10627 case SVE::BI__builtin_sve_svdup_neonq_f32:
10628 case SVE::BI__builtin_sve_svdup_neonq_f64:
10629 case SVE::BI__builtin_sve_svdup_neonq_bf16: {
10632 return Builder.CreateIntrinsic(Intrinsic::aarch64_sve_dupq_lane, {Ty},
10644 switch (BuiltinID) {
10647 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x1:
10650 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x2:
10651 case SME::BI__builtin_sme_svsudot_za32_s8_vg1x2:
10654 case SME::BI__builtin_sme_svsudot_za32_s8_vg1x4:
10655 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x4:
10661 for (
unsigned I = 0; I < MultiVec; ++I)
10662 std::swap(Ops[I + 1], Ops[I + 1 + MultiVec]);
10675 return EmitSMELd1St1(TypeFlags, Ops, Builtin->LLVMIntrinsic);
10678 else if (BuiltinID == SME::BI__builtin_sme_svzero_mask_za ||
10679 BuiltinID == SME::BI__builtin_sme_svzero_za)
10680 return EmitSMEZero(TypeFlags, Ops, Builtin->LLVMIntrinsic);
10681 else if (BuiltinID == SME::BI__builtin_sme_svldr_vnum_za ||
10682 BuiltinID == SME::BI__builtin_sme_svstr_vnum_za ||
10683 BuiltinID == SME::BI__builtin_sme_svldr_za ||
10684 BuiltinID == SME::BI__builtin_sme_svstr_za)
10685 return EmitSMELdrStr(TypeFlags, Ops, Builtin->LLVMIntrinsic);
10691 if (Builtin->LLVMIntrinsic == 0)
10695 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
10696 if (
auto PredTy = dyn_cast<llvm::VectorType>(Ops[i]->getType()))
10697 if (PredTy->getElementType()->isIntegerTy(1))
10711 llvm::Triple::ArchType Arch) {
10720 if (BuiltinID == Builtin::BI__builtin_cpu_supports)
10721 return EmitAArch64CpuSupports(E);
10723 unsigned HintID =
static_cast<unsigned>(-1);
10724 switch (BuiltinID) {
10726 case clang::AArch64::BI__builtin_arm_nop:
10729 case clang::AArch64::BI__builtin_arm_yield:
10730 case clang::AArch64::BI__yield:
10733 case clang::AArch64::BI__builtin_arm_wfe:
10734 case clang::AArch64::BI__wfe:
10737 case clang::AArch64::BI__builtin_arm_wfi:
10738 case clang::AArch64::BI__wfi:
10741 case clang::AArch64::BI__builtin_arm_sev:
10742 case clang::AArch64::BI__sev:
10745 case clang::AArch64::BI__builtin_arm_sevl:
10746 case clang::AArch64::BI__sevl:
10751 if (HintID !=
static_cast<unsigned>(-1)) {
10753 return Builder.CreateCall(F, llvm::ConstantInt::get(
Int32Ty, HintID));
10756 if (BuiltinID == clang::AArch64::BI__builtin_arm_trap) {
10762 if (BuiltinID == clang::AArch64::BI__builtin_arm_get_sme_state) {
10767 "__arm_sme_state"));
10769 "aarch64_pstate_sm_compatible");
10770 CI->setAttributes(Attrs);
10771 CI->setCallingConv(
10772 llvm::CallingConv::
10773 AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2);
10780 if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit) {
10782 "rbit of unusual size!");
10785 CGM.
getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
10787 if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit64) {
10789 "rbit of unusual size!");
10792 CGM.
getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
10795 if (BuiltinID == clang::AArch64::BI__builtin_arm_clz ||
10796 BuiltinID == clang::AArch64::BI__builtin_arm_clz64) {
10800 if (BuiltinID == clang::AArch64::BI__builtin_arm_clz64)
10805 if (BuiltinID == clang::AArch64::BI__builtin_arm_cls) {
10810 if (BuiltinID == clang::AArch64::BI__builtin_arm_cls64) {
10816 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint32zf ||
10817 BuiltinID == clang::AArch64::BI__builtin_arm_rint32z) {
10819 llvm::Type *Ty = Arg->getType();
10824 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint64zf ||
10825 BuiltinID == clang::AArch64::BI__builtin_arm_rint64z) {
10827 llvm::Type *Ty = Arg->getType();
10832 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint32xf ||
10833 BuiltinID == clang::AArch64::BI__builtin_arm_rint32x) {
10835 llvm::Type *Ty = Arg->getType();
10840 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint64xf ||
10841 BuiltinID == clang::AArch64::BI__builtin_arm_rint64x) {
10843 llvm::Type *Ty = Arg->getType();
10848 if (BuiltinID == clang::AArch64::BI__builtin_arm_jcvt) {
10850 "__jcvt of unusual size!");
10856 if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b ||
10857 BuiltinID == clang::AArch64::BI__builtin_arm_st64b ||
10858 BuiltinID == clang::AArch64::BI__builtin_arm_st64bv ||
10859 BuiltinID == clang::AArch64::BI__builtin_arm_st64bv0) {
10863 if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b) {
10867 llvm::Value *Val =
Builder.CreateCall(F, MemAddr);
10868 llvm::Value *ToRet;
10869 for (
size_t i = 0; i < 8; i++) {
10870 llvm::Value *ValOffsetPtr =
10881 Args.push_back(MemAddr);
10882 for (
size_t i = 0; i < 8; i++) {
10883 llvm::Value *ValOffsetPtr =
10890 auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_st64b
10891 ? Intrinsic::aarch64_st64b
10892 : BuiltinID == clang::AArch64::BI__builtin_arm_st64bv
10893 ? Intrinsic::aarch64_st64bv
10894 : Intrinsic::aarch64_st64bv0);
10896 return Builder.CreateCall(F, Args);
10900 if (BuiltinID == clang::AArch64::BI__builtin_arm_rndr ||
10901 BuiltinID == clang::AArch64::BI__builtin_arm_rndrrs) {
10903 auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_rndr
10904 ? Intrinsic::aarch64_rndr
10905 : Intrinsic::aarch64_rndrrs);
10907 llvm::Value *Val =
Builder.CreateCall(F);
10908 Value *RandomValue =
Builder.CreateExtractValue(Val, 0);
10917 if (BuiltinID == clang::AArch64::BI__clear_cache) {
10918 assert(E->
getNumArgs() == 2 &&
"__clear_cache takes 2 arguments");
10921 for (
unsigned i = 0; i < 2; i++)
10924 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
10925 StringRef Name = FD->
getName();
10929 if ((BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
10930 BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) &&
10934 ? Intrinsic::aarch64_ldaxp
10935 : Intrinsic::aarch64_ldxp);
10942 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
10943 Val0 =
Builder.CreateZExt(Val0, Int128Ty);
10944 Val1 =
Builder.CreateZExt(Val1, Int128Ty);
10946 Value *ShiftCst = llvm::ConstantInt::get(Int128Ty, 64);
10947 Val =
Builder.CreateShl(Val0, ShiftCst,
"shl",
true );
10948 Val =
Builder.CreateOr(Val, Val1);
10950 }
else if (BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
10951 BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) {
10956 llvm::Type *
IntTy =
10961 ? Intrinsic::aarch64_ldaxr
10962 : Intrinsic::aarch64_ldxr,
10964 CallInst *Val =
Builder.CreateCall(F, LoadAddr,
"ldxr");
10968 if (RealResTy->isPointerTy())
10969 return Builder.CreateIntToPtr(Val, RealResTy);
10971 llvm::Type *IntResTy = llvm::IntegerType::get(
10973 return Builder.CreateBitCast(
Builder.CreateTruncOrBitCast(Val, IntResTy),
10977 if ((BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
10978 BuiltinID == clang::AArch64::BI__builtin_arm_stlex) &&
10982 ? Intrinsic::aarch64_stlxp
10983 : Intrinsic::aarch64_stxp);
10995 return Builder.CreateCall(F, {Arg0, Arg1, StPtr},
"stxp");
10998 if (BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
10999 BuiltinID == clang::AArch64::BI__builtin_arm_stlex) {
11004 llvm::Type *StoreTy =
11007 if (StoreVal->
getType()->isPointerTy())
11010 llvm::Type *
IntTy = llvm::IntegerType::get(
11019 ? Intrinsic::aarch64_stlxr
11020 : Intrinsic::aarch64_stxr,
11022 CallInst *CI =
Builder.CreateCall(F, {StoreVal, StoreAddr},
"stxr");
11024 1, Attribute::get(
getLLVMContext(), Attribute::ElementType, StoreTy));
11028 if (BuiltinID == clang::AArch64::BI__getReg) {
11031 llvm_unreachable(
"Sema will ensure that the parameter is constant");
11037 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, Reg)};
11038 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
11039 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
11041 llvm::Function *F =
11043 return Builder.CreateCall(F, Metadata);
11046 if (BuiltinID == clang::AArch64::BI__break) {
11049 llvm_unreachable(
"Sema will ensure that the parameter is constant");
11051 llvm::Function *F =
CGM.
getIntrinsic(llvm::Intrinsic::aarch64_break);
11055 if (BuiltinID == clang::AArch64::BI__builtin_arm_clrex) {
11057 return Builder.CreateCall(F);
11060 if (BuiltinID == clang::AArch64::BI_ReadWriteBarrier)
11061 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
11062 llvm::SyncScope::SingleThread);
11065 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
11066 switch (BuiltinID) {
11067 case clang::AArch64::BI__builtin_arm_crc32b:
11068 CRCIntrinsicID = Intrinsic::aarch64_crc32b;
break;
11069 case clang::AArch64::BI__builtin_arm_crc32cb:
11070 CRCIntrinsicID = Intrinsic::aarch64_crc32cb;
break;
11071 case clang::AArch64::BI__builtin_arm_crc32h:
11072 CRCIntrinsicID = Intrinsic::aarch64_crc32h;
break;
11073 case clang::AArch64::BI__builtin_arm_crc32ch:
11074 CRCIntrinsicID = Intrinsic::aarch64_crc32ch;
break;
11075 case clang::AArch64::BI__builtin_arm_crc32w:
11076 CRCIntrinsicID = Intrinsic::aarch64_crc32w;
break;
11077 case clang::AArch64::BI__builtin_arm_crc32cw:
11078 CRCIntrinsicID = Intrinsic::aarch64_crc32cw;
break;
11079 case clang::AArch64::BI__builtin_arm_crc32d:
11080 CRCIntrinsicID = Intrinsic::aarch64_crc32x;
break;
11081 case clang::AArch64::BI__builtin_arm_crc32cd:
11082 CRCIntrinsicID = Intrinsic::aarch64_crc32cx;
break;
11085 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
11090 llvm::Type *DataTy = F->getFunctionType()->getParamType(1);
11091 Arg1 =
Builder.CreateZExtOrBitCast(Arg1, DataTy);
11093 return Builder.CreateCall(F, {Arg0, Arg1});
11097 if (BuiltinID == AArch64::BI__builtin_arm_mops_memset_tag) {
11105 CGM.
getIntrinsic(Intrinsic::aarch64_mops_memset_tag), {Dst, Val, Size});
11109 Intrinsic::ID MTEIntrinsicID = Intrinsic::not_intrinsic;
11110 switch (BuiltinID) {
11111 case clang::AArch64::BI__builtin_arm_irg:
11112 MTEIntrinsicID = Intrinsic::aarch64_irg;
break;
11113 case clang::AArch64::BI__builtin_arm_addg:
11114 MTEIntrinsicID = Intrinsic::aarch64_addg;
break;
11115 case clang::AArch64::BI__builtin_arm_gmi:
11116 MTEIntrinsicID = Intrinsic::aarch64_gmi;
break;
11117 case clang::AArch64::BI__builtin_arm_ldg:
11118 MTEIntrinsicID = Intrinsic::aarch64_ldg;
break;
11119 case clang::AArch64::BI__builtin_arm_stg:
11120 MTEIntrinsicID = Intrinsic::aarch64_stg;
break;
11121 case clang::AArch64::BI__builtin_arm_subp:
11122 MTEIntrinsicID = Intrinsic::aarch64_subp;
break;
11125 if (MTEIntrinsicID != Intrinsic::not_intrinsic) {
11128 if (MTEIntrinsicID == Intrinsic::aarch64_irg) {
11136 return Builder.CreatePointerCast(RV, T);
11138 if (MTEIntrinsicID == Intrinsic::aarch64_addg) {
11146 return Builder.CreatePointerCast(RV, T);
11148 if (MTEIntrinsicID == Intrinsic::aarch64_gmi) {
11160 if (MTEIntrinsicID == Intrinsic::aarch64_ldg) {
11165 return Builder.CreatePointerCast(RV, T);
11170 if (MTEIntrinsicID == Intrinsic::aarch64_stg) {
11176 if (MTEIntrinsicID == Intrinsic::aarch64_subp) {
11186 if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
11187 BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
11188 BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
11189 BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
11190 BuiltinID == clang::AArch64::BI__builtin_arm_wsr ||
11191 BuiltinID == clang::AArch64::BI__builtin_arm_wsr64 ||
11192 BuiltinID == clang::AArch64::BI__builtin_arm_wsr128 ||
11193 BuiltinID == clang::AArch64::BI__builtin_arm_wsrp) {
11196 if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
11197 BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
11198 BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
11199 BuiltinID == clang::AArch64::BI__builtin_arm_rsrp)
11202 bool IsPointerBuiltin = BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
11203 BuiltinID == clang::AArch64::BI__builtin_arm_wsrp;
11205 bool Is32Bit = BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
11206 BuiltinID == clang::AArch64::BI__builtin_arm_wsr;
11208 bool Is128Bit = BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
11209 BuiltinID == clang::AArch64::BI__builtin_arm_wsr128;
11211 llvm::Type *ValueType;
11212 llvm::Type *RegisterType =
Int64Ty;
11215 }
else if (Is128Bit) {
11216 llvm::Type *Int128Ty =
11218 ValueType = Int128Ty;
11219 RegisterType = Int128Ty;
11220 }
else if (IsPointerBuiltin) {
11230 if (BuiltinID == clang::AArch64::BI_ReadStatusReg ||
11231 BuiltinID == clang::AArch64::BI_WriteStatusReg) {
11237 std::string SysRegStr;
11238 llvm::raw_string_ostream(SysRegStr) <<
11239 ((1 << 1) | ((SysReg >> 14) & 1)) <<
":" <<
11240 ((SysReg >> 11) & 7) <<
":" <<
11241 ((SysReg >> 7) & 15) <<
":" <<
11242 ((SysReg >> 3) & 15) <<
":" <<
11245 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysRegStr) };
11246 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
11247 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
11249 llvm::Type *RegisterType =
Int64Ty;
11250 llvm::Type *Types[] = { RegisterType };
11252 if (BuiltinID == clang::AArch64::BI_ReadStatusReg) {
11253 llvm::Function *F =
CGM.
getIntrinsic(llvm::Intrinsic::read_register, Types);
11255 return Builder.CreateCall(F, Metadata);
11258 llvm::Function *F =
CGM.
getIntrinsic(llvm::Intrinsic::write_register, Types);
11261 return Builder.CreateCall(F, { Metadata, ArgValue });
11264 if (BuiltinID == clang::AArch64::BI_AddressOfReturnAddress) {
11265 llvm::Function *F =
11267 return Builder.CreateCall(F);
11270 if (BuiltinID == clang::AArch64::BI__builtin_sponentry) {
11272 return Builder.CreateCall(F);
11275 if (BuiltinID == clang::AArch64::BI__mulh ||
11276 BuiltinID == clang::AArch64::BI__umulh) {
11278 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
11280 bool IsSigned = BuiltinID == clang::AArch64::BI__mulh;
11286 Value *MulResult, *HigherBits;
11288 MulResult =
Builder.CreateNSWMul(LHS, RHS);
11289 HigherBits =
Builder.CreateAShr(MulResult, 64);
11291 MulResult =
Builder.CreateNUWMul(LHS, RHS);
11292 HigherBits =
Builder.CreateLShr(MulResult, 64);
11294 HigherBits =
Builder.CreateIntCast(HigherBits, ResType, IsSigned);
11299 if (BuiltinID == AArch64::BI__writex18byte ||
11300 BuiltinID == AArch64::BI__writex18word ||
11301 BuiltinID == AArch64::BI__writex18dword ||
11302 BuiltinID == AArch64::BI__writex18qword) {
11305 llvm::Metadata *Ops[] = {llvm::MDString::get(Context,
"x18")};
11306 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
11307 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
11308 llvm::Function *F =
11310 llvm::Value *X18 =
Builder.CreateCall(F, Metadata);
11321 if (BuiltinID == AArch64::BI__readx18byte ||
11322 BuiltinID == AArch64::BI__readx18word ||
11323 BuiltinID == AArch64::BI__readx18dword ||
11324 BuiltinID == AArch64::BI__readx18qword) {
11329 llvm::Metadata *Ops[] = {llvm::MDString::get(Context,
"x18")};
11330 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
11331 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
11332 llvm::Function *F =
11334 llvm::Value *X18 =
Builder.CreateCall(F, Metadata);
11344 if (BuiltinID == AArch64::BI_CopyDoubleFromInt64 ||
11345 BuiltinID == AArch64::BI_CopyFloatFromInt32 ||
11346 BuiltinID == AArch64::BI_CopyInt32FromFloat ||
11347 BuiltinID == AArch64::BI_CopyInt64FromDouble) {
11350 return Builder.CreateBitCast(Arg, RetTy);
11353 if (BuiltinID == AArch64::BI_CountLeadingOnes ||
11354 BuiltinID == AArch64::BI_CountLeadingOnes64 ||
11355 BuiltinID == AArch64::BI_CountLeadingZeros ||
11356 BuiltinID == AArch64::BI_CountLeadingZeros64) {
11358 llvm::Type *ArgType = Arg->
getType();
11360 if (BuiltinID == AArch64::BI_CountLeadingOnes ||
11361 BuiltinID == AArch64::BI_CountLeadingOnes64)
11362 Arg =
Builder.CreateXor(Arg, Constant::getAllOnesValue(ArgType));
11367 if (BuiltinID == AArch64::BI_CountLeadingOnes64 ||
11368 BuiltinID == AArch64::BI_CountLeadingZeros64)
11373 if (BuiltinID == AArch64::BI_CountLeadingSigns ||
11374 BuiltinID == AArch64::BI_CountLeadingSigns64) {
11377 Function *F = (BuiltinID == AArch64::BI_CountLeadingSigns)
11382 if (BuiltinID == AArch64::BI_CountLeadingSigns64)
11387 if (BuiltinID == AArch64::BI_CountOneBits ||
11388 BuiltinID == AArch64::BI_CountOneBits64) {
11390 llvm::Type *ArgType = ArgValue->
getType();
11394 if (BuiltinID == AArch64::BI_CountOneBits64)
11399 if (BuiltinID == AArch64::BI__prefetch) {
11410 if (std::optional<MSVCIntrin> MsvcIntId =
11416 return P.first == BuiltinID;
11419 BuiltinID = It->second;
11423 unsigned ICEArguments = 0;
11430 for (
unsigned i = 0, e = E->
getNumArgs() - 1; i != e; i++) {
11432 switch (BuiltinID) {
11433 case NEON::BI__builtin_neon_vld1_v:
11434 case NEON::BI__builtin_neon_vld1q_v:
11435 case NEON::BI__builtin_neon_vld1_dup_v:
11436 case NEON::BI__builtin_neon_vld1q_dup_v:
11437 case NEON::BI__builtin_neon_vld1_lane_v:
11438 case NEON::BI__builtin_neon_vld1q_lane_v:
11439 case NEON::BI__builtin_neon_vst1_v:
11440 case NEON::BI__builtin_neon_vst1q_v:
11441 case NEON::BI__builtin_neon_vst1_lane_v:
11442 case NEON::BI__builtin_neon_vst1q_lane_v:
11443 case NEON::BI__builtin_neon_vldap1_lane_s64:
11444 case NEON::BI__builtin_neon_vldap1q_lane_s64:
11445 case NEON::BI__builtin_neon_vstl1_lane_s64:
11446 case NEON::BI__builtin_neon_vstl1q_lane_s64:
11464 assert(
Result &&
"SISD intrinsic should have been handled");
11470 if (std::optional<llvm::APSInt>
Result =
11475 bool usgn =
Type.isUnsigned();
11476 bool quad =
Type.isQuad();
11479 switch (BuiltinID) {
11481 case NEON::BI__builtin_neon_vabsh_f16:
11484 case NEON::BI__builtin_neon_vaddq_p128: {
11487 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
11488 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
11489 Ops[0] =
Builder.CreateXor(Ops[0], Ops[1]);
11490 llvm::Type *Int128Ty = llvm::Type::getIntNTy(
getLLVMContext(), 128);
11491 return Builder.CreateBitCast(Ops[0], Int128Ty);
11493 case NEON::BI__builtin_neon_vldrq_p128: {
11494 llvm::Type *Int128Ty = llvm::Type::getIntNTy(
getLLVMContext(), 128);
11499 case NEON::BI__builtin_neon_vstrq_p128: {
11500 Value *Ptr = Ops[0];
11503 case NEON::BI__builtin_neon_vcvts_f32_u32:
11504 case NEON::BI__builtin_neon_vcvtd_f64_u64:
11507 case NEON::BI__builtin_neon_vcvts_f32_s32:
11508 case NEON::BI__builtin_neon_vcvtd_f64_s64: {
11510 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
11513 Ops[0] =
Builder.CreateBitCast(Ops[0], InTy);
11515 return Builder.CreateUIToFP(Ops[0], FTy);
11516 return Builder.CreateSIToFP(Ops[0], FTy);
11518 case NEON::BI__builtin_neon_vcvth_f16_u16:
11519 case NEON::BI__builtin_neon_vcvth_f16_u32:
11520 case NEON::BI__builtin_neon_vcvth_f16_u64:
11523 case NEON::BI__builtin_neon_vcvth_f16_s16:
11524 case NEON::BI__builtin_neon_vcvth_f16_s32:
11525 case NEON::BI__builtin_neon_vcvth_f16_s64: {
11527 llvm::Type *FTy =
HalfTy;
11529 if (Ops[0]->getType()->getPrimitiveSizeInBits() == 64)
11531 else if (Ops[0]->getType()->getPrimitiveSizeInBits() == 32)
11535 Ops[0] =
Builder.CreateBitCast(Ops[0], InTy);
11537 return Builder.CreateUIToFP(Ops[0], FTy);
11538 return Builder.CreateSIToFP(Ops[0], FTy);
11540 case NEON::BI__builtin_neon_vcvtah_u16_f16:
11541 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
11542 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
11543 case NEON::BI__builtin_neon_vcvtph_u16_f16:
11544 case NEON::BI__builtin_neon_vcvth_u16_f16:
11545 case NEON::BI__builtin_neon_vcvtah_s16_f16:
11546 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
11547 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
11548 case NEON::BI__builtin_neon_vcvtph_s16_f16:
11549 case NEON::BI__builtin_neon_vcvth_s16_f16: {
11552 llvm::Type* FTy =
HalfTy;
11553 llvm::Type *Tys[2] = {InTy, FTy};
11555 switch (BuiltinID) {
11556 default: llvm_unreachable(
"missing builtin ID in switch!");
11557 case NEON::BI__builtin_neon_vcvtah_u16_f16:
11558 Int = Intrinsic::aarch64_neon_fcvtau;
break;
11559 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
11560 Int = Intrinsic::aarch64_neon_fcvtmu;
break;
11561 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
11562 Int = Intrinsic::aarch64_neon_fcvtnu;
break;
11563 case NEON::BI__builtin_neon_vcvtph_u16_f16:
11564 Int = Intrinsic::aarch64_neon_fcvtpu;
break;
11565 case NEON::BI__builtin_neon_vcvth_u16_f16:
11566 Int = Intrinsic::aarch64_neon_fcvtzu;
break;
11567 case NEON::BI__builtin_neon_vcvtah_s16_f16:
11568 Int = Intrinsic::aarch64_neon_fcvtas;
break;
11569 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
11570 Int = Intrinsic::aarch64_neon_fcvtms;
break;
11571 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
11572 Int = Intrinsic::aarch64_neon_fcvtns;
break;
11573 case NEON::BI__builtin_neon_vcvtph_s16_f16:
11574 Int = Intrinsic::aarch64_neon_fcvtps;
break;
11575 case NEON::BI__builtin_neon_vcvth_s16_f16:
11576 Int = Intrinsic::aarch64_neon_fcvtzs;
break;
11581 case NEON::BI__builtin_neon_vcaleh_f16:
11582 case NEON::BI__builtin_neon_vcalth_f16:
11583 case NEON::BI__builtin_neon_vcageh_f16:
11584 case NEON::BI__builtin_neon_vcagth_f16: {
11587 llvm::Type* FTy =
HalfTy;
11588 llvm::Type *Tys[2] = {InTy, FTy};
11590 switch (BuiltinID) {
11591 default: llvm_unreachable(
"missing builtin ID in switch!");
11592 case NEON::BI__builtin_neon_vcageh_f16:
11593 Int = Intrinsic::aarch64_neon_facge;
break;
11594 case NEON::BI__builtin_neon_vcagth_f16:
11595 Int = Intrinsic::aarch64_neon_facgt;
break;
11596 case NEON::BI__builtin_neon_vcaleh_f16:
11597 Int = Intrinsic::aarch64_neon_facge; std::swap(Ops[0], Ops[1]);
break;
11598 case NEON::BI__builtin_neon_vcalth_f16:
11599 Int = Intrinsic::aarch64_neon_facgt; std::swap(Ops[0], Ops[1]);
break;
11604 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
11605 case NEON::BI__builtin_neon_vcvth_n_u16_f16: {
11608 llvm::Type* FTy =
HalfTy;
11609 llvm::Type *Tys[2] = {InTy, FTy};
11611 switch (BuiltinID) {
11612 default: llvm_unreachable(
"missing builtin ID in switch!");
11613 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
11614 Int = Intrinsic::aarch64_neon_vcvtfp2fxs;
break;
11615 case NEON::BI__builtin_neon_vcvth_n_u16_f16:
11616 Int = Intrinsic::aarch64_neon_vcvtfp2fxu;
break;
11621 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
11622 case NEON::BI__builtin_neon_vcvth_n_f16_u16: {
11624 llvm::Type* FTy =
HalfTy;
11626 llvm::Type *Tys[2] = {FTy, InTy};
11628 switch (BuiltinID) {
11629 default: llvm_unreachable(
"missing builtin ID in switch!");
11630 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
11631 Int = Intrinsic::aarch64_neon_vcvtfxs2fp;
11632 Ops[0] =
Builder.CreateSExt(Ops[0], InTy,
"sext");
11634 case NEON::BI__builtin_neon_vcvth_n_f16_u16:
11635 Int = Intrinsic::aarch64_neon_vcvtfxu2fp;
11636 Ops[0] =
Builder.CreateZExt(Ops[0], InTy);
11641 case NEON::BI__builtin_neon_vpaddd_s64: {
11642 auto *Ty = llvm::FixedVectorType::get(
Int64Ty, 2);
11645 Vec =
Builder.CreateBitCast(Vec, Ty,
"v2i64");
11646 llvm::Value *Idx0 = llvm::ConstantInt::get(
SizeTy, 0);
11647 llvm::Value *Idx1 = llvm::ConstantInt::get(
SizeTy, 1);
11648 Value *Op0 =
Builder.CreateExtractElement(Vec, Idx0,
"lane0");
11649 Value *Op1 =
Builder.CreateExtractElement(Vec, Idx1,
"lane1");
11651 return Builder.CreateAdd(Op0, Op1,
"vpaddd");
11653 case NEON::BI__builtin_neon_vpaddd_f64: {
11654 auto *Ty = llvm::FixedVectorType::get(
DoubleTy, 2);
11657 Vec =
Builder.CreateBitCast(Vec, Ty,
"v2f64");
11658 llvm::Value *Idx0 = llvm::ConstantInt::get(
SizeTy, 0);
11659 llvm::Value *Idx1 = llvm::ConstantInt::get(
SizeTy, 1);
11660 Value *Op0 =
Builder.CreateExtractElement(Vec, Idx0,
"lane0");
11661 Value *Op1 =
Builder.CreateExtractElement(Vec, Idx1,
"lane1");
11663 return Builder.CreateFAdd(Op0, Op1,
"vpaddd");
11665 case NEON::BI__builtin_neon_vpadds_f32: {
11666 auto *Ty = llvm::FixedVectorType::get(
FloatTy, 2);
11669 Vec =
Builder.CreateBitCast(Vec, Ty,
"v2f32");
11670 llvm::Value *Idx0 = llvm::ConstantInt::get(
SizeTy, 0);
11671 llvm::Value *Idx1 = llvm::ConstantInt::get(
SizeTy, 1);
11672 Value *Op0 =
Builder.CreateExtractElement(Vec, Idx0,
"lane0");
11673 Value *Op1 =
Builder.CreateExtractElement(Vec, Idx1,
"lane1");
11675 return Builder.CreateFAdd(Op0, Op1,
"vpaddd");
11677 case NEON::BI__builtin_neon_vceqzd_s64:
11678 case NEON::BI__builtin_neon_vceqzd_f64:
11679 case NEON::BI__builtin_neon_vceqzs_f32:
11680 case NEON::BI__builtin_neon_vceqzh_f16:
11684 ICmpInst::FCMP_OEQ, ICmpInst::ICMP_EQ,
"vceqz");
11685 case NEON::BI__builtin_neon_vcgezd_s64:
11686 case NEON::BI__builtin_neon_vcgezd_f64:
11687 case NEON::BI__builtin_neon_vcgezs_f32:
11688 case NEON::BI__builtin_neon_vcgezh_f16:
11692 ICmpInst::FCMP_OGE, ICmpInst::ICMP_SGE,
"vcgez");
11693 case NEON::BI__builtin_neon_vclezd_s64:
11694 case NEON::BI__builtin_neon_vclezd_f64:
11695 case NEON::BI__builtin_neon_vclezs_f32:
11696 case NEON::BI__builtin_neon_vclezh_f16:
11700 ICmpInst::FCMP_OLE, ICmpInst::ICMP_SLE,
"vclez");
11701 case NEON::BI__builtin_neon_vcgtzd_s64:
11702 case NEON::BI__builtin_neon_vcgtzd_f64:
11703 case NEON::BI__builtin_neon_vcgtzs_f32:
11704 case NEON::BI__builtin_neon_vcgtzh_f16:
11708 ICmpInst::FCMP_OGT, ICmpInst::ICMP_SGT,
"vcgtz");
11709 case NEON::BI__builtin_neon_vcltzd_s64:
11710 case NEON::BI__builtin_neon_vcltzd_f64:
11711 case NEON::BI__builtin_neon_vcltzs_f32:
11712 case NEON::BI__builtin_neon_vcltzh_f16:
11716 ICmpInst::FCMP_OLT, ICmpInst::ICMP_SLT,
"vcltz");
11718 case NEON::BI__builtin_neon_vceqzd_u64: {
11722 Builder.CreateICmpEQ(Ops[0], llvm::Constant::getNullValue(
Int64Ty));
11725 case NEON::BI__builtin_neon_vceqd_f64:
11726 case NEON::BI__builtin_neon_vcled_f64:
11727 case NEON::BI__builtin_neon_vcltd_f64:
11728 case NEON::BI__builtin_neon_vcged_f64:
11729 case NEON::BI__builtin_neon_vcgtd_f64: {
11730 llvm::CmpInst::Predicate
P;
11731 switch (BuiltinID) {
11732 default: llvm_unreachable(
"missing builtin ID in switch!");
11733 case NEON::BI__builtin_neon_vceqd_f64:
P = llvm::FCmpInst::FCMP_OEQ;
break;
11734 case NEON::BI__builtin_neon_vcled_f64:
P = llvm::FCmpInst::FCMP_OLE;
break;
11735 case NEON::BI__builtin_neon_vcltd_f64:
P = llvm::FCmpInst::FCMP_OLT;
break;
11736 case NEON::BI__builtin_neon_vcged_f64:
P = llvm::FCmpInst::FCMP_OGE;
break;
11737 case NEON::BI__builtin_neon_vcgtd_f64:
P = llvm::FCmpInst::FCMP_OGT;
break;
11742 if (
P == llvm::FCmpInst::FCMP_OEQ)
11743 Ops[0] =
Builder.CreateFCmp(
P, Ops[0], Ops[1]);
11745 Ops[0] =
Builder.CreateFCmpS(
P, Ops[0], Ops[1]);
11748 case NEON::BI__builtin_neon_vceqs_f32:
11749 case NEON::BI__builtin_neon_vcles_f32:
11750 case NEON::BI__builtin_neon_vclts_f32:
11751 case NEON::BI__builtin_neon_vcges_f32:
11752 case NEON::BI__builtin_neon_vcgts_f32: {
11753 llvm::CmpInst::Predicate
P;
11754 switch (BuiltinID) {
11755 default: llvm_unreachable(
"missing builtin ID in switch!");
11756 case NEON::BI__builtin_neon_vceqs_f32:
P = llvm::FCmpInst::FCMP_OEQ;
break;
11757 case NEON::BI__builtin_neon_vcles_f32:
P = llvm::FCmpInst::FCMP_OLE;
break;
11758 case NEON::BI__builtin_neon_vclts_f32:
P = llvm::FCmpInst::FCMP_OLT;
break;
11759 case NEON::BI__builtin_neon_vcges_f32:
P = llvm::FCmpInst::FCMP_OGE;
break;
11760 case NEON::BI__builtin_neon_vcgts_f32:
P = llvm::FCmpInst::FCMP_OGT;
break;
11765 if (
P == llvm::FCmpInst::FCMP_OEQ)
11766 Ops[0] =
Builder.CreateFCmp(
P, Ops[0], Ops[1]);
11768 Ops[0] =
Builder.CreateFCmpS(
P, Ops[0], Ops[1]);
11771 case NEON::BI__builtin_neon_vceqh_f16:
11772 case NEON::BI__builtin_neon_vcleh_f16:
11773 case NEON::BI__builtin_neon_vclth_f16:
11774 case NEON::BI__builtin_neon_vcgeh_f16:
11775 case NEON::BI__builtin_neon_vcgth_f16: {
11776 llvm::CmpInst::Predicate
P;
11777 switch (BuiltinID) {
11778 default: llvm_unreachable(
"missing builtin ID in switch!");
11779 case NEON::BI__builtin_neon_vceqh_f16:
P = llvm::FCmpInst::FCMP_OEQ;
break;
11780 case NEON::BI__builtin_neon_vcleh_f16:
P = llvm::FCmpInst::FCMP_OLE;
break;
11781 case NEON::BI__builtin_neon_vclth_f16:
P = llvm::FCmpInst::FCMP_OLT;
break;
11782 case NEON::BI__builtin_neon_vcgeh_f16:
P = llvm::FCmpInst::FCMP_OGE;
break;
11783 case NEON::BI__builtin_neon_vcgth_f16:
P = llvm::FCmpInst::FCMP_OGT;
break;
11788 if (
P == llvm::FCmpInst::FCMP_OEQ)
11789 Ops[0] =
Builder.CreateFCmp(
P, Ops[0], Ops[1]);
11791 Ops[0] =
Builder.CreateFCmpS(
P, Ops[0], Ops[1]);
11794 case NEON::BI__builtin_neon_vceqd_s64:
11795 case NEON::BI__builtin_neon_vceqd_u64:
11796 case NEON::BI__builtin_neon_vcgtd_s64:
11797 case NEON::BI__builtin_neon_vcgtd_u64:
11798 case NEON::BI__builtin_neon_vcltd_s64:
11799 case NEON::BI__builtin_neon_vcltd_u64:
11800 case NEON::BI__builtin_neon_vcged_u64:
11801 case NEON::BI__builtin_neon_vcged_s64:
11802 case NEON::BI__builtin_neon_vcled_u64:
11803 case NEON::BI__builtin_neon_vcled_s64: {
11804 llvm::CmpInst::Predicate
P;
11805 switch (BuiltinID) {
11806 default: llvm_unreachable(
"missing builtin ID in switch!");
11807 case NEON::BI__builtin_neon_vceqd_s64:
11808 case NEON::BI__builtin_neon_vceqd_u64:
P = llvm::ICmpInst::ICMP_EQ;
break;
11809 case NEON::BI__builtin_neon_vcgtd_s64:
P = llvm::ICmpInst::ICMP_SGT;
break;
11810 case NEON::BI__builtin_neon_vcgtd_u64:
P = llvm::ICmpInst::ICMP_UGT;
break;
11811 case NEON::BI__builtin_neon_vcltd_s64:
P = llvm::ICmpInst::ICMP_SLT;
break;
11812 case NEON::BI__builtin_neon_vcltd_u64:
P = llvm::ICmpInst::ICMP_ULT;
break;
11813 case NEON::BI__builtin_neon_vcged_u64:
P = llvm::ICmpInst::ICMP_UGE;
break;
11814 case NEON::BI__builtin_neon_vcged_s64:
P = llvm::ICmpInst::ICMP_SGE;
break;
11815 case NEON::BI__builtin_neon_vcled_u64:
P = llvm::ICmpInst::ICMP_ULE;
break;
11816 case NEON::BI__builtin_neon_vcled_s64:
P = llvm::ICmpInst::ICMP_SLE;
break;
11821 Ops[0] =
Builder.CreateICmp(
P, Ops[0], Ops[1]);
11824 case NEON::BI__builtin_neon_vtstd_s64:
11825 case NEON::BI__builtin_neon_vtstd_u64: {
11829 Ops[0] =
Builder.CreateAnd(Ops[0], Ops[1]);
11830 Ops[0] =
Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
11831 llvm::Constant::getNullValue(
Int64Ty));
11834 case NEON::BI__builtin_neon_vset_lane_i8:
11835 case NEON::BI__builtin_neon_vset_lane_i16:
11836 case NEON::BI__builtin_neon_vset_lane_i32:
11837 case NEON::BI__builtin_neon_vset_lane_i64:
11838 case NEON::BI__builtin_neon_vset_lane_bf16:
11839 case NEON::BI__builtin_neon_vset_lane_f32:
11840 case NEON::BI__builtin_neon_vsetq_lane_i8:
11841 case NEON::BI__builtin_neon_vsetq_lane_i16:
11842 case NEON::BI__builtin_neon_vsetq_lane_i32:
11843 case NEON::BI__builtin_neon_vsetq_lane_i64:
11844 case NEON::BI__builtin_neon_vsetq_lane_bf16:
11845 case NEON::BI__builtin_neon_vsetq_lane_f32:
11847 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
11848 case NEON::BI__builtin_neon_vset_lane_f64:
11851 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(
DoubleTy, 1));
11853 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
11854 case NEON::BI__builtin_neon_vsetq_lane_f64:
11857 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(
DoubleTy, 2));
11859 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
11861 case NEON::BI__builtin_neon_vget_lane_i8:
11862 case NEON::BI__builtin_neon_vdupb_lane_i8:
11864 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int8Ty, 8));
11867 case NEON::BI__builtin_neon_vgetq_lane_i8:
11868 case NEON::BI__builtin_neon_vdupb_laneq_i8:
11870 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int8Ty, 16));
11873 case NEON::BI__builtin_neon_vget_lane_i16:
11874 case NEON::BI__builtin_neon_vduph_lane_i16:
11876 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int16Ty, 4));
11879 case NEON::BI__builtin_neon_vgetq_lane_i16:
11880 case NEON::BI__builtin_neon_vduph_laneq_i16:
11882 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int16Ty, 8));
11885 case NEON::BI__builtin_neon_vget_lane_i32:
11886 case NEON::BI__builtin_neon_vdups_lane_i32:
11888 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int32Ty, 2));
11891 case NEON::BI__builtin_neon_vdups_lane_f32:
11893 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
FloatTy, 2));
11896 case NEON::BI__builtin_neon_vgetq_lane_i32:
11897 case NEON::BI__builtin_neon_vdups_laneq_i32:
11899 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int32Ty, 4));
11902 case NEON::BI__builtin_neon_vget_lane_i64:
11903 case NEON::BI__builtin_neon_vdupd_lane_i64:
11905 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int64Ty, 1));
11908 case NEON::BI__builtin_neon_vdupd_lane_f64:
11910 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
DoubleTy, 1));
11913 case NEON::BI__builtin_neon_vgetq_lane_i64:
11914 case NEON::BI__builtin_neon_vdupd_laneq_i64:
11916 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int64Ty, 2));
11919 case NEON::BI__builtin_neon_vget_lane_f32:
11921 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
FloatTy, 2));
11924 case NEON::BI__builtin_neon_vget_lane_f64:
11926 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
DoubleTy, 1));
11929 case NEON::BI__builtin_neon_vgetq_lane_f32:
11930 case NEON::BI__builtin_neon_vdups_laneq_f32:
11932 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
FloatTy, 4));
11935 case NEON::BI__builtin_neon_vgetq_lane_f64:
11936 case NEON::BI__builtin_neon_vdupd_laneq_f64:
11938 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
DoubleTy, 2));
11941 case NEON::BI__builtin_neon_vaddh_f16:
11943 return Builder.CreateFAdd(Ops[0], Ops[1],
"vaddh");
11944 case NEON::BI__builtin_neon_vsubh_f16:
11946 return Builder.CreateFSub(Ops[0], Ops[1],
"vsubh");
11947 case NEON::BI__builtin_neon_vmulh_f16:
11949 return Builder.CreateFMul(Ops[0], Ops[1],
"vmulh");
11950 case NEON::BI__builtin_neon_vdivh_f16:
11952 return Builder.CreateFDiv(Ops[0], Ops[1],
"vdivh");
11953 case NEON::BI__builtin_neon_vfmah_f16:
11956 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma,
HalfTy,
11958 case NEON::BI__builtin_neon_vfmsh_f16: {
11963 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma,
HalfTy,
11966 case NEON::BI__builtin_neon_vaddd_s64:
11967 case NEON::BI__builtin_neon_vaddd_u64:
11969 case NEON::BI__builtin_neon_vsubd_s64:
11970 case NEON::BI__builtin_neon_vsubd_u64:
11972 case NEON::BI__builtin_neon_vqdmlalh_s16:
11973 case NEON::BI__builtin_neon_vqdmlslh_s16: {
11977 auto *VTy = llvm::FixedVectorType::get(
Int32Ty, 4);
11979 ProductOps,
"vqdmlXl");
11980 Constant *CI = ConstantInt::get(
SizeTy, 0);
11981 Ops[1] =
Builder.CreateExtractElement(Ops[1], CI,
"lane0");
11983 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlalh_s16
11984 ? Intrinsic::aarch64_neon_sqadd
11985 : Intrinsic::aarch64_neon_sqsub;
11988 case NEON::BI__builtin_neon_vqshlud_n_s64: {
11994 case NEON::BI__builtin_neon_vqshld_n_u64:
11995 case NEON::BI__builtin_neon_vqshld_n_s64: {
11996 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vqshld_n_u64
11997 ? Intrinsic::aarch64_neon_uqshl
11998 : Intrinsic::aarch64_neon_sqshl;
12003 case NEON::BI__builtin_neon_vrshrd_n_u64:
12004 case NEON::BI__builtin_neon_vrshrd_n_s64: {
12005 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrshrd_n_u64
12006 ? Intrinsic::aarch64_neon_urshl
12007 : Intrinsic::aarch64_neon_srshl;
12009 int SV = cast<ConstantInt>(Ops[1])->getSExtValue();
12010 Ops[1] = ConstantInt::get(
Int64Ty, -SV);
12013 case NEON::BI__builtin_neon_vrsrad_n_u64:
12014 case NEON::BI__builtin_neon_vrsrad_n_s64: {
12015 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrsrad_n_u64
12016 ? Intrinsic::aarch64_neon_urshl
12017 : Intrinsic::aarch64_neon_srshl;
12021 {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)});
12024 case NEON::BI__builtin_neon_vshld_n_s64:
12025 case NEON::BI__builtin_neon_vshld_n_u64: {
12028 Ops[0], ConstantInt::get(
Int64Ty, Amt->getZExtValue()),
"shld_n");
12030 case NEON::BI__builtin_neon_vshrd_n_s64: {
12033 Ops[0], ConstantInt::get(
Int64Ty, std::min(
static_cast<uint64_t>(63),
12034 Amt->getZExtValue())),
12037 case NEON::BI__builtin_neon_vshrd_n_u64: {
12039 uint64_t ShiftAmt = Amt->getZExtValue();
12041 if (ShiftAmt == 64)
12042 return ConstantInt::get(
Int64Ty, 0);
12043 return Builder.CreateLShr(Ops[0], ConstantInt::get(
Int64Ty, ShiftAmt),
12046 case NEON::BI__builtin_neon_vsrad_n_s64: {
12049 Ops[1], ConstantInt::get(
Int64Ty, std::min(
static_cast<uint64_t>(63),
12050 Amt->getZExtValue())),
12052 return Builder.CreateAdd(Ops[0], Ops[1]);
12054 case NEON::BI__builtin_neon_vsrad_n_u64: {
12056 uint64_t ShiftAmt = Amt->getZExtValue();
12059 if (ShiftAmt == 64)
12061 Ops[1] =
Builder.CreateLShr(Ops[1], ConstantInt::get(
Int64Ty, ShiftAmt),
12063 return Builder.CreateAdd(Ops[0], Ops[1]);
12065 case NEON::BI__builtin_neon_vqdmlalh_lane_s16:
12066 case NEON::BI__builtin_neon_vqdmlalh_laneq_s16:
12067 case NEON::BI__builtin_neon_vqdmlslh_lane_s16:
12068 case NEON::BI__builtin_neon_vqdmlslh_laneq_s16: {
12074 auto *VTy = llvm::FixedVectorType::get(
Int32Ty, 4);
12076 ProductOps,
"vqdmlXl");
12077 Constant *CI = ConstantInt::get(
SizeTy, 0);
12078 Ops[1] =
Builder.CreateExtractElement(Ops[1], CI,
"lane0");
12081 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlalh_lane_s16 ||
12082 BuiltinID == NEON::BI__builtin_neon_vqdmlalh_laneq_s16)
12083 ? Intrinsic::aarch64_neon_sqadd
12084 : Intrinsic::aarch64_neon_sqsub;
12087 case NEON::BI__builtin_neon_vqdmlals_s32:
12088 case NEON::BI__builtin_neon_vqdmlsls_s32: {
12090 ProductOps.push_back(Ops[1]);
12094 ProductOps,
"vqdmlXl");
12096 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlals_s32
12097 ? Intrinsic::aarch64_neon_sqadd
12098 : Intrinsic::aarch64_neon_sqsub;
12101 case NEON::BI__builtin_neon_vqdmlals_lane_s32:
12102 case NEON::BI__builtin_neon_vqdmlals_laneq_s32:
12103 case NEON::BI__builtin_neon_vqdmlsls_lane_s32:
12104 case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: {
12108 ProductOps.push_back(Ops[1]);
12109 ProductOps.push_back(Ops[2]);
12112 ProductOps,
"vqdmlXl");
12115 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlals_lane_s32 ||
12116 BuiltinID == NEON::BI__builtin_neon_vqdmlals_laneq_s32)
12117 ? Intrinsic::aarch64_neon_sqadd
12118 : Intrinsic::aarch64_neon_sqsub;
12121 case NEON::BI__builtin_neon_vget_lane_bf16:
12122 case NEON::BI__builtin_neon_vduph_lane_bf16:
12123 case NEON::BI__builtin_neon_vduph_lane_f16: {
12127 case NEON::BI__builtin_neon_vgetq_lane_bf16:
12128 case NEON::BI__builtin_neon_vduph_laneq_bf16:
12129 case NEON::BI__builtin_neon_vduph_laneq_f16: {
12134 case clang::AArch64::BI_InterlockedAdd:
12135 case clang::AArch64::BI_InterlockedAdd64: {
12138 AtomicRMWInst *RMWI =
12140 llvm::AtomicOrdering::SequentiallyConsistent);
12141 return Builder.CreateAdd(RMWI, Val);
12146 llvm::Type *Ty = VTy;
12157 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
12158 Builtin->NameHint, Builtin->TypeModifier, E, Ops,
12165 switch (BuiltinID) {
12166 default:
return nullptr;
12167 case NEON::BI__builtin_neon_vbsl_v:
12168 case NEON::BI__builtin_neon_vbslq_v: {
12169 llvm::Type *BitTy = llvm::VectorType::getInteger(VTy);
12170 Ops[0] =
Builder.CreateBitCast(Ops[0], BitTy,
"vbsl");
12171 Ops[1] =
Builder.CreateBitCast(Ops[1], BitTy,
"vbsl");
12172 Ops[2] =
Builder.CreateBitCast(Ops[2], BitTy,
"vbsl");
12174 Ops[1] =
Builder.CreateAnd(Ops[0], Ops[1],
"vbsl");
12175 Ops[2] =
Builder.CreateAnd(
Builder.CreateNot(Ops[0]), Ops[2],
"vbsl");
12176 Ops[0] =
Builder.CreateOr(Ops[1], Ops[2],
"vbsl");
12177 return Builder.CreateBitCast(Ops[0], Ty);
12179 case NEON::BI__builtin_neon_vfma_lane_v:
12180 case NEON::BI__builtin_neon_vfmaq_lane_v: {
12183 Value *Addend = Ops[0];
12184 Value *Multiplicand = Ops[1];
12185 Value *LaneSource = Ops[2];
12186 Ops[0] = Multiplicand;
12187 Ops[1] = LaneSource;
12191 auto *SourceTy = BuiltinID == NEON::BI__builtin_neon_vfmaq_lane_v
12192 ? llvm::FixedVectorType::get(VTy->getElementType(),
12193 VTy->getNumElements() / 2)
12195 llvm::Constant *cst = cast<Constant>(Ops[3]);
12196 Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(), cst);
12197 Ops[1] =
Builder.CreateBitCast(Ops[1], SourceTy);
12198 Ops[1] =
Builder.CreateShuffleVector(Ops[1], Ops[1], SV,
"lane");
12201 Int =
Builder.getIsFPConstrained() ? Intrinsic::experimental_constrained_fma
12205 case NEON::BI__builtin_neon_vfma_laneq_v: {
12206 auto *VTy = cast<llvm::FixedVectorType>(Ty);
12208 if (VTy && VTy->getElementType() ==
DoubleTy) {
12211 llvm::FixedVectorType *VTy =
12213 Ops[2] =
Builder.CreateBitCast(Ops[2], VTy);
12214 Ops[2] =
Builder.CreateExtractElement(Ops[2], Ops[3],
"extract");
12217 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma,
12218 DoubleTy, {Ops[1], Ops[2], Ops[0]});
12221 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
12222 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
12224 auto *STy = llvm::FixedVectorType::get(VTy->getElementType(),
12225 VTy->getNumElements() * 2);
12226 Ops[2] =
Builder.CreateBitCast(Ops[2], STy);
12227 Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(),
12228 cast<ConstantInt>(Ops[3]));
12229 Ops[2] =
Builder.CreateShuffleVector(Ops[2], Ops[2], SV,
"lane");
12232 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
12233 {Ops[2], Ops[1], Ops[0]});
12235 case NEON::BI__builtin_neon_vfmaq_laneq_v: {
12236 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
12237 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
12239 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
12242 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
12243 {Ops[2], Ops[1], Ops[0]});
12245 case NEON::BI__builtin_neon_vfmah_lane_f16:
12246 case NEON::BI__builtin_neon_vfmas_lane_f32:
12247 case NEON::BI__builtin_neon_vfmah_laneq_f16:
12248 case NEON::BI__builtin_neon_vfmas_laneq_f32:
12249 case NEON::BI__builtin_neon_vfmad_lane_f64:
12250 case NEON::BI__builtin_neon_vfmad_laneq_f64: {
12253 Ops[2] =
Builder.CreateExtractElement(Ops[2], Ops[3],
"extract");
12255 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
12256 {Ops[1], Ops[2], Ops[0]});
12258 case NEON::BI__builtin_neon_vmull_v:
12260 Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull;
12261 if (
Type.isPoly()) Int = Intrinsic::aarch64_neon_pmull;
12263 case NEON::BI__builtin_neon_vmax_v:
12264 case NEON::BI__builtin_neon_vmaxq_v:
12266 Int = usgn ? Intrinsic::aarch64_neon_umax : Intrinsic::aarch64_neon_smax;
12267 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmax;
12269 case NEON::BI__builtin_neon_vmaxh_f16: {
12271 Int = Intrinsic::aarch64_neon_fmax;
12274 case NEON::BI__builtin_neon_vmin_v:
12275 case NEON::BI__builtin_neon_vminq_v:
12277 Int = usgn ? Intrinsic::aarch64_neon_umin : Intrinsic::aarch64_neon_smin;
12278 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmin;
12280 case NEON::BI__builtin_neon_vminh_f16: {
12282 Int = Intrinsic::aarch64_neon_fmin;
12285 case NEON::BI__builtin_neon_vabd_v:
12286 case NEON::BI__builtin_neon_vabdq_v:
12288 Int = usgn ? Intrinsic::aarch64_neon_uabd : Intrinsic::aarch64_neon_sabd;
12289 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fabd;
12291 case NEON::BI__builtin_neon_vpadal_v:
12292 case NEON::BI__builtin_neon_vpadalq_v: {
12293 unsigned ArgElts = VTy->getNumElements();
12294 llvm::IntegerType *EltTy = cast<IntegerType>(VTy->getElementType());
12295 unsigned BitWidth = EltTy->getBitWidth();
12296 auto *ArgTy = llvm::FixedVectorType::get(
12297 llvm::IntegerType::get(
getLLVMContext(), BitWidth / 2), 2 * ArgElts);
12298 llvm::Type* Tys[2] = { VTy, ArgTy };
12299 Int = usgn ? Intrinsic::aarch64_neon_uaddlp : Intrinsic::aarch64_neon_saddlp;
12301 TmpOps.push_back(Ops[1]);
12304 llvm::Value *addend =
Builder.CreateBitCast(Ops[0], tmp->getType());
12305 return Builder.CreateAdd(tmp, addend);
12307 case NEON::BI__builtin_neon_vpmin_v:
12308 case NEON::BI__builtin_neon_vpminq_v:
12310 Int = usgn ? Intrinsic::aarch64_neon_uminp : Intrinsic::aarch64_neon_sminp;
12311 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fminp;
12313 case NEON::BI__builtin_neon_vpmax_v:
12314 case NEON::BI__builtin_neon_vpmaxq_v:
12316 Int = usgn ? Intrinsic::aarch64_neon_umaxp : Intrinsic::aarch64_neon_smaxp;
12317 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmaxp;
12319 case NEON::BI__builtin_neon_vminnm_v:
12320 case NEON::BI__builtin_neon_vminnmq_v:
12321 Int = Intrinsic::aarch64_neon_fminnm;
12323 case NEON::BI__builtin_neon_vminnmh_f16:
12325 Int = Intrinsic::aarch64_neon_fminnm;
12327 case NEON::BI__builtin_neon_vmaxnm_v:
12328 case NEON::BI__builtin_neon_vmaxnmq_v:
12329 Int = Intrinsic::aarch64_neon_fmaxnm;
12331 case NEON::BI__builtin_neon_vmaxnmh_f16:
12333 Int = Intrinsic::aarch64_neon_fmaxnm;
12335 case NEON::BI__builtin_neon_vrecpss_f32: {
12340 case NEON::BI__builtin_neon_vrecpsd_f64:
12344 case NEON::BI__builtin_neon_vrecpsh_f16:
12348 case NEON::BI__builtin_neon_vqshrun_n_v:
12349 Int = Intrinsic::aarch64_neon_sqshrun;
12351 case NEON::BI__builtin_neon_vqrshrun_n_v:
12352 Int = Intrinsic::aarch64_neon_sqrshrun;
12354 case NEON::BI__builtin_neon_vqshrn_n_v:
12355 Int = usgn ? Intrinsic::aarch64_neon_uqshrn : Intrinsic::aarch64_neon_sqshrn;
12357 case NEON::BI__builtin_neon_vrshrn_n_v:
12358 Int = Intrinsic::aarch64_neon_rshrn;
12360 case NEON::BI__builtin_neon_vqrshrn_n_v:
12361 Int = usgn ? Intrinsic::aarch64_neon_uqrshrn : Intrinsic::aarch64_neon_sqrshrn;
12363 case NEON::BI__builtin_neon_vrndah_f16: {
12365 Int =
Builder.getIsFPConstrained()
12366 ? Intrinsic::experimental_constrained_round
12367 : Intrinsic::round;
12370 case NEON::BI__builtin_neon_vrnda_v:
12371 case NEON::BI__builtin_neon_vrndaq_v: {
12372 Int =
Builder.getIsFPConstrained()
12373 ? Intrinsic::experimental_constrained_round
12374 : Intrinsic::round;
12377 case NEON::BI__builtin_neon_vrndih_f16: {
12379 Int =
Builder.getIsFPConstrained()
12380 ? Intrinsic::experimental_constrained_nearbyint
12381 : Intrinsic::nearbyint;
12384 case NEON::BI__builtin_neon_vrndmh_f16: {
12386 Int =
Builder.getIsFPConstrained()
12387 ? Intrinsic::experimental_constrained_floor
12388 : Intrinsic::floor;
12391 case NEON::BI__builtin_neon_vrndm_v:
12392 case NEON::BI__builtin_neon_vrndmq_v: {
12393 Int =
Builder.getIsFPConstrained()
12394 ? Intrinsic::experimental_constrained_floor
12395 : Intrinsic::floor;
12398 case NEON::BI__builtin_neon_vrndnh_f16: {
12400 Int =
Builder.getIsFPConstrained()
12401 ? Intrinsic::experimental_constrained_roundeven
12402 : Intrinsic::roundeven;
12405 case NEON::BI__builtin_neon_vrndn_v:
12406 case NEON::BI__builtin_neon_vrndnq_v: {
12407 Int =
Builder.getIsFPConstrained()
12408 ? Intrinsic::experimental_constrained_roundeven
12409 : Intrinsic::roundeven;
12412 case NEON::BI__builtin_neon_vrndns_f32: {
12414 Int =
Builder.getIsFPConstrained()
12415 ? Intrinsic::experimental_constrained_roundeven
12416 : Intrinsic::roundeven;
12419 case NEON::BI__builtin_neon_vrndph_f16: {
12421 Int =
Builder.getIsFPConstrained()
12422 ? Intrinsic::experimental_constrained_ceil
12426 case NEON::BI__builtin_neon_vrndp_v:
12427 case NEON::BI__builtin_neon_vrndpq_v: {
12428 Int =
Builder.getIsFPConstrained()
12429 ? Intrinsic::experimental_constrained_ceil
12433 case NEON::BI__builtin_neon_vrndxh_f16: {
12435 Int =
Builder.getIsFPConstrained()
12436 ? Intrinsic::experimental_constrained_rint
12440 case NEON::BI__builtin_neon_vrndx_v:
12441 case NEON::BI__builtin_neon_vrndxq_v: {
12442 Int =
Builder.getIsFPConstrained()
12443 ? Intrinsic::experimental_constrained_rint
12447 case NEON::BI__builtin_neon_vrndh_f16: {
12449 Int =
Builder.getIsFPConstrained()
12450 ? Intrinsic::experimental_constrained_trunc
12451 : Intrinsic::trunc;
12454 case NEON::BI__builtin_neon_vrnd32x_f32:
12455 case NEON::BI__builtin_neon_vrnd32xq_f32:
12456 case NEON::BI__builtin_neon_vrnd32x_f64:
12457 case NEON::BI__builtin_neon_vrnd32xq_f64: {
12459 Int = Intrinsic::aarch64_neon_frint32x;
12462 case NEON::BI__builtin_neon_vrnd32z_f32:
12463 case NEON::BI__builtin_neon_vrnd32zq_f32:
12464 case NEON::BI__builtin_neon_vrnd32z_f64:
12465 case NEON::BI__builtin_neon_vrnd32zq_f64: {
12467 Int = Intrinsic::aarch64_neon_frint32z;
12470 case NEON::BI__builtin_neon_vrnd64x_f32:
12471 case NEON::BI__builtin_neon_vrnd64xq_f32:
12472 case NEON::BI__builtin_neon_vrnd64x_f64:
12473 case NEON::BI__builtin_neon_vrnd64xq_f64: {
12475 Int = Intrinsic::aarch64_neon_frint64x;
12478 case NEON::BI__builtin_neon_vrnd64z_f32:
12479 case NEON::BI__builtin_neon_vrnd64zq_f32:
12480 case NEON::BI__builtin_neon_vrnd64z_f64:
12481 case NEON::BI__builtin_neon_vrnd64zq_f64: {
12483 Int = Intrinsic::aarch64_neon_frint64z;
12486 case NEON::BI__builtin_neon_vrnd_v:
12487 case NEON::BI__builtin_neon_vrndq_v: {
12488 Int =
Builder.getIsFPConstrained()
12489 ? Intrinsic::experimental_constrained_trunc
12490 : Intrinsic::trunc;
12493 case NEON::BI__builtin_neon_vcvt_f64_v:
12494 case NEON::BI__builtin_neon_vcvtq_f64_v:
12495 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
12497 return usgn ?
Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
12498 :
Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
12499 case NEON::BI__builtin_neon_vcvt_f64_f32: {
12501 "unexpected vcvt_f64_f32 builtin");
12505 return Builder.CreateFPExt(Ops[0], Ty,
"vcvt");
12507 case NEON::BI__builtin_neon_vcvt_f32_f64: {
12509 "unexpected vcvt_f32_f64 builtin");
12513 return Builder.CreateFPTrunc(Ops[0], Ty,
"vcvt");
12515 case NEON::BI__builtin_neon_vcvt_s32_v:
12516 case NEON::BI__builtin_neon_vcvt_u32_v:
12517 case NEON::BI__builtin_neon_vcvt_s64_v:
12518 case NEON::BI__builtin_neon_vcvt_u64_v:
12519 case NEON::BI__builtin_neon_vcvt_s16_f16:
12520 case NEON::BI__builtin_neon_vcvt_u16_f16:
12521 case NEON::BI__builtin_neon_vcvtq_s32_v:
12522 case NEON::BI__builtin_neon_vcvtq_u32_v:
12523 case NEON::BI__builtin_neon_vcvtq_s64_v:
12524 case NEON::BI__builtin_neon_vcvtq_u64_v:
12525 case NEON::BI__builtin_neon_vcvtq_s16_f16:
12526 case NEON::BI__builtin_neon_vcvtq_u16_f16: {
12528 usgn ? Intrinsic::aarch64_neon_fcvtzu : Intrinsic::aarch64_neon_fcvtzs;
12532 case NEON::BI__builtin_neon_vcvta_s16_f16:
12533 case NEON::BI__builtin_neon_vcvta_u16_f16:
12534 case NEON::BI__builtin_neon_vcvta_s32_v:
12535 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
12536 case NEON::BI__builtin_neon_vcvtaq_s32_v:
12537 case NEON::BI__builtin_neon_vcvta_u32_v:
12538 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
12539 case NEON::BI__builtin_neon_vcvtaq_u32_v:
12540 case NEON::BI__builtin_neon_vcvta_s64_v:
12541 case NEON::BI__builtin_neon_vcvtaq_s64_v:
12542 case NEON::BI__builtin_neon_vcvta_u64_v:
12543 case NEON::BI__builtin_neon_vcvtaq_u64_v: {
12544 Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas;
12548 case NEON::BI__builtin_neon_vcvtm_s16_f16:
12549 case NEON::BI__builtin_neon_vcvtm_s32_v:
12550 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
12551 case NEON::BI__builtin_neon_vcvtmq_s32_v:
12552 case NEON::BI__builtin_neon_vcvtm_u16_f16:
12553 case NEON::BI__builtin_neon_vcvtm_u32_v:
12554 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
12555 case NEON::BI__builtin_neon_vcvtmq_u32_v:
12556 case NEON::BI__builtin_neon_vcvtm_s64_v:
12557 case NEON::BI__builtin_neon_vcvtmq_s64_v:
12558 case NEON::BI__builtin_neon_vcvtm_u64_v:
12559 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
12560 Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms;
12564 case NEON::BI__builtin_neon_vcvtn_s16_f16:
12565 case NEON::BI__builtin_neon_vcvtn_s32_v:
12566 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
12567 case NEON::BI__builtin_neon_vcvtnq_s32_v:
12568 case NEON::BI__builtin_neon_vcvtn_u16_f16:
12569 case NEON::BI__builtin_neon_vcvtn_u32_v:
12570 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
12571 case NEON::BI__builtin_neon_vcvtnq_u32_v:
12572 case NEON::BI__builtin_neon_vcvtn_s64_v:
12573 case NEON::BI__builtin_neon_vcvtnq_s64_v:
12574 case NEON::BI__builtin_neon_vcvtn_u64_v:
12575 case NEON::BI__builtin_neon_vcvtnq_u64_v: {
12576 Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns;
12580 case NEON::BI__builtin_neon_vcvtp_s16_f16:
12581 case NEON::BI__builtin_neon_vcvtp_s32_v:
12582 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
12583 case NEON::BI__builtin_neon_vcvtpq_s32_v:
12584 case NEON::BI__builtin_neon_vcvtp_u16_f16:
12585 case NEON::BI__builtin_neon_vcvtp_u32_v:
12586 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
12587 case NEON::BI__builtin_neon_vcvtpq_u32_v:
12588 case NEON::BI__builtin_neon_vcvtp_s64_v:
12589 case NEON::BI__builtin_neon_vcvtpq_s64_v:
12590 case NEON::BI__builtin_neon_vcvtp_u64_v:
12591 case NEON::BI__builtin_neon_vcvtpq_u64_v: {
12592 Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps;
12596 case NEON::BI__builtin_neon_vmulx_v:
12597 case NEON::BI__builtin_neon_vmulxq_v: {
12598 Int = Intrinsic::aarch64_neon_fmulx;
12601 case NEON::BI__builtin_neon_vmulxh_lane_f16:
12602 case NEON::BI__builtin_neon_vmulxh_laneq_f16: {
12606 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2],
"extract");
12608 Int = Intrinsic::aarch64_neon_fmulx;
12611 case NEON::BI__builtin_neon_vmul_lane_v:
12612 case NEON::BI__builtin_neon_vmul_laneq_v: {
12615 if (BuiltinID == NEON::BI__builtin_neon_vmul_laneq_v)
12618 llvm::FixedVectorType *VTy =
12620 Ops[1] =
Builder.CreateBitCast(Ops[1], VTy);
12621 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2],
"extract");
12625 case NEON::BI__builtin_neon_vnegd_s64:
12627 case NEON::BI__builtin_neon_vnegh_f16:
12629 case NEON::BI__builtin_neon_vpmaxnm_v:
12630 case NEON::BI__builtin_neon_vpmaxnmq_v: {
12631 Int = Intrinsic::aarch64_neon_fmaxnmp;
12634 case NEON::BI__builtin_neon_vpminnm_v:
12635 case NEON::BI__builtin_neon_vpminnmq_v: {
12636 Int = Intrinsic::aarch64_neon_fminnmp;
12639 case NEON::BI__builtin_neon_vsqrth_f16: {
12641 Int =
Builder.getIsFPConstrained()
12642 ? Intrinsic::experimental_constrained_sqrt
12646 case NEON::BI__builtin_neon_vsqrt_v:
12647 case NEON::BI__builtin_neon_vsqrtq_v: {
12648 Int =
Builder.getIsFPConstrained()
12649 ? Intrinsic::experimental_constrained_sqrt
12651 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
12654 case NEON::BI__builtin_neon_vrbit_v:
12655 case NEON::BI__builtin_neon_vrbitq_v: {
12656 Int = Intrinsic::bitreverse;
12659 case NEON::BI__builtin_neon_vaddv_u8:
12663 case NEON::BI__builtin_neon_vaddv_s8: {
12664 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
12666 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
12667 llvm::Type *Tys[2] = { Ty, VTy };
12672 case NEON::BI__builtin_neon_vaddv_u16:
12675 case NEON::BI__builtin_neon_vaddv_s16: {
12676 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
12678 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
12679 llvm::Type *Tys[2] = { Ty, VTy };
12684 case NEON::BI__builtin_neon_vaddvq_u8:
12687 case NEON::BI__builtin_neon_vaddvq_s8: {
12688 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
12690 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
12691 llvm::Type *Tys[2] = { Ty, VTy };
12696 case NEON::BI__builtin_neon_vaddvq_u16:
12699 case NEON::BI__builtin_neon_vaddvq_s16: {
12700 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
12702 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
12703 llvm::Type *Tys[2] = { Ty, VTy };
12708 case NEON::BI__builtin_neon_vmaxv_u8: {
12709 Int = Intrinsic::aarch64_neon_umaxv;
12711 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
12712 llvm::Type *Tys[2] = { Ty, VTy };
12717 case NEON::BI__builtin_neon_vmaxv_u16: {
12718 Int = Intrinsic::aarch64_neon_umaxv;
12720 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
12721 llvm::Type *Tys[2] = { Ty, VTy };
12726 case NEON::BI__builtin_neon_vmaxvq_u8: {
12727 Int = Intrinsic::aarch64_neon_umaxv;
12729 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
12730 llvm::Type *Tys[2] = { Ty, VTy };
12735 case NEON::BI__builtin_neon_vmaxvq_u16: {
12736 Int = Intrinsic::aarch64_neon_umaxv;
12738 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
12739 llvm::Type *Tys[2] = { Ty, VTy };
12744 case NEON::BI__builtin_neon_vmaxv_s8: {
12745 Int = Intrinsic::aarch64_neon_smaxv;
12747 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
12748 llvm::Type *Tys[2] = { Ty, VTy };
12753 case NEON::BI__builtin_neon_vmaxv_s16: {
12754 Int = Intrinsic::aarch64_neon_smaxv;
12756 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
12757 llvm::Type *Tys[2] = { Ty, VTy };
12762 case NEON::BI__builtin_neon_vmaxvq_s8: {
12763 Int = Intrinsic::aarch64_neon_smaxv;
12765 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
12766 llvm::Type *Tys[2] = { Ty, VTy };
12771 case NEON::BI__builtin_neon_vmaxvq_s16: {
12772 Int = Intrinsic::aarch64_neon_smaxv;
12774 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
12775 llvm::Type *Tys[2] = { Ty, VTy };
12780 case NEON::BI__builtin_neon_vmaxv_f16: {
12781 Int = Intrinsic::aarch64_neon_fmaxv;
12783 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
12784 llvm::Type *Tys[2] = { Ty, VTy };
12789 case NEON::BI__builtin_neon_vmaxvq_f16: {
12790 Int = Intrinsic::aarch64_neon_fmaxv;
12792 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
12793 llvm::Type *Tys[2] = { Ty, VTy };
12798 case NEON::BI__builtin_neon_vminv_u8: {
12799 Int = Intrinsic::aarch64_neon_uminv;
12801 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
12802 llvm::Type *Tys[2] = { Ty, VTy };
12807 case NEON::BI__builtin_neon_vminv_u16: {
12808 Int = Intrinsic::aarch64_neon_uminv;
12810 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
12811 llvm::Type *Tys[2] = { Ty, VTy };
12816 case NEON::BI__builtin_neon_vminvq_u8: {
12817 Int = Intrinsic::aarch64_neon_uminv;
12819 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
12820 llvm::Type *Tys[2] = { Ty, VTy };
12825 case NEON::BI__builtin_neon_vminvq_u16: {
12826 Int = Intrinsic::aarch64_neon_uminv;
12828 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
12829 llvm::Type *Tys[2] = { Ty, VTy };
12834 case NEON::BI__builtin_neon_vminv_s8: {
12835 Int = Intrinsic::aarch64_neon_sminv;
12837 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
12838 llvm::Type *Tys[2] = { Ty, VTy };
12843 case NEON::BI__builtin_neon_vminv_s16: {
12844 Int = Intrinsic::aarch64_neon_sminv;
12846 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
12847 llvm::Type *Tys[2] = { Ty, VTy };
12852 case NEON::BI__builtin_neon_vminvq_s8: {
12853 Int = Intrinsic::aarch64_neon_sminv;
12855 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
12856 llvm::Type *Tys[2] = { Ty, VTy };
12861 case NEON::BI__builtin_neon_vminvq_s16: {
12862 Int = Intrinsic::aarch64_neon_sminv;
12864 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
12865 llvm::Type *Tys[2] = { Ty, VTy };
12870 case NEON::BI__builtin_neon_vminv_f16: {
12871 Int = Intrinsic::aarch64_neon_fminv;
12873 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
12874 llvm::Type *Tys[2] = { Ty, VTy };
12879 case NEON::BI__builtin_neon_vminvq_f16: {
12880 Int = Intrinsic::aarch64_neon_fminv;
12882 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
12883 llvm::Type *Tys[2] = { Ty, VTy };
12888 case NEON::BI__builtin_neon_vmaxnmv_f16: {
12889 Int = Intrinsic::aarch64_neon_fmaxnmv;
12891 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
12892 llvm::Type *Tys[2] = { Ty, VTy };
12897 case NEON::BI__builtin_neon_vmaxnmvq_f16: {
12898 Int = Intrinsic::aarch64_neon_fmaxnmv;
12900 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
12901 llvm::Type *Tys[2] = { Ty, VTy };
12906 case NEON::BI__builtin_neon_vminnmv_f16: {
12907 Int = Intrinsic::aarch64_neon_fminnmv;
12909 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
12910 llvm::Type *Tys[2] = { Ty, VTy };
12915 case NEON::BI__builtin_neon_vminnmvq_f16: {
12916 Int = Intrinsic::aarch64_neon_fminnmv;
12918 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
12919 llvm::Type *Tys[2] = { Ty, VTy };
12924 case NEON::BI__builtin_neon_vmul_n_f64: {
12927 return Builder.CreateFMul(Ops[0], RHS);
12929 case NEON::BI__builtin_neon_vaddlv_u8: {
12930 Int = Intrinsic::aarch64_neon_uaddlv;
12932 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
12933 llvm::Type *Tys[2] = { Ty, VTy };
12938 case NEON::BI__builtin_neon_vaddlv_u16: {
12939 Int = Intrinsic::aarch64_neon_uaddlv;
12941 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
12942 llvm::Type *Tys[2] = { Ty, VTy };
12946 case NEON::BI__builtin_neon_vaddlvq_u8: {
12947 Int = Intrinsic::aarch64_neon_uaddlv;
12949 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
12950 llvm::Type *Tys[2] = { Ty, VTy };
12955 case NEON::BI__builtin_neon_vaddlvq_u16: {
12956 Int = Intrinsic::aarch64_neon_uaddlv;
12958 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
12959 llvm::Type *Tys[2] = { Ty, VTy };
12963 case NEON::BI__builtin_neon_vaddlv_s8: {
12964 Int = Intrinsic::aarch64_neon_saddlv;
12966 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
12967 llvm::Type *Tys[2] = { Ty, VTy };
12972 case NEON::BI__builtin_neon_vaddlv_s16: {
12973 Int = Intrinsic::aarch64_neon_saddlv;
12975 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
12976 llvm::Type *Tys[2] = { Ty, VTy };
12980 case NEON::BI__builtin_neon_vaddlvq_s8: {
12981 Int = Intrinsic::aarch64_neon_saddlv;
12983 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
12984 llvm::Type *Tys[2] = { Ty, VTy };
12989 case NEON::BI__builtin_neon_vaddlvq_s16: {
12990 Int = Intrinsic::aarch64_neon_saddlv;
12992 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
12993 llvm::Type *Tys[2] = { Ty, VTy };
12997 case NEON::BI__builtin_neon_vsri_n_v:
12998 case NEON::BI__builtin_neon_vsriq_n_v: {
12999 Int = Intrinsic::aarch64_neon_vsri;
13003 case NEON::BI__builtin_neon_vsli_n_v:
13004 case NEON::BI__builtin_neon_vsliq_n_v: {
13005 Int = Intrinsic::aarch64_neon_vsli;
13009 case NEON::BI__builtin_neon_vsra_n_v:
13010 case NEON::BI__builtin_neon_vsraq_n_v:
13011 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
13013 return Builder.CreateAdd(Ops[0], Ops[1]);
13014 case NEON::BI__builtin_neon_vrsra_n_v:
13015 case NEON::BI__builtin_neon_vrsraq_n_v: {
13016 Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl;
13018 TmpOps.push_back(Ops[1]);
13019 TmpOps.push_back(Ops[2]);
13021 llvm::Value *tmp =
EmitNeonCall(F, TmpOps,
"vrshr_n", 1,
true);
13022 Ops[0] =
Builder.CreateBitCast(Ops[0], VTy);
13023 return Builder.CreateAdd(Ops[0], tmp);
13025 case NEON::BI__builtin_neon_vld1_v:
13026 case NEON::BI__builtin_neon_vld1q_v: {
13029 case NEON::BI__builtin_neon_vst1_v:
13030 case NEON::BI__builtin_neon_vst1q_v:
13031 Ops[1] =
Builder.CreateBitCast(Ops[1], VTy);
13033 case NEON::BI__builtin_neon_vld1_lane_v:
13034 case NEON::BI__builtin_neon_vld1q_lane_v: {
13035 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13038 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vld1_lane");
13040 case NEON::BI__builtin_neon_vldap1_lane_s64:
13041 case NEON::BI__builtin_neon_vldap1q_lane_s64: {
13042 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13044 VTy->getElementType(), Ops[0], PtrOp0.
getAlignment());
13045 LI->setAtomic(llvm::AtomicOrdering::Acquire);
13047 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vldap1_lane");
13049 case NEON::BI__builtin_neon_vld1_dup_v:
13050 case NEON::BI__builtin_neon_vld1q_dup_v: {
13051 Value *
V = PoisonValue::get(Ty);
13054 llvm::Constant *CI = ConstantInt::get(
Int32Ty, 0);
13055 Ops[0] =
Builder.CreateInsertElement(
V, Ops[0], CI);
13058 case NEON::BI__builtin_neon_vst1_lane_v:
13059 case NEON::BI__builtin_neon_vst1q_lane_v:
13060 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13061 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2]);
13063 case NEON::BI__builtin_neon_vstl1_lane_s64:
13064 case NEON::BI__builtin_neon_vstl1q_lane_s64: {
13065 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13066 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2]);
13067 llvm::StoreInst *SI =
13069 SI->setAtomic(llvm::AtomicOrdering::Release);
13072 case NEON::BI__builtin_neon_vld2_v:
13073 case NEON::BI__builtin_neon_vld2q_v: {
13076 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld2");
13079 case NEON::BI__builtin_neon_vld3_v:
13080 case NEON::BI__builtin_neon_vld3q_v: {
13083 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld3");
13086 case NEON::BI__builtin_neon_vld4_v:
13087 case NEON::BI__builtin_neon_vld4q_v: {
13090 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld4");
13093 case NEON::BI__builtin_neon_vld2_dup_v:
13094 case NEON::BI__builtin_neon_vld2q_dup_v: {
13097 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld2");
13100 case NEON::BI__builtin_neon_vld3_dup_v:
13101 case NEON::BI__builtin_neon_vld3q_dup_v: {
13104 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld3");
13107 case NEON::BI__builtin_neon_vld4_dup_v:
13108 case NEON::BI__builtin_neon_vld4q_dup_v: {
13111 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld4");
13114 case NEON::BI__builtin_neon_vld2_lane_v:
13115 case NEON::BI__builtin_neon_vld2q_lane_v: {
13116 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
13118 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
13119 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13120 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13125 case NEON::BI__builtin_neon_vld3_lane_v:
13126 case NEON::BI__builtin_neon_vld3q_lane_v: {
13127 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
13129 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
13130 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13131 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13132 Ops[3] =
Builder.CreateBitCast(Ops[3], Ty);
13137 case NEON::BI__builtin_neon_vld4_lane_v:
13138 case NEON::BI__builtin_neon_vld4q_lane_v: {
13139 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
13141 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
13142 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13143 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13144 Ops[3] =
Builder.CreateBitCast(Ops[3], Ty);
13145 Ops[4] =
Builder.CreateBitCast(Ops[4], Ty);
13150 case NEON::BI__builtin_neon_vst2_v:
13151 case NEON::BI__builtin_neon_vst2q_v: {
13152 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13153 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() };
13157 case NEON::BI__builtin_neon_vst2_lane_v:
13158 case NEON::BI__builtin_neon_vst2q_lane_v: {
13159 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13161 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
13165 case NEON::BI__builtin_neon_vst3_v:
13166 case NEON::BI__builtin_neon_vst3q_v: {
13167 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13168 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
13172 case NEON::BI__builtin_neon_vst3_lane_v:
13173 case NEON::BI__builtin_neon_vst3q_lane_v: {
13174 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13176 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
13180 case NEON::BI__builtin_neon_vst4_v:
13181 case NEON::BI__builtin_neon_vst4q_v: {
13182 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13183 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
13187 case NEON::BI__builtin_neon_vst4_lane_v:
13188 case NEON::BI__builtin_neon_vst4q_lane_v: {
13189 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13191 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() };
13195 case NEON::BI__builtin_neon_vtrn_v:
13196 case NEON::BI__builtin_neon_vtrnq_v: {
13197 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13198 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13199 Value *SV =
nullptr;
13201 for (
unsigned vi = 0; vi != 2; ++vi) {
13203 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
13204 Indices.push_back(i+vi);
13205 Indices.push_back(i+e+vi);
13207 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
13208 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vtrn");
13213 case NEON::BI__builtin_neon_vuzp_v:
13214 case NEON::BI__builtin_neon_vuzpq_v: {
13215 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13216 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13217 Value *SV =
nullptr;
13219 for (
unsigned vi = 0; vi != 2; ++vi) {
13221 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
13222 Indices.push_back(2*i+vi);
13224 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
13225 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vuzp");
13230 case NEON::BI__builtin_neon_vzip_v:
13231 case NEON::BI__builtin_neon_vzipq_v: {
13232 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13233 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13234 Value *SV =
nullptr;
13236 for (
unsigned vi = 0; vi != 2; ++vi) {
13238 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
13239 Indices.push_back((i + vi*e) >> 1);
13240 Indices.push_back(((i + vi*e) >> 1)+e);
13242 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
13243 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vzip");
13248 case NEON::BI__builtin_neon_vqtbl1q_v: {
13252 case NEON::BI__builtin_neon_vqtbl2q_v: {
13256 case NEON::BI__builtin_neon_vqtbl3q_v: {
13260 case NEON::BI__builtin_neon_vqtbl4q_v: {
13264 case NEON::BI__builtin_neon_vqtbx1q_v: {
13268 case NEON::BI__builtin_neon_vqtbx2q_v: {
13272 case NEON::BI__builtin_neon_vqtbx3q_v: {
13276 case NEON::BI__builtin_neon_vqtbx4q_v: {
13280 case NEON::BI__builtin_neon_vsqadd_v:
13281 case NEON::BI__builtin_neon_vsqaddq_v: {
13282 Int = Intrinsic::aarch64_neon_usqadd;
13285 case NEON::BI__builtin_neon_vuqadd_v:
13286 case NEON::BI__builtin_neon_vuqaddq_v: {
13287 Int = Intrinsic::aarch64_neon_suqadd;
13295 assert((BuiltinID == BPF::BI__builtin_preserve_field_info ||
13296 BuiltinID == BPF::BI__builtin_btf_type_id ||
13297 BuiltinID == BPF::BI__builtin_preserve_type_info ||
13298 BuiltinID == BPF::BI__builtin_preserve_enum_value) &&
13299 "unexpected BPF builtin");
13304 static uint32_t BuiltinSeqNum;
13306 switch (BuiltinID) {
13308 llvm_unreachable(
"Unexpected BPF builtin");
13309 case BPF::BI__builtin_preserve_field_info: {
13315 "using __builtin_preserve_field_info() without -g");
13328 Value *InfoKind = ConstantInt::get(
Int64Ty,
C->getSExtValue());
13331 llvm::Function *FnGetFieldInfo = llvm::Intrinsic::getDeclaration(
13332 &
CGM.
getModule(), llvm::Intrinsic::bpf_preserve_field_info,
13333 {FieldAddr->getType()});
13334 return Builder.CreateCall(FnGetFieldInfo, {FieldAddr, InfoKind});
13336 case BPF::BI__builtin_btf_type_id:
13337 case BPF::BI__builtin_preserve_type_info: {
13348 Value *FlagValue = ConstantInt::get(
Int64Ty, Flag->getSExtValue());
13349 Value *SeqNumVal = ConstantInt::get(
Int32Ty, BuiltinSeqNum++);
13351 llvm::Function *FnDecl;
13352 if (BuiltinID == BPF::BI__builtin_btf_type_id)
13353 FnDecl = llvm::Intrinsic::getDeclaration(
13354 &
CGM.
getModule(), llvm::Intrinsic::bpf_btf_type_id, {});
13356 FnDecl = llvm::Intrinsic::getDeclaration(
13357 &
CGM.
getModule(), llvm::Intrinsic::bpf_preserve_type_info, {});
13358 CallInst *Fn =
Builder.CreateCall(FnDecl, {SeqNumVal, FlagValue});
13359 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
13362 case BPF::BI__builtin_preserve_enum_value: {
13373 const auto *UO = cast<UnaryOperator>(Arg0->
IgnoreParens());
13374 const auto *CE = cast<CStyleCastExpr>(UO->getSubExpr());
13375 const auto *DR = cast<DeclRefExpr>(CE->getSubExpr());
13376 const auto *Enumerator = cast<EnumConstantDecl>(DR->getDecl());
13378 auto InitVal = Enumerator->getInitVal();
13379 std::string InitValStr;
13380 if (InitVal.isNegative() || InitVal >
uint64_t(INT64_MAX))
13381 InitValStr = std::to_string(InitVal.getSExtValue());
13383 InitValStr = std::to_string(InitVal.getZExtValue());
13384 std::string EnumStr = Enumerator->getNameAsString() +
":" + InitValStr;
13385 Value *EnumStrVal =
Builder.CreateGlobalStringPtr(EnumStr);
13388 Value *FlagValue = ConstantInt::get(
Int64Ty, Flag->getSExtValue());
13389 Value *SeqNumVal = ConstantInt::get(
Int32Ty, BuiltinSeqNum++);
13391 llvm::Function *IntrinsicFn = llvm::Intrinsic::getDeclaration(
13392 &
CGM.
getModule(), llvm::Intrinsic::bpf_preserve_enum_value, {});
13394 Builder.CreateCall(IntrinsicFn, {SeqNumVal, EnumStrVal, FlagValue});
13395 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
13403 assert((Ops.size() & (Ops.size() - 1)) == 0 &&
13404 "Not a power-of-two sized vector!");
13405 bool AllConstants =
true;
13406 for (
unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i)
13407 AllConstants &= isa<Constant>(Ops[i]);
13410 if (AllConstants) {
13412 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
13413 CstOps.push_back(cast<Constant>(Ops[i]));
13414 return llvm::ConstantVector::get(CstOps);
13419 llvm::FixedVectorType::get(Ops[0]->getType(), Ops.size()));
13421 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
13429 unsigned NumElts) {
13431 auto *MaskTy = llvm::FixedVectorType::get(
13433 cast<IntegerType>(Mask->
getType())->getBitWidth());
13434 Value *MaskVec = CGF.
Builder.CreateBitCast(Mask, MaskTy);
13440 for (
unsigned i = 0; i != NumElts; ++i)
13442 MaskVec = CGF.
Builder.CreateShuffleVector(
13443 MaskVec, MaskVec,
ArrayRef(Indices, NumElts),
"extract");
13450 Value *Ptr = Ops[0];
13454 cast<llvm::FixedVectorType>(Ops[1]->getType())->getNumElements());
13456 return CGF.
Builder.CreateMaskedStore(Ops[1], Ptr, Alignment, MaskVec);
13461 llvm::Type *Ty = Ops[1]->getType();
13462 Value *Ptr = Ops[0];
13465 CGF, Ops[2], cast<llvm::FixedVectorType>(Ty)->getNumElements());
13467 return CGF.
Builder.CreateMaskedLoad(Ty, Ptr, Alignment, MaskVec, Ops[1]);
13472 auto *ResultTy = cast<llvm::VectorType>(Ops[1]->getType());
13473 Value *Ptr = Ops[0];
13476 CGF, Ops[2], cast<FixedVectorType>(ResultTy)->getNumElements());
13478 llvm::Function *F = CGF.
CGM.
getIntrinsic(Intrinsic::masked_expandload,
13480 return CGF.
Builder.CreateCall(F, { Ptr, MaskVec, Ops[1] });
13486 auto *ResultTy = cast<llvm::FixedVectorType>(Ops[1]->getType());
13490 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
13491 : Intrinsic::x86_avx512_mask_expand;
13493 return CGF.
Builder.CreateCall(F, { Ops[0], Ops[1], MaskVec });
13498 auto *ResultTy = cast<llvm::FixedVectorType>(Ops[1]->getType());
13499 Value *Ptr = Ops[0];
13503 llvm::Function *F = CGF.
CGM.
getIntrinsic(Intrinsic::masked_compressstore,
13505 return CGF.
Builder.CreateCall(F, { Ops[1], Ptr, MaskVec });
13510 bool InvertLHS =
false) {
13511 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
13516 LHS = CGF.
Builder.CreateNot(LHS);
13518 return CGF.
Builder.CreateBitCast(CGF.
Builder.CreateBinOp(Opc, LHS, RHS),
13519 Ops[0]->getType());
13523 Value *Amt,
bool IsRight) {
13524 llvm::Type *Ty = Op0->
getType();
13530 unsigned NumElts = cast<llvm::FixedVectorType>(Ty)->getNumElements();
13531 Amt = CGF.
Builder.CreateIntCast(Amt, Ty->getScalarType(),
false);
13532 Amt = CGF.
Builder.CreateVectorSplat(NumElts, Amt);
13535 unsigned IID = IsRight ? Intrinsic::fshr : Intrinsic::fshl;
13537 return CGF.
Builder.CreateCall(F, {Op0, Op1, Amt});
13542 Value *Op0 = Ops[0];
13543 Value *Op1 = Ops[1];
13544 llvm::Type *Ty = Op0->
getType();
13545 uint64_t Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
13547 CmpInst::Predicate Pred;
13550 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
13553 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
13556 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
13559 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
13562 Pred = ICmpInst::ICMP_EQ;
13565 Pred = ICmpInst::ICMP_NE;
13568 return llvm::Constant::getNullValue(Ty);
13570 return llvm::Constant::getAllOnesValue(Ty);
13572 llvm_unreachable(
"Unexpected XOP vpcom/vpcomu predicate");
13584 if (
const auto *
C = dyn_cast<Constant>(Mask))
13585 if (
C->isAllOnesValue())
13589 CGF, Mask, cast<llvm::FixedVectorType>(Op0->
getType())->getNumElements());
13591 return CGF.
Builder.CreateSelect(Mask, Op0, Op1);
13597 if (
const auto *
C = dyn_cast<Constant>(Mask))
13598 if (
C->isAllOnesValue())
13601 auto *MaskTy = llvm::FixedVectorType::get(
13602 CGF.
Builder.getInt1Ty(), Mask->
getType()->getIntegerBitWidth());
13603 Mask = CGF.
Builder.CreateBitCast(Mask, MaskTy);
13604 Mask = CGF.
Builder.CreateExtractElement(Mask, (uint64_t)0);
13605 return CGF.
Builder.CreateSelect(Mask, Op0, Op1);
13609 unsigned NumElts,
Value *MaskIn) {
13611 const auto *
C = dyn_cast<Constant>(MaskIn);
13612 if (!
C || !
C->isAllOnesValue())
13618 for (
unsigned i = 0; i != NumElts; ++i)
13620 for (
unsigned i = NumElts; i != 8; ++i)
13621 Indices[i] = i % NumElts + NumElts;
13622 Cmp = CGF.
Builder.CreateShuffleVector(
13623 Cmp, llvm::Constant::getNullValue(Cmp->getType()), Indices);
13626 return CGF.
Builder.CreateBitCast(Cmp,
13628 std::max(NumElts, 8U)));
13633 assert((Ops.size() == 2 || Ops.size() == 4) &&
13634 "Unexpected number of arguments");
13636 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
13640 Cmp = Constant::getNullValue(
13641 llvm::FixedVectorType::get(CGF.
Builder.getInt1Ty(), NumElts));
13642 }
else if (CC == 7) {
13643 Cmp = Constant::getAllOnesValue(
13644 llvm::FixedVectorType::get(CGF.
Builder.getInt1Ty(), NumElts));
13646 ICmpInst::Predicate Pred;
13648 default: llvm_unreachable(
"Unknown condition code");
13649 case 0: Pred = ICmpInst::ICMP_EQ;
break;
13650 case 1: Pred =
Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
break;
13651 case 2: Pred =
Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
break;
13652 case 4: Pred = ICmpInst::ICMP_NE;
break;
13653 case 5: Pred =
Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
break;
13654 case 6: Pred =
Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
break;
13656 Cmp = CGF.
Builder.CreateICmp(Pred, Ops[0], Ops[1]);
13659 Value *MaskIn =
nullptr;
13660 if (Ops.size() == 4)
13667 Value *Zero = Constant::getNullValue(In->getType());
13673 unsigned Rnd = cast<llvm::ConstantInt>(Ops[3])->getZExtValue();
13674 llvm::Type *Ty = Ops[1]->getType();
13678 Intrinsic::ID IID = IsSigned ? Intrinsic::x86_avx512_sitofp_round
13679 : Intrinsic::x86_avx512_uitofp_round;
13681 Res = CGF.
Builder.CreateCall(F, { Ops[0], Ops[3] });
13683 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
13684 Res = IsSigned ? CGF.
Builder.CreateSIToFP(Ops[0], Ty)
13685 : CGF.
Builder.CreateUIToFP(Ops[0], Ty);
13696 bool Subtract =
false;
13697 Intrinsic::ID IID = Intrinsic::not_intrinsic;
13698 switch (BuiltinID) {
13700 case clang::X86::BI__builtin_ia32_vfmsubph512_mask3:
13703 case clang::X86::BI__builtin_ia32_vfmaddph512_mask:
13704 case clang::X86::BI__builtin_ia32_vfmaddph512_maskz:
13705 case clang::X86::BI__builtin_ia32_vfmaddph512_mask3:
13706 IID = llvm::Intrinsic::x86_avx512fp16_vfmadd_ph_512;
13708 case clang::X86::BI__builtin_ia32_vfmsubaddph512_mask3:
13711 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask:
13712 case clang::X86::BI__builtin_ia32_vfmaddsubph512_maskz:
13713 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask3:
13714 IID = llvm::Intrinsic::x86_avx512fp16_vfmaddsub_ph_512;
13716 case clang::X86::BI__builtin_ia32_vfmsubps512_mask3:
13719 case clang::X86::BI__builtin_ia32_vfmaddps512_mask:
13720 case clang::X86::BI__builtin_ia32_vfmaddps512_maskz:
13721 case clang::X86::BI__builtin_ia32_vfmaddps512_mask3:
13722 IID = llvm::Intrinsic::x86_avx512_vfmadd_ps_512;
break;
13723 case clang::X86::BI__builtin_ia32_vfmsubpd512_mask3:
13726 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask:
13727 case clang::X86::BI__builtin_ia32_vfmaddpd512_maskz:
13728 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask3:
13729 IID = llvm::Intrinsic::x86_avx512_vfmadd_pd_512;
break;
13730 case clang::X86::BI__builtin_ia32_vfmsubaddps512_mask3:
13733 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask:
13734 case clang::X86::BI__builtin_ia32_vfmaddsubps512_maskz:
13735 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask3:
13736 IID = llvm::Intrinsic::x86_avx512_vfmaddsub_ps_512;
13738 case clang::X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
13741 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask:
13742 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
13743 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
13744 IID = llvm::Intrinsic::x86_avx512_vfmaddsub_pd_512;
13758 if (IID != Intrinsic::not_intrinsic &&
13759 (cast<llvm::ConstantInt>(Ops.back())->getZExtValue() != (uint64_t)4 ||
13762 Res = CGF.
Builder.CreateCall(Intr, {A, B,
C, Ops.back() });
13764 llvm::Type *Ty = A->
getType();
13766 if (CGF.
Builder.getIsFPConstrained()) {
13767 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
13768 FMA = CGF.
CGM.
getIntrinsic(Intrinsic::experimental_constrained_fma, Ty);
13769 Res = CGF.
Builder.CreateConstrainedFPCall(FMA, {A, B,
C});
13772 Res = CGF.
Builder.CreateCall(FMA, {A, B,
C});
13777 Value *MaskFalseVal =
nullptr;
13778 switch (BuiltinID) {
13779 case clang::X86::BI__builtin_ia32_vfmaddph512_mask:
13780 case clang::X86::BI__builtin_ia32_vfmaddps512_mask:
13781 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask:
13782 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask:
13783 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask:
13784 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask:
13785 MaskFalseVal = Ops[0];
13787 case clang::X86::BI__builtin_ia32_vfmaddph512_maskz:
13788 case clang::X86::BI__builtin_ia32_vfmaddps512_maskz:
13789 case clang::X86::BI__builtin_ia32_vfmaddpd512_maskz:
13790 case clang::X86::BI__builtin_ia32_vfmaddsubph512_maskz:
13791 case clang::X86::BI__builtin_ia32_vfmaddsubps512_maskz:
13792 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
13793 MaskFalseVal = Constant::getNullValue(Ops[0]->getType());
13795 case clang::X86::BI__builtin_ia32_vfmsubph512_mask3:
13796 case clang::X86::BI__builtin_ia32_vfmaddph512_mask3:
13797 case clang::X86::BI__builtin_ia32_vfmsubps512_mask3:
13798 case clang::X86::BI__builtin_ia32_vfmaddps512_mask3:
13799 case clang::X86::BI__builtin_ia32_vfmsubpd512_mask3:
13800 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask3:
13801 case clang::X86::BI__builtin_ia32_vfmsubaddph512_mask3:
13802 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask3:
13803 case clang::X86::BI__builtin_ia32_vfmsubaddps512_mask3:
13804 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask3:
13805 case clang::X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
13806 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
13807 MaskFalseVal = Ops[2];
13819 bool ZeroMask =
false,
unsigned PTIdx = 0,
13820 bool NegAcc =
false) {
13822 if (Ops.size() > 4)
13823 Rnd = cast<llvm::ConstantInt>(Ops[4])->getZExtValue();
13826 Ops[2] = CGF.
Builder.CreateFNeg(Ops[2]);
13828 Ops[0] = CGF.
Builder.CreateExtractElement(Ops[0], (uint64_t)0);
13829 Ops[1] = CGF.
Builder.CreateExtractElement(Ops[1], (uint64_t)0);
13830 Ops[2] = CGF.
Builder.CreateExtractElement(Ops[2], (uint64_t)0);
13835 switch (Ops[0]->getType()->getPrimitiveSizeInBits()) {
13837 IID = Intrinsic::x86_avx512fp16_vfmadd_f16;
13840 IID = Intrinsic::x86_avx512_vfmadd_f32;
13843 IID = Intrinsic::x86_avx512_vfmadd_f64;
13846 llvm_unreachable(
"Unexpected size");
13849 {Ops[0], Ops[1], Ops[2], Ops[4]});
13850 }
else if (CGF.
Builder.getIsFPConstrained()) {
13851 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
13853 Intrinsic::experimental_constrained_fma, Ops[0]->getType());
13854 Res = CGF.
Builder.CreateConstrainedFPCall(FMA, Ops.slice(0, 3));
13857 Res = CGF.
Builder.CreateCall(FMA, Ops.slice(0, 3));
13860 if (Ops.size() > 3) {
13861 Value *PassThru = ZeroMask ? Constant::getNullValue(Res->
getType())
13867 if (NegAcc && PTIdx == 2)
13868 PassThru = CGF.
Builder.CreateExtractElement(Upper, (uint64_t)0);
13872 return CGF.
Builder.CreateInsertElement(Upper, Res, (uint64_t)0);
13877 llvm::Type *Ty = Ops[0]->getType();
13879 Ty = llvm::FixedVectorType::get(CGF.
Int64Ty,
13880 Ty->getPrimitiveSizeInBits() / 64);
13886 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
13887 LHS = CGF.
Builder.CreateShl(LHS, ShiftAmt);
13888 LHS = CGF.
Builder.CreateAShr(LHS, ShiftAmt);
13889 RHS = CGF.
Builder.CreateShl(RHS, ShiftAmt);
13890 RHS = CGF.
Builder.CreateAShr(RHS, ShiftAmt);
13893 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
13894 LHS = CGF.
Builder.CreateAnd(LHS, Mask);
13895 RHS = CGF.
Builder.CreateAnd(RHS, Mask);
13898 return CGF.
Builder.CreateMul(LHS, RHS);
13906 llvm::Type *Ty = Ops[0]->getType();
13908 unsigned VecWidth = Ty->getPrimitiveSizeInBits();
13909 unsigned EltWidth = Ty->getScalarSizeInBits();
13911 if (VecWidth == 128 && EltWidth == 32)
13912 IID = Intrinsic::x86_avx512_pternlog_d_128;
13913 else if (VecWidth == 256 && EltWidth == 32)
13914 IID = Intrinsic::x86_avx512_pternlog_d_256;
13915 else if (VecWidth == 512 && EltWidth == 32)
13916 IID = Intrinsic::x86_avx512_pternlog_d_512;
13917 else if (VecWidth == 128 && EltWidth == 64)
13918 IID = Intrinsic::x86_avx512_pternlog_q_128;
13919 else if (VecWidth == 256 && EltWidth == 64)
13920 IID = Intrinsic::x86_avx512_pternlog_q_256;
13921 else if (VecWidth == 512 && EltWidth == 64)
13922 IID = Intrinsic::x86_avx512_pternlog_q_512;
13924 llvm_unreachable(
"Unexpected intrinsic");
13928 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty) : Ops[0];
13933 llvm::Type *DstTy) {
13934 unsigned NumberOfElements =
13935 cast<llvm::FixedVectorType>(DstTy)->getNumElements();
13937 return CGF.
Builder.CreateSExt(Mask, DstTy,
"vpmovm2");
13942 StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
13943 return EmitX86CpuIs(CPUStr);
13949 llvm::Type *DstTy) {
13950 assert((Ops.size() == 1 || Ops.size() == 3 || Ops.size() == 4) &&
13951 "Unknown cvtph2ps intrinsic");
13954 if (Ops.size() == 4 && cast<llvm::ConstantInt>(Ops[3])->getZExtValue() != 4) {
13957 return CGF.
Builder.CreateCall(F, {Ops[0], Ops[1], Ops[2], Ops[3]});
13960 unsigned NumDstElts = cast<llvm::FixedVectorType>(DstTy)->getNumElements();
13961 Value *Src = Ops[0];
13965 cast<llvm::FixedVectorType>(Src->
getType())->getNumElements()) {
13966 assert(NumDstElts == 4 &&
"Unexpected vector size");
13971 auto *HalfTy = llvm::FixedVectorType::get(
13973 Src = CGF.
Builder.CreateBitCast(Src, HalfTy);
13976 Value *Res = CGF.
Builder.CreateFPExt(Src, DstTy,
"cvtph2ps");
13978 if (Ops.size() >= 3)
13983Value *CodeGenFunction::EmitX86CpuIs(StringRef CPUStr) {
13994 llvm::ArrayType::get(
Int32Ty, 1));
13998 cast<llvm::GlobalValue>(CpuModel)->setDSOLocal(
true);
14004 std::tie(Index,
Value) = StringSwitch<std::pair<unsigned, unsigned>>(CPUStr)
14006 .Case(STRING, {0u,
static_cast<unsigned>(llvm::X86::ENUM)})
14008 .Case(
ALIAS, {1u,
static_cast<unsigned>(llvm::X86::ENUM)})
14010 .Case(STR, {1u,
static_cast<unsigned>(llvm::X86::ENUM)})
14012 .Case(
ALIAS, {2u,
static_cast<unsigned>(llvm::X86::ENUM)})
14014 .Case(STR, {2u,
static_cast<unsigned>(llvm::X86::ENUM)})
14015#include
"llvm/TargetParser/X86TargetParser.def"
14017 assert(
Value != 0 &&
"Invalid CPUStr passed to CpuIs");
14020 llvm::Value *Idxs[] = {ConstantInt::get(
Int32Ty, 0),
14021 ConstantInt::get(
Int32Ty, Index)};
14027 return Builder.CreateICmpEQ(CpuValue,
14031Value *CodeGenFunction::EmitX86CpuSupports(
const CallExpr *E) {
14033 StringRef FeatureStr = cast<StringLiteral>(FeatureExpr)->getString();
14034 if (!
getContext().getTargetInfo().validateCpuSupports(FeatureStr))
14036 return EmitX86CpuSupports(FeatureStr);
14040 return EmitX86CpuSupports(llvm::X86::getCpuSupportsMask(FeatureStrs));
14044CodeGenFunction::EmitX86CpuSupports(std::array<uint32_t, 4> FeatureMask) {
14046 if (FeatureMask[0] != 0) {
14054 llvm::ArrayType::get(
Int32Ty, 1));
14058 cast<llvm::GlobalValue>(CpuModel)->setDSOLocal(
true);
14075 llvm::Type *ATy = llvm::ArrayType::get(
Int32Ty, 3);
14076 llvm::Constant *CpuFeatures2 =
14078 cast<llvm::GlobalValue>(CpuFeatures2)->setDSOLocal(
true);
14079 for (
int i = 1; i != 4; ++i) {
14080 const uint32_t M = FeatureMask[i];
14097Value *CodeGenFunction::EmitAArch64CpuInit() {
14098 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy,
false);
14099 llvm::FunctionCallee
Func =
14101 cast<llvm::GlobalValue>(
Func.getCallee())->setDSOLocal(
true);
14102 cast<llvm::GlobalValue>(
Func.getCallee())
14103 ->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
14107Value *CodeGenFunction::EmitX86CpuInit() {
14108 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy,
14110 llvm::FunctionCallee
Func =
14112 cast<llvm::GlobalValue>(
Func.getCallee())->setDSOLocal(
true);
14113 cast<llvm::GlobalValue>(
Func.getCallee())
14114 ->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
14118Value *CodeGenFunction::EmitAArch64CpuSupports(
const CallExpr *E) {
14120 StringRef ArgStr = cast<StringLiteral>(ArgExpr)->getString();
14122 ArgStr.split(Features,
"+");
14123 for (
auto &Feature : Features) {
14124 Feature = Feature.trim();
14125 if (!llvm::AArch64::parseArchExtension(Feature))
14127 if (Feature !=
"default")
14128 Features.push_back(Feature);
14130 return EmitAArch64CpuSupports(Features);
14135 uint64_t FeaturesMask = llvm::AArch64::getCpuSupportsMask(FeaturesStrs);
14137 if (FeaturesMask != 0) {
14142 llvm::Type *STy = llvm::StructType::get(
Int64Ty);
14143 llvm::Constant *AArch64CPUFeatures =
14145 cast<llvm::GlobalValue>(AArch64CPUFeatures)->setDSOLocal(
true);
14147 STy, AArch64CPUFeatures,
14161 if (BuiltinID == Builtin::BI__builtin_cpu_is)
14162 return EmitX86CpuIs(E);
14163 if (BuiltinID == Builtin::BI__builtin_cpu_supports)
14164 return EmitX86CpuSupports(E);
14165 if (BuiltinID == Builtin::BI__builtin_cpu_init)
14166 return EmitX86CpuInit();
14174 bool IsMaskFCmp =
false;
14175 bool IsConjFMA =
false;
14178 unsigned ICEArguments = 0;
14183 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; i++) {
14193 auto getCmpIntrinsicCall = [
this, &Ops](Intrinsic::ID
ID,
unsigned Imm) {
14194 Ops.push_back(llvm::ConstantInt::get(
Int8Ty, Imm));
14196 return Builder.CreateCall(F, Ops);
14204 auto getVectorFCmpIR = [
this, &Ops, E](CmpInst::Predicate Pred,
14205 bool IsSignaling) {
14206 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
14209 Cmp =
Builder.CreateFCmpS(Pred, Ops[0], Ops[1]);
14211 Cmp =
Builder.CreateFCmp(Pred, Ops[0], Ops[1]);
14212 llvm::VectorType *FPVecTy = cast<llvm::VectorType>(Ops[0]->getType());
14213 llvm::VectorType *IntVecTy = llvm::VectorType::getInteger(FPVecTy);
14215 return Builder.CreateBitCast(Sext, FPVecTy);
14218 switch (BuiltinID) {
14219 default:
return nullptr;
14220 case X86::BI_mm_prefetch: {
14222 ConstantInt *
C = cast<ConstantInt>(Ops[1]);
14223 Value *RW = ConstantInt::get(
Int32Ty, (
C->getZExtValue() >> 2) & 0x1);
14224 Value *Locality = ConstantInt::get(
Int32Ty,
C->getZExtValue() & 0x3);
14229 case X86::BI_mm_clflush: {
14233 case X86::BI_mm_lfence: {
14236 case X86::BI_mm_mfence: {
14239 case X86::BI_mm_sfence: {
14242 case X86::BI_mm_pause: {
14245 case X86::BI__rdtsc: {
14248 case X86::BI__builtin_ia32_rdtscp: {
14254 case X86::BI__builtin_ia32_lzcnt_u16:
14255 case X86::BI__builtin_ia32_lzcnt_u32:
14256 case X86::BI__builtin_ia32_lzcnt_u64: {
14260 case X86::BI__builtin_ia32_tzcnt_u16:
14261 case X86::BI__builtin_ia32_tzcnt_u32:
14262 case X86::BI__builtin_ia32_tzcnt_u64: {
14266 case X86::BI__builtin_ia32_undef128:
14267 case X86::BI__builtin_ia32_undef256:
14268 case X86::BI__builtin_ia32_undef512:
14275 case X86::BI__builtin_ia32_vec_init_v8qi:
14276 case X86::BI__builtin_ia32_vec_init_v4hi:
14277 case X86::BI__builtin_ia32_vec_init_v2si:
14280 case X86::BI__builtin_ia32_vec_ext_v2si:
14281 case X86::BI__builtin_ia32_vec_ext_v16qi:
14282 case X86::BI__builtin_ia32_vec_ext_v8hi:
14283 case X86::BI__builtin_ia32_vec_ext_v4si:
14284 case X86::BI__builtin_ia32_vec_ext_v4sf:
14285 case X86::BI__builtin_ia32_vec_ext_v2di:
14286 case X86::BI__builtin_ia32_vec_ext_v32qi:
14287 case X86::BI__builtin_ia32_vec_ext_v16hi:
14288 case X86::BI__builtin_ia32_vec_ext_v8si:
14289 case X86::BI__builtin_ia32_vec_ext_v4di: {
14291 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
14292 uint64_t Index = cast<ConstantInt>(Ops[1])->getZExtValue();
14293 Index &= NumElts - 1;
14296 return Builder.CreateExtractElement(Ops[0], Index);
14298 case X86::BI__builtin_ia32_vec_set_v16qi:
14299 case X86::BI__builtin_ia32_vec_set_v8hi:
14300 case X86::BI__builtin_ia32_vec_set_v4si:
14301 case X86::BI__builtin_ia32_vec_set_v2di:
14302 case X86::BI__builtin_ia32_vec_set_v32qi:
14303 case X86::BI__builtin_ia32_vec_set_v16hi:
14304 case X86::BI__builtin_ia32_vec_set_v8si:
14305 case X86::BI__builtin_ia32_vec_set_v4di: {
14307 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
14308 unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue();
14309 Index &= NumElts - 1;
14312 return Builder.CreateInsertElement(Ops[0], Ops[1], Index);
14314 case X86::BI_mm_setcsr:
14315 case X86::BI__builtin_ia32_ldmxcsr: {
14321 case X86::BI_mm_getcsr:
14322 case X86::BI__builtin_ia32_stmxcsr: {
14328 case X86::BI__builtin_ia32_xsave:
14329 case X86::BI__builtin_ia32_xsave64:
14330 case X86::BI__builtin_ia32_xrstor:
14331 case X86::BI__builtin_ia32_xrstor64:
14332 case X86::BI__builtin_ia32_xsaveopt:
14333 case X86::BI__builtin_ia32_xsaveopt64:
14334 case X86::BI__builtin_ia32_xrstors:
14335 case X86::BI__builtin_ia32_xrstors64:
14336 case X86::BI__builtin_ia32_xsavec:
14337 case X86::BI__builtin_ia32_xsavec64:
14338 case X86::BI__builtin_ia32_xsaves:
14339 case X86::BI__builtin_ia32_xsaves64:
14340 case X86::BI__builtin_ia32_xsetbv:
14341 case X86::BI_xsetbv: {
14343#define INTRINSIC_X86_XSAVE_ID(NAME) \
14344 case X86::BI__builtin_ia32_##NAME: \
14345 ID = Intrinsic::x86_##NAME; \
14347 switch (BuiltinID) {
14348 default: llvm_unreachable(
"Unsupported intrinsic!");
14362 case X86::BI_xsetbv:
14363 ID = Intrinsic::x86_xsetbv;
14366#undef INTRINSIC_X86_XSAVE_ID
14371 Ops.push_back(Mlo);
14374 case X86::BI__builtin_ia32_xgetbv:
14375 case X86::BI_xgetbv:
14377 case X86::BI__builtin_ia32_storedqudi128_mask:
14378 case X86::BI__builtin_ia32_storedqusi128_mask:
14379 case X86::BI__builtin_ia32_storedquhi128_mask:
14380 case X86::BI__builtin_ia32_storedquqi128_mask:
14381 case X86::BI__builtin_ia32_storeupd128_mask:
14382 case X86::BI__builtin_ia32_storeups128_mask:
14383 case X86::BI__builtin_ia32_storedqudi256_mask:
14384 case X86::BI__builtin_ia32_storedqusi256_mask:
14385 case X86::BI__builtin_ia32_storedquhi256_mask:
14386 case X86::BI__builtin_ia32_storedquqi256_mask:
14387 case X86::BI__builtin_ia32_storeupd256_mask:
14388 case X86::BI__builtin_ia32_storeups256_mask:
14389 case X86::BI__builtin_ia32_storedqudi512_mask:
14390 case X86::BI__builtin_ia32_storedqusi512_mask:
14391 case X86::BI__builtin_ia32_storedquhi512_mask:
14392 case X86::BI__builtin_ia32_storedquqi512_mask:
14393 case X86::BI__builtin_ia32_storeupd512_mask:
14394 case X86::BI__builtin_ia32_storeups512_mask:
14397 case X86::BI__builtin_ia32_storesh128_mask:
14398 case X86::BI__builtin_ia32_storess128_mask:
14399 case X86::BI__builtin_ia32_storesd128_mask:
14402 case X86::BI__builtin_ia32_vpopcntb_128:
14403 case X86::BI__builtin_ia32_vpopcntd_128:
14404 case X86::BI__builtin_ia32_vpopcntq_128:
14405 case X86::BI__builtin_ia32_vpopcntw_128:
14406 case X86::BI__builtin_ia32_vpopcntb_256:
14407 case X86::BI__builtin_ia32_vpopcntd_256:
14408 case X86::BI__builtin_ia32_vpopcntq_256:
14409 case X86::BI__builtin_ia32_vpopcntw_256:
14410 case X86::BI__builtin_ia32_vpopcntb_512:
14411 case X86::BI__builtin_ia32_vpopcntd_512:
14412 case X86::BI__builtin_ia32_vpopcntq_512:
14413 case X86::BI__builtin_ia32_vpopcntw_512: {
14416 return Builder.CreateCall(F, Ops);
14418 case X86::BI__builtin_ia32_cvtmask2b128:
14419 case X86::BI__builtin_ia32_cvtmask2b256:
14420 case X86::BI__builtin_ia32_cvtmask2b512:
14421 case X86::BI__builtin_ia32_cvtmask2w128:
14422 case X86::BI__builtin_ia32_cvtmask2w256:
14423 case X86::BI__builtin_ia32_cvtmask2w512:
14424 case X86::BI__builtin_ia32_cvtmask2d128:
14425 case X86::BI__builtin_ia32_cvtmask2d256:
14426 case X86::BI__builtin_ia32_cvtmask2d512:
14427 case X86::BI__builtin_ia32_cvtmask2q128:
14428 case X86::BI__builtin_ia32_cvtmask2q256:
14429 case X86::BI__builtin_ia32_cvtmask2q512:
14432 case X86::BI__builtin_ia32_cvtb2mask128:
14433 case X86::BI__builtin_ia32_cvtb2mask256:
14434 case X86::BI__builtin_ia32_cvtb2mask512:
14435 case X86::BI__builtin_ia32_cvtw2mask128:
14436 case X86::BI__builtin_ia32_cvtw2mask256:
14437 case X86::BI__builtin_ia32_cvtw2mask512:
14438 case X86::BI__builtin_ia32_cvtd2mask128:
14439 case X86::BI__builtin_ia32_cvtd2mask256:
14440 case X86::BI__builtin_ia32_cvtd2mask512:
14441 case X86::BI__builtin_ia32_cvtq2mask128:
14442 case X86::BI__builtin_ia32_cvtq2mask256:
14443 case X86::BI__builtin_ia32_cvtq2mask512:
14446 case X86::BI__builtin_ia32_cvtdq2ps512_mask:
14447 case X86::BI__builtin_ia32_cvtqq2ps512_mask:
14448 case X86::BI__builtin_ia32_cvtqq2pd512_mask:
14449 case X86::BI__builtin_ia32_vcvtw2ph512_mask:
14450 case X86::BI__builtin_ia32_vcvtdq2ph512_mask:
14451 case X86::BI__builtin_ia32_vcvtqq2ph512_mask:
14453 case X86::BI__builtin_ia32_cvtudq2ps512_mask:
14454 case X86::BI__builtin_ia32_cvtuqq2ps512_mask:
14455 case X86::BI__builtin_ia32_cvtuqq2pd512_mask:
14456 case X86::BI__builtin_ia32_vcvtuw2ph512_mask:
14457 case X86::BI__builtin_ia32_vcvtudq2ph512_mask:
14458 case X86::BI__builtin_ia32_vcvtuqq2ph512_mask:
14461 case X86::BI__builtin_ia32_vfmaddss3:
14462 case X86::BI__builtin_ia32_vfmaddsd3:
14463 case X86::BI__builtin_ia32_vfmaddsh3_mask:
14464 case X86::BI__builtin_ia32_vfmaddss3_mask:
14465 case X86::BI__builtin_ia32_vfmaddsd3_mask:
14467 case X86::BI__builtin_ia32_vfmaddss:
14468 case X86::BI__builtin_ia32_vfmaddsd:
14470 Constant::getNullValue(Ops[0]->getType()));
14471 case X86::BI__builtin_ia32_vfmaddsh3_maskz:
14472 case X86::BI__builtin_ia32_vfmaddss3_maskz:
14473 case X86::BI__builtin_ia32_vfmaddsd3_maskz:
14475 case X86::BI__builtin_ia32_vfmaddsh3_mask3:
14476 case X86::BI__builtin_ia32_vfmaddss3_mask3:
14477 case X86::BI__builtin_ia32_vfmaddsd3_mask3:
14479 case X86::BI__builtin_ia32_vfmsubsh3_mask3:
14480 case X86::BI__builtin_ia32_vfmsubss3_mask3:
14481 case X86::BI__builtin_ia32_vfmsubsd3_mask3:
14484 case X86::BI__builtin_ia32_vfmaddph:
14485 case X86::BI__builtin_ia32_vfmaddps:
14486 case X86::BI__builtin_ia32_vfmaddpd:
14487 case X86::BI__builtin_ia32_vfmaddph256:
14488 case X86::BI__builtin_ia32_vfmaddps256:
14489 case X86::BI__builtin_ia32_vfmaddpd256:
14490 case X86::BI__builtin_ia32_vfmaddph512_mask:
14491 case X86::BI__builtin_ia32_vfmaddph512_maskz:
14492 case X86::BI__builtin_ia32_vfmaddph512_mask3:
14493 case X86::BI__builtin_ia32_vfmaddps512_mask:
14494 case X86::BI__builtin_ia32_vfmaddps512_maskz:
14495 case X86::BI__builtin_ia32_vfmaddps512_mask3:
14496 case X86::BI__builtin_ia32_vfmsubps512_mask3:
14497 case X86::BI__builtin_ia32_vfmaddpd512_mask:
14498 case X86::BI__builtin_ia32_vfmaddpd512_maskz:
14499 case X86::BI__builtin_ia32_vfmaddpd512_mask3:
14500 case X86::BI__builtin_ia32_vfmsubpd512_mask3:
14501 case X86::BI__builtin_ia32_vfmsubph512_mask3:
14503 case X86::BI__builtin_ia32_vfmaddsubph512_mask:
14504 case X86::BI__builtin_ia32_vfmaddsubph512_maskz:
14505 case X86::BI__builtin_ia32_vfmaddsubph512_mask3:
14506 case X86::BI__builtin_ia32_vfmsubaddph512_mask3:
14507 case X86::BI__builtin_ia32_vfmaddsubps512_mask:
14508 case X86::BI__builtin_ia32_vfmaddsubps512_maskz:
14509 case X86::BI__builtin_ia32_vfmaddsubps512_mask3:
14510 case X86::BI__builtin_ia32_vfmsubaddps512_mask3:
14511 case X86::BI__builtin_ia32_vfmaddsubpd512_mask:
14512 case X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
14513 case X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
14514 case X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
14517 case X86::BI__builtin_ia32_movdqa32store128_mask:
14518 case X86::BI__builtin_ia32_movdqa64store128_mask:
14519 case X86::BI__builtin_ia32_storeaps128_mask:
14520 case X86::BI__builtin_ia32_storeapd128_mask:
14521 case X86::BI__builtin_ia32_movdqa32store256_mask:
14522 case X86::BI__builtin_ia32_movdqa64store256_mask:
14523 case X86::BI__builtin_ia32_storeaps256_mask:
14524 case X86::BI__builtin_ia32_storeapd256_mask:
14525 case X86::BI__builtin_ia32_movdqa32store512_mask:
14526 case X86::BI__builtin_ia32_movdqa64store512_mask:
14527 case X86::BI__builtin_ia32_storeaps512_mask:
14528 case X86::BI__builtin_ia32_storeapd512_mask:
14533 case X86::BI__builtin_ia32_loadups128_mask:
14534 case X86::BI__builtin_ia32_loadups256_mask:
14535 case X86::BI__builtin_ia32_loadups512_mask:
14536 case X86::BI__builtin_ia32_loadupd128_mask:
14537 case X86::BI__builtin_ia32_loadupd256_mask:
14538 case X86::BI__builtin_ia32_loadupd512_mask:
14539 case X86::BI__builtin_ia32_loaddquqi128_mask:
14540 case X86::BI__builtin_ia32_loaddquqi256_mask:
14541 case X86::BI__builtin_ia32_loaddquqi512_mask:
14542 case X86::BI__builtin_ia32_loaddquhi128_mask:
14543 case X86::BI__builtin_ia32_loaddquhi256_mask:
14544 case X86::BI__builtin_ia32_loaddquhi512_mask:
14545 case X86::BI__builtin_ia32_loaddqusi128_mask:
14546 case X86::BI__builtin_ia32_loaddqusi256_mask:
14547 case X86::BI__builtin_ia32_loaddqusi512_mask:
14548 case X86::BI__builtin_ia32_loaddqudi128_mask:
14549 case X86::BI__builtin_ia32_loaddqudi256_mask:
14550 case X86::BI__builtin_ia32_loaddqudi512_mask:
14553 case X86::BI__builtin_ia32_loadsh128_mask:
14554 case X86::BI__builtin_ia32_loadss128_mask:
14555 case X86::BI__builtin_ia32_loadsd128_mask:
14558 case X86::BI__builtin_ia32_loadaps128_mask:
14559 case X86::BI__builtin_ia32_loadaps256_mask:
14560 case X86::BI__builtin_ia32_loadaps512_mask:
14561 case X86::BI__builtin_ia32_loadapd128_mask:
14562 case X86::BI__builtin_ia32_loadapd256_mask:
14563 case X86::BI__builtin_ia32_loadapd512_mask:
14564 case X86::BI__builtin_ia32_movdqa32load128_mask:
14565 case X86::BI__builtin_ia32_movdqa32load256_mask:
14566 case X86::BI__builtin_ia32_movdqa32load512_mask:
14567 case X86::BI__builtin_ia32_movdqa64load128_mask:
14568 case X86::BI__builtin_ia32_movdqa64load256_mask:
14569 case X86::BI__builtin_ia32_movdqa64load512_mask:
14574 case X86::BI__builtin_ia32_expandloaddf128_mask:
14575 case X86::BI__builtin_ia32_expandloaddf256_mask:
14576 case X86::BI__builtin_ia32_expandloaddf512_mask:
14577 case X86::BI__builtin_ia32_expandloadsf128_mask:
14578 case X86::BI__builtin_ia32_expandloadsf256_mask:
14579 case X86::BI__builtin_ia32_expandloadsf512_mask:
14580 case X86::BI__builtin_ia32_expandloaddi128_mask:
14581 case X86::BI__builtin_ia32_expandloaddi256_mask:
14582 case X86::BI__builtin_ia32_expandloaddi512_mask:
14583 case X86::BI__builtin_ia32_expandloadsi128_mask:
14584 case X86::BI__builtin_ia32_expandloadsi256_mask:
14585 case X86::BI__builtin_ia32_expandloadsi512_mask:
14586 case X86::BI__builtin_ia32_expandloadhi128_mask:
14587 case X86::BI__builtin_ia32_expandloadhi256_mask:
14588 case X86::BI__builtin_ia32_expandloadhi512_mask:
14589 case X86::BI__builtin_ia32_expandloadqi128_mask:
14590 case X86::BI__builtin_ia32_expandloadqi256_mask:
14591 case X86::BI__builtin_ia32_expandloadqi512_mask:
14594 case X86::BI__builtin_ia32_compressstoredf128_mask:
14595 case X86::BI__builtin_ia32_compressstoredf256_mask:
14596 case X86::BI__builtin_ia32_compressstoredf512_mask:
14597 case X86::BI__builtin_ia32_compressstoresf128_mask:
14598 case X86::BI__builtin_ia32_compressstoresf256_mask:
14599 case X86::BI__builtin_ia32_compressstoresf512_mask:
14600 case X86::BI__builtin_ia32_compressstoredi128_mask:
14601 case X86::BI__builtin_ia32_compressstoredi256_mask:
14602 case X86::BI__builtin_ia32_compressstoredi512_mask:
14603 case X86::BI__builtin_ia32_compressstoresi128_mask:
14604 case X86::BI__builtin_ia32_compressstoresi256_mask:
14605 case X86::BI__builtin_ia32_compressstoresi512_mask:
14606 case X86::BI__builtin_ia32_compressstorehi128_mask:
14607 case X86::BI__builtin_ia32_compressstorehi256_mask:
14608 case X86::BI__builtin_ia32_compressstorehi512_mask:
14609 case X86::BI__builtin_ia32_compressstoreqi128_mask:
14610 case X86::BI__builtin_ia32_compressstoreqi256_mask:
14611 case X86::BI__builtin_ia32_compressstoreqi512_mask:
14614 case X86::BI__builtin_ia32_expanddf128_mask:
14615 case X86::BI__builtin_ia32_expanddf256_mask:
14616 case X86::BI__builtin_ia32_expanddf512_mask:
14617 case X86::BI__builtin_ia32_expandsf128_mask:
14618 case X86::BI__builtin_ia32_expandsf256_mask:
14619 case X86::BI__builtin_ia32_expandsf512_mask:
14620 case X86::BI__builtin_ia32_expanddi128_mask:
14621 case X86::BI__builtin_ia32_expanddi256_mask:
14622 case X86::BI__builtin_ia32_expanddi512_mask:
14623 case X86::BI__builtin_ia32_expandsi128_mask:
14624 case X86::BI__builtin_ia32_expandsi256_mask:
14625 case X86::BI__builtin_ia32_expandsi512_mask:
14626 case X86::BI__builtin_ia32_expandhi128_mask:
14627 case X86::BI__builtin_ia32_expandhi256_mask:
14628 case X86::BI__builtin_ia32_expandhi512_mask:
14629 case X86::BI__builtin_ia32_expandqi128_mask:
14630 case X86::BI__builtin_ia32_expandqi256_mask:
14631 case X86::BI__builtin_ia32_expandqi512_mask:
14634 case X86::BI__builtin_ia32_compressdf128_mask:
14635 case X86::BI__builtin_ia32_compressdf256_mask:
14636 case X86::BI__builtin_ia32_compressdf512_mask:
14637 case X86::BI__builtin_ia32_compresssf128_mask:
14638 case X86::BI__builtin_ia32_compresssf256_mask:
14639 case X86::BI__builtin_ia32_compresssf512_mask:
14640 case X86::BI__builtin_ia32_compressdi128_mask:
14641 case X86::BI__builtin_ia32_compressdi256_mask:
14642 case X86::BI__builtin_ia32_compressdi512_mask:
14643 case X86::BI__builtin_ia32_compresssi128_mask:
14644 case X86::BI__builtin_ia32_compresssi256_mask:
14645 case X86::BI__builtin_ia32_compresssi512_mask:
14646 case X86::BI__builtin_ia32_compresshi128_mask:
14647 case X86::BI__builtin_ia32_compresshi256_mask:
14648 case X86::BI__builtin_ia32_compresshi512_mask:
14649 case X86::BI__builtin_ia32_compressqi128_mask:
14650 case X86::BI__builtin_ia32_compressqi256_mask:
14651 case X86::BI__builtin_ia32_compressqi512_mask:
14654 case X86::BI__builtin_ia32_gather3div2df:
14655 case X86::BI__builtin_ia32_gather3div2di:
14656 case X86::BI__builtin_ia32_gather3div4df:
14657 case X86::BI__builtin_ia32_gather3div4di:
14658 case X86::BI__builtin_ia32_gather3div4sf:
14659 case X86::BI__builtin_ia32_gather3div4si:
14660 case X86::BI__builtin_ia32_gather3div8sf:
14661 case X86::BI__builtin_ia32_gather3div8si:
14662 case X86::BI__builtin_ia32_gather3siv2df:
14663 case X86::BI__builtin_ia32_gather3siv2di:
14664 case X86::BI__builtin_ia32_gather3siv4df:
14665 case X86::BI__builtin_ia32_gather3siv4di:
14666 case X86::BI__builtin_ia32_gather3siv4sf:
14667 case X86::BI__builtin_ia32_gather3siv4si:
14668 case X86::BI__builtin_ia32_gather3siv8sf:
14669 case X86::BI__builtin_ia32_gather3siv8si:
14670 case X86::BI__builtin_ia32_gathersiv8df:
14671 case X86::BI__builtin_ia32_gathersiv16sf:
14672 case X86::BI__builtin_ia32_gatherdiv8df:
14673 case X86::BI__builtin_ia32_gatherdiv16sf:
14674 case X86::BI__builtin_ia32_gathersiv8di:
14675 case X86::BI__builtin_ia32_gathersiv16si:
14676 case X86::BI__builtin_ia32_gatherdiv8di:
14677 case X86::BI__builtin_ia32_gatherdiv16si: {
14679 switch (BuiltinID) {
14680 default: llvm_unreachable(
"Unexpected builtin");
14681 case X86::BI__builtin_ia32_gather3div2df:
14682 IID = Intrinsic::x86_avx512_mask_gather3div2_df;
14684 case X86::BI__builtin_ia32_gather3div2di:
14685 IID = Intrinsic::x86_avx512_mask_gather3div2_di;
14687 case X86::BI__builtin_ia32_gather3div4df:
14688 IID = Intrinsic::x86_avx512_mask_gather3div4_df;
14690 case X86::BI__builtin_ia32_gather3div4di:
14691 IID = Intrinsic::x86_avx512_mask_gather3div4_di;
14693 case X86::BI__builtin_ia32_gather3div4sf:
14694 IID = Intrinsic::x86_avx512_mask_gather3div4_sf;
14696 case X86::BI__builtin_ia32_gather3div4si:
14697 IID = Intrinsic::x86_avx512_mask_gather3div4_si;
14699 case X86::BI__builtin_ia32_gather3div8sf:
14700 IID = Intrinsic::x86_avx512_mask_gather3div8_sf;
14702 case X86::BI__builtin_ia32_gather3div8si:
14703 IID = Intrinsic::x86_avx512_mask_gather3div8_si;
14705 case X86::BI__builtin_ia32_gather3siv2df:
14706 IID = Intrinsic::x86_avx512_mask_gather3siv2_df;
14708 case X86::BI__builtin_ia32_gather3siv2di:
14709 IID = Intrinsic::x86_avx512_mask_gather3siv2_di;
14711 case X86::BI__builtin_ia32_gather3siv4df:
14712 IID = Intrinsic::x86_avx512_mask_gather3siv4_df;
14714 case X86::BI__builtin_ia32_gather3siv4di:
14715 IID = Intrinsic::x86_avx512_mask_gather3siv4_di;
14717 case X86::BI__builtin_ia32_gather3siv4sf:
14718 IID = Intrinsic::x86_avx512_mask_gather3siv4_sf;
14720 case X86::BI__builtin_ia32_gather3siv4si:
14721 IID = Intrinsic::x86_avx512_mask_gather3siv4_si;
14723 case X86::BI__builtin_ia32_gather3siv8sf:
14724 IID = Intrinsic::x86_avx512_mask_gather3siv8_sf;
14726 case X86::BI__builtin_ia32_gather3siv8si:
14727 IID = Intrinsic::x86_avx512_mask_gather3siv8_si;
14729 case X86::BI__builtin_ia32_gathersiv8df:
14730 IID = Intrinsic::x86_avx512_mask_gather_dpd_512;
14732 case X86::BI__builtin_ia32_gathersiv16sf:
14733 IID = Intrinsic::x86_avx512_mask_gather_dps_512;
14735 case X86::BI__builtin_ia32_gatherdiv8df:
14736 IID = Intrinsic::x86_avx512_mask_gather_qpd_512;
14738 case X86::BI__builtin_ia32_gatherdiv16sf:
14739 IID = Intrinsic::x86_avx512_mask_gather_qps_512;
14741 case X86::BI__builtin_ia32_gathersiv8di:
14742 IID = Intrinsic::x86_avx512_mask_gather_dpq_512;
14744 case X86::BI__builtin_ia32_gathersiv16si:
14745 IID = Intrinsic::x86_avx512_mask_gather_dpi_512;
14747 case X86::BI__builtin_ia32_gatherdiv8di:
14748 IID = Intrinsic::x86_avx512_mask_gather_qpq_512;
14750 case X86::BI__builtin_ia32_gatherdiv16si:
14751 IID = Intrinsic::x86_avx512_mask_gather_qpi_512;
14755 unsigned MinElts = std::min(
14756 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(),
14757 cast<llvm::FixedVectorType>(Ops[2]->getType())->getNumElements());
14760 return Builder.CreateCall(Intr, Ops);
14763 case X86::BI__builtin_ia32_scattersiv8df:
14764 case X86::BI__builtin_ia32_scattersiv16sf:
14765 case X86::BI__builtin_ia32_scatterdiv8df:
14766 case X86::BI__builtin_ia32_scatterdiv16sf:
14767 case X86::BI__builtin_ia32_scattersiv8di:
14768 case X86::BI__builtin_ia32_scattersiv16si:
14769 case X86::BI__builtin_ia32_scatterdiv8di:
14770 case X86::BI__builtin_ia32_scatterdiv16si:
14771 case X86::BI__builtin_ia32_scatterdiv2df:
14772 case X86::BI__builtin_ia32_scatterdiv2di:
14773 case X86::BI__builtin_ia32_scatterdiv4df:
14774 case X86::BI__builtin_ia32_scatterdiv4di:
14775 case X86::BI__builtin_ia32_scatterdiv4sf:
14776 case X86::BI__builtin_ia32_scatterdiv4si:
14777 case X86::BI__builtin_ia32_scatterdiv8sf:
14778 case X86::BI__builtin_ia32_scatterdiv8si:
14779 case X86::BI__builtin_ia32_scattersiv2df:
14780 case X86::BI__builtin_ia32_scattersiv2di:
14781 case X86::BI__builtin_ia32_scattersiv4df:
14782 case X86::BI__builtin_ia32_scattersiv4di:
14783 case X86::BI__builtin_ia32_scattersiv4sf:
14784 case X86::BI__builtin_ia32_scattersiv4si:
14785 case X86::BI__builtin_ia32_scattersiv8sf:
14786 case X86::BI__builtin_ia32_scattersiv8si: {
14788 switch (BuiltinID) {
14789 default: llvm_unreachable(
"Unexpected builtin");
14790 case X86::BI__builtin_ia32_scattersiv8df:
14791 IID = Intrinsic::x86_avx512_mask_scatter_dpd_512;
14793 case X86::BI__builtin_ia32_scattersiv16sf:
14794 IID = Intrinsic::x86_avx512_mask_scatter_dps_512;
14796 case X86::BI__builtin_ia32_scatterdiv8df:
14797 IID = Intrinsic::x86_avx512_mask_scatter_qpd_512;
14799 case X86::BI__builtin_ia32_scatterdiv16sf:
14800 IID = Intrinsic::x86_avx512_mask_scatter_qps_512;
14802 case X86::BI__builtin_ia32_scattersiv8di:
14803 IID = Intrinsic::x86_avx512_mask_scatter_dpq_512;
14805 case X86::BI__builtin_ia32_scattersiv16si:
14806 IID = Intrinsic::x86_avx512_mask_scatter_dpi_512;
14808 case X86::BI__builtin_ia32_scatterdiv8di:
14809 IID = Intrinsic::x86_avx512_mask_scatter_qpq_512;
14811 case X86::BI__builtin_ia32_scatterdiv16si:
14812 IID = Intrinsic::x86_avx512_mask_scatter_qpi_512;
14814 case X86::BI__builtin_ia32_scatterdiv2df:
14815 IID = Intrinsic::x86_avx512_mask_scatterdiv2_df;
14817 case X86::BI__builtin_ia32_scatterdiv2di:
14818 IID = Intrinsic::x86_avx512_mask_scatterdiv2_di;
14820 case X86::BI__builtin_ia32_scatterdiv4df:
14821 IID = Intrinsic::x86_avx512_mask_scatterdiv4_df;
14823 case X86::BI__builtin_ia32_scatterdiv4di:
14824 IID = Intrinsic::x86_avx512_mask_scatterdiv4_di;
14826 case X86::BI__builtin_ia32_scatterdiv4sf:
14827 IID = Intrinsic::x86_avx512_mask_scatterdiv4_sf;
14829 case X86::BI__builtin_ia32_scatterdiv4si:
14830 IID = Intrinsic::x86_avx512_mask_scatterdiv4_si;
14832 case X86::BI__builtin_ia32_scatterdiv8sf:
14833 IID = Intrinsic::x86_avx512_mask_scatterdiv8_sf;
14835 case X86::BI__builtin_ia32_scatterdiv8si:
14836 IID = Intrinsic::x86_avx512_mask_scatterdiv8_si;
14838 case X86::BI__builtin_ia32_scattersiv2df:
14839 IID = Intrinsic::x86_avx512_mask_scattersiv2_df;
14841 case X86::BI__builtin_ia32_scattersiv2di:
14842 IID = Intrinsic::x86_avx512_mask_scattersiv2_di;
14844 case X86::BI__builtin_ia32_scattersiv4df:
14845 IID = Intrinsic::x86_avx512_mask_scattersiv4_df;
14847 case X86::BI__builtin_ia32_scattersiv4di:
14848 IID = Intrinsic::x86_avx512_mask_scattersiv4_di;
14850 case X86::BI__builtin_ia32_scattersiv4sf:
14851 IID = Intrinsic::x86_avx512_mask_scattersiv4_sf;
14853 case X86::BI__builtin_ia32_scattersiv4si:
14854 IID = Intrinsic::x86_avx512_mask_scattersiv4_si;
14856 case X86::BI__builtin_ia32_scattersiv8sf:
14857 IID = Intrinsic::x86_avx512_mask_scattersiv8_sf;
14859 case X86::BI__builtin_ia32_scattersiv8si:
14860 IID = Intrinsic::x86_avx512_mask_scattersiv8_si;
14864 unsigned MinElts = std::min(
14865 cast<llvm::FixedVectorType>(Ops[2]->getType())->getNumElements(),
14866 cast<llvm::FixedVectorType>(Ops[3]->getType())->getNumElements());
14869 return Builder.CreateCall(Intr, Ops);
14872 case X86::BI__builtin_ia32_vextractf128_pd256:
14873 case X86::BI__builtin_ia32_vextractf128_ps256:
14874 case X86::BI__builtin_ia32_vextractf128_si256:
14875 case X86::BI__builtin_ia32_extract128i256:
14876 case X86::BI__builtin_ia32_extractf64x4_mask:
14877 case X86::BI__builtin_ia32_extractf32x4_mask:
14878 case X86::BI__builtin_ia32_extracti64x4_mask:
14879 case X86::BI__builtin_ia32_extracti32x4_mask:
14880 case X86::BI__builtin_ia32_extractf32x8_mask:
14881 case X86::BI__builtin_ia32_extracti32x8_mask:
14882 case X86::BI__builtin_ia32_extractf32x4_256_mask:
14883 case X86::BI__builtin_ia32_extracti32x4_256_mask:
14884 case X86::BI__builtin_ia32_extractf64x2_256_mask:
14885 case X86::BI__builtin_ia32_extracti64x2_256_mask:
14886 case X86::BI__builtin_ia32_extractf64x2_512_mask:
14887 case X86::BI__builtin_ia32_extracti64x2_512_mask: {
14889 unsigned NumElts = DstTy->getNumElements();
14890 unsigned SrcNumElts =
14891 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
14892 unsigned SubVectors = SrcNumElts / NumElts;
14893 unsigned Index = cast<ConstantInt>(Ops[1])->getZExtValue();
14894 assert(llvm::isPowerOf2_32(SubVectors) &&
"Expected power of 2 subvectors");
14895 Index &= SubVectors - 1;
14899 for (
unsigned i = 0; i != NumElts; ++i)
14900 Indices[i] = i + Index;
14905 if (Ops.size() == 4)
14910 case X86::BI__builtin_ia32_vinsertf128_pd256:
14911 case X86::BI__builtin_ia32_vinsertf128_ps256:
14912 case X86::BI__builtin_ia32_vinsertf128_si256:
14913 case X86::BI__builtin_ia32_insert128i256:
14914 case X86::BI__builtin_ia32_insertf64x4:
14915 case X86::BI__builtin_ia32_insertf32x4:
14916 case X86::BI__builtin_ia32_inserti64x4:
14917 case X86::BI__builtin_ia32_inserti32x4:
14918 case X86::BI__builtin_ia32_insertf32x8:
14919 case X86::BI__builtin_ia32_inserti32x8:
14920 case X86::BI__builtin_ia32_insertf32x4_256:
14921 case X86::BI__builtin_ia32_inserti32x4_256:
14922 case X86::BI__builtin_ia32_insertf64x2_256:
14923 case X86::BI__builtin_ia32_inserti64x2_256:
14924 case X86::BI__builtin_ia32_insertf64x2_512:
14925 case X86::BI__builtin_ia32_inserti64x2_512: {
14926 unsigned DstNumElts =
14927 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
14928 unsigned SrcNumElts =
14929 cast<llvm::FixedVectorType>(Ops[1]->getType())->getNumElements();
14930 unsigned SubVectors = DstNumElts / SrcNumElts;
14931 unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue();
14932 assert(llvm::isPowerOf2_32(SubVectors) &&
"Expected power of 2 subvectors");
14933 Index &= SubVectors - 1;
14934 Index *= SrcNumElts;
14937 for (
unsigned i = 0; i != DstNumElts; ++i)
14938 Indices[i] = (i >= SrcNumElts) ? SrcNumElts + (i % SrcNumElts) : i;
14941 Ops[1],
ArrayRef(Indices, DstNumElts),
"widen");
14943 for (
unsigned i = 0; i != DstNumElts; ++i) {
14944 if (i >= Index && i < (Index + SrcNumElts))
14945 Indices[i] = (i - Index) + DstNumElts;
14950 return Builder.CreateShuffleVector(Ops[0], Op1,
14951 ArrayRef(Indices, DstNumElts),
"insert");
14953 case X86::BI__builtin_ia32_pmovqd512_mask:
14954 case X86::BI__builtin_ia32_pmovwb512_mask: {
14955 Value *Res =
Builder.CreateTrunc(Ops[0], Ops[1]->getType());
14958 case X86::BI__builtin_ia32_pmovdb512_mask:
14959 case X86::BI__builtin_ia32_pmovdw512_mask:
14960 case X86::BI__builtin_ia32_pmovqw512_mask: {
14961 if (
const auto *
C = dyn_cast<Constant>(Ops[2]))
14962 if (
C->isAllOnesValue())
14963 return Builder.CreateTrunc(Ops[0], Ops[1]->getType());
14966 switch (BuiltinID) {
14967 default: llvm_unreachable(
"Unsupported intrinsic!");
14968 case X86::BI__builtin_ia32_pmovdb512_mask:
14969 IID = Intrinsic::x86_avx512_mask_pmov_db_512;
14971 case X86::BI__builtin_ia32_pmovdw512_mask:
14972 IID = Intrinsic::x86_avx512_mask_pmov_dw_512;
14974 case X86::BI__builtin_ia32_pmovqw512_mask:
14975 IID = Intrinsic::x86_avx512_mask_pmov_qw_512;
14980 return Builder.CreateCall(Intr, Ops);
14982 case X86::BI__builtin_ia32_pblendw128:
14983 case X86::BI__builtin_ia32_blendpd:
14984 case X86::BI__builtin_ia32_blendps:
14985 case X86::BI__builtin_ia32_blendpd256:
14986 case X86::BI__builtin_ia32_blendps256:
14987 case X86::BI__builtin_ia32_pblendw256:
14988 case X86::BI__builtin_ia32_pblendd128:
14989 case X86::BI__builtin_ia32_pblendd256: {
14991 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
14992 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
14997 for (
unsigned i = 0; i != NumElts; ++i)
14998 Indices[i] = ((Imm >> (i % 8)) & 0x1) ? NumElts + i : i;
15000 return Builder.CreateShuffleVector(Ops[0], Ops[1],
15001 ArrayRef(Indices, NumElts),
"blend");
15003 case X86::BI__builtin_ia32_pshuflw:
15004 case X86::BI__builtin_ia32_pshuflw256:
15005 case X86::BI__builtin_ia32_pshuflw512: {
15006 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
15007 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15008 unsigned NumElts = Ty->getNumElements();
15011 Imm = (Imm & 0xff) * 0x01010101;
15014 for (
unsigned l = 0; l != NumElts; l += 8) {
15015 for (
unsigned i = 0; i != 4; ++i) {
15016 Indices[l + i] = l + (Imm & 3);
15019 for (
unsigned i = 4; i != 8; ++i)
15020 Indices[l + i] = l + i;
15023 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
15026 case X86::BI__builtin_ia32_pshufhw:
15027 case X86::BI__builtin_ia32_pshufhw256:
15028 case X86::BI__builtin_ia32_pshufhw512: {
15029 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
15030 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15031 unsigned NumElts = Ty->getNumElements();
15034 Imm = (Imm & 0xff) * 0x01010101;
15037 for (
unsigned l = 0; l != NumElts; l += 8) {
15038 for (
unsigned i = 0; i != 4; ++i)
15039 Indices[l + i] = l + i;
15040 for (
unsigned i = 4; i != 8; ++i) {
15041 Indices[l + i] = l + 4 + (Imm & 3);
15046 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
15049 case X86::BI__builtin_ia32_pshufd:
15050 case X86::BI__builtin_ia32_pshufd256:
15051 case X86::BI__builtin_ia32_pshufd512:
15052 case X86::BI__builtin_ia32_vpermilpd:
15053 case X86::BI__builtin_ia32_vpermilps:
15054 case X86::BI__builtin_ia32_vpermilpd256:
15055 case X86::BI__builtin_ia32_vpermilps256:
15056 case X86::BI__builtin_ia32_vpermilpd512:
15057 case X86::BI__builtin_ia32_vpermilps512: {
15058 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
15059 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15060 unsigned NumElts = Ty->getNumElements();
15061 unsigned NumLanes = Ty->getPrimitiveSizeInBits() / 128;
15062 unsigned NumLaneElts = NumElts / NumLanes;
15065 Imm = (Imm & 0xff) * 0x01010101;
15068 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
15069 for (
unsigned i = 0; i != NumLaneElts; ++i) {
15070 Indices[i + l] = (Imm % NumLaneElts) + l;
15071 Imm /= NumLaneElts;
15075 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
15078 case X86::BI__builtin_ia32_shufpd:
15079 case X86::BI__builtin_ia32_shufpd256:
15080 case X86::BI__builtin_ia32_shufpd512:
15081 case X86::BI__builtin_ia32_shufps:
15082 case X86::BI__builtin_ia32_shufps256:
15083 case X86::BI__builtin_ia32_shufps512: {
15084 uint32_t Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
15085 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15086 unsigned NumElts = Ty->getNumElements();
15087 unsigned NumLanes = Ty->getPrimitiveSizeInBits() / 128;
15088 unsigned NumLaneElts = NumElts / NumLanes;
15091 Imm = (Imm & 0xff) * 0x01010101;
15094 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
15095 for (
unsigned i = 0; i != NumLaneElts; ++i) {
15096 unsigned Index = Imm % NumLaneElts;
15097 Imm /= NumLaneElts;
15098 if (i >= (NumLaneElts / 2))
15100 Indices[l + i] = l + Index;
15104 return Builder.CreateShuffleVector(Ops[0], Ops[1],
15105 ArrayRef(Indices, NumElts),
"shufp");
15107 case X86::BI__builtin_ia32_permdi256:
15108 case X86::BI__builtin_ia32_permdf256:
15109 case X86::BI__builtin_ia32_permdi512:
15110 case X86::BI__builtin_ia32_permdf512: {
15111 unsigned Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
15112 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15113 unsigned NumElts = Ty->getNumElements();
15117 for (
unsigned l = 0; l != NumElts; l += 4)
15118 for (
unsigned i = 0; i != 4; ++i)
15119 Indices[l + i] = l + ((Imm >> (2 * i)) & 0x3);
15121 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
15124 case X86::BI__builtin_ia32_palignr128:
15125 case X86::BI__builtin_ia32_palignr256:
15126 case X86::BI__builtin_ia32_palignr512: {
15127 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff;
15130 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15131 assert(NumElts % 16 == 0);
15135 if (ShiftVal >= 32)
15140 if (ShiftVal > 16) {
15143 Ops[0] = llvm::Constant::getNullValue(Ops[0]->getType());
15148 for (
unsigned l = 0; l != NumElts; l += 16) {
15149 for (
unsigned i = 0; i != 16; ++i) {
15150 unsigned Idx = ShiftVal + i;
15152 Idx += NumElts - 16;
15153 Indices[l + i] = Idx + l;
15157 return Builder.CreateShuffleVector(Ops[1], Ops[0],
15158 ArrayRef(Indices, NumElts),
"palignr");
15160 case X86::BI__builtin_ia32_alignd128:
15161 case X86::BI__builtin_ia32_alignd256:
15162 case X86::BI__builtin_ia32_alignd512:
15163 case X86::BI__builtin_ia32_alignq128:
15164 case X86::BI__builtin_ia32_alignq256:
15165 case X86::BI__builtin_ia32_alignq512: {
15167 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15168 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff;
15171 ShiftVal &= NumElts - 1;
15174 for (
unsigned i = 0; i != NumElts; ++i)
15175 Indices[i] = i + ShiftVal;
15177 return Builder.CreateShuffleVector(Ops[1], Ops[0],
15178 ArrayRef(Indices, NumElts),
"valign");
15180 case X86::BI__builtin_ia32_shuf_f32x4_256:
15181 case X86::BI__builtin_ia32_shuf_f64x2_256:
15182 case X86::BI__builtin_ia32_shuf_i32x4_256:
15183 case X86::BI__builtin_ia32_shuf_i64x2_256:
15184 case X86::BI__builtin_ia32_shuf_f32x4:
15185 case X86::BI__builtin_ia32_shuf_f64x2:
15186 case X86::BI__builtin_ia32_shuf_i32x4:
15187 case X86::BI__builtin_ia32_shuf_i64x2: {
15188 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
15189 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15190 unsigned NumElts = Ty->getNumElements();
15191 unsigned NumLanes = Ty->getPrimitiveSizeInBits() == 512 ? 4 : 2;
15192 unsigned NumLaneElts = NumElts / NumLanes;
15195 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
15196 unsigned Index = (Imm % NumLanes) * NumLaneElts;
15198 if (l >= (NumElts / 2))
15200 for (
unsigned i = 0; i != NumLaneElts; ++i) {
15201 Indices[l + i] = Index + i;
15205 return Builder.CreateShuffleVector(Ops[0], Ops[1],
15206 ArrayRef(Indices, NumElts),
"shuf");
15209 case X86::BI__builtin_ia32_vperm2f128_pd256:
15210 case X86::BI__builtin_ia32_vperm2f128_ps256:
15211 case X86::BI__builtin_ia32_vperm2f128_si256:
15212 case X86::BI__builtin_ia32_permti256: {
15213 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
15215 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15224 for (
unsigned l = 0; l != 2; ++l) {
15226 if (Imm & (1 << ((l * 4) + 3)))
15227 OutOps[l] = llvm::ConstantAggregateZero::get(Ops[0]->getType());
15228 else if (Imm & (1 << ((l * 4) + 1)))
15229 OutOps[l] = Ops[1];
15231 OutOps[l] = Ops[0];
15233 for (
unsigned i = 0; i != NumElts/2; ++i) {
15235 unsigned Idx = (l * NumElts) + i;
15238 if (Imm & (1 << (l * 4)))
15240 Indices[(l * (NumElts/2)) + i] = Idx;
15244 return Builder.CreateShuffleVector(OutOps[0], OutOps[1],
15245 ArrayRef(Indices, NumElts),
"vperm");
15248 case X86::BI__builtin_ia32_pslldqi128_byteshift:
15249 case X86::BI__builtin_ia32_pslldqi256_byteshift:
15250 case X86::BI__builtin_ia32_pslldqi512_byteshift: {
15251 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
15252 auto *ResultType = cast<llvm::FixedVectorType>(Ops[0]->getType());
15254 unsigned NumElts = ResultType->getNumElements() * 8;
15257 if (ShiftVal >= 16)
15258 return llvm::Constant::getNullValue(ResultType);
15262 for (
unsigned l = 0; l != NumElts; l += 16) {
15263 for (
unsigned i = 0; i != 16; ++i) {
15264 unsigned Idx = NumElts + i - ShiftVal;
15265 if (Idx < NumElts) Idx -= NumElts - 16;
15266 Indices[l + i] = Idx + l;
15270 auto *VecTy = llvm::FixedVectorType::get(
Int8Ty, NumElts);
15272 Value *
Zero = llvm::Constant::getNullValue(VecTy);
15274 Zero, Cast,
ArrayRef(Indices, NumElts),
"pslldq");
15275 return Builder.CreateBitCast(SV, Ops[0]->getType(),
"cast");
15277 case X86::BI__builtin_ia32_psrldqi128_byteshift:
15278 case X86::BI__builtin_ia32_psrldqi256_byteshift:
15279 case X86::BI__builtin_ia32_psrldqi512_byteshift: {
15280 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
15281 auto *ResultType = cast<llvm::FixedVectorType>(Ops[0]->getType());
15283 unsigned NumElts = ResultType->getNumElements() * 8;
15286 if (ShiftVal >= 16)
15287 return llvm::Constant::getNullValue(ResultType);
15291 for (
unsigned l = 0; l != NumElts; l += 16) {
15292 for (
unsigned i = 0; i != 16; ++i) {
15293 unsigned Idx = i + ShiftVal;
15294 if (Idx >= 16) Idx += NumElts - 16;
15295 Indices[l + i] = Idx + l;
15299 auto *VecTy = llvm::FixedVectorType::get(
Int8Ty, NumElts);
15301 Value *
Zero = llvm::Constant::getNullValue(VecTy);
15303 Cast, Zero,
ArrayRef(Indices, NumElts),
"psrldq");
15304 return Builder.CreateBitCast(SV, ResultType,
"cast");
15306 case X86::BI__builtin_ia32_kshiftliqi:
15307 case X86::BI__builtin_ia32_kshiftlihi:
15308 case X86::BI__builtin_ia32_kshiftlisi:
15309 case X86::BI__builtin_ia32_kshiftlidi: {
15310 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
15311 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15313 if (ShiftVal >= NumElts)
15314 return llvm::Constant::getNullValue(Ops[0]->getType());
15319 for (
unsigned i = 0; i != NumElts; ++i)
15320 Indices[i] = NumElts + i - ShiftVal;
15322 Value *
Zero = llvm::Constant::getNullValue(
In->getType());
15324 Zero, In,
ArrayRef(Indices, NumElts),
"kshiftl");
15325 return Builder.CreateBitCast(SV, Ops[0]->getType());
15327 case X86::BI__builtin_ia32_kshiftriqi:
15328 case X86::BI__builtin_ia32_kshiftrihi:
15329 case X86::BI__builtin_ia32_kshiftrisi:
15330 case X86::BI__builtin_ia32_kshiftridi: {
15331 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
15332 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15334 if (ShiftVal >= NumElts)
15335 return llvm::Constant::getNullValue(Ops[0]->getType());
15340 for (
unsigned i = 0; i != NumElts; ++i)
15341 Indices[i] = i + ShiftVal;
15343 Value *
Zero = llvm::Constant::getNullValue(
In->getType());
15345 In, Zero,
ArrayRef(Indices, NumElts),
"kshiftr");
15346 return Builder.CreateBitCast(SV, Ops[0]->getType());
15348 case X86::BI__builtin_ia32_movnti:
15349 case X86::BI__builtin_ia32_movnti64:
15350 case X86::BI__builtin_ia32_movntsd:
15351 case X86::BI__builtin_ia32_movntss: {
15352 llvm::MDNode *
Node = llvm::MDNode::get(
15355 Value *Ptr = Ops[0];
15356 Value *Src = Ops[1];
15359 if (BuiltinID == X86::BI__builtin_ia32_movntsd ||
15360 BuiltinID == X86::BI__builtin_ia32_movntss)
15361 Src =
Builder.CreateExtractElement(Src, (uint64_t)0,
"extract");
15365 SI->setMetadata(llvm::LLVMContext::MD_nontemporal,
Node);
15366 SI->setAlignment(llvm::Align(1));
15370 case X86::BI__builtin_ia32_vprotb:
15371 case X86::BI__builtin_ia32_vprotw:
15372 case X86::BI__builtin_ia32_vprotd:
15373 case X86::BI__builtin_ia32_vprotq:
15374 case X86::BI__builtin_ia32_vprotbi:
15375 case X86::BI__builtin_ia32_vprotwi:
15376 case X86::BI__builtin_ia32_vprotdi:
15377 case X86::BI__builtin_ia32_vprotqi:
15378 case X86::BI__builtin_ia32_prold128:
15379 case X86::BI__builtin_ia32_prold256:
15380 case X86::BI__builtin_ia32_prold512:
15381 case X86::BI__builtin_ia32_prolq128:
15382 case X86::BI__builtin_ia32_prolq256:
15383 case X86::BI__builtin_ia32_prolq512:
15384 case X86::BI__builtin_ia32_prolvd128:
15385 case X86::BI__builtin_ia32_prolvd256:
15386 case X86::BI__builtin_ia32_prolvd512:
15387 case X86::BI__builtin_ia32_prolvq128:
15388 case X86::BI__builtin_ia32_prolvq256:
15389 case X86::BI__builtin_ia32_prolvq512:
15391 case X86::BI__builtin_ia32_prord128:
15392 case X86::BI__builtin_ia32_prord256:
15393 case X86::BI__builtin_ia32_prord512:
15394 case X86::BI__builtin_ia32_prorq128:
15395 case X86::BI__builtin_ia32_prorq256:
15396 case X86::BI__builtin_ia32_prorq512:
15397 case X86::BI__builtin_ia32_prorvd128:
15398 case X86::BI__builtin_ia32_prorvd256:
15399 case X86::BI__builtin_ia32_prorvd512:
15400 case X86::BI__builtin_ia32_prorvq128:
15401 case X86::BI__builtin_ia32_prorvq256:
15402 case X86::BI__builtin_ia32_prorvq512:
15404 case X86::BI__builtin_ia32_selectb_128:
15405 case X86::BI__builtin_ia32_selectb_256:
15406 case X86::BI__builtin_ia32_selectb_512:
15407 case X86::BI__builtin_ia32_selectw_128:
15408 case X86::BI__builtin_ia32_selectw_256:
15409 case X86::BI__builtin_ia32_selectw_512:
15410 case X86::BI__builtin_ia32_selectd_128:
15411 case X86::BI__builtin_ia32_selectd_256:
15412 case X86::BI__builtin_ia32_selectd_512:
15413 case X86::BI__builtin_ia32_selectq_128:
15414 case X86::BI__builtin_ia32_selectq_256:
15415 case X86::BI__builtin_ia32_selectq_512:
15416 case X86::BI__builtin_ia32_selectph_128:
15417 case X86::BI__builtin_ia32_selectph_256:
15418 case X86::BI__builtin_ia32_selectph_512:
15419 case X86::BI__builtin_ia32_selectpbf_128:
15420 case X86::BI__builtin_ia32_selectpbf_256:
15421 case X86::BI__builtin_ia32_selectpbf_512:
15422 case X86::BI__builtin_ia32_selectps_128:
15423 case X86::BI__builtin_ia32_selectps_256:
15424 case X86::BI__builtin_ia32_selectps_512:
15425 case X86::BI__builtin_ia32_selectpd_128:
15426 case X86::BI__builtin_ia32_selectpd_256:
15427 case X86::BI__builtin_ia32_selectpd_512:
15429 case X86::BI__builtin_ia32_selectsh_128:
15430 case X86::BI__builtin_ia32_selectsbf_128:
15431 case X86::BI__builtin_ia32_selectss_128:
15432 case X86::BI__builtin_ia32_selectsd_128: {
15433 Value *A =
Builder.CreateExtractElement(Ops[1], (uint64_t)0);
15434 Value *B =
Builder.CreateExtractElement(Ops[2], (uint64_t)0);
15436 return Builder.CreateInsertElement(Ops[1], A, (uint64_t)0);
15438 case X86::BI__builtin_ia32_cmpb128_mask:
15439 case X86::BI__builtin_ia32_cmpb256_mask:
15440 case X86::BI__builtin_ia32_cmpb512_mask:
15441 case X86::BI__builtin_ia32_cmpw128_mask:
15442 case X86::BI__builtin_ia32_cmpw256_mask:
15443 case X86::BI__builtin_ia32_cmpw512_mask:
15444 case X86::BI__builtin_ia32_cmpd128_mask:
15445 case X86::BI__builtin_ia32_cmpd256_mask:
15446 case X86::BI__builtin_ia32_cmpd512_mask:
15447 case X86::BI__builtin_ia32_cmpq128_mask:
15448 case X86::BI__builtin_ia32_cmpq256_mask:
15449 case X86::BI__builtin_ia32_cmpq512_mask: {
15450 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
15453 case X86::BI__builtin_ia32_ucmpb128_mask:
15454 case X86::BI__builtin_ia32_ucmpb256_mask:
15455 case X86::BI__builtin_ia32_ucmpb512_mask:
15456 case X86::BI__builtin_ia32_ucmpw128_mask:
15457 case X86::BI__builtin_ia32_ucmpw256_mask:
15458 case X86::BI__builtin_ia32_ucmpw512_mask:
15459 case X86::BI__builtin_ia32_ucmpd128_mask:
15460 case X86::BI__builtin_ia32_ucmpd256_mask:
15461 case X86::BI__builtin_ia32_ucmpd512_mask:
15462 case X86::BI__builtin_ia32_ucmpq128_mask:
15463 case X86::BI__builtin_ia32_ucmpq256_mask:
15464 case X86::BI__builtin_ia32_ucmpq512_mask: {
15465 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
15468 case X86::BI__builtin_ia32_vpcomb:
15469 case X86::BI__builtin_ia32_vpcomw:
15470 case X86::BI__builtin_ia32_vpcomd:
15471 case X86::BI__builtin_ia32_vpcomq:
15473 case X86::BI__builtin_ia32_vpcomub:
15474 case X86::BI__builtin_ia32_vpcomuw:
15475 case X86::BI__builtin_ia32_vpcomud:
15476 case X86::BI__builtin_ia32_vpcomuq:
15479 case X86::BI__builtin_ia32_kortestcqi:
15480 case X86::BI__builtin_ia32_kortestchi:
15481 case X86::BI__builtin_ia32_kortestcsi:
15482 case X86::BI__builtin_ia32_kortestcdi: {
15484 Value *
C = llvm::Constant::getAllOnesValue(Ops[0]->getType());
15488 case X86::BI__builtin_ia32_kortestzqi:
15489 case X86::BI__builtin_ia32_kortestzhi:
15490 case X86::BI__builtin_ia32_kortestzsi:
15491 case X86::BI__builtin_ia32_kortestzdi: {
15493 Value *
C = llvm::Constant::getNullValue(Ops[0]->getType());
15498 case X86::BI__builtin_ia32_ktestcqi:
15499 case X86::BI__builtin_ia32_ktestzqi:
15500 case X86::BI__builtin_ia32_ktestchi:
15501 case X86::BI__builtin_ia32_ktestzhi:
15502 case X86::BI__builtin_ia32_ktestcsi:
15503 case X86::BI__builtin_ia32_ktestzsi:
15504 case X86::BI__builtin_ia32_ktestcdi:
15505 case X86::BI__builtin_ia32_ktestzdi: {
15507 switch (BuiltinID) {
15508 default: llvm_unreachable(
"Unsupported intrinsic!");
15509 case X86::BI__builtin_ia32_ktestcqi:
15510 IID = Intrinsic::x86_avx512_ktestc_b;
15512 case X86::BI__builtin_ia32_ktestzqi:
15513 IID = Intrinsic::x86_avx512_ktestz_b;
15515 case X86::BI__builtin_ia32_ktestchi:
15516 IID = Intrinsic::x86_avx512_ktestc_w;
15518 case X86::BI__builtin_ia32_ktestzhi:
15519 IID = Intrinsic::x86_avx512_ktestz_w;
15521 case X86::BI__builtin_ia32_ktestcsi:
15522 IID = Intrinsic::x86_avx512_ktestc_d;
15524 case X86::BI__builtin_ia32_ktestzsi:
15525 IID = Intrinsic::x86_avx512_ktestz_d;
15527 case X86::BI__builtin_ia32_ktestcdi:
15528 IID = Intrinsic::x86_avx512_ktestc_q;
15530 case X86::BI__builtin_ia32_ktestzdi:
15531 IID = Intrinsic::x86_avx512_ktestz_q;
15535 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15539 return Builder.CreateCall(Intr, {LHS, RHS});
15542 case X86::BI__builtin_ia32_kaddqi:
15543 case X86::BI__builtin_ia32_kaddhi:
15544 case X86::BI__builtin_ia32_kaddsi:
15545 case X86::BI__builtin_ia32_kadddi: {
15547 switch (BuiltinID) {
15548 default: llvm_unreachable(
"Unsupported intrinsic!");
15549 case X86::BI__builtin_ia32_kaddqi:
15550 IID = Intrinsic::x86_avx512_kadd_b;
15552 case X86::BI__builtin_ia32_kaddhi:
15553 IID = Intrinsic::x86_avx512_kadd_w;
15555 case X86::BI__builtin_ia32_kaddsi:
15556 IID = Intrinsic::x86_avx512_kadd_d;
15558 case X86::BI__builtin_ia32_kadddi:
15559 IID = Intrinsic::x86_avx512_kadd_q;
15563 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15568 return Builder.CreateBitCast(Res, Ops[0]->getType());
15570 case X86::BI__builtin_ia32_kandqi:
15571 case X86::BI__builtin_ia32_kandhi:
15572 case X86::BI__builtin_ia32_kandsi:
15573 case X86::BI__builtin_ia32_kanddi:
15575 case X86::BI__builtin_ia32_kandnqi:
15576 case X86::BI__builtin_ia32_kandnhi:
15577 case X86::BI__builtin_ia32_kandnsi:
15578 case X86::BI__builtin_ia32_kandndi:
15580 case X86::BI__builtin_ia32_korqi:
15581 case X86::BI__builtin_ia32_korhi:
15582 case X86::BI__builtin_ia32_korsi:
15583 case X86::BI__builtin_ia32_kordi:
15585 case X86::BI__builtin_ia32_kxnorqi:
15586 case X86::BI__builtin_ia32_kxnorhi:
15587 case X86::BI__builtin_ia32_kxnorsi:
15588 case X86::BI__builtin_ia32_kxnordi:
15590 case X86::BI__builtin_ia32_kxorqi:
15591 case X86::BI__builtin_ia32_kxorhi:
15592 case X86::BI__builtin_ia32_kxorsi:
15593 case X86::BI__builtin_ia32_kxordi:
15595 case X86::BI__builtin_ia32_knotqi:
15596 case X86::BI__builtin_ia32_knothi:
15597 case X86::BI__builtin_ia32_knotsi:
15598 case X86::BI__builtin_ia32_knotdi: {
15599 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15602 Ops[0]->getType());
15604 case X86::BI__builtin_ia32_kmovb:
15605 case X86::BI__builtin_ia32_kmovw:
15606 case X86::BI__builtin_ia32_kmovd:
15607 case X86::BI__builtin_ia32_kmovq: {
15611 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15613 return Builder.CreateBitCast(Res, Ops[0]->getType());
15616 case X86::BI__builtin_ia32_kunpckdi:
15617 case X86::BI__builtin_ia32_kunpcksi:
15618 case X86::BI__builtin_ia32_kunpckhi: {
15619 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15623 for (
unsigned i = 0; i != NumElts; ++i)
15628 LHS =
Builder.CreateShuffleVector(LHS, LHS,
ArrayRef(Indices, NumElts / 2));
15629 RHS =
Builder.CreateShuffleVector(RHS, RHS,
ArrayRef(Indices, NumElts / 2));
15634 return Builder.CreateBitCast(Res, Ops[0]->getType());
15637 case X86::BI__builtin_ia32_vplzcntd_128:
15638 case X86::BI__builtin_ia32_vplzcntd_256:
15639 case X86::BI__builtin_ia32_vplzcntd_512:
15640 case X86::BI__builtin_ia32_vplzcntq_128:
15641 case X86::BI__builtin_ia32_vplzcntq_256:
15642 case X86::BI__builtin_ia32_vplzcntq_512: {
15646 case X86::BI__builtin_ia32_sqrtss:
15647 case X86::BI__builtin_ia32_sqrtsd: {
15648 Value *A =
Builder.CreateExtractElement(Ops[0], (uint64_t)0);
15650 if (
Builder.getIsFPConstrained()) {
15651 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
15654 A =
Builder.CreateConstrainedFPCall(F, {A});
15657 A =
Builder.CreateCall(F, {A});
15659 return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0);
15661 case X86::BI__builtin_ia32_sqrtsh_round_mask:
15662 case X86::BI__builtin_ia32_sqrtsd_round_mask:
15663 case X86::BI__builtin_ia32_sqrtss_round_mask: {
15664 unsigned CC = cast<llvm::ConstantInt>(Ops[4])->getZExtValue();
15670 switch (BuiltinID) {
15672 llvm_unreachable(
"Unsupported intrinsic!");
15673 case X86::BI__builtin_ia32_sqrtsh_round_mask:
15674 IID = Intrinsic::x86_avx512fp16_mask_sqrt_sh;
15676 case X86::BI__builtin_ia32_sqrtsd_round_mask:
15677 IID = Intrinsic::x86_avx512_mask_sqrt_sd;
15679 case X86::BI__builtin_ia32_sqrtss_round_mask:
15680 IID = Intrinsic::x86_avx512_mask_sqrt_ss;
15685 Value *A =
Builder.CreateExtractElement(Ops[1], (uint64_t)0);
15687 if (
Builder.getIsFPConstrained()) {
15688 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
15691 A =
Builder.CreateConstrainedFPCall(F, A);
15694 A =
Builder.CreateCall(F, A);
15696 Value *Src =
Builder.CreateExtractElement(Ops[2], (uint64_t)0);
15698 return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0);
15700 case X86::BI__builtin_ia32_sqrtpd256:
15701 case X86::BI__builtin_ia32_sqrtpd:
15702 case X86::BI__builtin_ia32_sqrtps256:
15703 case X86::BI__builtin_ia32_sqrtps:
15704 case X86::BI__builtin_ia32_sqrtph256:
15705 case X86::BI__builtin_ia32_sqrtph:
15706 case X86::BI__builtin_ia32_sqrtph512:
15707 case X86::BI__builtin_ia32_sqrtps512:
15708 case X86::BI__builtin_ia32_sqrtpd512: {
15709 if (Ops.size() == 2) {
15710 unsigned CC = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
15716 switch (BuiltinID) {
15718 llvm_unreachable(
"Unsupported intrinsic!");
15719 case X86::BI__builtin_ia32_sqrtph512:
15720 IID = Intrinsic::x86_avx512fp16_sqrt_ph_512;
15722 case X86::BI__builtin_ia32_sqrtps512:
15723 IID = Intrinsic::x86_avx512_sqrt_ps_512;
15725 case X86::BI__builtin_ia32_sqrtpd512:
15726 IID = Intrinsic::x86_avx512_sqrt_pd_512;
15732 if (
Builder.getIsFPConstrained()) {
15733 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
15735 Ops[0]->getType());
15736 return Builder.CreateConstrainedFPCall(F, Ops[0]);
15739 return Builder.CreateCall(F, Ops[0]);
15743 case X86::BI__builtin_ia32_pmuludq128:
15744 case X86::BI__builtin_ia32_pmuludq256:
15745 case X86::BI__builtin_ia32_pmuludq512:
15748 case X86::BI__builtin_ia32_pmuldq128:
15749 case X86::BI__builtin_ia32_pmuldq256:
15750 case X86::BI__builtin_ia32_pmuldq512:
15753 case X86::BI__builtin_ia32_pternlogd512_mask:
15754 case X86::BI__builtin_ia32_pternlogq512_mask:
15755 case X86::BI__builtin_ia32_pternlogd128_mask:
15756 case X86::BI__builtin_ia32_pternlogd256_mask:
15757 case X86::BI__builtin_ia32_pternlogq128_mask:
15758 case X86::BI__builtin_ia32_pternlogq256_mask:
15761 case X86::BI__builtin_ia32_pternlogd512_maskz:
15762 case X86::BI__builtin_ia32_pternlogq512_maskz:
15763 case X86::BI__builtin_ia32_pternlogd128_maskz:
15764 case X86::BI__builtin_ia32_pternlogd256_maskz:
15765 case X86::BI__builtin_ia32_pternlogq128_maskz:
15766 case X86::BI__builtin_ia32_pternlogq256_maskz:
15769 case X86::BI__builtin_ia32_vpshldd128:
15770 case X86::BI__builtin_ia32_vpshldd256:
15771 case X86::BI__builtin_ia32_vpshldd512:
15772 case X86::BI__builtin_ia32_vpshldq128:
15773 case X86::BI__builtin_ia32_vpshldq256:
15774 case X86::BI__builtin_ia32_vpshldq512:
15775 case X86::BI__builtin_ia32_vpshldw128:
15776 case X86::BI__builtin_ia32_vpshldw256:
15777 case X86::BI__builtin_ia32_vpshldw512:
15780 case X86::BI__builtin_ia32_vpshrdd128:
15781 case X86::BI__builtin_ia32_vpshrdd256:
15782 case X86::BI__builtin_ia32_vpshrdd512:
15783 case X86::BI__builtin_ia32_vpshrdq128:
15784 case X86::BI__builtin_ia32_vpshrdq256:
15785 case X86::BI__builtin_ia32_vpshrdq512:
15786 case X86::BI__builtin_ia32_vpshrdw128:
15787 case X86::BI__builtin_ia32_vpshrdw256:
15788 case X86::BI__builtin_ia32_vpshrdw512:
15792 case X86::BI__builtin_ia32_vpshldvd128:
15793 case X86::BI__builtin_ia32_vpshldvd256:
15794 case X86::BI__builtin_ia32_vpshldvd512:
15795 case X86::BI__builtin_ia32_vpshldvq128:
15796 case X86::BI__builtin_ia32_vpshldvq256:
15797 case X86::BI__builtin_ia32_vpshldvq512:
15798 case X86::BI__builtin_ia32_vpshldvw128:
15799 case X86::BI__builtin_ia32_vpshldvw256:
15800 case X86::BI__builtin_ia32_vpshldvw512:
15803 case X86::BI__builtin_ia32_vpshrdvd128:
15804 case X86::BI__builtin_ia32_vpshrdvd256:
15805 case X86::BI__builtin_ia32_vpshrdvd512:
15806 case X86::BI__builtin_ia32_vpshrdvq128:
15807 case X86::BI__builtin_ia32_vpshrdvq256:
15808 case X86::BI__builtin_ia32_vpshrdvq512:
15809 case X86::BI__builtin_ia32_vpshrdvw128:
15810 case X86::BI__builtin_ia32_vpshrdvw256:
15811 case X86::BI__builtin_ia32_vpshrdvw512:
15816 case X86::BI__builtin_ia32_reduce_fadd_pd512:
15817 case X86::BI__builtin_ia32_reduce_fadd_ps512:
15818 case X86::BI__builtin_ia32_reduce_fadd_ph512:
15819 case X86::BI__builtin_ia32_reduce_fadd_ph256:
15820 case X86::BI__builtin_ia32_reduce_fadd_ph128: {
15823 IRBuilder<>::FastMathFlagGuard FMFGuard(
Builder);
15824 Builder.getFastMathFlags().setAllowReassoc();
15825 return Builder.CreateCall(F, {Ops[0], Ops[1]});
15827 case X86::BI__builtin_ia32_reduce_fmul_pd512:
15828 case X86::BI__builtin_ia32_reduce_fmul_ps512:
15829 case X86::BI__builtin_ia32_reduce_fmul_ph512:
15830 case X86::BI__builtin_ia32_reduce_fmul_ph256:
15831 case X86::BI__builtin_ia32_reduce_fmul_ph128: {
15834 IRBuilder<>::FastMathFlagGuard FMFGuard(
Builder);
15835 Builder.getFastMathFlags().setAllowReassoc();
15836 return Builder.CreateCall(F, {Ops[0], Ops[1]});
15838 case X86::BI__builtin_ia32_reduce_fmax_pd512:
15839 case X86::BI__builtin_ia32_reduce_fmax_ps512:
15840 case X86::BI__builtin_ia32_reduce_fmax_ph512:
15841 case X86::BI__builtin_ia32_reduce_fmax_ph256:
15842 case X86::BI__builtin_ia32_reduce_fmax_ph128: {
15845 IRBuilder<>::FastMathFlagGuard FMFGuard(
Builder);
15846 Builder.getFastMathFlags().setNoNaNs();
15847 return Builder.CreateCall(F, {Ops[0]});
15849 case X86::BI__builtin_ia32_reduce_fmin_pd512:
15850 case X86::BI__builtin_ia32_reduce_fmin_ps512:
15851 case X86::BI__builtin_ia32_reduce_fmin_ph512:
15852 case X86::BI__builtin_ia32_reduce_fmin_ph256:
15853 case X86::BI__builtin_ia32_reduce_fmin_ph128: {
15856 IRBuilder<>::FastMathFlagGuard FMFGuard(
Builder);
15857 Builder.getFastMathFlags().setNoNaNs();
15858 return Builder.CreateCall(F, {Ops[0]});
15862 case X86::BI__builtin_ia32_pswapdsf:
15863 case X86::BI__builtin_ia32_pswapdsi: {
15865 Ops[0] =
Builder.CreateBitCast(Ops[0], MMXTy,
"cast");
15867 return Builder.CreateCall(F, Ops,
"pswapd");
15869 case X86::BI__builtin_ia32_rdrand16_step:
15870 case X86::BI__builtin_ia32_rdrand32_step:
15871 case X86::BI__builtin_ia32_rdrand64_step:
15872 case X86::BI__builtin_ia32_rdseed16_step:
15873 case X86::BI__builtin_ia32_rdseed32_step:
15874 case X86::BI__builtin_ia32_rdseed64_step: {
15876 switch (BuiltinID) {
15877 default: llvm_unreachable(
"Unsupported intrinsic!");
15878 case X86::BI__builtin_ia32_rdrand16_step:
15879 ID = Intrinsic::x86_rdrand_16;
15881 case X86::BI__builtin_ia32_rdrand32_step:
15882 ID = Intrinsic::x86_rdrand_32;
15884 case X86::BI__builtin_ia32_rdrand64_step:
15885 ID = Intrinsic::x86_rdrand_64;
15887 case X86::BI__builtin_ia32_rdseed16_step:
15888 ID = Intrinsic::x86_rdseed_16;
15890 case X86::BI__builtin_ia32_rdseed32_step:
15891 ID = Intrinsic::x86_rdseed_32;
15893 case X86::BI__builtin_ia32_rdseed64_step:
15894 ID = Intrinsic::x86_rdseed_64;
15903 case X86::BI__builtin_ia32_addcarryx_u32:
15904 case X86::BI__builtin_ia32_addcarryx_u64:
15905 case X86::BI__builtin_ia32_subborrow_u32:
15906 case X86::BI__builtin_ia32_subborrow_u64: {
15908 switch (BuiltinID) {
15909 default: llvm_unreachable(
"Unsupported intrinsic!");
15910 case X86::BI__builtin_ia32_addcarryx_u32:
15911 IID = Intrinsic::x86_addcarry_32;
15913 case X86::BI__builtin_ia32_addcarryx_u64:
15914 IID = Intrinsic::x86_addcarry_64;
15916 case X86::BI__builtin_ia32_subborrow_u32:
15917 IID = Intrinsic::x86_subborrow_32;
15919 case X86::BI__builtin_ia32_subborrow_u64:
15920 IID = Intrinsic::x86_subborrow_64;
15925 { Ops[0], Ops[1], Ops[2] });
15931 case X86::BI__builtin_ia32_fpclassps128_mask:
15932 case X86::BI__builtin_ia32_fpclassps256_mask:
15933 case X86::BI__builtin_ia32_fpclassps512_mask:
15934 case X86::BI__builtin_ia32_fpclassph128_mask:
15935 case X86::BI__builtin_ia32_fpclassph256_mask:
15936 case X86::BI__builtin_ia32_fpclassph512_mask:
15937 case X86::BI__builtin_ia32_fpclasspd128_mask:
15938 case X86::BI__builtin_ia32_fpclasspd256_mask:
15939 case X86::BI__builtin_ia32_fpclasspd512_mask: {
15941 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15942 Value *MaskIn = Ops[2];
15943 Ops.erase(&Ops[2]);
15946 switch (BuiltinID) {
15947 default: llvm_unreachable(
"Unsupported intrinsic!");
15948 case X86::BI__builtin_ia32_fpclassph128_mask:
15949 ID = Intrinsic::x86_avx512fp16_fpclass_ph_128;
15951 case X86::BI__builtin_ia32_fpclassph256_mask:
15952 ID = Intrinsic::x86_avx512fp16_fpclass_ph_256;
15954 case X86::BI__builtin_ia32_fpclassph512_mask:
15955 ID = Intrinsic::x86_avx512fp16_fpclass_ph_512;
15957 case X86::BI__builtin_ia32_fpclassps128_mask:
15958 ID = Intrinsic::x86_avx512_fpclass_ps_128;
15960 case X86::BI__builtin_ia32_fpclassps256_mask:
15961 ID = Intrinsic::x86_avx512_fpclass_ps_256;
15963 case X86::BI__builtin_ia32_fpclassps512_mask:
15964 ID = Intrinsic::x86_avx512_fpclass_ps_512;
15966 case X86::BI__builtin_ia32_fpclasspd128_mask:
15967 ID = Intrinsic::x86_avx512_fpclass_pd_128;
15969 case X86::BI__builtin_ia32_fpclasspd256_mask:
15970 ID = Intrinsic::x86_avx512_fpclass_pd_256;
15972 case X86::BI__builtin_ia32_fpclasspd512_mask:
15973 ID = Intrinsic::x86_avx512_fpclass_pd_512;
15981 case X86::BI__builtin_ia32_vp2intersect_q_512:
15982 case X86::BI__builtin_ia32_vp2intersect_q_256:
15983 case X86::BI__builtin_ia32_vp2intersect_q_128:
15984 case X86::BI__builtin_ia32_vp2intersect_d_512:
15985 case X86::BI__builtin_ia32_vp2intersect_d_256:
15986 case X86::BI__builtin_ia32_vp2intersect_d_128: {
15988 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15991 switch (BuiltinID) {
15992 default: llvm_unreachable(
"Unsupported intrinsic!");
15993 case X86::BI__builtin_ia32_vp2intersect_q_512:
15994 ID = Intrinsic::x86_avx512_vp2intersect_q_512;
15996 case X86::BI__builtin_ia32_vp2intersect_q_256:
15997 ID = Intrinsic::x86_avx512_vp2intersect_q_256;
15999 case X86::BI__builtin_ia32_vp2intersect_q_128:
16000 ID = Intrinsic::x86_avx512_vp2intersect_q_128;
16002 case X86::BI__builtin_ia32_vp2intersect_d_512:
16003 ID = Intrinsic::x86_avx512_vp2intersect_d_512;
16005 case X86::BI__builtin_ia32_vp2intersect_d_256:
16006 ID = Intrinsic::x86_avx512_vp2intersect_d_256;
16008 case X86::BI__builtin_ia32_vp2intersect_d_128:
16009 ID = Intrinsic::x86_avx512_vp2intersect_d_128;
16023 case X86::BI__builtin_ia32_vpmultishiftqb128:
16024 case X86::BI__builtin_ia32_vpmultishiftqb256:
16025 case X86::BI__builtin_ia32_vpmultishiftqb512: {
16027 switch (BuiltinID) {
16028 default: llvm_unreachable(
"Unsupported intrinsic!");
16029 case X86::BI__builtin_ia32_vpmultishiftqb128:
16030 ID = Intrinsic::x86_avx512_pmultishift_qb_128;
16032 case X86::BI__builtin_ia32_vpmultishiftqb256:
16033 ID = Intrinsic::x86_avx512_pmultishift_qb_256;
16035 case X86::BI__builtin_ia32_vpmultishiftqb512:
16036 ID = Intrinsic::x86_avx512_pmultishift_qb_512;
16043 case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
16044 case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
16045 case X86::BI__builtin_ia32_vpshufbitqmb512_mask: {
16047 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16048 Value *MaskIn = Ops[2];
16049 Ops.erase(&Ops[2]);
16052 switch (BuiltinID) {
16053 default: llvm_unreachable(
"Unsupported intrinsic!");
16054 case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
16055 ID = Intrinsic::x86_avx512_vpshufbitqmb_128;
16057 case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
16058 ID = Intrinsic::x86_avx512_vpshufbitqmb_256;
16060 case X86::BI__builtin_ia32_vpshufbitqmb512_mask:
16061 ID = Intrinsic::x86_avx512_vpshufbitqmb_512;
16070 case X86::BI__builtin_ia32_cmpeqps:
16071 case X86::BI__builtin_ia32_cmpeqpd:
16072 return getVectorFCmpIR(CmpInst::FCMP_OEQ,
false);
16073 case X86::BI__builtin_ia32_cmpltps:
16074 case X86::BI__builtin_ia32_cmpltpd:
16075 return getVectorFCmpIR(CmpInst::FCMP_OLT,
true);
16076 case X86::BI__builtin_ia32_cmpleps:
16077 case X86::BI__builtin_ia32_cmplepd:
16078 return getVectorFCmpIR(CmpInst::FCMP_OLE,
true);
16079 case X86::BI__builtin_ia32_cmpunordps:
16080 case X86::BI__builtin_ia32_cmpunordpd:
16081 return getVectorFCmpIR(CmpInst::FCMP_UNO,
false);
16082 case X86::BI__builtin_ia32_cmpneqps:
16083 case X86::BI__builtin_ia32_cmpneqpd:
16084 return getVectorFCmpIR(CmpInst::FCMP_UNE,
false);
16085 case X86::BI__builtin_ia32_cmpnltps:
16086 case X86::BI__builtin_ia32_cmpnltpd:
16087 return getVectorFCmpIR(CmpInst::FCMP_UGE,
true);
16088 case X86::BI__builtin_ia32_cmpnleps:
16089 case X86::BI__builtin_ia32_cmpnlepd:
16090 return getVectorFCmpIR(CmpInst::FCMP_UGT,
true);
16091 case X86::BI__builtin_ia32_cmpordps:
16092 case X86::BI__builtin_ia32_cmpordpd:
16093 return getVectorFCmpIR(CmpInst::FCMP_ORD,
false);
16094 case X86::BI__builtin_ia32_cmpph128_mask:
16095 case X86::BI__builtin_ia32_cmpph256_mask:
16096 case X86::BI__builtin_ia32_cmpph512_mask:
16097 case X86::BI__builtin_ia32_cmpps128_mask:
16098 case X86::BI__builtin_ia32_cmpps256_mask:
16099 case X86::BI__builtin_ia32_cmpps512_mask:
16100 case X86::BI__builtin_ia32_cmppd128_mask:
16101 case X86::BI__builtin_ia32_cmppd256_mask:
16102 case X86::BI__builtin_ia32_cmppd512_mask:
16105 case X86::BI__builtin_ia32_cmpps:
16106 case X86::BI__builtin_ia32_cmpps256:
16107 case X86::BI__builtin_ia32_cmppd:
16108 case X86::BI__builtin_ia32_cmppd256: {
16116 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x1f;
16121 FCmpInst::Predicate Pred;
16125 switch (CC & 0xf) {
16126 case 0x00: Pred = FCmpInst::FCMP_OEQ; IsSignaling =
false;
break;
16127 case 0x01: Pred = FCmpInst::FCMP_OLT; IsSignaling =
true;
break;
16128 case 0x02: Pred = FCmpInst::FCMP_OLE; IsSignaling =
true;
break;
16129 case 0x03: Pred = FCmpInst::FCMP_UNO; IsSignaling =
false;
break;
16130 case 0x04: Pred = FCmpInst::FCMP_UNE; IsSignaling =
false;
break;
16131 case 0x05: Pred = FCmpInst::FCMP_UGE; IsSignaling =
true;
break;
16132 case 0x06: Pred = FCmpInst::FCMP_UGT; IsSignaling =
true;
break;
16133 case 0x07: Pred = FCmpInst::FCMP_ORD; IsSignaling =
false;
break;
16134 case 0x08: Pred = FCmpInst::FCMP_UEQ; IsSignaling =
false;
break;
16135 case 0x09: Pred = FCmpInst::FCMP_ULT; IsSignaling =
true;
break;
16136 case 0x0a: Pred = FCmpInst::FCMP_ULE; IsSignaling =
true;
break;
16137 case 0x0b: Pred = FCmpInst::FCMP_FALSE; IsSignaling =
false;
break;
16138 case 0x0c: Pred = FCmpInst::FCMP_ONE; IsSignaling =
false;
break;
16139 case 0x0d: Pred = FCmpInst::FCMP_OGE; IsSignaling =
true;
break;
16140 case 0x0e: Pred = FCmpInst::FCMP_OGT; IsSignaling =
true;
break;
16141 case 0x0f: Pred = FCmpInst::FCMP_TRUE; IsSignaling =
false;
break;
16142 default: llvm_unreachable(
"Unhandled CC");
16147 IsSignaling = !IsSignaling;
16154 if (
Builder.getIsFPConstrained() &&
16155 (Pred == FCmpInst::FCMP_TRUE || Pred == FCmpInst::FCMP_FALSE ||
16159 switch (BuiltinID) {
16160 default: llvm_unreachable(
"Unexpected builtin");
16161 case X86::BI__builtin_ia32_cmpps:
16162 IID = Intrinsic::x86_sse_cmp_ps;
16164 case X86::BI__builtin_ia32_cmpps256:
16165 IID = Intrinsic::x86_avx_cmp_ps_256;
16167 case X86::BI__builtin_ia32_cmppd:
16168 IID = Intrinsic::x86_sse2_cmp_pd;
16170 case X86::BI__builtin_ia32_cmppd256:
16171 IID = Intrinsic::x86_avx_cmp_pd_256;
16173 case X86::BI__builtin_ia32_cmpph128_mask:
16174 IID = Intrinsic::x86_avx512fp16_mask_cmp_ph_128;
16176 case X86::BI__builtin_ia32_cmpph256_mask:
16177 IID = Intrinsic::x86_avx512fp16_mask_cmp_ph_256;
16179 case X86::BI__builtin_ia32_cmpph512_mask:
16180 IID = Intrinsic::x86_avx512fp16_mask_cmp_ph_512;
16182 case X86::BI__builtin_ia32_cmpps512_mask:
16183 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
16185 case X86::BI__builtin_ia32_cmppd512_mask:
16186 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
16188 case X86::BI__builtin_ia32_cmpps128_mask:
16189 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
16191 case X86::BI__builtin_ia32_cmpps256_mask:
16192 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
16194 case X86::BI__builtin_ia32_cmppd128_mask:
16195 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
16197 case X86::BI__builtin_ia32_cmppd256_mask:
16198 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
16205 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16211 return Builder.CreateCall(Intr, Ops);
16222 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16225 Cmp =
Builder.CreateFCmpS(Pred, Ops[0], Ops[1]);
16227 Cmp =
Builder.CreateFCmp(Pred, Ops[0], Ops[1]);
16231 return getVectorFCmpIR(Pred, IsSignaling);
16235 case X86::BI__builtin_ia32_cmpeqss:
16236 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 0);
16237 case X86::BI__builtin_ia32_cmpltss:
16238 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 1);
16239 case X86::BI__builtin_ia32_cmpless:
16240 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 2);
16241 case X86::BI__builtin_ia32_cmpunordss:
16242 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 3);
16243 case X86::BI__builtin_ia32_cmpneqss:
16244 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 4);
16245 case X86::BI__builtin_ia32_cmpnltss:
16246 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 5);
16247 case X86::BI__builtin_ia32_cmpnless:
16248 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 6);
16249 case X86::BI__builtin_ia32_cmpordss:
16250 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 7);
16251 case X86::BI__builtin_ia32_cmpeqsd:
16252 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 0);
16253 case X86::BI__builtin_ia32_cmpltsd:
16254 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 1);
16255 case X86::BI__builtin_ia32_cmplesd:
16256 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 2);
16257 case X86::BI__builtin_ia32_cmpunordsd:
16258 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 3);
16259 case X86::BI__builtin_ia32_cmpneqsd:
16260 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 4);
16261 case X86::BI__builtin_ia32_cmpnltsd:
16262 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 5);
16263 case X86::BI__builtin_ia32_cmpnlesd:
16264 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 6);
16265 case X86::BI__builtin_ia32_cmpordsd:
16266 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 7);
16269 case X86::BI__builtin_ia32_vcvtph2ps:
16270 case X86::BI__builtin_ia32_vcvtph2ps256:
16271 case X86::BI__builtin_ia32_vcvtph2ps_mask:
16272 case X86::BI__builtin_ia32_vcvtph2ps256_mask:
16273 case X86::BI__builtin_ia32_vcvtph2ps512_mask: {
16274 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
16279 case X86::BI__builtin_ia32_cvtneps2bf16_128_mask: {
16282 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements());
16283 Intrinsic::ID IID = Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128;
16286 case X86::BI__builtin_ia32_cvtsbf162ss_32:
16289 case X86::BI__builtin_ia32_cvtneps2bf16_256_mask:
16290 case X86::BI__builtin_ia32_cvtneps2bf16_512_mask: {
16292 switch (BuiltinID) {
16293 default: llvm_unreachable(
"Unsupported intrinsic!");
16294 case X86::BI__builtin_ia32_cvtneps2bf16_256_mask:
16295 IID = Intrinsic::x86_avx512bf16_cvtneps2bf16_256;
16297 case X86::BI__builtin_ia32_cvtneps2bf16_512_mask:
16298 IID = Intrinsic::x86_avx512bf16_cvtneps2bf16_512;
16305 case X86::BI__cpuid:
16306 case X86::BI__cpuidex: {
16308 Value *SubFuncId = BuiltinID == X86::BI__cpuidex
16312 llvm::StructType *CpuidRetTy =
16314 llvm::FunctionType *FTy =
16317 StringRef
Asm, Constraints;
16318 if (
getTarget().getTriple().getArch() == llvm::Triple::x86) {
16320 Constraints =
"={ax},={bx},={cx},={dx},{ax},{cx}";
16323 Asm =
"xchgq %rbx, ${1:q}\n"
16325 "xchgq %rbx, ${1:q}";
16326 Constraints =
"={ax},=r,={cx},={dx},0,2";
16329 llvm::InlineAsm *IA = llvm::InlineAsm::get(FTy,
Asm, Constraints,
16331 Value *IACall =
Builder.CreateCall(IA, {FuncId, SubFuncId});
16334 for (
unsigned i = 0; i < 4; i++) {
16335 Value *Extracted =
Builder.CreateExtractValue(IACall, i);
16345 case X86::BI__emul:
16346 case X86::BI__emulu: {
16348 bool isSigned = (BuiltinID == X86::BI__emul);
16351 return Builder.CreateMul(LHS, RHS,
"", !isSigned, isSigned);
16353 case X86::BI__mulh:
16354 case X86::BI__umulh:
16355 case X86::BI_mul128:
16356 case X86::BI_umul128: {
16358 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
16360 bool IsSigned = (BuiltinID == X86::BI__mulh || BuiltinID == X86::BI_mul128);
16361 Value *LHS =
Builder.CreateIntCast(Ops[0], Int128Ty, IsSigned);
16362 Value *RHS =
Builder.CreateIntCast(Ops[1], Int128Ty, IsSigned);
16364 Value *MulResult, *HigherBits;
16366 MulResult =
Builder.CreateNSWMul(LHS, RHS);
16367 HigherBits =
Builder.CreateAShr(MulResult, 64);
16369 MulResult =
Builder.CreateNUWMul(LHS, RHS);
16370 HigherBits =
Builder.CreateLShr(MulResult, 64);
16372 HigherBits =
Builder.CreateIntCast(HigherBits, ResType, IsSigned);
16374 if (BuiltinID == X86::BI__mulh || BuiltinID == X86::BI__umulh)
16379 return Builder.CreateIntCast(MulResult, ResType, IsSigned);
16382 case X86::BI__faststorefence: {
16383 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
16384 llvm::SyncScope::System);
16386 case X86::BI__shiftleft128:
16387 case X86::BI__shiftright128: {
16389 BuiltinID == X86::BI__shiftleft128 ? Intrinsic::fshl : Intrinsic::fshr,
16394 std::swap(Ops[0], Ops[1]);
16396 return Builder.CreateCall(F, Ops);
16398 case X86::BI_ReadWriteBarrier:
16399 case X86::BI_ReadBarrier:
16400 case X86::BI_WriteBarrier: {
16401 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
16402 llvm::SyncScope::SingleThread);
16405 case X86::BI_AddressOfReturnAddress: {
16408 return Builder.CreateCall(F);
16410 case X86::BI__stosb: {
16418 case X86::BI__int2c: {
16420 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy,
false);
16421 llvm::InlineAsm *IA =
16422 llvm::InlineAsm::get(FTy,
"int $$0x2c",
"",
true);
16423 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
16425 llvm::Attribute::NoReturn);
16426 llvm::CallInst *CI =
Builder.CreateCall(IA);
16427 CI->setAttributes(NoReturnAttr);
16430 case X86::BI__readfsbyte:
16431 case X86::BI__readfsword:
16432 case X86::BI__readfsdword:
16433 case X86::BI__readfsqword: {
16439 Load->setVolatile(
true);
16442 case X86::BI__readgsbyte:
16443 case X86::BI__readgsword:
16444 case X86::BI__readgsdword:
16445 case X86::BI__readgsqword: {
16451 Load->setVolatile(
true);
16454 case X86::BI__builtin_ia32_encodekey128_u32: {
16455 Intrinsic::ID IID = Intrinsic::x86_encodekey128;
16459 for (
int i = 0; i < 3; ++i) {
16467 case X86::BI__builtin_ia32_encodekey256_u32: {
16468 Intrinsic::ID IID = Intrinsic::x86_encodekey256;
16473 for (
int i = 0; i < 4; ++i) {
16481 case X86::BI__builtin_ia32_aesenc128kl_u8:
16482 case X86::BI__builtin_ia32_aesdec128kl_u8:
16483 case X86::BI__builtin_ia32_aesenc256kl_u8:
16484 case X86::BI__builtin_ia32_aesdec256kl_u8: {
16486 StringRef BlockName;
16487 switch (BuiltinID) {
16489 llvm_unreachable(
"Unexpected builtin");
16490 case X86::BI__builtin_ia32_aesenc128kl_u8:
16491 IID = Intrinsic::x86_aesenc128kl;
16492 BlockName =
"aesenc128kl";
16494 case X86::BI__builtin_ia32_aesdec128kl_u8:
16495 IID = Intrinsic::x86_aesdec128kl;
16496 BlockName =
"aesdec128kl";
16498 case X86::BI__builtin_ia32_aesenc256kl_u8:
16499 IID = Intrinsic::x86_aesenc256kl;
16500 BlockName =
"aesenc256kl";
16502 case X86::BI__builtin_ia32_aesdec256kl_u8:
16503 IID = Intrinsic::x86_aesdec256kl;
16504 BlockName =
"aesdec256kl";
16510 BasicBlock *NoError =
16518 Builder.CreateCondBr(Succ, NoError, Error);
16520 Builder.SetInsertPoint(NoError);
16524 Builder.SetInsertPoint(Error);
16525 Constant *
Zero = llvm::Constant::getNullValue(Out->getType());
16532 case X86::BI__builtin_ia32_aesencwide128kl_u8:
16533 case X86::BI__builtin_ia32_aesdecwide128kl_u8:
16534 case X86::BI__builtin_ia32_aesencwide256kl_u8:
16535 case X86::BI__builtin_ia32_aesdecwide256kl_u8: {
16537 StringRef BlockName;
16538 switch (BuiltinID) {
16539 case X86::BI__builtin_ia32_aesencwide128kl_u8:
16540 IID = Intrinsic::x86_aesencwide128kl;
16541 BlockName =
"aesencwide128kl";
16543 case X86::BI__builtin_ia32_aesdecwide128kl_u8:
16544 IID = Intrinsic::x86_aesdecwide128kl;
16545 BlockName =
"aesdecwide128kl";
16547 case X86::BI__builtin_ia32_aesencwide256kl_u8:
16548 IID = Intrinsic::x86_aesencwide256kl;
16549 BlockName =
"aesencwide256kl";
16551 case X86::BI__builtin_ia32_aesdecwide256kl_u8:
16552 IID = Intrinsic::x86_aesdecwide256kl;
16553 BlockName =
"aesdecwide256kl";
16557 llvm::Type *Ty = FixedVectorType::get(
Builder.getInt64Ty(), 2);
16560 for (
int i = 0; i != 8; ++i) {
16561 Value *Ptr =
Builder.CreateConstGEP1_32(Ty, Ops[1], i);
16567 BasicBlock *NoError =
16574 Builder.CreateCondBr(Succ, NoError, Error);
16576 Builder.SetInsertPoint(NoError);
16577 for (
int i = 0; i != 8; ++i) {
16584 Builder.SetInsertPoint(Error);
16585 for (
int i = 0; i != 8; ++i) {
16587 Constant *
Zero = llvm::Constant::getNullValue(Out->getType());
16588 Value *Ptr =
Builder.CreateConstGEP1_32(Out->getType(), Ops[0], i);
16596 case X86::BI__builtin_ia32_vfcmaddcph512_mask:
16599 case X86::BI__builtin_ia32_vfmaddcph512_mask: {
16600 Intrinsic::ID IID = IsConjFMA
16601 ? Intrinsic::x86_avx512fp16_mask_vfcmadd_cph_512
16602 : Intrinsic::x86_avx512fp16_mask_vfmadd_cph_512;
16606 case X86::BI__builtin_ia32_vfcmaddcsh_round_mask:
16609 case X86::BI__builtin_ia32_vfmaddcsh_round_mask: {
16610 Intrinsic::ID IID = IsConjFMA ? Intrinsic::x86_avx512fp16_mask_vfcmadd_csh
16611 : Intrinsic::x86_avx512fp16_mask_vfmadd_csh;
16616 case X86::BI__builtin_ia32_vfcmaddcsh_round_mask3:
16619 case X86::BI__builtin_ia32_vfmaddcsh_round_mask3: {
16620 Intrinsic::ID IID = IsConjFMA ? Intrinsic::x86_avx512fp16_mask_vfcmadd_csh
16621 : Intrinsic::x86_avx512fp16_mask_vfmadd_csh;
16623 static constexpr int Mask[] = {0, 5, 6, 7};
16624 return Builder.CreateShuffleVector(
Call, Ops[2], Mask);
16626 case X86::BI__builtin_ia32_prefetchi:
16629 {Ops[0], llvm::ConstantInt::get(Int32Ty, 0), Ops[1],
16630 llvm::ConstantInt::get(Int32Ty, 0)});
16648 Intrinsic::ID
ID = Intrinsic::not_intrinsic;
16650#include "llvm/TargetParser/PPCTargetParser.def"
16651 auto GenAIXPPCBuiltinCpuExpr = [&](
unsigned SupportMethod,
unsigned FieldIdx,
16653 unsigned OpValue) ->
Value * {
16654 if (SupportMethod == AIX_BUILTIN_PPC_FALSE)
16657 if (SupportMethod == AIX_BUILTIN_PPC_TRUE)
16660 assert(SupportMethod <= USE_SYS_CONF &&
"Invalid value for SupportMethod.");
16661 assert((CompOp == COMP_EQ) &&
"Only equal comparisons are supported.");
16663 llvm::Type *STy = llvm::StructType::get(PPC_SYSTEMCONFIG_TYPE);
16664 llvm::Constant *SysConf =
16668 llvm::Value *Idxs[] = {ConstantInt::get(
Int32Ty, 0),
16669 ConstantInt::get(
Int32Ty, FieldIdx)};
16674 assert(FieldValue->getType()->isIntegerTy(32) &&
16675 "Only 32-bit integers are supported in GenAIXPPCBuiltinCpuExpr().");
16676 return Builder.CreateICmp(ICmpInst::ICMP_EQ, FieldValue,
16677 ConstantInt::get(
Int32Ty, OpValue));
16680 switch (BuiltinID) {
16681 default:
return nullptr;
16683 case Builtin::BI__builtin_cpu_is: {
16685 StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
16688 if (Triple.isOSAIX()) {
16689 unsigned IsCpuSupport, FieldIdx, CompareOp, CpuIdValue;
16690 typedef std::tuple<unsigned, unsigned, unsigned, unsigned> CPUType;
16691 std::tie(IsCpuSupport, FieldIdx, CompareOp, CpuIdValue) =
16692 static_cast<CPUType
>(StringSwitch<CPUType>(CPUStr)
16693#define PPC_AIX_CPU(NAME, SUPPORT_MAGIC, INDEX, COMPARE_OP, VALUE) \
16694 .Case(NAME, {SUPPORT_MAGIC, INDEX, COMPARE_OP, VALUE})
16695#include "llvm/TargetParser/PPCTargetParser.def"
16697 return GenAIXPPCBuiltinCpuExpr(IsCpuSupport, FieldIdx, CompareOp,
16701 assert(Triple.isOSLinux() &&
16702 "__builtin_cpu_is() is only supported for AIX and Linux.");
16703 unsigned NumCPUID = StringSwitch<unsigned>(CPUStr)
16704#define PPC_LNX_CPU(Name, NumericID) .Case(Name, NumericID)
16705#include "llvm/TargetParser/PPCTargetParser.def"
16707 assert(NumCPUID < -1U &&
"Invalid CPU name. Missed by SemaChecking?");
16708 Value *Op0 = llvm::ConstantInt::get(
Int32Ty, PPC_FAWORD_CPUID);
16710 Value *TheCall =
Builder.CreateCall(F, {Op0},
"cpu_is");
16711 return Builder.CreateICmpEQ(TheCall,
16712 llvm::ConstantInt::get(
Int32Ty, NumCPUID));
16714 case Builtin::BI__builtin_cpu_supports: {
16715 unsigned FeatureWord;
16718 StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
16719 std::tie(FeatureWord, BitMask) =
16720 StringSwitch<std::pair<unsigned, unsigned>>(CPUStr)
16721#define
PPC_LNX_FEATURE(Name, Description, EnumName, Bitmask, FA_WORD) \
16722 .Case(Name, {FA_WORD, Bitmask})
16723#include
"llvm/TargetParser/PPCTargetParser.def"
16727 Value *Op0 = llvm::ConstantInt::get(
Int32Ty, FeatureWord);
16729 Value *TheCall =
Builder.CreateCall(F, {Op0},
"cpu_supports");
16731 Builder.CreateAnd(TheCall, llvm::ConstantInt::get(
Int32Ty, BitMask));
16732 return Builder.CreateICmpNE(Mask, llvm::Constant::getNullValue(
Int32Ty));
16733#undef PPC_FAWORD_HWCAP
16734#undef PPC_FAWORD_HWCAP2
16735#undef PPC_FAWORD_CPUID
16740 case PPC::BI__builtin_ppc_get_timebase:
16744 case PPC::BI__builtin_altivec_lvx:
16745 case PPC::BI__builtin_altivec_lvxl:
16746 case PPC::BI__builtin_altivec_lvebx:
16747 case PPC::BI__builtin_altivec_lvehx:
16748 case PPC::BI__builtin_altivec_lvewx:
16749 case PPC::BI__builtin_altivec_lvsl:
16750 case PPC::BI__builtin_altivec_lvsr:
16751 case PPC::BI__builtin_vsx_lxvd2x:
16752 case PPC::BI__builtin_vsx_lxvw4x:
16753 case PPC::BI__builtin_vsx_lxvd2x_be:
16754 case PPC::BI__builtin_vsx_lxvw4x_be:
16755 case PPC::BI__builtin_vsx_lxvl:
16756 case PPC::BI__builtin_vsx_lxvll:
16761 if (!(BuiltinID == PPC::BI__builtin_vsx_lxvl ||
16762 BuiltinID == PPC::BI__builtin_vsx_lxvll)) {
16767 switch (BuiltinID) {
16768 default: llvm_unreachable(
"Unsupported ld/lvsl/lvsr intrinsic!");
16769 case PPC::BI__builtin_altivec_lvx:
16770 ID = Intrinsic::ppc_altivec_lvx;
16772 case PPC::BI__builtin_altivec_lvxl:
16773 ID = Intrinsic::ppc_altivec_lvxl;
16775 case PPC::BI__builtin_altivec_lvebx:
16776 ID = Intrinsic::ppc_altivec_lvebx;
16778 case PPC::BI__builtin_altivec_lvehx:
16779 ID = Intrinsic::ppc_altivec_lvehx;
16781 case PPC::BI__builtin_altivec_lvewx:
16782 ID = Intrinsic::ppc_altivec_lvewx;
16784 case PPC::BI__builtin_altivec_lvsl:
16785 ID = Intrinsic::ppc_altivec_lvsl;
16787 case PPC::BI__builtin_altivec_lvsr:
16788 ID = Intrinsic::ppc_altivec_lvsr;
16790 case PPC::BI__builtin_vsx_lxvd2x:
16791 ID = Intrinsic::ppc_vsx_lxvd2x;
16793 case PPC::BI__builtin_vsx_lxvw4x:
16794 ID = Intrinsic::ppc_vsx_lxvw4x;
16796 case PPC::BI__builtin_vsx_lxvd2x_be:
16797 ID = Intrinsic::ppc_vsx_lxvd2x_be;
16799 case PPC::BI__builtin_vsx_lxvw4x_be:
16800 ID = Intrinsic::ppc_vsx_lxvw4x_be;
16802 case PPC::BI__builtin_vsx_lxvl:
16803 ID = Intrinsic::ppc_vsx_lxvl;
16805 case PPC::BI__builtin_vsx_lxvll:
16806 ID = Intrinsic::ppc_vsx_lxvll;
16810 return Builder.CreateCall(F, Ops,
"");
16814 case PPC::BI__builtin_altivec_stvx:
16815 case PPC::BI__builtin_altivec_stvxl:
16816 case PPC::BI__builtin_altivec_stvebx:
16817 case PPC::BI__builtin_altivec_stvehx:
16818 case PPC::BI__builtin_altivec_stvewx:
16819 case PPC::BI__builtin_vsx_stxvd2x:
16820 case PPC::BI__builtin_vsx_stxvw4x:
16821 case PPC::BI__builtin_vsx_stxvd2x_be:
16822 case PPC::BI__builtin_vsx_stxvw4x_be:
16823 case PPC::BI__builtin_vsx_stxvl:
16824 case PPC::BI__builtin_vsx_stxvll:
16830 if (!(BuiltinID == PPC::BI__builtin_vsx_stxvl ||
16831 BuiltinID == PPC::BI__builtin_vsx_stxvll)) {
16836 switch (BuiltinID) {
16837 default: llvm_unreachable(
"Unsupported st intrinsic!");
16838 case PPC::BI__builtin_altivec_stvx:
16839 ID = Intrinsic::ppc_altivec_stvx;
16841 case PPC::BI__builtin_altivec_stvxl:
16842 ID = Intrinsic::ppc_altivec_stvxl;
16844 case PPC::BI__builtin_altivec_stvebx:
16845 ID = Intrinsic::ppc_altivec_stvebx;
16847 case PPC::BI__builtin_altivec_stvehx:
16848 ID = Intrinsic::ppc_altivec_stvehx;
16850 case PPC::BI__builtin_altivec_stvewx:
16851 ID = Intrinsic::ppc_altivec_stvewx;
16853 case PPC::BI__builtin_vsx_stxvd2x:
16854 ID = Intrinsic::ppc_vsx_stxvd2x;
16856 case PPC::BI__builtin_vsx_stxvw4x:
16857 ID = Intrinsic::ppc_vsx_stxvw4x;
16859 case PPC::BI__builtin_vsx_stxvd2x_be:
16860 ID = Intrinsic::ppc_vsx_stxvd2x_be;
16862 case PPC::BI__builtin_vsx_stxvw4x_be:
16863 ID = Intrinsic::ppc_vsx_stxvw4x_be;
16865 case PPC::BI__builtin_vsx_stxvl:
16866 ID = Intrinsic::ppc_vsx_stxvl;
16868 case PPC::BI__builtin_vsx_stxvll:
16869 ID = Intrinsic::ppc_vsx_stxvll;
16873 return Builder.CreateCall(F, Ops,
"");
16875 case PPC::BI__builtin_vsx_ldrmb: {
16881 int64_t NumBytes = cast<ConstantInt>(Op1)->getZExtValue();
16886 if (NumBytes == 16) {
16894 for (
int Idx = 0; Idx < 16; Idx++)
16895 RevMask.push_back(15 - Idx);
16896 return Builder.CreateShuffleVector(LD, LD, RevMask);
16900 llvm::Function *Lvs =
CGM.
getIntrinsic(IsLE ? Intrinsic::ppc_altivec_lvsr
16901 : Intrinsic::ppc_altivec_lvsl);
16902 llvm::Function *Vperm =
CGM.
getIntrinsic(Intrinsic::ppc_altivec_vperm);
16904 Int8Ty, Op0, ConstantInt::get(Op1->
getType(), NumBytes - 1));
16906 Value *HiLd =
Builder.CreateCall(Lvx, HiMem,
"ld.hi");
16909 Op0 = IsLE ? HiLd : LoLd;
16910 Op1 = IsLE ? LoLd : HiLd;
16911 Value *AllElts =
Builder.CreateCall(Vperm, {Op0, Op1, Mask1},
"shuffle1");
16912 Constant *
Zero = llvm::Constant::getNullValue(IsLE ? ResTy : AllElts->
getType());
16916 for (
int Idx = 0; Idx < 16; Idx++) {
16917 int Val = (NumBytes - Idx - 1 >= 0) ? (NumBytes - Idx - 1)
16918 : 16 - (NumBytes - Idx);
16919 Consts.push_back(Val);
16921 return Builder.CreateShuffleVector(
Builder.CreateBitCast(AllElts, ResTy),
16925 for (
int Idx = 0; Idx < 16; Idx++)
16926 Consts.push_back(
Builder.getInt8(NumBytes + Idx));
16927 Value *Mask2 = ConstantVector::get(Consts);
16928 return Builder.CreateBitCast(
16929 Builder.CreateCall(Vperm, {Zero, AllElts, Mask2},
"shuffle2"), ResTy);
16931 case PPC::BI__builtin_vsx_strmb: {
16935 int64_t NumBytes = cast<ConstantInt>(Op1)->getZExtValue();
16937 auto StoreSubVec = [&](
unsigned Width,
unsigned Offset,
unsigned EltNo) {
16941 Value *StVec = Op2;
16944 for (
int Idx = 0; Idx < 16; Idx++)
16945 RevMask.push_back(15 - Idx);
16946 StVec =
Builder.CreateShuffleVector(Op2, Op2, RevMask);
16952 unsigned NumElts = 0;
16955 llvm_unreachable(
"width for stores must be a power of 2");
16974 Op2, llvm::FixedVectorType::get(ConvTy, NumElts));
16977 Value *Elt =
Builder.CreateExtractElement(Vec, EltNo);
16978 if (IsLE && Width > 1) {
16980 Elt =
Builder.CreateCall(F, Elt);
16985 unsigned Stored = 0;
16986 unsigned RemainingBytes = NumBytes;
16988 if (NumBytes == 16)
16989 return StoreSubVec(16, 0, 0);
16990 if (NumBytes >= 8) {
16991 Result = StoreSubVec(8, NumBytes - 8, IsLE ? 0 : 1);
16992 RemainingBytes -= 8;
16995 if (RemainingBytes >= 4) {
16996 Result = StoreSubVec(4, NumBytes - Stored - 4,
16997 IsLE ? (Stored >> 2) : 3 - (Stored >> 2));
16998 RemainingBytes -= 4;
17001 if (RemainingBytes >= 2) {
17002 Result = StoreSubVec(2, NumBytes - Stored - 2,
17003 IsLE ? (Stored >> 1) : 7 - (Stored >> 1));
17004 RemainingBytes -= 2;
17007 if (RemainingBytes)
17009 StoreSubVec(1, NumBytes - Stored - 1, IsLE ? Stored : 15 - Stored);
17013 case PPC::BI__builtin_vsx_xvsqrtsp:
17014 case PPC::BI__builtin_vsx_xvsqrtdp: {
17017 if (
Builder.getIsFPConstrained()) {
17019 Intrinsic::experimental_constrained_sqrt, ResultType);
17020 return Builder.CreateConstrainedFPCall(F,
X);
17027 case PPC::BI__builtin_altivec_vclzb:
17028 case PPC::BI__builtin_altivec_vclzh:
17029 case PPC::BI__builtin_altivec_vclzw:
17030 case PPC::BI__builtin_altivec_vclzd: {
17033 Value *Undef = ConstantInt::get(
Builder.getInt1Ty(),
false);
17035 return Builder.CreateCall(F, {
X, Undef});
17037 case PPC::BI__builtin_altivec_vctzb:
17038 case PPC::BI__builtin_altivec_vctzh:
17039 case PPC::BI__builtin_altivec_vctzw:
17040 case PPC::BI__builtin_altivec_vctzd: {
17043 Value *Undef = ConstantInt::get(
Builder.getInt1Ty(),
false);
17045 return Builder.CreateCall(F, {
X, Undef});
17047 case PPC::BI__builtin_altivec_vinsd:
17048 case PPC::BI__builtin_altivec_vinsw:
17049 case PPC::BI__builtin_altivec_vinsd_elt:
17050 case PPC::BI__builtin_altivec_vinsw_elt: {
17056 bool IsUnaligned = (BuiltinID == PPC::BI__builtin_altivec_vinsw ||
17057 BuiltinID == PPC::BI__builtin_altivec_vinsd);
17059 bool Is32bit = (BuiltinID == PPC::BI__builtin_altivec_vinsw ||
17060 BuiltinID == PPC::BI__builtin_altivec_vinsw_elt);
17063 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
17065 "Third Arg to vinsw/vinsd intrinsic must be a constant integer!");
17069 int ValidMaxValue = 0;
17071 ValidMaxValue = (Is32bit) ? 12 : 8;
17073 ValidMaxValue = (Is32bit) ? 3 : 1;
17076 int64_t ConstArg = ArgCI->getSExtValue();
17079 std::string RangeErrMsg = IsUnaligned ?
"byte" :
"element";
17080 RangeErrMsg +=
" number " + llvm::to_string(ConstArg);
17081 RangeErrMsg +=
" is outside of the valid range [0, ";
17082 RangeErrMsg += llvm::to_string(ValidMaxValue) +
"]";
17085 if (ConstArg < 0 || ConstArg > ValidMaxValue)
17089 if (!IsUnaligned) {
17090 ConstArg *= Is32bit ? 4 : 8;
17093 ConstArg = (Is32bit ? 12 : 8) - ConstArg;
17096 ID = Is32bit ? Intrinsic::ppc_altivec_vinsw : Intrinsic::ppc_altivec_vinsd;
17097 Op2 = ConstantInt::getSigned(
Int32Ty, ConstArg);
17101 ?
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int32Ty, 4))
17103 llvm::FixedVectorType::get(
Int64Ty, 2));
17104 return Builder.CreateBitCast(
17107 case PPC::BI__builtin_altivec_vpopcntb:
17108 case PPC::BI__builtin_altivec_vpopcnth:
17109 case PPC::BI__builtin_altivec_vpopcntw:
17110 case PPC::BI__builtin_altivec_vpopcntd: {
17116 case PPC::BI__builtin_altivec_vadduqm:
17117 case PPC::BI__builtin_altivec_vsubuqm: {
17120 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
17121 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(Int128Ty, 1));
17122 Op1 =
Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(Int128Ty, 1));
17123 if (BuiltinID == PPC::BI__builtin_altivec_vadduqm)
17124 return Builder.CreateAdd(Op0, Op1,
"vadduqm");
17126 return Builder.CreateSub(Op0, Op1,
"vsubuqm");
17128 case PPC::BI__builtin_altivec_vaddcuq_c:
17129 case PPC::BI__builtin_altivec_vsubcuq_c: {
17133 llvm::Type *V1I128Ty = llvm::FixedVectorType::get(
17135 Ops.push_back(
Builder.CreateBitCast(Op0, V1I128Ty));
17136 Ops.push_back(
Builder.CreateBitCast(Op1, V1I128Ty));
17137 ID = (BuiltinID == PPC::BI__builtin_altivec_vaddcuq_c)
17138 ? Intrinsic::ppc_altivec_vaddcuq
17139 : Intrinsic::ppc_altivec_vsubcuq;
17142 case PPC::BI__builtin_altivec_vaddeuqm_c:
17143 case PPC::BI__builtin_altivec_vaddecuq_c:
17144 case PPC::BI__builtin_altivec_vsubeuqm_c:
17145 case PPC::BI__builtin_altivec_vsubecuq_c: {
17150 llvm::Type *V1I128Ty = llvm::FixedVectorType::get(
17152 Ops.push_back(
Builder.CreateBitCast(Op0, V1I128Ty));
17153 Ops.push_back(
Builder.CreateBitCast(Op1, V1I128Ty));
17154 Ops.push_back(
Builder.CreateBitCast(Op2, V1I128Ty));
17155 switch (BuiltinID) {
17157 llvm_unreachable(
"Unsupported intrinsic!");
17158 case PPC::BI__builtin_altivec_vaddeuqm_c:
17159 ID = Intrinsic::ppc_altivec_vaddeuqm;
17161 case PPC::BI__builtin_altivec_vaddecuq_c:
17162 ID = Intrinsic::ppc_altivec_vaddecuq;
17164 case PPC::BI__builtin_altivec_vsubeuqm_c:
17165 ID = Intrinsic::ppc_altivec_vsubeuqm;
17167 case PPC::BI__builtin_altivec_vsubecuq_c:
17168 ID = Intrinsic::ppc_altivec_vsubecuq;
17173 case PPC::BI__builtin_ppc_rldimi:
17174 case PPC::BI__builtin_ppc_rlwimi: {
17181 ? Intrinsic::ppc_rldimi
17182 : Intrinsic::ppc_rlwimi),
17183 {Op0, Op1, Op2, Op3});
17185 case PPC::BI__builtin_ppc_rlwnm: {
17192 case PPC::BI__builtin_ppc_poppar4:
17193 case PPC::BI__builtin_ppc_poppar8: {
17195 llvm::Type *ArgType = Op0->
getType();
17201 if (
Result->getType() != ResultType)
17206 case PPC::BI__builtin_ppc_cmpb: {
17209 if (
getTarget().getTriple().isPPC64()) {
17212 return Builder.CreateCall(F, {Op0, Op1},
"cmpb");
17232 Constant *ShiftAmt = ConstantInt::get(
Int64Ty, 32);
17241 Value *ResHi =
Builder.CreateShl(ResHiShift, ShiftAmt);
17242 return Builder.CreateOr(ResLo, ResHi);
17245 case PPC::BI__builtin_vsx_xvcpsgnsp:
17246 case PPC::BI__builtin_vsx_xvcpsgndp: {
17250 ID = Intrinsic::copysign;
17252 return Builder.CreateCall(F, {
X, Y});
17255 case PPC::BI__builtin_vsx_xvrspip:
17256 case PPC::BI__builtin_vsx_xvrdpip:
17257 case PPC::BI__builtin_vsx_xvrdpim:
17258 case PPC::BI__builtin_vsx_xvrspim:
17259 case PPC::BI__builtin_vsx_xvrdpi:
17260 case PPC::BI__builtin_vsx_xvrspi:
17261 case PPC::BI__builtin_vsx_xvrdpic:
17262 case PPC::BI__builtin_vsx_xvrspic:
17263 case PPC::BI__builtin_vsx_xvrdpiz:
17264 case PPC::BI__builtin_vsx_xvrspiz: {
17267 if (BuiltinID == PPC::BI__builtin_vsx_xvrdpim ||
17268 BuiltinID == PPC::BI__builtin_vsx_xvrspim)
17270 ? Intrinsic::experimental_constrained_floor
17271 : Intrinsic::floor;
17272 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpi ||
17273 BuiltinID == PPC::BI__builtin_vsx_xvrspi)
17275 ? Intrinsic::experimental_constrained_round
17276 : Intrinsic::round;
17277 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpic ||
17278 BuiltinID == PPC::BI__builtin_vsx_xvrspic)
17280 ? Intrinsic::experimental_constrained_rint
17282 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpip ||
17283 BuiltinID == PPC::BI__builtin_vsx_xvrspip)
17285 ? Intrinsic::experimental_constrained_ceil
17287 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpiz ||
17288 BuiltinID == PPC::BI__builtin_vsx_xvrspiz)
17290 ? Intrinsic::experimental_constrained_trunc
17291 : Intrinsic::trunc;
17293 return Builder.getIsFPConstrained() ?
Builder.CreateConstrainedFPCall(F,
X)
17298 case PPC::BI__builtin_vsx_xvabsdp:
17299 case PPC::BI__builtin_vsx_xvabssp: {
17307 case PPC::BI__builtin_ppc_recipdivf:
17308 case PPC::BI__builtin_ppc_recipdivd:
17309 case PPC::BI__builtin_ppc_rsqrtf:
17310 case PPC::BI__builtin_ppc_rsqrtd: {
17311 FastMathFlags FMF =
Builder.getFastMathFlags();
17312 Builder.getFastMathFlags().setFast();
17316 if (BuiltinID == PPC::BI__builtin_ppc_recipdivf ||
17317 BuiltinID == PPC::BI__builtin_ppc_recipdivd) {
17320 Builder.getFastMathFlags() &= (FMF);
17323 auto *One = ConstantFP::get(ResultType, 1.0);
17326 Builder.getFastMathFlags() &= (FMF);
17329 case PPC::BI__builtin_ppc_alignx: {
17332 ConstantInt *AlignmentCI = cast<ConstantInt>(Op0);
17333 if (AlignmentCI->getValue().ugt(llvm::Value::MaximumAlignment))
17334 AlignmentCI = ConstantInt::get(AlignmentCI->getIntegerType(),
17335 llvm::Value::MaximumAlignment);
17339 AlignmentCI,
nullptr);
17342 case PPC::BI__builtin_ppc_rdlam: {
17346 llvm::Type *Ty = Op0->
getType();
17347 Value *ShiftAmt =
Builder.CreateIntCast(Op1, Ty,
false);
17349 Value *Rotate =
Builder.CreateCall(F, {Op0, Op0, ShiftAmt});
17350 return Builder.CreateAnd(Rotate, Op2);
17352 case PPC::BI__builtin_ppc_load2r: {
17359 case PPC::BI__builtin_ppc_fnmsub:
17360 case PPC::BI__builtin_ppc_fnmsubs:
17361 case PPC::BI__builtin_vsx_xvmaddadp:
17362 case PPC::BI__builtin_vsx_xvmaddasp:
17363 case PPC::BI__builtin_vsx_xvnmaddadp:
17364 case PPC::BI__builtin_vsx_xvnmaddasp:
17365 case PPC::BI__builtin_vsx_xvmsubadp:
17366 case PPC::BI__builtin_vsx_xvmsubasp:
17367 case PPC::BI__builtin_vsx_xvnmsubadp:
17368 case PPC::BI__builtin_vsx_xvnmsubasp: {
17374 if (
Builder.getIsFPConstrained())
17375 F =
CGM.
getIntrinsic(Intrinsic::experimental_constrained_fma, ResultType);
17378 switch (BuiltinID) {
17379 case PPC::BI__builtin_vsx_xvmaddadp:
17380 case PPC::BI__builtin_vsx_xvmaddasp:
17381 if (
Builder.getIsFPConstrained())
17382 return Builder.CreateConstrainedFPCall(F, {
X, Y, Z});
17384 return Builder.CreateCall(F, {
X, Y, Z});
17385 case PPC::BI__builtin_vsx_xvnmaddadp:
17386 case PPC::BI__builtin_vsx_xvnmaddasp:
17387 if (
Builder.getIsFPConstrained())
17389 Builder.CreateConstrainedFPCall(F, {X, Y, Z}),
"neg");
17391 return Builder.CreateFNeg(
Builder.CreateCall(F, {X, Y, Z}),
"neg");
17392 case PPC::BI__builtin_vsx_xvmsubadp:
17393 case PPC::BI__builtin_vsx_xvmsubasp:
17394 if (
Builder.getIsFPConstrained())
17395 return Builder.CreateConstrainedFPCall(
17396 F, {
X, Y,
Builder.CreateFNeg(Z,
"neg")});
17399 case PPC::BI__builtin_ppc_fnmsub:
17400 case PPC::BI__builtin_ppc_fnmsubs:
17401 case PPC::BI__builtin_vsx_xvnmsubadp:
17402 case PPC::BI__builtin_vsx_xvnmsubasp:
17403 if (
Builder.getIsFPConstrained())
17405 Builder.CreateConstrainedFPCall(
17406 F, {X, Y, Builder.CreateFNeg(Z,
"neg")}),
17412 llvm_unreachable(
"Unknown FMA operation");
17416 case PPC::BI__builtin_vsx_insertword: {
17424 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
17426 "Third arg to xxinsertw intrinsic must be constant integer");
17428 int64_t Index = std::clamp(ArgCI->getSExtValue(), (int64_t)0, MaxIndex);
17435 std::swap(Op0, Op1);
17439 Op1 =
Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(
Int64Ty, 2));
17443 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int64Ty, 2));
17447 Index = MaxIndex - Index;
17451 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int32Ty, 4));
17452 Op2 = ConstantInt::getSigned(
Int32Ty, Index);
17453 return Builder.CreateCall(F, {Op0, Op1, Op2});
17456 case PPC::BI__builtin_vsx_extractuword: {
17459 llvm::Function *F =
CGM.
getIntrinsic(Intrinsic::ppc_vsx_xxextractuw);
17462 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int64Ty, 2));
17466 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op1);
17468 "Second Arg to xxextractuw intrinsic must be a constant integer!");
17470 int64_t Index = std::clamp(ArgCI->getSExtValue(), (int64_t)0, MaxIndex);
17474 Index = MaxIndex - Index;
17475 Op1 = ConstantInt::getSigned(
Int32Ty, Index);
17480 Value *ShuffleCall =
17482 return ShuffleCall;
17484 Op1 = ConstantInt::getSigned(
Int32Ty, Index);
17485 return Builder.CreateCall(F, {Op0, Op1});
17489 case PPC::BI__builtin_vsx_xxpermdi: {
17493 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
17494 assert(ArgCI &&
"Third arg must be constant integer!");
17496 unsigned Index = ArgCI->getZExtValue();
17497 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int64Ty, 2));
17498 Op1 =
Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(
Int64Ty, 2));
17503 int ElemIdx0 = (Index & 2) >> 1;
17504 int ElemIdx1 = 2 + (Index & 1);
17506 int ShuffleElts[2] = {ElemIdx0, ElemIdx1};
17507 Value *ShuffleCall =
Builder.CreateShuffleVector(Op0, Op1, ShuffleElts);
17510 return Builder.CreateBitCast(ShuffleCall, RetTy);
17513 case PPC::BI__builtin_vsx_xxsldwi: {
17517 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
17518 assert(ArgCI &&
"Third argument must be a compile time constant");
17519 unsigned Index = ArgCI->getZExtValue() & 0x3;
17520 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int32Ty, 4));
17521 Op1 =
Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(
Int32Ty, 4));
17532 ElemIdx0 = (8 - Index) % 8;
17533 ElemIdx1 = (9 - Index) % 8;
17534 ElemIdx2 = (10 - Index) % 8;
17535 ElemIdx3 = (11 - Index) % 8;
17539 ElemIdx1 = Index + 1;
17540 ElemIdx2 = Index + 2;
17541 ElemIdx3 = Index + 3;
17544 int ShuffleElts[4] = {ElemIdx0, ElemIdx1, ElemIdx2, ElemIdx3};
17545 Value *ShuffleCall =
Builder.CreateShuffleVector(Op0, Op1, ShuffleElts);
17548 return Builder.CreateBitCast(ShuffleCall, RetTy);
17551 case PPC::BI__builtin_pack_vector_int128: {
17555 Value *PoisonValue =
17556 llvm::PoisonValue::get(llvm::FixedVectorType::get(Op0->
getType(), 2));
17558 PoisonValue, Op0, (uint64_t)(isLittleEndian ? 1 : 0));
17559 Res =
Builder.CreateInsertElement(Res, Op1,
17560 (uint64_t)(isLittleEndian ? 0 : 1));
17564 case PPC::BI__builtin_unpack_vector_int128: {
17567 ConstantInt *Index = cast<ConstantInt>(Op1);
17573 ConstantInt::get(Index->getIntegerType(), 1 - Index->getZExtValue());
17575 return Builder.CreateExtractElement(Unpacked, Index);
17578 case PPC::BI__builtin_ppc_sthcx: {
17582 return Builder.CreateCall(F, {Op0, Op1});
17591#define CUSTOM_BUILTIN(Name, Intr, Types, Accumulate, Feature) \
17592 case PPC::BI__builtin_##Name:
17593#include "clang/Basic/BuiltinsPPC.def"
17596 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; i++)
17605 if (BuiltinID == PPC::BI__builtin_mma_disassemble_acc ||
17606 BuiltinID == PPC::BI__builtin_vsx_disassemble_pair ||
17607 BuiltinID == PPC::BI__builtin_mma_disassemble_pair) {
17608 unsigned NumVecs = 2;
17609 auto Intrinsic = Intrinsic::ppc_vsx_disassemble_pair;
17610 if (BuiltinID == PPC::BI__builtin_mma_disassemble_acc) {
17612 Intrinsic = Intrinsic::ppc_mma_disassemble_acc;
17618 llvm::Type *VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
17619 Value *Ptr = Ops[0];
17620 for (
unsigned i=0; i<NumVecs; i++) {
17622 llvm::ConstantInt* Index = llvm::ConstantInt::get(
IntTy, i);
17623 Value *GEP =
Builder.CreateInBoundsGEP(VTy, Ptr, Index);
17628 if (BuiltinID == PPC::BI__builtin_vsx_build_pair ||
17629 BuiltinID == PPC::BI__builtin_mma_build_acc) {
17637 std::reverse(Ops.begin() + 1, Ops.end());
17640 switch (BuiltinID) {
17641 #define CUSTOM_BUILTIN(Name, Intr, Types, Acc, Feature) \
17642 case PPC::BI__builtin_##Name: \
17643 ID = Intrinsic::ppc_##Intr; \
17644 Accumulate = Acc; \
17646 #include "clang/Basic/BuiltinsPPC.def"
17648 if (BuiltinID == PPC::BI__builtin_vsx_lxvp ||
17649 BuiltinID == PPC::BI__builtin_vsx_stxvp ||
17650 BuiltinID == PPC::BI__builtin_mma_lxvp ||
17651 BuiltinID == PPC::BI__builtin_mma_stxvp) {
17652 if (BuiltinID == PPC::BI__builtin_vsx_lxvp ||
17653 BuiltinID == PPC::BI__builtin_mma_lxvp) {
17660 return Builder.CreateCall(F, Ops,
"");
17666 CallOps.push_back(Acc);
17668 for (
unsigned i=1; i<Ops.size(); i++)
17669 CallOps.push_back(Ops[i]);
17675 case PPC::BI__builtin_ppc_compare_and_swap:
17676 case PPC::BI__builtin_ppc_compare_and_swaplp: {
17685 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Monotonic,
true);
17693 Value *LoadedVal = Pair.first.getScalarVal();
17697 case PPC::BI__builtin_ppc_fetch_and_add:
17698 case PPC::BI__builtin_ppc_fetch_and_addlp: {
17700 llvm::AtomicOrdering::Monotonic);
17702 case PPC::BI__builtin_ppc_fetch_and_and:
17703 case PPC::BI__builtin_ppc_fetch_and_andlp: {
17705 llvm::AtomicOrdering::Monotonic);
17708 case PPC::BI__builtin_ppc_fetch_and_or:
17709 case PPC::BI__builtin_ppc_fetch_and_orlp: {
17711 llvm::AtomicOrdering::Monotonic);
17713 case PPC::BI__builtin_ppc_fetch_and_swap:
17714 case PPC::BI__builtin_ppc_fetch_and_swaplp: {
17716 llvm::AtomicOrdering::Monotonic);
17718 case PPC::BI__builtin_ppc_ldarx:
17719 case PPC::BI__builtin_ppc_lwarx:
17720 case PPC::BI__builtin_ppc_lharx:
17721 case PPC::BI__builtin_ppc_lbarx:
17723 case PPC::BI__builtin_ppc_mfspr: {
17729 return Builder.CreateCall(F, {Op0});
17731 case PPC::BI__builtin_ppc_mtspr: {
17738 return Builder.CreateCall(F, {Op0, Op1});
17740 case PPC::BI__builtin_ppc_popcntb: {
17742 llvm::Type *ArgType = ArgValue->
getType();
17744 return Builder.CreateCall(F, {ArgValue},
"popcntb");
17746 case PPC::BI__builtin_ppc_mtfsf: {
17756 case PPC::BI__builtin_ppc_swdiv_nochk:
17757 case PPC::BI__builtin_ppc_swdivs_nochk: {
17760 FastMathFlags FMF =
Builder.getFastMathFlags();
17761 Builder.getFastMathFlags().setFast();
17762 Value *FDiv =
Builder.CreateFDiv(Op0, Op1,
"swdiv_nochk");
17763 Builder.getFastMathFlags() &= (FMF);
17766 case PPC::BI__builtin_ppc_fric:
17768 *
this, E, Intrinsic::rint,
17769 Intrinsic::experimental_constrained_rint))
17771 case PPC::BI__builtin_ppc_frim:
17772 case PPC::BI__builtin_ppc_frims:
17774 *
this, E, Intrinsic::floor,
17775 Intrinsic::experimental_constrained_floor))
17777 case PPC::BI__builtin_ppc_frin:
17778 case PPC::BI__builtin_ppc_frins:
17780 *
this, E, Intrinsic::round,
17781 Intrinsic::experimental_constrained_round))
17783 case PPC::BI__builtin_ppc_frip:
17784 case PPC::BI__builtin_ppc_frips:
17786 *
this, E, Intrinsic::ceil,
17787 Intrinsic::experimental_constrained_ceil))
17789 case PPC::BI__builtin_ppc_friz:
17790 case PPC::BI__builtin_ppc_frizs:
17792 *
this, E, Intrinsic::trunc,
17793 Intrinsic::experimental_constrained_trunc))
17795 case PPC::BI__builtin_ppc_fsqrt:
17796 case PPC::BI__builtin_ppc_fsqrts:
17798 *
this, E, Intrinsic::sqrt,
17799 Intrinsic::experimental_constrained_sqrt))
17801 case PPC::BI__builtin_ppc_test_data_class: {
17806 {Op0, Op1},
"test_data_class");
17808 case PPC::BI__builtin_ppc_maxfe: {
17814 {Op0, Op1, Op2, Op3});
17816 case PPC::BI__builtin_ppc_maxfl: {
17822 {Op0, Op1, Op2, Op3});
17824 case PPC::BI__builtin_ppc_maxfs: {
17830 {Op0, Op1, Op2, Op3});
17832 case PPC::BI__builtin_ppc_minfe: {
17838 {Op0, Op1, Op2, Op3});
17840 case PPC::BI__builtin_ppc_minfl: {
17846 {Op0, Op1, Op2, Op3});
17848 case PPC::BI__builtin_ppc_minfs: {
17854 {Op0, Op1, Op2, Op3});
17856 case PPC::BI__builtin_ppc_swdiv:
17857 case PPC::BI__builtin_ppc_swdivs: {
17860 return Builder.CreateFDiv(Op0, Op1,
"swdiv");
17862 case PPC::BI__builtin_ppc_set_fpscr_rn:
17864 {EmitScalarExpr(E->getArg(0))});
17865 case PPC::BI__builtin_ppc_mffs:
17878 Attribute::getWithDereferenceableBytes(
Call->getContext(), 64));
17879 Call->addRetAttr(Attribute::getWithAlignment(
Call->getContext(), Align(4)));
17883 auto *RetTy = cast<llvm::PointerType>(CGF.
ConvertType(BuiltinRetType));
17884 if (RetTy ==
Call->getType())
17893 Attribute::getWithDereferenceableBytes(
Call->getContext(), 256));
17894 Call->addRetAttr(Attribute::getWithAlignment(
Call->getContext(), Align(8)));
17909 llvm::LoadInst *LD;
17913 if (Cov == CodeObjectVersionKind::COV_None) {
17914 StringRef Name =
"__oclc_ABI_version";
17915 auto *ABIVersionC = CGF.
CGM.
getModule().getNamedGlobal(Name);
17917 ABIVersionC =
new llvm::GlobalVariable(
17919 llvm::GlobalValue::ExternalLinkage,
nullptr, Name,
nullptr,
17920 llvm::GlobalVariable::NotThreadLocal,
17931 llvm::ConstantInt::get(CGF.
Int32Ty, CodeObjectVersionKind::COV_5));
17935 CGF.
Int8Ty, EmitAMDGPUImplicitArgPtr(CGF), 12 + Index * 2);
17939 CGF.
Int8Ty, EmitAMDGPUDispatchPtr(CGF), 4 + Index * 2);
17941 auto Result = CGF.
Builder.CreateSelect(IsCOV5, ImplicitGEP, DispatchGEP);
17945 Value *GEP =
nullptr;
17946 if (Cov >= CodeObjectVersionKind::COV_5) {
17948 GEP = CGF.
Builder.CreateConstGEP1_32(
17949 CGF.
Int8Ty, EmitAMDGPUImplicitArgPtr(CGF), 12 + Index * 2);
17952 GEP = CGF.
Builder.CreateConstGEP1_32(
17953 CGF.
Int8Ty, EmitAMDGPUDispatchPtr(CGF), 4 + Index * 2);
17960 llvm::MDNode *RNode = MDHelper.createRange(
APInt(16, 1),
17962 LD->setMetadata(llvm::LLVMContext::MD_range, RNode);
17963 LD->setMetadata(llvm::LLVMContext::MD_noundef,
17965 LD->setMetadata(llvm::LLVMContext::MD_invariant_load,
17972 const unsigned XOffset = 12;
17973 auto *DP = EmitAMDGPUDispatchPtr(CGF);
17975 auto *Offset = llvm::ConstantInt::get(CGF.
Int32Ty, XOffset + Index * 4);
17979 LD->setMetadata(llvm::LLVMContext::MD_invariant_load,
17992 llvm::AtomicOrdering &AO,
17993 llvm::SyncScope::ID &SSID) {
17994 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
17997 assert(llvm::isValidAtomicOrderingCABI(ord));
17998 switch (
static_cast<llvm::AtomicOrderingCABI
>(ord)) {
17999 case llvm::AtomicOrderingCABI::acquire:
18000 case llvm::AtomicOrderingCABI::consume:
18001 AO = llvm::AtomicOrdering::Acquire;
18003 case llvm::AtomicOrderingCABI::release:
18004 AO = llvm::AtomicOrdering::Release;
18006 case llvm::AtomicOrderingCABI::acq_rel:
18007 AO = llvm::AtomicOrdering::AcquireRelease;
18009 case llvm::AtomicOrderingCABI::seq_cst:
18010 AO = llvm::AtomicOrdering::SequentiallyConsistent;
18012 case llvm::AtomicOrderingCABI::relaxed:
18013 AO = llvm::AtomicOrdering::Monotonic;
18018 llvm::getConstantStringInfo(
Scope, scp);
18025 llvm::Value *Arg =
nullptr;
18026 if ((ICEArguments & (1 << Idx)) == 0) {
18031 std::optional<llvm::APSInt>
Result =
18033 assert(
Result &&
"Expected argument to be a constant");
18044 switch (BuiltinID) {
18045 case Builtin::BI__builtin_hlsl_elementwise_any: {
18047 return Builder.CreateIntrinsic(
18051 case Builtin::BI__builtin_hlsl_elementwise_clamp: {
18057 bool IsUnsigned =
false;
18059 Ty = VecTy->getElementType();
18061 return Builder.CreateIntrinsic(
18063 IsUnsigned ? Intrinsic::dx_uclamp : Intrinsic::dx_clamp,
18066 case Builtin::BI__builtin_hlsl_dot: {
18069 llvm::Type *T0 = Op0->
getType();
18070 llvm::Type *T1 = Op1->
getType();
18071 if (!T0->isVectorTy() && !T1->isVectorTy()) {
18072 if (T0->isFloatingPointTy())
18073 return Builder.CreateFMul(Op0, Op1,
"dx.dot");
18075 if (T0->isIntegerTy())
18076 return Builder.CreateMul(Op0, Op1,
"dx.dot");
18080 "Scalar dot product is only supported on ints and floats.");
18083 assert(T0->isVectorTy() && T1->isVectorTy() &&
18084 "Dot product of vector and scalar is not supported.");
18087 assert(T0->getScalarType() == T1->getScalarType() &&
18088 "Dot product of vectors need the same element types.");
18090 [[maybe_unused]]
auto *VecTy0 =
18092 [[maybe_unused]]
auto *VecTy1 =
18096 "Dot product requires vectors to be of the same size.");
18098 return Builder.CreateIntrinsic(
18099 T0->getScalarType(), Intrinsic::dx_dot,
18102 case Builtin::BI__builtin_hlsl_lerp: {
18107 llvm_unreachable(
"lerp operand must have a float representation");
18108 return Builder.CreateIntrinsic(
18109 X->getType(), Intrinsic::dx_lerp,
18112 case Builtin::BI__builtin_hlsl_elementwise_frac: {
18115 llvm_unreachable(
"frac operand must have a float representation");
18116 return Builder.CreateIntrinsic(
18117 Op0->
getType(), Intrinsic::dx_frac,
18120 case Builtin::BI__builtin_hlsl_elementwise_isinf: {
18122 llvm::Type *Xty = Op0->
getType();
18123 llvm::Type *retType = llvm::Type::getInt1Ty(this->
getLLVMContext());
18124 if (Xty->isVectorTy()) {
18126 retType = llvm::VectorType::get(
18127 retType, ElementCount::getFixed(XVecTy->getNumElements()));
18130 llvm_unreachable(
"isinf operand must have a float representation");
18131 return Builder.CreateIntrinsic(retType, Intrinsic::dx_isinf,
18134 case Builtin::BI__builtin_hlsl_mad: {
18139 return Builder.CreateIntrinsic(
18140 M->
getType(), Intrinsic::fmuladd,
18144 return Builder.CreateIntrinsic(
18145 M->
getType(), Intrinsic::dx_imad,
18149 return Builder.CreateIntrinsic(
18150 M->
getType(), Intrinsic::dx_umad,
18153 case Builtin::BI__builtin_hlsl_elementwise_rcp: {
18156 llvm_unreachable(
"rcp operand must have a float representation");
18157 return Builder.CreateIntrinsic(
18158 Op0->
getType(), Intrinsic::dx_rcp,
18161 case Builtin::BI__builtin_hlsl_elementwise_rsqrt: {
18164 llvm_unreachable(
"rsqrt operand must have a float representation");
18165 return Builder.CreateIntrinsic(
18166 Op0->
getType(), Intrinsic::dx_rsqrt,
18175 llvm::AtomicOrdering AO = llvm::AtomicOrdering::SequentiallyConsistent;
18176 llvm::SyncScope::ID SSID;
18177 switch (BuiltinID) {
18178 case AMDGPU::BI__builtin_amdgcn_div_scale:
18179 case AMDGPU::BI__builtin_amdgcn_div_scalef: {
18192 llvm::Value *Tmp =
Builder.CreateCall(Callee, {
X, Y, Z});
18195 llvm::Value *Flag =
Builder.CreateExtractValue(Tmp, 1);
18199 llvm::Value *FlagExt =
Builder.CreateZExt(Flag, RealFlagType);
18203 case AMDGPU::BI__builtin_amdgcn_div_fmas:
18204 case AMDGPU::BI__builtin_amdgcn_div_fmasf: {
18212 llvm::Value *Src3ToBool =
Builder.CreateIsNotNull(Src3);
18213 return Builder.CreateCall(F, {Src0, Src1, Src2, Src3ToBool});
18216 case AMDGPU::BI__builtin_amdgcn_ds_swizzle:
18218 case AMDGPU::BI__builtin_amdgcn_mov_dpp8:
18220 case AMDGPU::BI__builtin_amdgcn_mov_dpp:
18221 case AMDGPU::BI__builtin_amdgcn_update_dpp: {
18225 unsigned ICEArguments = 0;
18229 for (
unsigned I = 0; I != E->
getNumArgs(); ++I) {
18232 assert(Args.size() == 5 || Args.size() == 6);
18233 if (Args.size() == 5)
18234 Args.insert(Args.begin(), llvm::PoisonValue::get(Args[0]->getType()));
18237 return Builder.CreateCall(F, Args);
18239 case AMDGPU::BI__builtin_amdgcn_div_fixup:
18240 case AMDGPU::BI__builtin_amdgcn_div_fixupf:
18241 case AMDGPU::BI__builtin_amdgcn_div_fixuph:
18243 case AMDGPU::BI__builtin_amdgcn_trig_preop:
18244 case AMDGPU::BI__builtin_amdgcn_trig_preopf:
18246 case AMDGPU::BI__builtin_amdgcn_rcp:
18247 case AMDGPU::BI__builtin_amdgcn_rcpf:
18248 case AMDGPU::BI__builtin_amdgcn_rcph:
18250 case AMDGPU::BI__builtin_amdgcn_sqrt:
18251 case AMDGPU::BI__builtin_amdgcn_sqrtf:
18252 case AMDGPU::BI__builtin_amdgcn_sqrth:
18254 case AMDGPU::BI__builtin_amdgcn_rsq:
18255 case AMDGPU::BI__builtin_amdgcn_rsqf:
18256 case AMDGPU::BI__builtin_amdgcn_rsqh:
18258 case AMDGPU::BI__builtin_amdgcn_rsq_clamp:
18259 case AMDGPU::BI__builtin_amdgcn_rsq_clampf:
18261 case AMDGPU::BI__builtin_amdgcn_sinf:
18262 case AMDGPU::BI__builtin_amdgcn_sinh:
18264 case AMDGPU::BI__builtin_amdgcn_cosf:
18265 case AMDGPU::BI__builtin_amdgcn_cosh:
18267 case AMDGPU::BI__builtin_amdgcn_dispatch_ptr:
18268 return EmitAMDGPUDispatchPtr(*
this, E);
18269 case AMDGPU::BI__builtin_amdgcn_logf:
18271 case AMDGPU::BI__builtin_amdgcn_exp2f:
18273 case AMDGPU::BI__builtin_amdgcn_log_clampf:
18275 case AMDGPU::BI__builtin_amdgcn_ldexp:
18276 case AMDGPU::BI__builtin_amdgcn_ldexpf: {
18279 llvm::Function *F =
18280 CGM.
getIntrinsic(Intrinsic::ldexp, {Src0->getType(), Src1->getType()});
18281 return Builder.CreateCall(F, {Src0, Src1});
18283 case AMDGPU::BI__builtin_amdgcn_ldexph: {
18288 llvm::Function *F =
18292 case AMDGPU::BI__builtin_amdgcn_frexp_mant:
18293 case AMDGPU::BI__builtin_amdgcn_frexp_mantf:
18294 case AMDGPU::BI__builtin_amdgcn_frexp_manth:
18296 case AMDGPU::BI__builtin_amdgcn_frexp_exp:
18297 case AMDGPU::BI__builtin_amdgcn_frexp_expf: {
18301 return Builder.CreateCall(F, Src0);
18303 case AMDGPU::BI__builtin_amdgcn_frexp_exph: {
18307 return Builder.CreateCall(F, Src0);
18309 case AMDGPU::BI__builtin_amdgcn_fract:
18310 case AMDGPU::BI__builtin_amdgcn_fractf:
18311 case AMDGPU::BI__builtin_amdgcn_fracth:
18313 case AMDGPU::BI__builtin_amdgcn_lerp:
18315 case AMDGPU::BI__builtin_amdgcn_ubfe:
18317 case AMDGPU::BI__builtin_amdgcn_sbfe:
18319 case AMDGPU::BI__builtin_amdgcn_ballot_w32:
18320 case AMDGPU::BI__builtin_amdgcn_ballot_w64: {
18324 return Builder.CreateCall(F, { Src });
18326 case AMDGPU::BI__builtin_amdgcn_uicmp:
18327 case AMDGPU::BI__builtin_amdgcn_uicmpl:
18328 case AMDGPU::BI__builtin_amdgcn_sicmp:
18329 case AMDGPU::BI__builtin_amdgcn_sicmpl: {
18336 {
Builder.getInt64Ty(), Src0->getType() });
18337 return Builder.CreateCall(F, { Src0, Src1, Src2 });
18339 case AMDGPU::BI__builtin_amdgcn_fcmp:
18340 case AMDGPU::BI__builtin_amdgcn_fcmpf: {
18347 {
Builder.getInt64Ty(), Src0->getType() });
18348 return Builder.CreateCall(F, { Src0, Src1, Src2 });
18350 case AMDGPU::BI__builtin_amdgcn_class:
18351 case AMDGPU::BI__builtin_amdgcn_classf:
18352 case AMDGPU::BI__builtin_amdgcn_classh:
18354 case AMDGPU::BI__builtin_amdgcn_fmed3f:
18355 case AMDGPU::BI__builtin_amdgcn_fmed3h:
18357 case AMDGPU::BI__builtin_amdgcn_ds_append:
18358 case AMDGPU::BI__builtin_amdgcn_ds_consume: {
18359 Intrinsic::ID Intrin = BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_append ?
18360 Intrinsic::amdgcn_ds_append : Intrinsic::amdgcn_ds_consume;
18365 case AMDGPU::BI__builtin_amdgcn_ds_faddf:
18366 case AMDGPU::BI__builtin_amdgcn_ds_fminf:
18367 case AMDGPU::BI__builtin_amdgcn_ds_fmaxf: {
18368 Intrinsic::ID Intrin;
18369 switch (BuiltinID) {
18370 case AMDGPU::BI__builtin_amdgcn_ds_faddf:
18371 Intrin = Intrinsic::amdgcn_ds_fadd;
18373 case AMDGPU::BI__builtin_amdgcn_ds_fminf:
18374 Intrin = Intrinsic::amdgcn_ds_fmin;
18376 case AMDGPU::BI__builtin_amdgcn_ds_fmaxf:
18377 Intrin = Intrinsic::amdgcn_ds_fmax;
18386 llvm::FunctionType *FTy = F->getFunctionType();
18387 llvm::Type *PTy = FTy->getParamType(0);
18389 return Builder.CreateCall(F, { Src0, Src1, Src2, Src3, Src4 });
18391 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f64:
18392 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f32:
18393 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2f16:
18394 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmin_f64:
18395 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmax_f64:
18396 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f64:
18397 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmin_f64:
18398 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmax_f64:
18399 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f32:
18400 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2f16: {
18403 switch (BuiltinID) {
18404 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f32:
18406 IID = Intrinsic::amdgcn_global_atomic_fadd;
18408 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2f16:
18409 ArgTy = llvm::FixedVectorType::get(
18411 IID = Intrinsic::amdgcn_global_atomic_fadd;
18413 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f64:
18414 IID = Intrinsic::amdgcn_global_atomic_fadd;
18416 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmin_f64:
18417 IID = Intrinsic::amdgcn_global_atomic_fmin;
18419 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmax_f64:
18420 IID = Intrinsic::amdgcn_global_atomic_fmax;
18422 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f64:
18423 IID = Intrinsic::amdgcn_flat_atomic_fadd;
18425 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmin_f64:
18426 IID = Intrinsic::amdgcn_flat_atomic_fmin;
18428 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmax_f64:
18429 IID = Intrinsic::amdgcn_flat_atomic_fmax;
18431 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f32:
18433 IID = Intrinsic::amdgcn_flat_atomic_fadd;
18435 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2f16:
18436 ArgTy = llvm::FixedVectorType::get(
18438 IID = Intrinsic::amdgcn_flat_atomic_fadd;
18443 llvm::Function *F =
18445 return Builder.CreateCall(F, {Addr, Val});
18447 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2bf16:
18448 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2bf16: {
18450 switch (BuiltinID) {
18451 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2bf16:
18452 IID = Intrinsic::amdgcn_global_atomic_fadd_v2bf16;
18454 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2bf16:
18455 IID = Intrinsic::amdgcn_flat_atomic_fadd_v2bf16;
18461 return Builder.CreateCall(F, {Addr, Val});
18463 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f64:
18464 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f32:
18465 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2f16: {
18468 switch (BuiltinID) {
18469 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f32:
18471 IID = Intrinsic::amdgcn_ds_fadd;
18473 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f64:
18475 IID = Intrinsic::amdgcn_ds_fadd;
18477 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2f16:
18478 ArgTy = llvm::FixedVectorType::get(
18480 IID = Intrinsic::amdgcn_ds_fadd;
18485 llvm::Constant *ZeroI32 = llvm::ConstantInt::getIntegerValue(
18487 llvm::Constant *ZeroI1 = llvm::ConstantInt::getIntegerValue(
18490 return Builder.CreateCall(F, {Addr, Val, ZeroI32, ZeroI32, ZeroI1});
18492 case AMDGPU::BI__builtin_amdgcn_global_load_tr_i32:
18493 case AMDGPU::BI__builtin_amdgcn_global_load_tr_v2i32:
18494 case AMDGPU::BI__builtin_amdgcn_global_load_tr_v4f16:
18495 case AMDGPU::BI__builtin_amdgcn_global_load_tr_v4i16:
18496 case AMDGPU::BI__builtin_amdgcn_global_load_tr_v8f16:
18497 case AMDGPU::BI__builtin_amdgcn_global_load_tr_v8i16: {
18500 switch (BuiltinID) {
18501 case AMDGPU::BI__builtin_amdgcn_global_load_tr_i32:
18504 case AMDGPU::BI__builtin_amdgcn_global_load_tr_v2i32:
18505 ArgTy = llvm::FixedVectorType::get(
18508 case AMDGPU::BI__builtin_amdgcn_global_load_tr_v4f16:
18509 ArgTy = llvm::FixedVectorType::get(
18512 case AMDGPU::BI__builtin_amdgcn_global_load_tr_v4i16:
18513 ArgTy = llvm::FixedVectorType::get(
18516 case AMDGPU::BI__builtin_amdgcn_global_load_tr_v8f16:
18517 ArgTy = llvm::FixedVectorType::get(
18520 case AMDGPU::BI__builtin_amdgcn_global_load_tr_v8i16:
18521 ArgTy = llvm::FixedVectorType::get(
18527 llvm::Function *F =
18529 return Builder.CreateCall(F, {Addr});
18531 case AMDGPU::BI__builtin_amdgcn_get_fpenv: {
18534 return Builder.CreateCall(F);
18536 case AMDGPU::BI__builtin_amdgcn_set_fpenv: {
18542 case AMDGPU::BI__builtin_amdgcn_read_exec:
18544 case AMDGPU::BI__builtin_amdgcn_read_exec_lo:
18546 case AMDGPU::BI__builtin_amdgcn_read_exec_hi:
18548 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray:
18549 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_h:
18550 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_l:
18551 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_lh: {
18561 RayOrigin =
Builder.CreateShuffleVector(RayOrigin, RayOrigin,
18565 RayInverseDir =
Builder.CreateShuffleVector(RayInverseDir, RayInverseDir,
18569 {NodePtr->getType(), RayDir->getType()});
18570 return Builder.CreateCall(F, {NodePtr, RayExtent, RayOrigin, RayDir,
18571 RayInverseDir, TextureDescr});
18574 case AMDGPU::BI__builtin_amdgcn_ds_bvh_stack_rtn: {
18576 for (
int i = 0, e = E->
getNumArgs(); i != e; ++i)
18584 Value *I0 =
Builder.CreateInsertElement(PoisonValue::get(RetTy), Rtn,
18586 return Builder.CreateInsertElement(I0, A, 1);
18589 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32:
18590 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w32:
18591 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64:
18592 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w64:
18593 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32:
18594 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w32:
18595 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64:
18596 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w64:
18597 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32:
18598 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64:
18599 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32:
18600 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64:
18601 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32:
18602 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64:
18603 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32:
18604 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64:
18605 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32_gfx12:
18606 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64_gfx12:
18607 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32_gfx12:
18608 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64_gfx12:
18609 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32_gfx12:
18610 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64_gfx12:
18611 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12:
18612 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64_gfx12:
18613 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32_gfx12:
18614 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64_gfx12:
18615 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12:
18616 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64_gfx12:
18617 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w32_gfx12:
18618 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w64_gfx12:
18619 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w32_gfx12:
18620 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w64_gfx12:
18621 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w32_gfx12:
18622 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w64_gfx12:
18623 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w32_gfx12:
18624 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w64_gfx12:
18625 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w32_gfx12:
18626 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w64_gfx12:
18627 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w32:
18628 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w64:
18629 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w32:
18630 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w64:
18631 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w32:
18632 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w64:
18633 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w32:
18634 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w64:
18635 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w32:
18636 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w64:
18637 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w32:
18638 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w64:
18639 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w32:
18640 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w64:
18641 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w32:
18642 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w64:
18643 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w32:
18644 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w64:
18645 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w32:
18646 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w64:
18647 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w32:
18648 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w64: {
18661 bool AppendFalseForOpselArg =
false;
18662 unsigned BuiltinWMMAOp;
18664 switch (BuiltinID) {
18665 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32:
18666 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64:
18667 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12:
18668 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64_gfx12:
18669 ArgsForMatchingMatrixTypes = {2, 0};
18670 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_f16;
18672 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32:
18673 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64:
18674 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32_gfx12:
18675 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64_gfx12:
18676 ArgsForMatchingMatrixTypes = {2, 0};
18677 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_bf16;
18679 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32_gfx12:
18680 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64_gfx12:
18681 AppendFalseForOpselArg =
true;
18683 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32:
18684 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64:
18685 ArgsForMatchingMatrixTypes = {2, 0};
18686 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f16_16x16x16_f16;
18688 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32_gfx12:
18689 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64_gfx12:
18690 AppendFalseForOpselArg =
true;
18692 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32:
18693 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64:
18694 ArgsForMatchingMatrixTypes = {2, 0};
18695 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_bf16_16x16x16_bf16;
18697 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w32:
18698 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w64:
18699 ArgsForMatchingMatrixTypes = {2, 0};
18700 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f16_16x16x16_f16_tied;
18702 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w32:
18703 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w64:
18704 ArgsForMatchingMatrixTypes = {2, 0};
18705 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_bf16_16x16x16_bf16_tied;
18707 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32:
18708 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64:
18709 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12:
18710 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64_gfx12:
18711 ArgsForMatchingMatrixTypes = {4, 1};
18712 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x16_iu8;
18714 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32:
18715 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64:
18716 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32_gfx12:
18717 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64_gfx12:
18718 ArgsForMatchingMatrixTypes = {4, 1};
18719 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x16_iu4;
18721 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w32_gfx12:
18722 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w64_gfx12:
18723 ArgsForMatchingMatrixTypes = {2, 0};
18724 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_fp8_fp8;
18726 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w32_gfx12:
18727 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w64_gfx12:
18728 ArgsForMatchingMatrixTypes = {2, 0};
18729 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_fp8_bf8;
18731 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w32_gfx12:
18732 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w64_gfx12:
18733 ArgsForMatchingMatrixTypes = {2, 0};
18734 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_bf8_fp8;
18736 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w32_gfx12:
18737 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w64_gfx12:
18738 ArgsForMatchingMatrixTypes = {2, 0};
18739 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_bf8_bf8;
18741 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w32_gfx12:
18742 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w64_gfx12:
18743 ArgsForMatchingMatrixTypes = {4, 1};
18744 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x32_iu4;
18746 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w32:
18747 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w64:
18748 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
18749 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_f16;
18751 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w32:
18752 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w64:
18753 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
18754 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_bf16;
18756 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w32:
18757 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w64:
18758 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
18759 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f16_16x16x32_f16;
18761 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w32:
18762 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w64:
18763 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
18764 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_bf16_16x16x32_bf16;
18766 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w32:
18767 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w64:
18768 ArgsForMatchingMatrixTypes = {4, 1, 3, 5};
18769 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_i32_16x16x32_iu8;
18771 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w32:
18772 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w64:
18773 ArgsForMatchingMatrixTypes = {4, 1, 3, 5};
18774 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_i32_16x16x32_iu4;
18776 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w32:
18777 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w64:
18778 ArgsForMatchingMatrixTypes = {4, 1, 3, 5};
18779 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_i32_16x16x64_iu4;
18781 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w32:
18782 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w64:
18783 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
18784 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_fp8;
18786 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w32:
18787 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w64:
18788 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
18789 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_bf8;
18791 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w32:
18792 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w64:
18793 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
18794 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_fp8;
18796 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w32:
18797 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w64:
18798 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
18799 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_bf8;
18804 for (
int i = 0, e = E->
getNumArgs(); i != e; ++i)
18806 if (AppendFalseForOpselArg)
18807 Args.push_back(
Builder.getFalse());
18810 for (
auto ArgIdx : ArgsForMatchingMatrixTypes)
18811 ArgTypes.push_back(Args[ArgIdx]->getType());
18814 return Builder.CreateCall(F, Args);
18818 case AMDGPU::BI__builtin_amdgcn_workitem_id_x:
18820 case AMDGPU::BI__builtin_amdgcn_workitem_id_y:
18822 case AMDGPU::BI__builtin_amdgcn_workitem_id_z:
18826 case AMDGPU::BI__builtin_amdgcn_workgroup_size_x:
18827 return EmitAMDGPUWorkGroupSize(*
this, 0);
18828 case AMDGPU::BI__builtin_amdgcn_workgroup_size_y:
18829 return EmitAMDGPUWorkGroupSize(*
this, 1);
18830 case AMDGPU::BI__builtin_amdgcn_workgroup_size_z:
18831 return EmitAMDGPUWorkGroupSize(*
this, 2);
18834 case AMDGPU::BI__builtin_amdgcn_grid_size_x:
18835 return EmitAMDGPUGridSize(*
this, 0);
18836 case AMDGPU::BI__builtin_amdgcn_grid_size_y:
18837 return EmitAMDGPUGridSize(*
this, 1);
18838 case AMDGPU::BI__builtin_amdgcn_grid_size_z:
18839 return EmitAMDGPUGridSize(*
this, 2);
18842 case AMDGPU::BI__builtin_r600_recipsqrt_ieee:
18843 case AMDGPU::BI__builtin_r600_recipsqrt_ieeef:
18845 case AMDGPU::BI__builtin_r600_read_tidig_x:
18847 case AMDGPU::BI__builtin_r600_read_tidig_y:
18849 case AMDGPU::BI__builtin_r600_read_tidig_z:
18851 case AMDGPU::BI__builtin_amdgcn_alignbit: {
18856 return Builder.CreateCall(F, { Src0, Src1, Src2 });
18858 case AMDGPU::BI__builtin_amdgcn_fence: {
18861 return Builder.CreateFence(AO, SSID);
18863 case AMDGPU::BI__builtin_amdgcn_atomic_inc32:
18864 case AMDGPU::BI__builtin_amdgcn_atomic_inc64:
18865 case AMDGPU::BI__builtin_amdgcn_atomic_dec32:
18866 case AMDGPU::BI__builtin_amdgcn_atomic_dec64: {
18867 llvm::AtomicRMWInst::BinOp BinOp;
18868 switch (BuiltinID) {
18869 case AMDGPU::BI__builtin_amdgcn_atomic_inc32:
18870 case AMDGPU::BI__builtin_amdgcn_atomic_inc64:
18871 BinOp = llvm::AtomicRMWInst::UIncWrap;
18873 case AMDGPU::BI__builtin_amdgcn_atomic_dec32:
18874 case AMDGPU::BI__builtin_amdgcn_atomic_dec64:
18875 BinOp = llvm::AtomicRMWInst::UDecWrap;
18889 llvm::AtomicRMWInst *RMW =
18892 RMW->setVolatile(
true);
18895 case AMDGPU::BI__builtin_amdgcn_s_sendmsg_rtn:
18896 case AMDGPU::BI__builtin_amdgcn_s_sendmsg_rtnl: {
18902 return Builder.CreateCall(F, {Arg});
18913 unsigned IntrinsicID,
18917 for (
unsigned I = 0; I < NumArgs; ++I)
18929 switch (BuiltinID) {
18930 case SystemZ::BI__builtin_tbegin: {
18932 Value *Control = llvm::ConstantInt::get(
Int32Ty, 0xff0c);
18934 return Builder.CreateCall(F, {TDB, Control});
18936 case SystemZ::BI__builtin_tbegin_nofloat: {
18938 Value *Control = llvm::ConstantInt::get(
Int32Ty, 0xff0c);
18940 return Builder.CreateCall(F, {TDB, Control});
18942 case SystemZ::BI__builtin_tbeginc: {
18944 Value *Control = llvm::ConstantInt::get(
Int32Ty, 0xff08);
18946 return Builder.CreateCall(F, {TDB, Control});
18948 case SystemZ::BI__builtin_tabort: {
18953 case SystemZ::BI__builtin_non_tx_store: {
18965 case SystemZ::BI__builtin_s390_vpopctb:
18966 case SystemZ::BI__builtin_s390_vpopcth:
18967 case SystemZ::BI__builtin_s390_vpopctf:
18968 case SystemZ::BI__builtin_s390_vpopctg: {
18975 case SystemZ::BI__builtin_s390_vclzb:
18976 case SystemZ::BI__builtin_s390_vclzh:
18977 case SystemZ::BI__builtin_s390_vclzf:
18978 case SystemZ::BI__builtin_s390_vclzg: {
18981 Value *Undef = ConstantInt::get(
Builder.getInt1Ty(),
false);
18983 return Builder.CreateCall(F, {
X, Undef});
18986 case SystemZ::BI__builtin_s390_vctzb:
18987 case SystemZ::BI__builtin_s390_vctzh:
18988 case SystemZ::BI__builtin_s390_vctzf:
18989 case SystemZ::BI__builtin_s390_vctzg: {
18992 Value *Undef = ConstantInt::get(
Builder.getInt1Ty(),
false);
18994 return Builder.CreateCall(F, {
X, Undef});
18997 case SystemZ::BI__builtin_s390_verllb:
18998 case SystemZ::BI__builtin_s390_verllh:
18999 case SystemZ::BI__builtin_s390_verllf:
19000 case SystemZ::BI__builtin_s390_verllg: {
19005 unsigned NumElts = cast<llvm::FixedVectorType>(ResultType)->getNumElements();
19006 Amt =
Builder.CreateIntCast(Amt, ResultType->getScalarType(),
false);
19007 Amt =
Builder.CreateVectorSplat(NumElts, Amt);
19009 return Builder.CreateCall(F, { Src, Src, Amt });
19012 case SystemZ::BI__builtin_s390_verllvb:
19013 case SystemZ::BI__builtin_s390_verllvh:
19014 case SystemZ::BI__builtin_s390_verllvf:
19015 case SystemZ::BI__builtin_s390_verllvg: {
19020 return Builder.CreateCall(F, { Src, Src, Amt });
19023 case SystemZ::BI__builtin_s390_vfsqsb:
19024 case SystemZ::BI__builtin_s390_vfsqdb: {
19027 if (
Builder.getIsFPConstrained()) {
19029 return Builder.CreateConstrainedFPCall(F, {
X });
19035 case SystemZ::BI__builtin_s390_vfmasb:
19036 case SystemZ::BI__builtin_s390_vfmadb: {
19041 if (
Builder.getIsFPConstrained()) {
19043 return Builder.CreateConstrainedFPCall(F, {
X, Y, Z});
19046 return Builder.CreateCall(F, {
X, Y, Z});
19049 case SystemZ::BI__builtin_s390_vfmssb:
19050 case SystemZ::BI__builtin_s390_vfmsdb: {
19055 if (
Builder.getIsFPConstrained()) {
19057 return Builder.CreateConstrainedFPCall(F, {
X, Y,
Builder.CreateFNeg(Z,
"neg")});
19063 case SystemZ::BI__builtin_s390_vfnmasb:
19064 case SystemZ::BI__builtin_s390_vfnmadb: {
19069 if (
Builder.getIsFPConstrained()) {
19071 return Builder.CreateFNeg(
Builder.CreateConstrainedFPCall(F, {X, Y, Z}),
"neg");
19074 return Builder.CreateFNeg(
Builder.CreateCall(F, {X, Y, Z}),
"neg");
19077 case SystemZ::BI__builtin_s390_vfnmssb:
19078 case SystemZ::BI__builtin_s390_vfnmsdb: {
19083 if (
Builder.getIsFPConstrained()) {
19086 return Builder.CreateFNeg(
Builder.CreateConstrainedFPCall(F, {X, Y, NegZ}));
19093 case SystemZ::BI__builtin_s390_vflpsb:
19094 case SystemZ::BI__builtin_s390_vflpdb: {
19100 case SystemZ::BI__builtin_s390_vflnsb:
19101 case SystemZ::BI__builtin_s390_vflndb: {
19107 case SystemZ::BI__builtin_s390_vfisb:
19108 case SystemZ::BI__builtin_s390_vfidb: {
19116 Intrinsic::ID
ID = Intrinsic::not_intrinsic;
19118 switch (M4.getZExtValue()) {
19121 switch (M5.getZExtValue()) {
19123 case 0:
ID = Intrinsic::rint;
19124 CI = Intrinsic::experimental_constrained_rint;
break;
19128 switch (M5.getZExtValue()) {
19130 case 0:
ID = Intrinsic::nearbyint;
19131 CI = Intrinsic::experimental_constrained_nearbyint;
break;
19132 case 1:
ID = Intrinsic::round;
19133 CI = Intrinsic::experimental_constrained_round;
break;
19134 case 5:
ID = Intrinsic::trunc;
19135 CI = Intrinsic::experimental_constrained_trunc;
break;
19136 case 6:
ID = Intrinsic::ceil;
19137 CI = Intrinsic::experimental_constrained_ceil;
break;
19138 case 7:
ID = Intrinsic::floor;
19139 CI = Intrinsic::experimental_constrained_floor;
break;
19143 if (ID != Intrinsic::not_intrinsic) {
19144 if (
Builder.getIsFPConstrained()) {
19146 return Builder.CreateConstrainedFPCall(F,
X);
19152 switch (BuiltinID) {
19153 case SystemZ::BI__builtin_s390_vfisb:
ID = Intrinsic::s390_vfisb;
break;
19154 case SystemZ::BI__builtin_s390_vfidb:
ID = Intrinsic::s390_vfidb;
break;
19155 default: llvm_unreachable(
"Unknown BuiltinID");
19160 return Builder.CreateCall(F, {
X, M4Value, M5Value});
19162 case SystemZ::BI__builtin_s390_vfmaxsb:
19163 case SystemZ::BI__builtin_s390_vfmaxdb: {
19171 Intrinsic::ID
ID = Intrinsic::not_intrinsic;
19173 switch (M4.getZExtValue()) {
19175 case 4:
ID = Intrinsic::maxnum;
19176 CI = Intrinsic::experimental_constrained_maxnum;
break;
19178 if (ID != Intrinsic::not_intrinsic) {
19179 if (
Builder.getIsFPConstrained()) {
19181 return Builder.CreateConstrainedFPCall(F, {
X, Y});
19184 return Builder.CreateCall(F, {
X, Y});
19187 switch (BuiltinID) {
19188 case SystemZ::BI__builtin_s390_vfmaxsb:
ID = Intrinsic::s390_vfmaxsb;
break;
19189 case SystemZ::BI__builtin_s390_vfmaxdb:
ID = Intrinsic::s390_vfmaxdb;
break;
19190 default: llvm_unreachable(
"Unknown BuiltinID");
19194 return Builder.CreateCall(F, {
X, Y, M4Value});
19196 case SystemZ::BI__builtin_s390_vfminsb:
19197 case SystemZ::BI__builtin_s390_vfmindb: {
19205 Intrinsic::ID
ID = Intrinsic::not_intrinsic;
19207 switch (M4.getZExtValue()) {
19209 case 4:
ID = Intrinsic::minnum;
19210 CI = Intrinsic::experimental_constrained_minnum;
break;
19212 if (ID != Intrinsic::not_intrinsic) {
19213 if (
Builder.getIsFPConstrained()) {
19215 return Builder.CreateConstrainedFPCall(F, {
X, Y});
19218 return Builder.CreateCall(F, {
X, Y});
19221 switch (BuiltinID) {
19222 case SystemZ::BI__builtin_s390_vfminsb:
ID = Intrinsic::s390_vfminsb;
break;
19223 case SystemZ::BI__builtin_s390_vfmindb:
ID = Intrinsic::s390_vfmindb;
break;
19224 default: llvm_unreachable(
"Unknown BuiltinID");
19228 return Builder.CreateCall(F, {
X, Y, M4Value});
19231 case SystemZ::BI__builtin_s390_vlbrh:
19232 case SystemZ::BI__builtin_s390_vlbrf:
19233 case SystemZ::BI__builtin_s390_vlbrg: {
19242#define INTRINSIC_WITH_CC(NAME) \
19243 case SystemZ::BI__builtin_##NAME: \
19244 return EmitSystemZIntrinsicWithCC(*this, Intrinsic::NAME, E)
19323#undef INTRINSIC_WITH_CC
19332struct NVPTXMmaLdstInfo {
19333 unsigned NumResults;
19339#define MMA_INTR(geom_op_type, layout) \
19340 Intrinsic::nvvm_wmma_##geom_op_type##_##layout##_stride
19341#define MMA_LDST(n, geom_op_type) \
19342 { n, MMA_INTR(geom_op_type, col), MMA_INTR(geom_op_type, row) }
19344static NVPTXMmaLdstInfo getNVPTXMmaLdstInfo(
unsigned BuiltinID) {
19345 switch (BuiltinID) {
19347 case NVPTX::BI__hmma_m16n16k16_ld_a:
19348 return MMA_LDST(8, m16n16k16_load_a_f16);
19349 case NVPTX::BI__hmma_m16n16k16_ld_b:
19350 return MMA_LDST(8, m16n16k16_load_b_f16);
19351 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
19352 return MMA_LDST(4, m16n16k16_load_c_f16);
19353 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
19354 return MMA_LDST(8, m16n16k16_load_c_f32);
19355 case NVPTX::BI__hmma_m32n8k16_ld_a:
19356 return MMA_LDST(8, m32n8k16_load_a_f16);
19357 case NVPTX::BI__hmma_m32n8k16_ld_b:
19358 return MMA_LDST(8, m32n8k16_load_b_f16);
19359 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
19360 return MMA_LDST(4, m32n8k16_load_c_f16);
19361 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
19362 return MMA_LDST(8, m32n8k16_load_c_f32);
19363 case NVPTX::BI__hmma_m8n32k16_ld_a:
19364 return MMA_LDST(8, m8n32k16_load_a_f16);
19365 case NVPTX::BI__hmma_m8n32k16_ld_b:
19366 return MMA_LDST(8, m8n32k16_load_b_f16);
19367 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
19368 return MMA_LDST(4, m8n32k16_load_c_f16);
19369 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
19370 return MMA_LDST(8, m8n32k16_load_c_f32);
19373 case NVPTX::BI__imma_m16n16k16_ld_a_s8:
19374 return MMA_LDST(2, m16n16k16_load_a_s8);
19375 case NVPTX::BI__imma_m16n16k16_ld_a_u8:
19376 return MMA_LDST(2, m16n16k16_load_a_u8);
19377 case NVPTX::BI__imma_m16n16k16_ld_b_s8:
19378 return MMA_LDST(2, m16n16k16_load_b_s8);
19379 case NVPTX::BI__imma_m16n16k16_ld_b_u8:
19380 return MMA_LDST(2, m16n16k16_load_b_u8);
19381 case NVPTX::BI__imma_m16n16k16_ld_c:
19382 return MMA_LDST(8, m16n16k16_load_c_s32);
19383 case NVPTX::BI__imma_m32n8k16_ld_a_s8:
19384 return MMA_LDST(4, m32n8k16_load_a_s8);
19385 case NVPTX::BI__imma_m32n8k16_ld_a_u8:
19386 return MMA_LDST(4, m32n8k16_load_a_u8);
19387 case NVPTX::BI__imma_m32n8k16_ld_b_s8:
19388 return MMA_LDST(1, m32n8k16_load_b_s8);
19389 case NVPTX::BI__imma_m32n8k16_ld_b_u8:
19390 return MMA_LDST(1, m32n8k16_load_b_u8);
19391 case NVPTX::BI__imma_m32n8k16_ld_c:
19392 return MMA_LDST(8, m32n8k16_load_c_s32);
19393 case NVPTX::BI__imma_m8n32k16_ld_a_s8:
19394 return MMA_LDST(1, m8n32k16_load_a_s8);
19395 case NVPTX::BI__imma_m8n32k16_ld_a_u8:
19396 return MMA_LDST(1, m8n32k16_load_a_u8);
19397 case NVPTX::BI__imma_m8n32k16_ld_b_s8:
19398 return MMA_LDST(4, m8n32k16_load_b_s8);
19399 case NVPTX::BI__imma_m8n32k16_ld_b_u8:
19400 return MMA_LDST(4, m8n32k16_load_b_u8);
19401 case NVPTX::BI__imma_m8n32k16_ld_c:
19402 return MMA_LDST(8, m8n32k16_load_c_s32);
19406 case NVPTX::BI__imma_m8n8k32_ld_a_s4:
19407 return {1, 0,
MMA_INTR(m8n8k32_load_a_s4, row)};
19408 case NVPTX::BI__imma_m8n8k32_ld_a_u4:
19409 return {1, 0,
MMA_INTR(m8n8k32_load_a_u4, row)};
19410 case NVPTX::BI__imma_m8n8k32_ld_b_s4:
19411 return {1,
MMA_INTR(m8n8k32_load_b_s4, col), 0};
19412 case NVPTX::BI__imma_m8n8k32_ld_b_u4:
19413 return {1,
MMA_INTR(m8n8k32_load_b_u4, col), 0};
19414 case NVPTX::BI__imma_m8n8k32_ld_c:
19415 return MMA_LDST(2, m8n8k32_load_c_s32);
19416 case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
19417 return {1, 0,
MMA_INTR(m8n8k128_load_a_b1, row)};
19418 case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
19419 return {1,
MMA_INTR(m8n8k128_load_b_b1, col), 0};
19420 case NVPTX::BI__bmma_m8n8k128_ld_c:
19421 return MMA_LDST(2, m8n8k128_load_c_s32);
19424 case NVPTX::BI__dmma_m8n8k4_ld_a:
19425 return MMA_LDST(1, m8n8k4_load_a_f64);
19426 case NVPTX::BI__dmma_m8n8k4_ld_b:
19427 return MMA_LDST(1, m8n8k4_load_b_f64);
19428 case NVPTX::BI__dmma_m8n8k4_ld_c:
19429 return MMA_LDST(2, m8n8k4_load_c_f64);
19432 case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
19433 return MMA_LDST(4, m16n16k16_load_a_bf16);
19434 case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
19435 return MMA_LDST(4, m16n16k16_load_b_bf16);
19436 case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
19437 return MMA_LDST(2, m8n32k16_load_a_bf16);
19438 case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
19439 return MMA_LDST(8, m8n32k16_load_b_bf16);
19440 case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
19441 return MMA_LDST(8, m32n8k16_load_a_bf16);
19442 case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
19443 return MMA_LDST(2, m32n8k16_load_b_bf16);
19444 case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
19445 return MMA_LDST(4, m16n16k8_load_a_tf32);
19446 case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
19447 return MMA_LDST(4, m16n16k8_load_b_tf32);
19448 case NVPTX::BI__mma_tf32_m16n16k8_ld_c:
19449 return MMA_LDST(8, m16n16k8_load_c_f32);
19455 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
19456 return MMA_LDST(4, m16n16k16_store_d_f16);
19457 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
19458 return MMA_LDST(8, m16n16k16_store_d_f32);
19459 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
19460 return MMA_LDST(4, m32n8k16_store_d_f16);
19461 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
19462 return MMA_LDST(8, m32n8k16_store_d_f32);
19463 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
19464 return MMA_LDST(4, m8n32k16_store_d_f16);
19465 case NVPTX::BI__hmma_m8n32k16_st_c_f32:
19466 return MMA_LDST(8, m8n32k16_store_d_f32);
19471 case NVPTX::BI__imma_m16n16k16_st_c_i32:
19472 return MMA_LDST(8, m16n16k16_store_d_s32);
19473 case NVPTX::BI__imma_m32n8k16_st_c_i32:
19474 return MMA_LDST(8, m32n8k16_store_d_s32);
19475 case NVPTX::BI__imma_m8n32k16_st_c_i32:
19476 return MMA_LDST(8, m8n32k16_store_d_s32);
19477 case NVPTX::BI__imma_m8n8k32_st_c_i32:
19478 return MMA_LDST(2, m8n8k32_store_d_s32);
19479 case NVPTX::BI__bmma_m8n8k128_st_c_i32:
19480 return MMA_LDST(2, m8n8k128_store_d_s32);
19483 case NVPTX::BI__dmma_m8n8k4_st_c_f64:
19484 return MMA_LDST(2, m8n8k4_store_d_f64);
19487 case NVPTX::BI__mma_m16n16k8_st_c_f32:
19488 return MMA_LDST(8, m16n16k8_store_d_f32);
19491 llvm_unreachable(
"Unknown MMA builtin");
19498struct NVPTXMmaInfo {
19507 std::array<unsigned, 8> Variants;
19509 unsigned getMMAIntrinsic(
int Layout,
bool Satf) {
19510 unsigned Index = Layout + 4 * Satf;
19511 if (Index >= Variants.size())
19513 return Variants[Index];
19519static NVPTXMmaInfo getNVPTXMmaInfo(
unsigned BuiltinID) {
19521#define MMA_VARIANTS(geom, type) \
19522 Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type, \
19523 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type, \
19524 Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type, \
19525 Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type
19526#define MMA_SATF_VARIANTS(geom, type) \
19527 MMA_VARIANTS(geom, type), \
19528 Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type##_satfinite, \
19529 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type##_satfinite, \
19530 Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type##_satfinite, \
19531 Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type##_satfinite
19533#define MMA_VARIANTS_I4(geom, type) \
19535 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type, \
19539 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type##_satfinite, \
19543#define MMA_VARIANTS_B1_XOR(geom, type) \
19545 Intrinsic::nvvm_wmma_##geom##_mma_xor_popc_row_col_##type, \
19552#define MMA_VARIANTS_B1_AND(geom, type) \
19554 Intrinsic::nvvm_wmma_##geom##_mma_and_popc_row_col_##type, \
19562 switch (BuiltinID) {
19566 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
19568 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
19570 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
19572 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
19574 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
19576 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
19578 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
19580 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
19582 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
19584 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
19586 case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
19588 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
19592 case NVPTX::BI__imma_m16n16k16_mma_s8:
19594 case NVPTX::BI__imma_m16n16k16_mma_u8:
19596 case NVPTX::BI__imma_m32n8k16_mma_s8:
19598 case NVPTX::BI__imma_m32n8k16_mma_u8:
19600 case NVPTX::BI__imma_m8n32k16_mma_s8:
19602 case NVPTX::BI__imma_m8n32k16_mma_u8:
19606 case NVPTX::BI__imma_m8n8k32_mma_s4:
19608 case NVPTX::BI__imma_m8n8k32_mma_u4:
19610 case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1:
19612 case NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1:
19616 case NVPTX::BI__dmma_m8n8k4_mma_f64:
19620 case NVPTX::BI__mma_bf16_m16n16k16_mma_f32:
19621 return {4, 4, 8, 8, {{
MMA_VARIANTS(m16n16k16, bf16)}}};
19622 case NVPTX::BI__mma_bf16_m8n32k16_mma_f32:
19623 return {2, 8, 8, 8, {{
MMA_VARIANTS(m8n32k16, bf16)}}};
19624 case NVPTX::BI__mma_bf16_m32n8k16_mma_f32:
19625 return {8, 2, 8, 8, {{
MMA_VARIANTS(m32n8k16, bf16)}}};
19626 case NVPTX::BI__mma_tf32_m16n16k8_mma_f32:
19627 return {4, 4, 8, 8, {{
MMA_VARIANTS(m16n16k8, tf32)}}};
19629 llvm_unreachable(
"Unexpected builtin ID.");
19632#undef MMA_SATF_VARIANTS
19633#undef MMA_VARIANTS_I4
19634#undef MMA_VARIANTS_B1_AND
19635#undef MMA_VARIANTS_B1_XOR
19644 return CGF.
Builder.CreateCall(
19646 {Ptr, ConstantInt::get(CGF.Builder.getInt32Ty(), Align.getQuantity())});
19652 llvm::Type *ElemTy =
19654 return CGF.
Builder.CreateCall(
19656 {Ptr, CGF.EmitScalarExpr(E->getArg(1))});
19659static Value *MakeCpAsync(
unsigned IntrinsicID,
unsigned IntrinsicIDS,
19664 {CGF.EmitScalarExpr(E->getArg(0)),
19665 CGF.EmitScalarExpr(E->getArg(1)),
19666 CGF.EmitScalarExpr(E->getArg(2))})
19668 {CGF.EmitScalarExpr(E->getArg(0)),
19669 CGF.EmitScalarExpr(E->getArg(1))});
19672static Value *MakeHalfType(
unsigned IntrinsicID,
unsigned BuiltinID,
19675 if (!(
C.getLangOpts().NativeHalfType ||
19676 !
C.getTargetInfo().useFP16ConversionIntrinsics())) {
19678 " requires native half type support.");
19682 if (IntrinsicID == Intrinsic::nvvm_ldg_global_f ||
19683 IntrinsicID == Intrinsic::nvvm_ldu_global_f)
19684 return MakeLdgLdu(IntrinsicID, CGF, E);
19688 auto *FTy = F->getFunctionType();
19689 unsigned ICEArguments = 0;
19691 C.GetBuiltinType(BuiltinID, Error, &ICEArguments);
19693 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; ++i) {
19694 assert((ICEArguments & (1 << i)) == 0);
19696 auto *PTy = FTy->getParamType(i);
19697 if (PTy != ArgValue->
getType())
19698 ArgValue = CGF.
Builder.CreateBitCast(ArgValue, PTy);
19699 Args.push_back(ArgValue);
19702 return CGF.
Builder.CreateCall(F, Args);
19708 switch (BuiltinID) {
19709 case NVPTX::BI__nvvm_atom_add_gen_i:
19710 case NVPTX::BI__nvvm_atom_add_gen_l:
19711 case NVPTX::BI__nvvm_atom_add_gen_ll:
19714 case NVPTX::BI__nvvm_atom_sub_gen_i:
19715 case NVPTX::BI__nvvm_atom_sub_gen_l:
19716 case NVPTX::BI__nvvm_atom_sub_gen_ll:
19719 case NVPTX::BI__nvvm_atom_and_gen_i:
19720 case NVPTX::BI__nvvm_atom_and_gen_l:
19721 case NVPTX::BI__nvvm_atom_and_gen_ll:
19724 case NVPTX::BI__nvvm_atom_or_gen_i:
19725 case NVPTX::BI__nvvm_atom_or_gen_l:
19726 case NVPTX::BI__nvvm_atom_or_gen_ll:
19729 case NVPTX::BI__nvvm_atom_xor_gen_i:
19730 case NVPTX::BI__nvvm_atom_xor_gen_l:
19731 case NVPTX::BI__nvvm_atom_xor_gen_ll:
19734 case NVPTX::BI__nvvm_atom_xchg_gen_i:
19735 case NVPTX::BI__nvvm_atom_xchg_gen_l:
19736 case NVPTX::BI__nvvm_atom_xchg_gen_ll:
19739 case NVPTX::BI__nvvm_atom_max_gen_i:
19740 case NVPTX::BI__nvvm_atom_max_gen_l:
19741 case NVPTX::BI__nvvm_atom_max_gen_ll:
19744 case NVPTX::BI__nvvm_atom_max_gen_ui:
19745 case NVPTX::BI__nvvm_atom_max_gen_ul:
19746 case NVPTX::BI__nvvm_atom_max_gen_ull:
19749 case NVPTX::BI__nvvm_atom_min_gen_i:
19750 case NVPTX::BI__nvvm_atom_min_gen_l:
19751 case NVPTX::BI__nvvm_atom_min_gen_ll:
19754 case NVPTX::BI__nvvm_atom_min_gen_ui:
19755 case NVPTX::BI__nvvm_atom_min_gen_ul:
19756 case NVPTX::BI__nvvm_atom_min_gen_ull:
19759 case NVPTX::BI__nvvm_atom_cas_gen_i:
19760 case NVPTX::BI__nvvm_atom_cas_gen_l:
19761 case NVPTX::BI__nvvm_atom_cas_gen_ll:
19766 case NVPTX::BI__nvvm_atom_add_gen_f:
19767 case NVPTX::BI__nvvm_atom_add_gen_d: {
19772 AtomicOrdering::SequentiallyConsistent);
19775 case NVPTX::BI__nvvm_atom_inc_gen_ui: {
19780 return Builder.CreateCall(FnALI32, {Ptr, Val});
19783 case NVPTX::BI__nvvm_atom_dec_gen_ui: {
19788 return Builder.CreateCall(FnALD32, {Ptr, Val});
19791 case NVPTX::BI__nvvm_ldg_c:
19792 case NVPTX::BI__nvvm_ldg_sc:
19793 case NVPTX::BI__nvvm_ldg_c2:
19794 case NVPTX::BI__nvvm_ldg_sc2:
19795 case NVPTX::BI__nvvm_ldg_c4:
19796 case NVPTX::BI__nvvm_ldg_sc4:
19797 case NVPTX::BI__nvvm_ldg_s:
19798 case NVPTX::BI__nvvm_ldg_s2:
19799 case NVPTX::BI__nvvm_ldg_s4:
19800 case NVPTX::BI__nvvm_ldg_i:
19801 case NVPTX::BI__nvvm_ldg_i2:
19802 case NVPTX::BI__nvvm_ldg_i4:
19803 case NVPTX::BI__nvvm_ldg_l:
19804 case NVPTX::BI__nvvm_ldg_l2:
19805 case NVPTX::BI__nvvm_ldg_ll:
19806 case NVPTX::BI__nvvm_ldg_ll2:
19807 case NVPTX::BI__nvvm_ldg_uc:
19808 case NVPTX::BI__nvvm_ldg_uc2:
19809 case NVPTX::BI__nvvm_ldg_uc4:
19810 case NVPTX::BI__nvvm_ldg_us:
19811 case NVPTX::BI__nvvm_ldg_us2:
19812 case NVPTX::BI__nvvm_ldg_us4:
19813 case NVPTX::BI__nvvm_ldg_ui:
19814 case NVPTX::BI__nvvm_ldg_ui2:
19815 case NVPTX::BI__nvvm_ldg_ui4:
19816 case NVPTX::BI__nvvm_ldg_ul:
19817 case NVPTX::BI__nvvm_ldg_ul2:
19818 case NVPTX::BI__nvvm_ldg_ull:
19819 case NVPTX::BI__nvvm_ldg_ull2:
19823 return MakeLdgLdu(Intrinsic::nvvm_ldg_global_i, *
this, E);
19824 case NVPTX::BI__nvvm_ldg_f:
19825 case NVPTX::BI__nvvm_ldg_f2:
19826 case NVPTX::BI__nvvm_ldg_f4:
19827 case NVPTX::BI__nvvm_ldg_d:
19828 case NVPTX::BI__nvvm_ldg_d2:
19829 return MakeLdgLdu(Intrinsic::nvvm_ldg_global_f, *
this, E);
19831 case NVPTX::BI__nvvm_ldu_c:
19832 case NVPTX::BI__nvvm_ldu_sc:
19833 case NVPTX::BI__nvvm_ldu_c2:
19834 case NVPTX::BI__nvvm_ldu_sc2:
19835 case NVPTX::BI__nvvm_ldu_c4:
19836 case NVPTX::BI__nvvm_ldu_sc4:
19837 case NVPTX::BI__nvvm_ldu_s:
19838 case NVPTX::BI__nvvm_ldu_s2:
19839 case NVPTX::BI__nvvm_ldu_s4:
19840 case NVPTX::BI__nvvm_ldu_i:
19841 case NVPTX::BI__nvvm_ldu_i2:
19842 case NVPTX::BI__nvvm_ldu_i4:
19843 case NVPTX::BI__nvvm_ldu_l:
19844 case NVPTX::BI__nvvm_ldu_l2:
19845 case NVPTX::BI__nvvm_ldu_ll:
19846 case NVPTX::BI__nvvm_ldu_ll2:
19847 case NVPTX::BI__nvvm_ldu_uc:
19848 case NVPTX::BI__nvvm_ldu_uc2:
19849 case NVPTX::BI__nvvm_ldu_uc4:
19850 case NVPTX::BI__nvvm_ldu_us:
19851 case NVPTX::BI__nvvm_ldu_us2:
19852 case NVPTX::BI__nvvm_ldu_us4:
19853 case NVPTX::BI__nvvm_ldu_ui:
19854 case NVPTX::BI__nvvm_ldu_ui2:
19855 case NVPTX::BI__nvvm_ldu_ui4:
19856 case NVPTX::BI__nvvm_ldu_ul:
19857 case NVPTX::BI__nvvm_ldu_ul2:
19858 case NVPTX::BI__nvvm_ldu_ull:
19859 case NVPTX::BI__nvvm_ldu_ull2:
19860 return MakeLdgLdu(Intrinsic::nvvm_ldu_global_i, *
this, E);
19861 case NVPTX::BI__nvvm_ldu_f:
19862 case NVPTX::BI__nvvm_ldu_f2:
19863 case NVPTX::BI__nvvm_ldu_f4:
19864 case NVPTX::BI__nvvm_ldu_d:
19865 case NVPTX::BI__nvvm_ldu_d2:
19866 return MakeLdgLdu(Intrinsic::nvvm_ldu_global_f, *
this, E);
19868 case NVPTX::BI__nvvm_atom_cta_add_gen_i:
19869 case NVPTX::BI__nvvm_atom_cta_add_gen_l:
19870 case NVPTX::BI__nvvm_atom_cta_add_gen_ll:
19871 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_cta, *
this, E);
19872 case NVPTX::BI__nvvm_atom_sys_add_gen_i:
19873 case NVPTX::BI__nvvm_atom_sys_add_gen_l:
19874 case NVPTX::BI__nvvm_atom_sys_add_gen_ll:
19875 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_sys, *
this, E);
19876 case NVPTX::BI__nvvm_atom_cta_add_gen_f:
19877 case NVPTX::BI__nvvm_atom_cta_add_gen_d:
19878 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_cta, *
this, E);
19879 case NVPTX::BI__nvvm_atom_sys_add_gen_f:
19880 case NVPTX::BI__nvvm_atom_sys_add_gen_d:
19881 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_sys, *
this, E);
19882 case NVPTX::BI__nvvm_atom_cta_xchg_gen_i:
19883 case NVPTX::BI__nvvm_atom_cta_xchg_gen_l:
19884 case NVPTX::BI__nvvm_atom_cta_xchg_gen_ll:
19885 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_cta, *
this, E);
19886 case NVPTX::BI__nvvm_atom_sys_xchg_gen_i:
19887 case NVPTX::BI__nvvm_atom_sys_xchg_gen_l:
19888 case NVPTX::BI__nvvm_atom_sys_xchg_gen_ll:
19889 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_sys, *
this, E);
19890 case NVPTX::BI__nvvm_atom_cta_max_gen_i:
19891 case NVPTX::BI__nvvm_atom_cta_max_gen_ui:
19892 case NVPTX::BI__nvvm_atom_cta_max_gen_l:
19893 case NVPTX::BI__nvvm_atom_cta_max_gen_ul:
19894 case NVPTX::BI__nvvm_atom_cta_max_gen_ll:
19895 case NVPTX::BI__nvvm_atom_cta_max_gen_ull:
19896 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_cta, *
this, E);
19897 case NVPTX::BI__nvvm_atom_sys_max_gen_i:
19898 case NVPTX::BI__nvvm_atom_sys_max_gen_ui:
19899 case NVPTX::BI__nvvm_atom_sys_max_gen_l:
19900 case NVPTX::BI__nvvm_atom_sys_max_gen_ul:
19901 case NVPTX::BI__nvvm_atom_sys_max_gen_ll:
19902 case NVPTX::BI__nvvm_atom_sys_max_gen_ull:
19903 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_sys, *
this, E);
19904 case NVPTX::BI__nvvm_atom_cta_min_gen_i:
19905 case NVPTX::BI__nvvm_atom_cta_min_gen_ui:
19906 case NVPTX::BI__nvvm_atom_cta_min_gen_l:
19907 case NVPTX::BI__nvvm_atom_cta_min_gen_ul:
19908 case NVPTX::BI__nvvm_atom_cta_min_gen_ll:
19909 case NVPTX::BI__nvvm_atom_cta_min_gen_ull:
19910 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_cta, *
this, E);
19911 case NVPTX::BI__nvvm_atom_sys_min_gen_i:
19912 case NVPTX::BI__nvvm_atom_sys_min_gen_ui:
19913 case NVPTX::BI__nvvm_atom_sys_min_gen_l:
19914 case NVPTX::BI__nvvm_atom_sys_min_gen_ul:
19915 case NVPTX::BI__nvvm_atom_sys_min_gen_ll:
19916 case NVPTX::BI__nvvm_atom_sys_min_gen_ull:
19917 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_sys, *
this, E);
19918 case NVPTX::BI__nvvm_atom_cta_inc_gen_ui:
19919 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_cta, *
this, E);
19920 case NVPTX::BI__nvvm_atom_cta_dec_gen_ui:
19921 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_cta, *
this, E);
19922 case NVPTX::BI__nvvm_atom_sys_inc_gen_ui:
19923 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_sys, *
this, E);
19924 case NVPTX::BI__nvvm_atom_sys_dec_gen_ui:
19925 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_sys, *
this, E);
19926 case NVPTX::BI__nvvm_atom_cta_and_gen_i:
19927 case NVPTX::BI__nvvm_atom_cta_and_gen_l:
19928 case NVPTX::BI__nvvm_atom_cta_and_gen_ll:
19929 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_cta, *
this, E);
19930 case NVPTX::BI__nvvm_atom_sys_and_gen_i:
19931 case NVPTX::BI__nvvm_atom_sys_and_gen_l:
19932 case NVPTX::BI__nvvm_atom_sys_and_gen_ll:
19933 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_sys, *
this, E);
19934 case NVPTX::BI__nvvm_atom_cta_or_gen_i:
19935 case NVPTX::BI__nvvm_atom_cta_or_gen_l:
19936 case NVPTX::BI__nvvm_atom_cta_or_gen_ll:
19937 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_cta, *
this, E);
19938 case NVPTX::BI__nvvm_atom_sys_or_gen_i:
19939 case NVPTX::BI__nvvm_atom_sys_or_gen_l:
19940 case NVPTX::BI__nvvm_atom_sys_or_gen_ll:
19941 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_sys, *
this, E);
19942 case NVPTX::BI__nvvm_atom_cta_xor_gen_i:
19943 case NVPTX::BI__nvvm_atom_cta_xor_gen_l:
19944 case NVPTX::BI__nvvm_atom_cta_xor_gen_ll:
19945 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_cta, *
this, E);
19946 case NVPTX::BI__nvvm_atom_sys_xor_gen_i:
19947 case NVPTX::BI__nvvm_atom_sys_xor_gen_l:
19948 case NVPTX::BI__nvvm_atom_sys_xor_gen_ll:
19949 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_sys, *
this, E);
19950 case NVPTX::BI__nvvm_atom_cta_cas_gen_i:
19951 case NVPTX::BI__nvvm_atom_cta_cas_gen_l:
19952 case NVPTX::BI__nvvm_atom_cta_cas_gen_ll: {
19954 llvm::Type *ElemTy =
19958 Intrinsic::nvvm_atomic_cas_gen_i_cta, {ElemTy, Ptr->getType()}),
19959 {Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))});
19961 case NVPTX::BI__nvvm_atom_sys_cas_gen_i:
19962 case NVPTX::BI__nvvm_atom_sys_cas_gen_l:
19963 case NVPTX::BI__nvvm_atom_sys_cas_gen_ll: {
19965 llvm::Type *ElemTy =
19969 Intrinsic::nvvm_atomic_cas_gen_i_sys, {ElemTy, Ptr->getType()}),
19970 {Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))});
19972 case NVPTX::BI__nvvm_match_all_sync_i32p:
19973 case NVPTX::BI__nvvm_match_all_sync_i64p: {
19979 ? Intrinsic::nvvm_match_all_sync_i32p
19980 : Intrinsic::nvvm_match_all_sync_i64p),
19985 return Builder.CreateExtractValue(ResultPair, 0);
19989 case NVPTX::BI__hmma_m16n16k16_ld_a:
19990 case NVPTX::BI__hmma_m16n16k16_ld_b:
19991 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
19992 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
19993 case NVPTX::BI__hmma_m32n8k16_ld_a:
19994 case NVPTX::BI__hmma_m32n8k16_ld_b:
19995 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
19996 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
19997 case NVPTX::BI__hmma_m8n32k16_ld_a:
19998 case NVPTX::BI__hmma_m8n32k16_ld_b:
19999 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
20000 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
20002 case NVPTX::BI__imma_m16n16k16_ld_a_s8:
20003 case NVPTX::BI__imma_m16n16k16_ld_a_u8:
20004 case NVPTX::BI__imma_m16n16k16_ld_b_s8:
20005 case NVPTX::BI__imma_m16n16k16_ld_b_u8:
20006 case NVPTX::BI__imma_m16n16k16_ld_c:
20007 case NVPTX::BI__imma_m32n8k16_ld_a_s8:
20008 case NVPTX::BI__imma_m32n8k16_ld_a_u8:
20009 case NVPTX::BI__imma_m32n8k16_ld_b_s8:
20010 case NVPTX::BI__imma_m32n8k16_ld_b_u8:
20011 case NVPTX::BI__imma_m32n8k16_ld_c:
20012 case NVPTX::BI__imma_m8n32k16_ld_a_s8:
20013 case NVPTX::BI__imma_m8n32k16_ld_a_u8:
20014 case NVPTX::BI__imma_m8n32k16_ld_b_s8:
20015 case NVPTX::BI__imma_m8n32k16_ld_b_u8:
20016 case NVPTX::BI__imma_m8n32k16_ld_c:
20018 case NVPTX::BI__imma_m8n8k32_ld_a_s4:
20019 case NVPTX::BI__imma_m8n8k32_ld_a_u4:
20020 case NVPTX::BI__imma_m8n8k32_ld_b_s4:
20021 case NVPTX::BI__imma_m8n8k32_ld_b_u4:
20022 case NVPTX::BI__imma_m8n8k32_ld_c:
20023 case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
20024 case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
20025 case NVPTX::BI__bmma_m8n8k128_ld_c:
20027 case NVPTX::BI__dmma_m8n8k4_ld_a:
20028 case NVPTX::BI__dmma_m8n8k4_ld_b:
20029 case NVPTX::BI__dmma_m8n8k4_ld_c:
20031 case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
20032 case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
20033 case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
20034 case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
20035 case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
20036 case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
20037 case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
20038 case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
20039 case NVPTX::BI__mma_tf32_m16n16k8_ld_c: {
20043 std::optional<llvm::APSInt> isColMajorArg =
20045 if (!isColMajorArg)
20047 bool isColMajor = isColMajorArg->getSExtValue();
20048 NVPTXMmaLdstInfo II = getNVPTXMmaLdstInfo(BuiltinID);
20049 unsigned IID = isColMajor ? II.IID_col : II.IID_row;
20057 assert(II.NumResults);
20058 if (II.NumResults == 1) {
20062 for (
unsigned i = 0; i < II.NumResults; ++i) {
20067 llvm::ConstantInt::get(
IntTy, i)),
20074 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
20075 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
20076 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
20077 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
20078 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
20079 case NVPTX::BI__hmma_m8n32k16_st_c_f32:
20080 case NVPTX::BI__imma_m16n16k16_st_c_i32:
20081 case NVPTX::BI__imma_m32n8k16_st_c_i32:
20082 case NVPTX::BI__imma_m8n32k16_st_c_i32:
20083 case NVPTX::BI__imma_m8n8k32_st_c_i32:
20084 case NVPTX::BI__bmma_m8n8k128_st_c_i32:
20085 case NVPTX::BI__dmma_m8n8k4_st_c_f64:
20086 case NVPTX::BI__mma_m16n16k8_st_c_f32: {
20090 std::optional<llvm::APSInt> isColMajorArg =
20092 if (!isColMajorArg)
20094 bool isColMajor = isColMajorArg->getSExtValue();
20095 NVPTXMmaLdstInfo II = getNVPTXMmaLdstInfo(BuiltinID);
20096 unsigned IID = isColMajor ? II.IID_col : II.IID_row;
20101 llvm::Type *ParamType = Intrinsic->getFunctionType()->getParamType(1);
20103 for (
unsigned i = 0; i < II.NumResults; ++i) {
20107 llvm::ConstantInt::get(
IntTy, i)),
20109 Values.push_back(
Builder.CreateBitCast(
V, ParamType));
20111 Values.push_back(Ldm);
20118 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
20119 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
20120 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
20121 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
20122 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
20123 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
20124 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
20125 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
20126 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
20127 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
20128 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
20129 case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
20130 case NVPTX::BI__imma_m16n16k16_mma_s8:
20131 case NVPTX::BI__imma_m16n16k16_mma_u8:
20132 case NVPTX::BI__imma_m32n8k16_mma_s8:
20133 case NVPTX::BI__imma_m32n8k16_mma_u8:
20134 case NVPTX::BI__imma_m8n32k16_mma_s8:
20135 case NVPTX::BI__imma_m8n32k16_mma_u8:
20136 case NVPTX::BI__imma_m8n8k32_mma_s4:
20137 case NVPTX::BI__imma_m8n8k32_mma_u4:
20138 case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1:
20139 case NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1:
20140 case NVPTX::BI__dmma_m8n8k4_mma_f64:
20141 case NVPTX::BI__mma_bf16_m16n16k16_mma_f32:
20142 case NVPTX::BI__mma_bf16_m8n32k16_mma_f32:
20143 case NVPTX::BI__mma_bf16_m32n8k16_mma_f32:
20144 case NVPTX::BI__mma_tf32_m16n16k8_mma_f32: {
20149 std::optional<llvm::APSInt> LayoutArg =
20153 int Layout = LayoutArg->getSExtValue();
20154 if (Layout < 0 || Layout > 3)
20156 llvm::APSInt SatfArg;
20157 if (BuiltinID == NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1 ||
20158 BuiltinID == NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1)
20160 else if (std::optional<llvm::APSInt> OptSatfArg =
20162 SatfArg = *OptSatfArg;
20165 bool Satf = SatfArg.getSExtValue();
20166 NVPTXMmaInfo MI = getNVPTXMmaInfo(BuiltinID);
20167 unsigned IID = MI.getMMAIntrinsic(Layout, Satf);
20173 llvm::Type *AType = Intrinsic->getFunctionType()->getParamType(0);
20175 for (
unsigned i = 0; i < MI.NumEltsA; ++i) {
20179 llvm::ConstantInt::get(
IntTy, i)),
20181 Values.push_back(
Builder.CreateBitCast(
V, AType));
20184 llvm::Type *BType = Intrinsic->getFunctionType()->getParamType(MI.NumEltsA);
20185 for (
unsigned i = 0; i < MI.NumEltsB; ++i) {
20189 llvm::ConstantInt::get(
IntTy, i)),
20191 Values.push_back(
Builder.CreateBitCast(
V, BType));
20194 llvm::Type *CType =
20195 Intrinsic->getFunctionType()->getParamType(MI.NumEltsA + MI.NumEltsB);
20196 for (
unsigned i = 0; i < MI.NumEltsC; ++i) {
20200 llvm::ConstantInt::get(
IntTy, i)),
20202 Values.push_back(
Builder.CreateBitCast(
V, CType));
20206 for (
unsigned i = 0; i < MI.NumEltsD; ++i)
20210 llvm::ConstantInt::get(
IntTy, i)),
20215 case NVPTX::BI__nvvm_ex2_approx_f16:
20216 return MakeHalfType(Intrinsic::nvvm_ex2_approx_f16, BuiltinID, E, *
this);
20217 case NVPTX::BI__nvvm_ex2_approx_f16x2:
20218 return MakeHalfType(Intrinsic::nvvm_ex2_approx_f16x2, BuiltinID, E, *
this);
20219 case NVPTX::BI__nvvm_ff2f16x2_rn:
20220 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rn, BuiltinID, E, *
this);
20221 case NVPTX::BI__nvvm_ff2f16x2_rn_relu:
20222 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rn_relu, BuiltinID, E, *
this);
20223 case NVPTX::BI__nvvm_ff2f16x2_rz:
20224 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rz, BuiltinID, E, *
this);
20225 case NVPTX::BI__nvvm_ff2f16x2_rz_relu:
20226 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rz_relu, BuiltinID, E, *
this);
20227 case NVPTX::BI__nvvm_fma_rn_f16:
20228 return MakeHalfType(Intrinsic::nvvm_fma_rn_f16, BuiltinID, E, *
this);
20229 case NVPTX::BI__nvvm_fma_rn_f16x2:
20230 return MakeHalfType(Intrinsic::nvvm_fma_rn_f16x2, BuiltinID, E, *
this);
20231 case NVPTX::BI__nvvm_fma_rn_ftz_f16:
20232 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_f16, BuiltinID, E, *
this);
20233 case NVPTX::BI__nvvm_fma_rn_ftz_f16x2:
20234 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_f16x2, BuiltinID, E, *
this);
20235 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16:
20236 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_relu_f16, BuiltinID, E,
20238 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16x2:
20239 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_relu_f16x2, BuiltinID, E,
20241 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16:
20242 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_sat_f16, BuiltinID, E,
20244 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16x2:
20245 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_sat_f16x2, BuiltinID, E,
20247 case NVPTX::BI__nvvm_fma_rn_relu_f16:
20248 return MakeHalfType(Intrinsic::nvvm_fma_rn_relu_f16, BuiltinID, E, *
this);
20249 case NVPTX::BI__nvvm_fma_rn_relu_f16x2:
20250 return MakeHalfType(Intrinsic::nvvm_fma_rn_relu_f16x2, BuiltinID, E, *
this);
20251 case NVPTX::BI__nvvm_fma_rn_sat_f16:
20252 return MakeHalfType(Intrinsic::nvvm_fma_rn_sat_f16, BuiltinID, E, *
this);
20253 case NVPTX::BI__nvvm_fma_rn_sat_f16x2:
20254 return MakeHalfType(Intrinsic::nvvm_fma_rn_sat_f16x2, BuiltinID, E, *
this);
20255 case NVPTX::BI__nvvm_fmax_f16:
20256 return MakeHalfType(Intrinsic::nvvm_fmax_f16, BuiltinID, E, *
this);
20257 case NVPTX::BI__nvvm_fmax_f16x2:
20258 return MakeHalfType(Intrinsic::nvvm_fmax_f16x2, BuiltinID, E, *
this);
20259 case NVPTX::BI__nvvm_fmax_ftz_f16:
20260 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_f16, BuiltinID, E, *
this);
20261 case NVPTX::BI__nvvm_fmax_ftz_f16x2:
20262 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_f16x2, BuiltinID, E, *
this);
20263 case NVPTX::BI__nvvm_fmax_ftz_nan_f16:
20264 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_f16, BuiltinID, E, *
this);
20265 case NVPTX::BI__nvvm_fmax_ftz_nan_f16x2:
20266 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_f16x2, BuiltinID, E,
20268 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16:
20269 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f16, BuiltinID,
20271 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16x2:
20272 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f16x2,
20273 BuiltinID, E, *
this);
20274 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16:
20275 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_xorsign_abs_f16, BuiltinID, E,
20277 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16x2:
20278 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_xorsign_abs_f16x2, BuiltinID,
20280 case NVPTX::BI__nvvm_fmax_nan_f16:
20281 return MakeHalfType(Intrinsic::nvvm_fmax_nan_f16, BuiltinID, E, *
this);
20282 case NVPTX::BI__nvvm_fmax_nan_f16x2:
20283 return MakeHalfType(Intrinsic::nvvm_fmax_nan_f16x2, BuiltinID, E, *
this);
20284 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16:
20285 return MakeHalfType(Intrinsic::nvvm_fmax_nan_xorsign_abs_f16, BuiltinID, E,
20287 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16x2:
20288 return MakeHalfType(Intrinsic::nvvm_fmax_nan_xorsign_abs_f16x2, BuiltinID,
20290 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16:
20291 return MakeHalfType(Intrinsic::nvvm_fmax_xorsign_abs_f16, BuiltinID, E,
20293 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16x2:
20294 return MakeHalfType(Intrinsic::nvvm_fmax_xorsign_abs_f16x2, BuiltinID, E,
20296 case NVPTX::BI__nvvm_fmin_f16:
20297 return MakeHalfType(Intrinsic::nvvm_fmin_f16, BuiltinID, E, *
this);
20298 case NVPTX::BI__nvvm_fmin_f16x2:
20299 return MakeHalfType(Intrinsic::nvvm_fmin_f16x2, BuiltinID, E, *
this);
20300 case NVPTX::BI__nvvm_fmin_ftz_f16:
20301 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_f16, BuiltinID, E, *
this);
20302 case NVPTX::BI__nvvm_fmin_ftz_f16x2:
20303 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_f16x2, BuiltinID, E, *
this);
20304 case NVPTX::BI__nvvm_fmin_ftz_nan_f16:
20305 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_f16, BuiltinID, E, *
this);
20306 case NVPTX::BI__nvvm_fmin_ftz_nan_f16x2:
20307 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_f16x2, BuiltinID, E,
20309 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16:
20310 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f16, BuiltinID,
20312 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16x2:
20313 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f16x2,
20314 BuiltinID, E, *
this);
20315 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16:
20316 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_xorsign_abs_f16, BuiltinID, E,
20318 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16x2:
20319 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_xorsign_abs_f16x2, BuiltinID,
20321 case NVPTX::BI__nvvm_fmin_nan_f16:
20322 return MakeHalfType(Intrinsic::nvvm_fmin_nan_f16, BuiltinID, E, *
this);
20323 case NVPTX::BI__nvvm_fmin_nan_f16x2:
20324 return MakeHalfType(Intrinsic::nvvm_fmin_nan_f16x2, BuiltinID, E, *
this);
20325 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16:
20326 return MakeHalfType(Intrinsic::nvvm_fmin_nan_xorsign_abs_f16, BuiltinID, E,
20328 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16x2:
20329 return MakeHalfType(Intrinsic::nvvm_fmin_nan_xorsign_abs_f16x2, BuiltinID,
20331 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16:
20332 return MakeHalfType(Intrinsic::nvvm_fmin_xorsign_abs_f16, BuiltinID, E,
20334 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16x2:
20335 return MakeHalfType(Intrinsic::nvvm_fmin_xorsign_abs_f16x2, BuiltinID, E,
20337 case NVPTX::BI__nvvm_ldg_h:
20338 return MakeHalfType(Intrinsic::nvvm_ldg_global_f, BuiltinID, E, *
this);
20339 case NVPTX::BI__nvvm_ldg_h2:
20340 return MakeHalfType(Intrinsic::nvvm_ldg_global_f, BuiltinID, E, *
this);
20341 case NVPTX::BI__nvvm_ldu_h:
20342 return MakeHalfType(Intrinsic::nvvm_ldu_global_f, BuiltinID, E, *
this);
20343 case NVPTX::BI__nvvm_ldu_h2: {
20344 return MakeHalfType(Intrinsic::nvvm_ldu_global_f, BuiltinID, E, *
this);
20346 case NVPTX::BI__nvvm_cp_async_ca_shared_global_4:
20347 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_4,
20348 Intrinsic::nvvm_cp_async_ca_shared_global_4_s, *
this, E,
20350 case NVPTX::BI__nvvm_cp_async_ca_shared_global_8:
20351 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_8,
20352 Intrinsic::nvvm_cp_async_ca_shared_global_8_s, *
this, E,
20354 case NVPTX::BI__nvvm_cp_async_ca_shared_global_16:
20355 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_16,
20356 Intrinsic::nvvm_cp_async_ca_shared_global_16_s, *
this, E,
20358 case NVPTX::BI__nvvm_cp_async_cg_shared_global_16:
20359 return MakeCpAsync(Intrinsic::nvvm_cp_async_cg_shared_global_16,
20360 Intrinsic::nvvm_cp_async_cg_shared_global_16_s, *
this, E,
20362 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_x:
20365 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_y:
20368 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_z:
20371 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_w:
20374 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_x:
20377 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_y:
20380 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_z:
20383 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_w:
20386 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_x:
20389 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_y:
20392 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_z:
20395 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_w:
20398 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_x:
20401 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_y:
20404 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_z:
20407 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_w:
20410 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctarank:
20413 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctarank:
20416 case NVPTX::BI__nvvm_is_explicit_cluster:
20419 case NVPTX::BI__nvvm_isspacep_shared_cluster:
20423 case NVPTX::BI__nvvm_mapa:
20426 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
20427 case NVPTX::BI__nvvm_mapa_shared_cluster:
20430 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
20431 case NVPTX::BI__nvvm_getctarank:
20435 case NVPTX::BI__nvvm_getctarank_shared_cluster:
20439 case NVPTX::BI__nvvm_barrier_cluster_arrive:
20442 case NVPTX::BI__nvvm_barrier_cluster_arrive_relaxed:
20445 case NVPTX::BI__nvvm_barrier_cluster_wait:
20448 case NVPTX::BI__nvvm_fence_sc_cluster:
20457struct BuiltinAlignArgs {
20458 llvm::Value *Src =
nullptr;
20459 llvm::Type *SrcType =
nullptr;
20460 llvm::Value *Alignment =
nullptr;
20461 llvm::Value *Mask =
nullptr;
20462 llvm::IntegerType *IntType =
nullptr;
20470 SrcType = Src->getType();
20471 if (SrcType->isPointerTy()) {
20472 IntType = IntegerType::get(
20476 assert(SrcType->isIntegerTy());
20477 IntType = cast<llvm::IntegerType>(SrcType);
20480 Alignment = CGF.
Builder.CreateZExtOrTrunc(Alignment, IntType,
"alignment");
20481 auto *One = llvm::ConstantInt::get(IntType, 1);
20482 Mask = CGF.
Builder.CreateSub(Alignment, One,
"mask");
20489 BuiltinAlignArgs Args(E, *
this);
20490 llvm::Value *SrcAddress = Args.Src;
20491 if (Args.SrcType->isPointerTy())
20493 Builder.CreateBitOrPointerCast(Args.Src, Args.IntType,
"src_addr");
20495 Builder.CreateAnd(SrcAddress, Args.Mask,
"set_bits"),
20496 llvm::Constant::getNullValue(Args.IntType),
"is_aligned"));
20503 BuiltinAlignArgs Args(E, *
this);
20504 llvm::Value *SrcForMask = Args.Src;
20510 if (Args.Src->getType()->isPointerTy()) {
20520 SrcForMask =
Builder.CreateAdd(SrcForMask, Args.Mask,
"over_boundary");
20524 llvm::Value *InvertedMask =
Builder.CreateNot(Args.Mask,
"inverted_mask");
20525 llvm::Value *
Result =
nullptr;
20526 if (Args.Src->getType()->isPointerTy()) {
20528 Intrinsic::ptrmask, {Args.SrcType, Args.IntType},
20529 {SrcForMask, InvertedMask},
nullptr,
"aligned_result");
20531 Result =
Builder.CreateAnd(SrcForMask, InvertedMask,
"aligned_result");
20533 assert(
Result->getType() == Args.SrcType);
20539 switch (BuiltinID) {
20540 case WebAssembly::BI__builtin_wasm_memory_size: {
20545 return Builder.CreateCall(Callee, I);
20547 case WebAssembly::BI__builtin_wasm_memory_grow: {
20553 return Builder.CreateCall(Callee, Args);
20555 case WebAssembly::BI__builtin_wasm_tls_size: {
20558 return Builder.CreateCall(Callee);
20560 case WebAssembly::BI__builtin_wasm_tls_align: {
20563 return Builder.CreateCall(Callee);
20565 case WebAssembly::BI__builtin_wasm_tls_base: {
20567 return Builder.CreateCall(Callee);
20569 case WebAssembly::BI__builtin_wasm_throw: {
20573 return Builder.CreateCall(Callee, {Tag, Obj});
20575 case WebAssembly::BI__builtin_wasm_rethrow: {
20577 return Builder.CreateCall(Callee);
20579 case WebAssembly::BI__builtin_wasm_memory_atomic_wait32: {
20586 case WebAssembly::BI__builtin_wasm_memory_atomic_wait64: {
20593 case WebAssembly::BI__builtin_wasm_memory_atomic_notify: {
20597 return Builder.CreateCall(Callee, {Addr, Count});
20599 case WebAssembly::BI__builtin_wasm_trunc_s_i32_f32:
20600 case WebAssembly::BI__builtin_wasm_trunc_s_i32_f64:
20601 case WebAssembly::BI__builtin_wasm_trunc_s_i64_f32:
20602 case WebAssembly::BI__builtin_wasm_trunc_s_i64_f64: {
20607 return Builder.CreateCall(Callee, {Src});
20609 case WebAssembly::BI__builtin_wasm_trunc_u_i32_f32:
20610 case WebAssembly::BI__builtin_wasm_trunc_u_i32_f64:
20611 case WebAssembly::BI__builtin_wasm_trunc_u_i64_f32:
20612 case WebAssembly::BI__builtin_wasm_trunc_u_i64_f64: {
20617 return Builder.CreateCall(Callee, {Src});
20619 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32_f32:
20620 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32_f64:
20621 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i64_f32:
20622 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i64_f64:
20623 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32x4_f32x4: {
20628 return Builder.CreateCall(Callee, {Src});
20630 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32_f32:
20631 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32_f64:
20632 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i64_f32:
20633 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i64_f64:
20634 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32x4_f32x4: {
20639 return Builder.CreateCall(Callee, {Src});
20641 case WebAssembly::BI__builtin_wasm_min_f32:
20642 case WebAssembly::BI__builtin_wasm_min_f64:
20643 case WebAssembly::BI__builtin_wasm_min_f32x4:
20644 case WebAssembly::BI__builtin_wasm_min_f64x2: {
20649 return Builder.CreateCall(Callee, {LHS, RHS});
20651 case WebAssembly::BI__builtin_wasm_max_f32:
20652 case WebAssembly::BI__builtin_wasm_max_f64:
20653 case WebAssembly::BI__builtin_wasm_max_f32x4:
20654 case WebAssembly::BI__builtin_wasm_max_f64x2: {
20659 return Builder.CreateCall(Callee, {LHS, RHS});
20661 case WebAssembly::BI__builtin_wasm_pmin_f32x4:
20662 case WebAssembly::BI__builtin_wasm_pmin_f64x2: {
20667 return Builder.CreateCall(Callee, {LHS, RHS});
20669 case WebAssembly::BI__builtin_wasm_pmax_f32x4:
20670 case WebAssembly::BI__builtin_wasm_pmax_f64x2: {
20675 return Builder.CreateCall(Callee, {LHS, RHS});
20677 case WebAssembly::BI__builtin_wasm_ceil_f32x4:
20678 case WebAssembly::BI__builtin_wasm_floor_f32x4:
20679 case WebAssembly::BI__builtin_wasm_trunc_f32x4:
20680 case WebAssembly::BI__builtin_wasm_nearest_f32x4:
20681 case WebAssembly::BI__builtin_wasm_ceil_f64x2:
20682 case WebAssembly::BI__builtin_wasm_floor_f64x2:
20683 case WebAssembly::BI__builtin_wasm_trunc_f64x2:
20684 case WebAssembly::BI__builtin_wasm_nearest_f64x2: {
20686 switch (BuiltinID) {
20687 case WebAssembly::BI__builtin_wasm_ceil_f32x4:
20688 case WebAssembly::BI__builtin_wasm_ceil_f64x2:
20689 IntNo = Intrinsic::ceil;
20691 case WebAssembly::BI__builtin_wasm_floor_f32x4:
20692 case WebAssembly::BI__builtin_wasm_floor_f64x2:
20693 IntNo = Intrinsic::floor;
20695 case WebAssembly::BI__builtin_wasm_trunc_f32x4:
20696 case WebAssembly::BI__builtin_wasm_trunc_f64x2:
20697 IntNo = Intrinsic::trunc;
20699 case WebAssembly::BI__builtin_wasm_nearest_f32x4:
20700 case WebAssembly::BI__builtin_wasm_nearest_f64x2:
20701 IntNo = Intrinsic::nearbyint;
20704 llvm_unreachable(
"unexpected builtin ID");
20710 case WebAssembly::BI__builtin_wasm_ref_null_extern: {
20712 return Builder.CreateCall(Callee);
20714 case WebAssembly::BI__builtin_wasm_ref_null_func: {
20716 return Builder.CreateCall(Callee);
20718 case WebAssembly::BI__builtin_wasm_swizzle_i8x16: {
20722 return Builder.CreateCall(Callee, {Src, Indices});
20724 case WebAssembly::BI__builtin_wasm_add_sat_s_i8x16:
20725 case WebAssembly::BI__builtin_wasm_add_sat_u_i8x16:
20726 case WebAssembly::BI__builtin_wasm_add_sat_s_i16x8:
20727 case WebAssembly::BI__builtin_wasm_add_sat_u_i16x8:
20728 case WebAssembly::BI__builtin_wasm_sub_sat_s_i8x16:
20729 case WebAssembly::BI__builtin_wasm_sub_sat_u_i8x16:
20730 case WebAssembly::BI__builtin_wasm_sub_sat_s_i16x8:
20731 case WebAssembly::BI__builtin_wasm_sub_sat_u_i16x8: {
20733 switch (BuiltinID) {
20734 case WebAssembly::BI__builtin_wasm_add_sat_s_i8x16:
20735 case WebAssembly::BI__builtin_wasm_add_sat_s_i16x8:
20736 IntNo = Intrinsic::sadd_sat;
20738 case WebAssembly::BI__builtin_wasm_add_sat_u_i8x16:
20739 case WebAssembly::BI__builtin_wasm_add_sat_u_i16x8:
20740 IntNo = Intrinsic::uadd_sat;
20742 case WebAssembly::BI__builtin_wasm_sub_sat_s_i8x16:
20743 case WebAssembly::BI__builtin_wasm_sub_sat_s_i16x8:
20744 IntNo = Intrinsic::wasm_sub_sat_signed;
20746 case WebAssembly::BI__builtin_wasm_sub_sat_u_i8x16:
20747 case WebAssembly::BI__builtin_wasm_sub_sat_u_i16x8:
20748 IntNo = Intrinsic::wasm_sub_sat_unsigned;
20751 llvm_unreachable(
"unexpected builtin ID");
20756 return Builder.CreateCall(Callee, {LHS, RHS});
20758 case WebAssembly::BI__builtin_wasm_abs_i8x16:
20759 case WebAssembly::BI__builtin_wasm_abs_i16x8:
20760 case WebAssembly::BI__builtin_wasm_abs_i32x4:
20761 case WebAssembly::BI__builtin_wasm_abs_i64x2: {
20764 Constant *
Zero = llvm::Constant::getNullValue(Vec->
getType());
20765 Value *ICmp =
Builder.CreateICmpSLT(Vec, Zero,
"abscond");
20766 return Builder.CreateSelect(ICmp, Neg, Vec,
"abs");
20768 case WebAssembly::BI__builtin_wasm_min_s_i8x16:
20769 case WebAssembly::BI__builtin_wasm_min_u_i8x16:
20770 case WebAssembly::BI__builtin_wasm_max_s_i8x16:
20771 case WebAssembly::BI__builtin_wasm_max_u_i8x16:
20772 case WebAssembly::BI__builtin_wasm_min_s_i16x8:
20773 case WebAssembly::BI__builtin_wasm_min_u_i16x8:
20774 case WebAssembly::BI__builtin_wasm_max_s_i16x8:
20775 case WebAssembly::BI__builtin_wasm_max_u_i16x8:
20776 case WebAssembly::BI__builtin_wasm_min_s_i32x4:
20777 case WebAssembly::BI__builtin_wasm_min_u_i32x4:
20778 case WebAssembly::BI__builtin_wasm_max_s_i32x4:
20779 case WebAssembly::BI__builtin_wasm_max_u_i32x4: {
20783 switch (BuiltinID) {
20784 case WebAssembly::BI__builtin_wasm_min_s_i8x16:
20785 case WebAssembly::BI__builtin_wasm_min_s_i16x8:
20786 case WebAssembly::BI__builtin_wasm_min_s_i32x4:
20787 ICmp =
Builder.CreateICmpSLT(LHS, RHS);
20789 case WebAssembly::BI__builtin_wasm_min_u_i8x16:
20790 case WebAssembly::BI__builtin_wasm_min_u_i16x8:
20791 case WebAssembly::BI__builtin_wasm_min_u_i32x4:
20792 ICmp =
Builder.CreateICmpULT(LHS, RHS);
20794 case WebAssembly::BI__builtin_wasm_max_s_i8x16:
20795 case WebAssembly::BI__builtin_wasm_max_s_i16x8:
20796 case WebAssembly::BI__builtin_wasm_max_s_i32x4:
20797 ICmp =
Builder.CreateICmpSGT(LHS, RHS);
20799 case WebAssembly::BI__builtin_wasm_max_u_i8x16:
20800 case WebAssembly::BI__builtin_wasm_max_u_i16x8:
20801 case WebAssembly::BI__builtin_wasm_max_u_i32x4:
20802 ICmp =
Builder.CreateICmpUGT(LHS, RHS);
20805 llvm_unreachable(
"unexpected builtin ID");
20807 return Builder.CreateSelect(ICmp, LHS, RHS);
20809 case WebAssembly::BI__builtin_wasm_avgr_u_i8x16:
20810 case WebAssembly::BI__builtin_wasm_avgr_u_i16x8: {
20815 return Builder.CreateCall(Callee, {LHS, RHS});
20817 case WebAssembly::BI__builtin_wasm_q15mulr_sat_s_i16x8: {
20821 return Builder.CreateCall(Callee, {LHS, RHS});
20823 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_s_i16x8:
20824 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_u_i16x8:
20825 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_s_i32x4:
20826 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_u_i32x4: {
20829 switch (BuiltinID) {
20830 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_s_i16x8:
20831 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_s_i32x4:
20832 IntNo = Intrinsic::wasm_extadd_pairwise_signed;
20834 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_u_i16x8:
20835 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_u_i32x4:
20836 IntNo = Intrinsic::wasm_extadd_pairwise_unsigned;
20839 llvm_unreachable(
"unexpected builtin ID");
20843 return Builder.CreateCall(Callee, Vec);
20845 case WebAssembly::BI__builtin_wasm_bitselect: {
20851 return Builder.CreateCall(Callee, {V1, V2,
C});
20853 case WebAssembly::BI__builtin_wasm_dot_s_i32x4_i16x8: {
20857 return Builder.CreateCall(Callee, {LHS, RHS});
20859 case WebAssembly::BI__builtin_wasm_popcnt_i8x16: {
20863 return Builder.CreateCall(Callee, {Vec});
20865 case WebAssembly::BI__builtin_wasm_any_true_v128:
20866 case WebAssembly::BI__builtin_wasm_all_true_i8x16:
20867 case WebAssembly::BI__builtin_wasm_all_true_i16x8:
20868 case WebAssembly::BI__builtin_wasm_all_true_i32x4:
20869 case WebAssembly::BI__builtin_wasm_all_true_i64x2: {
20871 switch (BuiltinID) {
20872 case WebAssembly::BI__builtin_wasm_any_true_v128:
20873 IntNo = Intrinsic::wasm_anytrue;
20875 case WebAssembly::BI__builtin_wasm_all_true_i8x16:
20876 case WebAssembly::BI__builtin_wasm_all_true_i16x8:
20877 case WebAssembly::BI__builtin_wasm_all_true_i32x4:
20878 case WebAssembly::BI__builtin_wasm_all_true_i64x2:
20879 IntNo = Intrinsic::wasm_alltrue;
20882 llvm_unreachable(
"unexpected builtin ID");
20886 return Builder.CreateCall(Callee, {Vec});
20888 case WebAssembly::BI__builtin_wasm_bitmask_i8x16:
20889 case WebAssembly::BI__builtin_wasm_bitmask_i16x8:
20890 case WebAssembly::BI__builtin_wasm_bitmask_i32x4:
20891 case WebAssembly::BI__builtin_wasm_bitmask_i64x2: {
20895 return Builder.CreateCall(Callee, {Vec});
20897 case WebAssembly::BI__builtin_wasm_abs_f32x4:
20898 case WebAssembly::BI__builtin_wasm_abs_f64x2: {
20901 return Builder.CreateCall(Callee, {Vec});
20903 case WebAssembly::BI__builtin_wasm_sqrt_f32x4:
20904 case WebAssembly::BI__builtin_wasm_sqrt_f64x2: {
20907 return Builder.CreateCall(Callee, {Vec});
20909 case WebAssembly::BI__builtin_wasm_narrow_s_i8x16_i16x8:
20910 case WebAssembly::BI__builtin_wasm_narrow_u_i8x16_i16x8:
20911 case WebAssembly::BI__builtin_wasm_narrow_s_i16x8_i32x4:
20912 case WebAssembly::BI__builtin_wasm_narrow_u_i16x8_i32x4: {
20916 switch (BuiltinID) {
20917 case WebAssembly::BI__builtin_wasm_narrow_s_i8x16_i16x8:
20918 case WebAssembly::BI__builtin_wasm_narrow_s_i16x8_i32x4:
20919 IntNo = Intrinsic::wasm_narrow_signed;
20921 case WebAssembly::BI__builtin_wasm_narrow_u_i8x16_i16x8:
20922 case WebAssembly::BI__builtin_wasm_narrow_u_i16x8_i32x4:
20923 IntNo = Intrinsic::wasm_narrow_unsigned;
20926 llvm_unreachable(
"unexpected builtin ID");
20930 return Builder.CreateCall(Callee, {Low, High});
20932 case WebAssembly::BI__builtin_wasm_trunc_sat_s_zero_f64x2_i32x4:
20933 case WebAssembly::BI__builtin_wasm_trunc_sat_u_zero_f64x2_i32x4: {
20936 switch (BuiltinID) {
20937 case WebAssembly::BI__builtin_wasm_trunc_sat_s_zero_f64x2_i32x4:
20938 IntNo = Intrinsic::fptosi_sat;
20940 case WebAssembly::BI__builtin_wasm_trunc_sat_u_zero_f64x2_i32x4:
20941 IntNo = Intrinsic::fptoui_sat;
20944 llvm_unreachable(
"unexpected builtin ID");
20946 llvm::Type *SrcT = Vec->
getType();
20947 llvm::Type *TruncT = SrcT->getWithNewType(
Builder.getInt32Ty());
20950 Value *Splat = Constant::getNullValue(TruncT);
20953 case WebAssembly::BI__builtin_wasm_shuffle_i8x16: {
20958 while (OpIdx < 18) {
20959 std::optional<llvm::APSInt> LaneConst =
20961 assert(LaneConst &&
"Constant arg isn't actually constant?");
20962 Ops[OpIdx++] = llvm::ConstantInt::get(
getLLVMContext(), *LaneConst);
20965 return Builder.CreateCall(Callee, Ops);
20967 case WebAssembly::BI__builtin_wasm_relaxed_madd_f32x4:
20968 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f32x4:
20969 case WebAssembly::BI__builtin_wasm_relaxed_madd_f64x2:
20970 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f64x2: {
20975 switch (BuiltinID) {
20976 case WebAssembly::BI__builtin_wasm_relaxed_madd_f32x4:
20977 case WebAssembly::BI__builtin_wasm_relaxed_madd_f64x2:
20978 IntNo = Intrinsic::wasm_relaxed_madd;
20980 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f32x4:
20981 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f64x2:
20982 IntNo = Intrinsic::wasm_relaxed_nmadd;
20985 llvm_unreachable(
"unexpected builtin ID");
20988 return Builder.CreateCall(Callee, {A, B,
C});
20990 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i8x16:
20991 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i16x8:
20992 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i32x4:
20993 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i64x2: {
20999 return Builder.CreateCall(Callee, {A, B,
C});
21001 case WebAssembly::BI__builtin_wasm_relaxed_swizzle_i8x16: {
21005 return Builder.CreateCall(Callee, {Src, Indices});
21007 case WebAssembly::BI__builtin_wasm_relaxed_min_f32x4:
21008 case WebAssembly::BI__builtin_wasm_relaxed_max_f32x4:
21009 case WebAssembly::BI__builtin_wasm_relaxed_min_f64x2:
21010 case WebAssembly::BI__builtin_wasm_relaxed_max_f64x2: {
21014 switch (BuiltinID) {
21015 case WebAssembly::BI__builtin_wasm_relaxed_min_f32x4:
21016 case WebAssembly::BI__builtin_wasm_relaxed_min_f64x2:
21017 IntNo = Intrinsic::wasm_relaxed_min;
21019 case WebAssembly::BI__builtin_wasm_relaxed_max_f32x4:
21020 case WebAssembly::BI__builtin_wasm_relaxed_max_f64x2:
21021 IntNo = Intrinsic::wasm_relaxed_max;
21024 llvm_unreachable(
"unexpected builtin ID");
21027 return Builder.CreateCall(Callee, {LHS, RHS});
21029 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_i32x4_f32x4:
21030 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_i32x4_f32x4:
21031 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_zero_i32x4_f64x2:
21032 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_zero_i32x4_f64x2: {
21035 switch (BuiltinID) {
21036 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_i32x4_f32x4:
21037 IntNo = Intrinsic::wasm_relaxed_trunc_signed;
21039 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_i32x4_f32x4:
21040 IntNo = Intrinsic::wasm_relaxed_trunc_unsigned;
21042 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_zero_i32x4_f64x2:
21043 IntNo = Intrinsic::wasm_relaxed_trunc_signed_zero;
21045 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_zero_i32x4_f64x2:
21046 IntNo = Intrinsic::wasm_relaxed_trunc_unsigned_zero;
21049 llvm_unreachable(
"unexpected builtin ID");
21052 return Builder.CreateCall(Callee, {Vec});
21054 case WebAssembly::BI__builtin_wasm_relaxed_q15mulr_s_i16x8: {
21058 return Builder.CreateCall(Callee, {LHS, RHS});
21060 case WebAssembly::BI__builtin_wasm_relaxed_dot_i8x16_i7x16_s_i16x8: {
21065 return Builder.CreateCall(Callee, {LHS, RHS});
21067 case WebAssembly::BI__builtin_wasm_relaxed_dot_i8x16_i7x16_add_s_i32x4: {
21072 CGM.
getIntrinsic(Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed);
21073 return Builder.CreateCall(Callee, {LHS, RHS, Acc});
21075 case WebAssembly::BI__builtin_wasm_relaxed_dot_bf16x8_add_f32_f32x4: {
21081 return Builder.CreateCall(Callee, {LHS, RHS, Acc});
21083 case WebAssembly::BI__builtin_wasm_table_get: {
21094 "Unexpected reference type for __builtin_wasm_table_get");
21095 return Builder.CreateCall(Callee, {Table, Index});
21097 case WebAssembly::BI__builtin_wasm_table_set: {
21109 "Unexpected reference type for __builtin_wasm_table_set");
21110 return Builder.CreateCall(Callee, {Table, Index, Val});
21112 case WebAssembly::BI__builtin_wasm_table_size: {
21118 case WebAssembly::BI__builtin_wasm_table_grow: {
21131 "Unexpected reference type for __builtin_wasm_table_grow");
21133 return Builder.CreateCall(Callee, {Table, Val, NElems});
21135 case WebAssembly::BI__builtin_wasm_table_fill: {
21149 "Unexpected reference type for __builtin_wasm_table_fill");
21151 return Builder.CreateCall(Callee, {Table, Index, Val, NElems});
21153 case WebAssembly::BI__builtin_wasm_table_copy: {
21163 return Builder.CreateCall(Callee, {TableX, TableY, SrcIdx, DstIdx, NElems});
21170static std::pair<Intrinsic::ID, unsigned>
21173 unsigned BuiltinID;
21174 Intrinsic::ID IntrinsicID;
21177 static Info Infos[] = {
21178#define CUSTOM_BUILTIN_MAPPING(x,s) \
21179 { Hexagon::BI__builtin_HEXAGON_##x, Intrinsic::hexagon_##x, s },
21211#include "clang/Basic/BuiltinsHexagonMapCustomDep.def"
21212#undef CUSTOM_BUILTIN_MAPPING
21215 auto CmpInfo = [] (Info A, Info B) {
return A.BuiltinID < B.BuiltinID; };
21216 static const bool SortOnce = (llvm::sort(Infos, CmpInfo),
true);
21219 const Info *F = llvm::lower_bound(Infos, Info{BuiltinID, 0, 0}, CmpInfo);
21220 if (F == std::end(Infos) || F->BuiltinID != BuiltinID)
21221 return {Intrinsic::not_intrinsic, 0};
21223 return {F->IntrinsicID, F->VecLen};
21232 auto MakeCircOp = [
this, E](
unsigned IntID,
bool IsLoad) {
21246 for (
unsigned i = 1, e = E->
getNumArgs(); i != e; ++i)
21252 llvm::Value *NewBase = IsLoad ?
Builder.CreateExtractValue(
Result, 1)
21256 llvm::Value *RetVal =
21266 auto MakeBrevLd = [
this, E](
unsigned IntID, llvm::Type *DestTy) {
21277 llvm::Value *DestAddress = DestAddr.
getPointer();
21283 CGM.
getIntrinsic(IntID), {BaseAddress, EmitScalarExpr(E->getArg(2))});
21286 llvm::Value *DestVal =
Builder.CreateExtractValue(
Result, 0);
21291 DestVal =
Builder.CreateTrunc(DestVal, DestTy);
21298 auto V2Q = [
this, VecLen] (llvm::Value *Vec) {
21299 Intrinsic::ID
ID = VecLen == 128 ? Intrinsic::hexagon_V6_vandvrt_128B
21300 : Intrinsic::hexagon_V6_vandvrt;
21302 {Vec,
Builder.getInt32(-1)});
21304 auto Q2V = [
this, VecLen] (llvm::Value *Pred) {
21305 Intrinsic::ID
ID = VecLen == 128 ? Intrinsic::hexagon_V6_vandqrt_128B
21306 : Intrinsic::hexagon_V6_vandqrt;
21308 {Pred,
Builder.getInt32(-1)});
21311 switch (BuiltinID) {
21315 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarry:
21316 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarry_128B:
21317 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarry:
21318 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarry_128B: {
21325 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1)), PredIn});
21327 llvm::Value *PredOut =
Builder.CreateExtractValue(
Result, 1);
21335 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarryo:
21336 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarryo_128B:
21337 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarryo:
21338 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarryo_128B: {
21344 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
21346 llvm::Value *PredOut =
Builder.CreateExtractValue(
Result, 1);
21352 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstoreq:
21353 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorenq:
21354 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentq:
21355 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentnq:
21356 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstoreq_128B:
21357 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorenq_128B:
21358 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentq_128B:
21359 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentnq_128B: {
21363 if (
auto *Cast = dyn_cast<ImplicitCastExpr>(PredOp)) {
21364 if (
Cast->getCastKind() == CK_BitCast)
21365 PredOp =
Cast->getSubExpr();
21368 for (
int i = 1, e = E->
getNumArgs(); i != e; ++i)
21373 case Hexagon::BI__builtin_HEXAGON_L2_loadrub_pci:
21374 case Hexagon::BI__builtin_HEXAGON_L2_loadrb_pci:
21375 case Hexagon::BI__builtin_HEXAGON_L2_loadruh_pci:
21376 case Hexagon::BI__builtin_HEXAGON_L2_loadrh_pci:
21377 case Hexagon::BI__builtin_HEXAGON_L2_loadri_pci:
21378 case Hexagon::BI__builtin_HEXAGON_L2_loadrd_pci:
21379 case Hexagon::BI__builtin_HEXAGON_L2_loadrub_pcr:
21380 case Hexagon::BI__builtin_HEXAGON_L2_loadrb_pcr:
21381 case Hexagon::BI__builtin_HEXAGON_L2_loadruh_pcr:
21382 case Hexagon::BI__builtin_HEXAGON_L2_loadrh_pcr:
21383 case Hexagon::BI__builtin_HEXAGON_L2_loadri_pcr:
21384 case Hexagon::BI__builtin_HEXAGON_L2_loadrd_pcr:
21385 return MakeCircOp(ID,
true);
21386 case Hexagon::BI__builtin_HEXAGON_S2_storerb_pci:
21387 case Hexagon::BI__builtin_HEXAGON_S2_storerh_pci:
21388 case Hexagon::BI__builtin_HEXAGON_S2_storerf_pci:
21389 case Hexagon::BI__builtin_HEXAGON_S2_storeri_pci:
21390 case Hexagon::BI__builtin_HEXAGON_S2_storerd_pci:
21391 case Hexagon::BI__builtin_HEXAGON_S2_storerb_pcr:
21392 case Hexagon::BI__builtin_HEXAGON_S2_storerh_pcr:
21393 case Hexagon::BI__builtin_HEXAGON_S2_storerf_pcr:
21394 case Hexagon::BI__builtin_HEXAGON_S2_storeri_pcr:
21395 case Hexagon::BI__builtin_HEXAGON_S2_storerd_pcr:
21396 return MakeCircOp(ID,
false);
21397 case Hexagon::BI__builtin_brev_ldub:
21398 return MakeBrevLd(Intrinsic::hexagon_L2_loadrub_pbr,
Int8Ty);
21399 case Hexagon::BI__builtin_brev_ldb:
21400 return MakeBrevLd(Intrinsic::hexagon_L2_loadrb_pbr,
Int8Ty);
21401 case Hexagon::BI__builtin_brev_lduh:
21402 return MakeBrevLd(Intrinsic::hexagon_L2_loadruh_pbr,
Int16Ty);
21403 case Hexagon::BI__builtin_brev_ldh:
21404 return MakeBrevLd(Intrinsic::hexagon_L2_loadrh_pbr,
Int16Ty);
21405 case Hexagon::BI__builtin_brev_ldw:
21406 return MakeBrevLd(Intrinsic::hexagon_L2_loadri_pbr,
Int32Ty);
21407 case Hexagon::BI__builtin_brev_ldd:
21408 return MakeBrevLd(Intrinsic::hexagon_L2_loadrd_pbr,
Int64Ty);
21421 unsigned ICEArguments = 0;
21429 if (BuiltinID == RISCVVector::BI__builtin_rvv_vget_v ||
21430 BuiltinID == RISCVVector::BI__builtin_rvv_vset_v)
21431 ICEArguments = 1 << 1;
21436 if (BuiltinID == RISCV::BI__builtin_riscv_ntl_load)
21437 ICEArguments |= (1 << 1);
21438 if (BuiltinID == RISCV::BI__builtin_riscv_ntl_store)
21439 ICEArguments |= (1 << 2);
21441 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; i++) {
21446 Ops.push_back(AggValue);
21452 Intrinsic::ID
ID = Intrinsic::not_intrinsic;
21456 constexpr unsigned RVV_VTA = 0x1;
21457 constexpr unsigned RVV_VMA = 0x2;
21458 int PolicyAttrs = 0;
21459 bool IsMasked =
false;
21463 switch (BuiltinID) {
21464 default: llvm_unreachable(
"unexpected builtin ID");
21465 case RISCV::BI__builtin_riscv_orc_b_32:
21466 case RISCV::BI__builtin_riscv_orc_b_64:
21467 case RISCV::BI__builtin_riscv_clz_32:
21468 case RISCV::BI__builtin_riscv_clz_64:
21469 case RISCV::BI__builtin_riscv_ctz_32:
21470 case RISCV::BI__builtin_riscv_ctz_64:
21471 case RISCV::BI__builtin_riscv_clmul_32:
21472 case RISCV::BI__builtin_riscv_clmul_64:
21473 case RISCV::BI__builtin_riscv_clmulh_32:
21474 case RISCV::BI__builtin_riscv_clmulh_64:
21475 case RISCV::BI__builtin_riscv_clmulr_32:
21476 case RISCV::BI__builtin_riscv_clmulr_64:
21477 case RISCV::BI__builtin_riscv_xperm4_32:
21478 case RISCV::BI__builtin_riscv_xperm4_64:
21479 case RISCV::BI__builtin_riscv_xperm8_32:
21480 case RISCV::BI__builtin_riscv_xperm8_64:
21481 case RISCV::BI__builtin_riscv_brev8_32:
21482 case RISCV::BI__builtin_riscv_brev8_64:
21483 case RISCV::BI__builtin_riscv_zip_32:
21484 case RISCV::BI__builtin_riscv_unzip_32: {
21485 switch (BuiltinID) {
21486 default: llvm_unreachable(
"unexpected builtin ID");
21488 case RISCV::BI__builtin_riscv_orc_b_32:
21489 case RISCV::BI__builtin_riscv_orc_b_64:
21490 ID = Intrinsic::riscv_orc_b;
21492 case RISCV::BI__builtin_riscv_clz_32:
21493 case RISCV::BI__builtin_riscv_clz_64: {
21496 if (
Result->getType() != ResultType)
21501 case RISCV::BI__builtin_riscv_ctz_32:
21502 case RISCV::BI__builtin_riscv_ctz_64: {
21505 if (
Result->getType() != ResultType)
21512 case RISCV::BI__builtin_riscv_clmul_32:
21513 case RISCV::BI__builtin_riscv_clmul_64:
21514 ID = Intrinsic::riscv_clmul;
21516 case RISCV::BI__builtin_riscv_clmulh_32:
21517 case RISCV::BI__builtin_riscv_clmulh_64:
21518 ID = Intrinsic::riscv_clmulh;
21520 case RISCV::BI__builtin_riscv_clmulr_32:
21521 case RISCV::BI__builtin_riscv_clmulr_64:
21522 ID = Intrinsic::riscv_clmulr;
21526 case RISCV::BI__builtin_riscv_xperm8_32:
21527 case RISCV::BI__builtin_riscv_xperm8_64:
21528 ID = Intrinsic::riscv_xperm8;
21530 case RISCV::BI__builtin_riscv_xperm4_32:
21531 case RISCV::BI__builtin_riscv_xperm4_64:
21532 ID = Intrinsic::riscv_xperm4;
21536 case RISCV::BI__builtin_riscv_brev8_32:
21537 case RISCV::BI__builtin_riscv_brev8_64:
21538 ID = Intrinsic::riscv_brev8;
21540 case RISCV::BI__builtin_riscv_zip_32:
21541 ID = Intrinsic::riscv_zip;
21543 case RISCV::BI__builtin_riscv_unzip_32:
21544 ID = Intrinsic::riscv_unzip;
21548 IntrinsicTypes = {ResultType};
21555 case RISCV::BI__builtin_riscv_sha256sig0:
21556 ID = Intrinsic::riscv_sha256sig0;
21558 case RISCV::BI__builtin_riscv_sha256sig1:
21559 ID = Intrinsic::riscv_sha256sig1;
21561 case RISCV::BI__builtin_riscv_sha256sum0:
21562 ID = Intrinsic::riscv_sha256sum0;
21564 case RISCV::BI__builtin_riscv_sha256sum1:
21565 ID = Intrinsic::riscv_sha256sum1;
21569 case RISCV::BI__builtin_riscv_sm4ks:
21570 ID = Intrinsic::riscv_sm4ks;
21572 case RISCV::BI__builtin_riscv_sm4ed:
21573 ID = Intrinsic::riscv_sm4ed;
21577 case RISCV::BI__builtin_riscv_sm3p0:
21578 ID = Intrinsic::riscv_sm3p0;
21580 case RISCV::BI__builtin_riscv_sm3p1:
21581 ID = Intrinsic::riscv_sm3p1;
21585 case RISCV::BI__builtin_riscv_ntl_load: {
21587 unsigned DomainVal = 5;
21588 if (Ops.size() == 2)
21589 DomainVal = cast<ConstantInt>(Ops[1])->getZExtValue();
21591 llvm::MDNode *RISCVDomainNode = llvm::MDNode::get(
21593 llvm::ConstantAsMetadata::get(
Builder.getInt32(DomainVal)));
21594 llvm::MDNode *NontemporalNode = llvm::MDNode::get(
21598 if(ResTy->isScalableTy()) {
21599 const ScalableVectorType *SVTy = cast<ScalableVectorType>(ResTy);
21600 llvm::Type *ScalarTy = ResTy->getScalarType();
21601 Width = ScalarTy->getPrimitiveSizeInBits() *
21602 SVTy->getElementCount().getKnownMinValue();
21604 Width = ResTy->getPrimitiveSizeInBits();
21608 Load->setMetadata(llvm::LLVMContext::MD_nontemporal, NontemporalNode);
21609 Load->setMetadata(
CGM.
getModule().getMDKindID(
"riscv-nontemporal-domain"),
21614 case RISCV::BI__builtin_riscv_ntl_store: {
21615 unsigned DomainVal = 5;
21616 if (Ops.size() == 3)
21617 DomainVal = cast<ConstantInt>(Ops[2])->getZExtValue();
21619 llvm::MDNode *RISCVDomainNode = llvm::MDNode::get(
21621 llvm::ConstantAsMetadata::get(
Builder.getInt32(DomainVal)));
21622 llvm::MDNode *NontemporalNode = llvm::MDNode::get(
21626 Store->setMetadata(llvm::LLVMContext::MD_nontemporal, NontemporalNode);
21634#include "clang/Basic/riscv_vector_builtin_cg.inc"
21636#include "clang/Basic/riscv_sifive_vector_builtin_cg.inc"
21639 assert(ID != Intrinsic::not_intrinsic);
21642 return Builder.CreateCall(F, Ops,
"");
Defines the clang::ASTContext interface.
#define PPC_LNX_FEATURE(NAME, DESC, ENUMNAME, ENUMVAL, HWCAPN)
#define X86_CPU_SUBTYPE(ENUM, STR)
#define X86_CPU_SUBTYPE_ALIAS(ENUM, ALIAS)
#define X86_VENDOR(ENUM, STRING)
#define X86_CPU_TYPE_ALIAS(ENUM, ALIAS)
#define X86_CPU_TYPE(ENUM, STR)
static constexpr Builtin::Info BuiltinInfo[]
static void Accumulate(SMap &SM, CFGBlock *B)
static Value * EmitSpecialRegisterBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Type *RegisterType, llvm::Type *ValueType, SpecialRegisterAccessKind AccessKind, StringRef SysReg="")
static llvm::Value * ARMMVEVectorReinterpret(CGBuilderTy &Builder, CodeGenFunction *CGF, llvm::Value *V, llvm::Type *DestType)
static Value * MakeBinaryAtomicValue(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E, AtomicOrdering Ordering=AtomicOrdering::SequentiallyConsistent)
Utility to insert an atomic instruction based on Intrinsic::ID and the expression node.
static char bitActionToX86BTCode(BitTest::ActionKind A)
#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier)
static Value * EmitAtomicCmpXchg128ForMSIntrin(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering SuccessOrdering)
#define INTRINSIC_X86_XSAVE_ID(NAME)
static CanQualType getOSLogArgType(ASTContext &C, int Size)
Get the argument type for arguments to os_log_helper.
static Value * EmitOverflowCheckedAbs(CodeGenFunction &CGF, const CallExpr *E, bool SanitizeOverflow)
static llvm::VectorType * GetFloatNeonType(CodeGenFunction *CGF, NeonTypeFlags IntTypeFlags)
static Value * tryUseTestFPKind(CodeGenFunction &CGF, unsigned BuiltinID, Value *V)
static llvm::Value * MVEImmediateShr(CGBuilderTy &Builder, llvm::Value *V, uint32_t Shift, bool Unsigned)
static bool areBOSTypesCompatible(int From, int To)
Checks if using the result of __builtin_object_size(p, From) in place of __builtin_object_size(p,...
static llvm::Value * SignOrZeroExtend(CGBuilderTy &Builder, llvm::Value *V, llvm::Type *T, bool Unsigned)
static void InsertExplicitZeroOperand(CGBuilderTy &Builder, llvm::Type *Ty, SmallVectorImpl< Value * > &Ops)
static const ARMVectorIntrinsicInfo AArch64SMEIntrinsicMap[]
static Value * EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E, SmallVectorImpl< Value * > &Ops, llvm::Triple::ArchType Arch)
#define MMA_VARIANTS_B1_AND(geom, type)
static void swapCommutativeSMEOperands(unsigned BuiltinID, SmallVectorImpl< Value * > &Ops)
static bool AArch64SISDIntrinsicsProvenSorted
static Value * EmitX86CompressExpand(CodeGenFunction &CGF, ArrayRef< Value * > Ops, bool IsCompress)
static const ARMVectorIntrinsicInfo AArch64SVEIntrinsicMap[]
static bool HasExtraNeonArgument(unsigned BuiltinID)
Return true if BuiltinID is an overloaded Neon intrinsic with an extra argument that specifies the ve...
static bool TypeRequiresBuiltinLaunderImp(const ASTContext &Ctx, QualType Ty, llvm::SmallPtrSetImpl< const Decl * > &Seen)
static Value * EmitAtomicIncrementValue(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering Ordering=AtomicOrdering::SequentiallyConsistent)
static std::pair< Intrinsic::ID, unsigned > getIntrinsicForHexagonNonClangBuiltin(unsigned BuiltinID)
static Value * emitRangedBuiltin(CodeGenFunction &CGF, unsigned IntrinsicID, int low, int high)
#define MMA_INTR(geom_op_type, layout)
static Value * EmitX86MaskedCompare(CodeGenFunction &CGF, unsigned CC, bool Signed, ArrayRef< Value * > Ops)
static Value * emitUnaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier)
static bool AArch64SVEIntrinsicsProvenSorted
static RValue EmitMSVCRTSetJmp(CodeGenFunction &CGF, MSVCSetJmpKind SJKind, const CallExpr *E)
MSVC handles setjmp a bit differently on different platforms.
static const ARMVectorIntrinsicInfo * findARMVectorIntrinsicInMap(ArrayRef< ARMVectorIntrinsicInfo > IntrinsicMap, unsigned BuiltinID, bool &MapProvenSorted)
static Value * EmitScalarFMAExpr(CodeGenFunction &CGF, const CallExpr *E, MutableArrayRef< Value * > Ops, Value *Upper, bool ZeroMask=false, unsigned PTIdx=0, bool NegAcc=false)
#define MUTATE_LDBL(func)
static Value * EmitX86ExpandLoad(CodeGenFunction &CGF, ArrayRef< Value * > Ops)
static unsigned CountCountedByAttrs(const RecordDecl *RD)
static Value * emitMaybeConstrainedFPToIntRoundBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
static Value * EmitX86MaskedLoad(CodeGenFunction &CGF, ArrayRef< Value * > Ops, Align Alignment)
static bool TypeRequiresBuiltinLaunder(CodeGenModule &CGM, QualType Ty)
Determine if the specified type requires laundering by checking if it is a dynamic class type or cont...
static void InsertExplicitUndefOperand(CGBuilderTy &Builder, llvm::Type *Ty, SmallVectorImpl< Value * > &Ops)
static Value * EmitISOVolatileLoad(CodeGenFunction &CGF, const CallExpr *E)
static struct WidthAndSignedness EncompassingIntegerType(ArrayRef< struct WidthAndSignedness > Types)
static Integer GetIntegerConstantValue(const Expr *E, ASTContext &Context)
#define MMA_VARIANTS(geom, type)
static bool AArch64SMEIntrinsicsProvenSorted
static llvm::Value * VectorZip(CGBuilderTy &Builder, llvm::Value *V0, llvm::Value *V1)
static Value * EmitTargetArchBuiltinExpr(CodeGenFunction *CGF, unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
constexpr unsigned SVEBitsPerBlock
static std::optional< CodeGenFunction::MSVCIntrin > translateX86ToMsvcIntrin(unsigned BuiltinID)
static const std::pair< unsigned, unsigned > NEONEquivalentIntrinsicMap[]
#define NEONMAP0(NameBase)
static Value * EmitX86MaskedStore(CodeGenFunction &CGF, ArrayRef< Value * > Ops, Align Alignment)
static Value * emitBinaryBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
static Value * emitBinaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E, Instruction::BinaryOps Op, bool Invert=false)
Utility to insert an atomic instruction based Intrinsic::ID and the expression node,...
static Value * EmitX86Muldq(CodeGenFunction &CGF, bool IsSigned, ArrayRef< Value * > Ops)
static llvm::Value * ARMMVEVectorElementReverse(CGBuilderTy &Builder, llvm::Value *V, unsigned ReverseWidth)
#define MMA_SATF_VARIANTS(geom, type)
static std::optional< CodeGenFunction::MSVCIntrin > translateAarch64ToMsvcIntrin(unsigned BuiltinID)
static std::optional< CodeGenFunction::MSVCIntrin > translateArmToMsvcIntrin(unsigned BuiltinID)
static llvm::Value * EmitBitTestIntrinsic(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E)
Emit a _bittest* intrinsic.
static const ARMVectorIntrinsicInfo ARMSIMDIntrinsicMap[]
static Value * EmitSignBit(CodeGenFunction &CGF, Value *V)
Emit the computation of the sign bit for a floating point value.
static Value * EmitFAbs(CodeGenFunction &CGF, Value *V)
EmitFAbs - Emit a call to @llvm.fabs().
#define CUSTOM_BUILTIN_MAPPING(x, s)
static Value * EmitX86CvtF16ToFloatExpr(CodeGenFunction &CGF, ArrayRef< Value * > Ops, llvm::Type *DstTy)
static bool isSpecialUnsignedMultiplySignedResult(unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info, WidthAndSignedness ResultInfo)
static llvm::Value * getDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType)
static llvm::Value * emitPPCLoadReserveIntrinsic(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E)
static llvm::Value * VectorUnzip(CGBuilderTy &Builder, llvm::Value *V, bool Odd)
static Value * EmitX86SExtMask(CodeGenFunction &CGF, Value *Op, llvm::Type *DstTy)
static Value * emitTernaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
static WidthAndSignedness getIntegerWidthAndSignedness(const clang::ASTContext &context, const clang::QualType Type)
static Value * EmitX86FunnelShift(CodeGenFunction &CGF, Value *Op0, Value *Op1, Value *Amt, bool IsRight)
static RValue EmitCheckedMixedSignMultiply(CodeGenFunction &CGF, const clang::Expr *Op1, WidthAndSignedness Op1Info, const clang::Expr *Op2, WidthAndSignedness Op2Info, const clang::Expr *ResultArg, QualType ResultQTy, WidthAndSignedness ResultInfo)
Emit a checked mixed-sign multiply.
static llvm::ScalableVectorType * getSVEVectorForElementType(llvm::Type *EltTy)
static unsigned mutateLongDoubleBuiltin(unsigned BuiltinID)
#define INTRINSIC_WITH_CC(NAME)
static llvm::FixedVectorType * GetNeonType(CodeGenFunction *CGF, NeonTypeFlags TypeFlags, bool HasLegalHalfType=true, bool V1Ty=false, bool AllowBFloatArgsAndRet=true)
static RValue EmitBinaryAtomic(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E)
static llvm::Value * ARMMVEConstantSplat(CGBuilderTy &Builder, llvm::Type *VT)
static Value * EmitX86MaskLogic(CodeGenFunction &CGF, Instruction::BinaryOps Opc, ArrayRef< Value * > Ops, bool InvertLHS=false)
static Value * EmitFromInt(CodeGenFunction &CGF, llvm::Value *V, QualType T, llvm::Type *ResultType)
static Value * EmitAMDGCNBallotForExec(CodeGenFunction &CGF, const CallExpr *E, llvm::Type *RegisterType, llvm::Type *ValueType, bool isExecHi)
static void initializeAlloca(CodeGenFunction &CGF, AllocaInst *AI, Value *Size, Align AlignmentInBytes)
static Value * EmitX86Select(CodeGenFunction &CGF, Value *Mask, Value *Op0, Value *Op1)
SpecialRegisterAccessKind
static Value * EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering SuccessOrdering=AtomicOrdering::SequentiallyConsistent)
This function should be invoked to emit atomic cmpxchg for Microsoft's _InterlockedCompareExchange* i...
static Address CheckAtomicAlignment(CodeGenFunction &CGF, const CallExpr *E)
static Value * EmitX86FMAExpr(CodeGenFunction &CGF, const CallExpr *E, ArrayRef< Value * > Ops, unsigned BuiltinID, bool IsAddSub)
static Value * getMaskVecValue(CodeGenFunction &CGF, Value *Mask, unsigned NumElts)
static bool isSpecialMixedSignMultiply(unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info, WidthAndSignedness ResultInfo)
Determine if a binop is a checked mixed-sign multiply we can specialize.
static Value * MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E, bool ReturnBool)
Utility to insert an atomic cmpxchg instruction.
static Value * emitBinaryExpMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Intrinsic::ID IntrinsicID, llvm::Intrinsic::ID ConstrainedIntrinsicID)
static Value * EmitToInt(CodeGenFunction &CGF, llvm::Value *V, QualType T, llvm::IntegerType *IntType)
Emit the conversions required to turn the given value into an integer of the given size.
static llvm::Value * ARMMVEVectorSplat(CGBuilderTy &Builder, llvm::Value *V)
static Value * EmitX86MaskedCompareResult(CodeGenFunction &CGF, Value *Cmp, unsigned NumElts, Value *MaskIn)
static Value * EmitX86CompressStore(CodeGenFunction &CGF, ArrayRef< Value * > Ops)
static bool NEONSIMDIntrinsicsProvenSorted
static const ARMVectorIntrinsicInfo AArch64SIMDIntrinsicMap[]
static Value * EmitNontemporalStore(CodeGenFunction &CGF, const CallExpr *E)
static llvm::Value * EmitOverflowIntrinsic(CodeGenFunction &CGF, const llvm::Intrinsic::ID IntrinsicID, llvm::Value *X, llvm::Value *Y, llvm::Value *&Carry)
Emit a call to llvm.
static Value * EmitCommonNeonSISDBuiltinExpr(CodeGenFunction &CGF, const ARMVectorIntrinsicInfo &SISDInfo, SmallVectorImpl< Value * > &Ops, const CallExpr *E)
static Value * emitFPIntBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
static Value * emitTernaryBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
#define MMA_LDST(n, geom_op_type)
static Value * EmitX86vpcom(CodeGenFunction &CGF, ArrayRef< Value * > Ops, bool IsSigned)
static Value * emitFrexpBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Intrinsic::ID IntrinsicID)
@ _InterlockedExchangeAdd_rel
@ _InterlockedIncrement_acq
@ _InterlockedExchange_nf
@ _InterlockedIncrement_nf
@ _InterlockedExchange_acq
@ _InterlockedCompareExchange128_rel
@ _InterlockedCompareExchange128_acq
@ _InterlockedCompareExchange_acq
@ _InterlockedExchangeAdd_nf
@ _InterlockedCompareExchange_nf
@ _InterlockedDecrement_rel
@ _InterlockedExchangeSub
@ _InterlockedExchangeAdd_acq
@ _InterlockedIncrement_rel
@ _InterlockedCompareExchange128_nf
@ _InterlockedCompareExchange128
@ _InterlockedExchange_rel
@ _InterlockedDecrement_nf
@ _InterlockedExchangeAdd
@ _InterlockedDecrement_acq
@ _InterlockedCompareExchange_rel
static Value * EmitX86ConvertToMask(CodeGenFunction &CGF, Value *In)
static Value * EmitISOVolatileStore(CodeGenFunction &CGF, const CallExpr *E)
static Value * EmitSystemZIntrinsicWithCC(CodeGenFunction &CGF, unsigned IntrinsicID, const CallExpr *E)
Handle a SystemZ function in which the final argument is a pointer to an int that receives the post-i...
static RValue EmitHipStdParUnsupportedBuiltin(CodeGenFunction *CGF, const FunctionDecl *FD)
static llvm::Value * EmitX86BitTestIntrinsic(CodeGenFunction &CGF, BitTest BT, const CallExpr *E, Value *BitBase, Value *BitPos)
static RValue EmitCheckedUnsignedMultiplySignedResult(CodeGenFunction &CGF, const clang::Expr *Op1, WidthAndSignedness Op1Info, const clang::Expr *Op2, WidthAndSignedness Op2Info, const clang::Expr *ResultArg, QualType ResultQTy, WidthAndSignedness ResultInfo)
static Value * emitUnaryBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, llvm::StringRef Name="")
static Value * emitCallMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID, llvm::Type *Ty, ArrayRef< Value * > Args)
static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *FD, const CallExpr *E, llvm::Constant *calleeValue)
static Value * EmitNontemporalLoad(CodeGenFunction &CGF, const CallExpr *E)
static Value * EmitX86Ternlog(CodeGenFunction &CGF, bool ZeroMask, ArrayRef< Value * > Ops)
static llvm::AtomicOrdering getBitTestAtomicOrdering(BitTest::InterlockingKind I)
#define MMA_VARIANTS_B1_XOR(geom, type)
#define MMA_VARIANTS_I4(geom, type)
static Value * EmitX86ConvertIntToFp(CodeGenFunction &CGF, const CallExpr *E, ArrayRef< Value * > Ops, bool IsSigned)
static Value * packTBLDVectorList(CodeGenFunction &CGF, ArrayRef< Value * > Ops, Value *ExtOp, Value *IndexOp, llvm::Type *ResTy, unsigned IntID, const char *Name)
static Value * EmitAbs(CodeGenFunction &CGF, Value *ArgValue, bool HasNSW)
static Value * EmitX86ScalarSelect(CodeGenFunction &CGF, Value *Mask, Value *Op0, Value *Op1)
static Value * EmitAtomicDecrementValue(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering Ordering=AtomicOrdering::SequentiallyConsistent)
static bool AArch64SIMDIntrinsicsProvenSorted
static const ARMVectorIntrinsicInfo AArch64SISDIntrinsicMap[]
CodeGenFunction::ComplexPairTy ComplexPairTy
#define ALIAS(NAME, TOK, FLAGS)
llvm::MachO::Record Record
static std::string getName(const CallEvent &Call)
static std::string toString(const clang::SanitizerSet &Sanitizers)
Produce a string containing comma-separated names of sanitizers in Sanitizers set.
Enumerates target-specific builtins in their own namespaces within namespace clang.
Defines the clang::TargetOptions class.
__DEVICE__ float modf(float __x, float *__iptr)
__DEVICE__ double nan(const char *)
__device__ __2f16 float __ockl_bool s
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
CharUnits getTypeAlignInChars(QualType T) const
Return the ABI-specified alignment of a (complete) type T, in characters.
unsigned getIntWidth(QualType T) const
const ASTRecordLayout & getASTRecordLayout(const RecordDecl *D) const
Get or compute information about the layout of the specified record (struct/union/class) D,...
Builtin::Context & BuiltinInfo
QualType getConstantArrayType(QualType EltTy, const llvm::APInt &ArySize, const Expr *SizeExpr, ArraySizeModifier ASM, unsigned IndexTypeQuals) const
Return the unique reference to the type for a constant array of the specified element type.
QualType getBaseElementType(const ArrayType *VAT) const
Return the innermost element type of an array type.
TypeInfo getTypeInfo(const Type *T) const
Get the size and alignment of the specified complete type in bits.
QualType getObjCIdType() const
Represents the Objective-CC id type.
bool hasSameUnqualifiedType(QualType T1, QualType T2) const
Determine whether the given types are equivalent after cvr-qualifiers have been removed.
const ArrayType * getAsArrayType(QualType T) const
Type Query functions.
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
QualType GetBuiltinType(unsigned ID, GetBuiltinTypeError &Error, unsigned *IntegerConstantArgs=nullptr) const
Return the type for the specified builtin.
const TargetInfo & getTargetInfo() const
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
unsigned getTargetAddressSpace(LangAS AS) const
@ GE_Missing_type
Missing a type.
ASTRecordLayout - This class contains layout information for one RecordDecl, which is a struct/union/...
CharUnits getSize() const
getSize - Get the record size in characters.
uint64_t getFieldOffset(unsigned FieldNo) const
getFieldOffset - Get the offset of the given field index, in bits.
Represents an array type, per C99 6.7.5.2 - Array Declarators.
QualType getElementType() const
bool isLibFunction(unsigned ID) const
Return true if this is a builtin for a libc/libm function, with a "__builtin_" prefix (e....
llvm::StringRef getName(unsigned ID) const
Return the identifier name for the specified builtin, e.g.
bool isConstWithoutErrnoAndExceptions(unsigned ID) const
Return true if this function has no side effects and doesn't read memory, except for possibly errno o...
bool isConstWithoutExceptions(unsigned ID) const
bool isConst(unsigned ID) const
Return true if this function has no side effects and doesn't read memory.
CallExpr - Represents a function call (C99 6.5.2.2, C++ [expr.call]).
Expr * getArg(unsigned Arg)
getArg - Return the specified argument.
bool hasStoredFPFeatures() const
SourceLocation getBeginLoc() const LLVM_READONLY
FunctionDecl * getDirectCallee()
If the callee is a FunctionDecl, return it. Otherwise return null.
FPOptionsOverride getFPFeatures() const
unsigned getNumArgs() const
getNumArgs - Return the number of actual arguments to this call.
QualType getCallReturnType(const ASTContext &Ctx) const
getCallReturnType - Get the return type of the call expr.
CharUnits - This is an opaque type for sizes expressed in character units.
llvm::Align getAsAlign() const
getAsAlign - Returns Quantity as a valid llvm::Align, Beware llvm::Align assumes power of two 8-bit b...
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
static CharUnits One()
One - Construct a CharUnits quantity of one.
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
XRayInstrSet XRayInstrumentationBundle
Set of XRay instrumentation kinds to emit.
CharUnits getAlignment() const
Return the alignment of this pointer.
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
Address withAlignment(CharUnits NewAlignment) const
Return address with different alignment, but same pointer and element type.
llvm::Value * getPointer() const
llvm::PointerType * getType() const
Return the type of the pointer value.
Address getAddress() const
static ApplyDebugLocation CreateArtificial(CodeGenFunction &CGF)
Apply TemporaryLocation if it is valid.
static ApplyDebugLocation CreateEmpty(CodeGenFunction &CGF)
Set the IRBuilder to not attach debug locations.
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
llvm::StoreInst * CreateAlignedStore(llvm::Value *Val, llvm::Value *Addr, CharUnits Align, bool IsVolatile=false)
llvm::CallInst * CreateMemMove(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
llvm::CallInst * CreateMemCpyInline(Address Dest, Address Src, uint64_t Size)
llvm::AtomicRMWInst * CreateAtomicRMW(llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val, llvm::AtomicOrdering Ordering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
llvm::CallInst * CreateMemSetInline(Address Dest, llvm::Value *Value, uint64_t Size)
llvm::StoreInst * CreateDefaultAlignedStore(llvm::Value *Val, llvm::Value *Addr, bool IsVolatile=false)
llvm::CallInst * CreateMemSet(Address Dest, llvm::Value *Value, llvm::Value *Size, bool IsVolatile=false)
llvm::AtomicCmpXchgInst * CreateAtomicCmpXchg(Address Addr, llvm::Value *Cmp, llvm::Value *New, llvm::AtomicOrdering SuccessOrdering, llvm::AtomicOrdering FailureOrdering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
Address CreateConstByteGEP(Address Addr, CharUnits Offset, const llvm::Twine &Name="")
Address CreateLaunderInvariantGroup(Address Addr)
llvm::CallInst * CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
llvm::LoadInst * CreateAlignedLoad(llvm::Type *Ty, llvm::Value *Addr, CharUnits Align, const llvm::Twine &Name="")
Address CreateConstInBoundsGEP(Address Addr, uint64_t Index, const llvm::Twine &Name="")
Given addr = T* ... produce name = getelementptr inbounds addr, i64 index where i64 is actually the t...
Address CreateAddrSpaceCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
Address CreateGEP(Address Addr, llvm::Value *Index, const llvm::Twine &Name="")
virtual std::string getDeviceSideName(const NamedDecl *ND)=0
Returns function or variable name on device side even if the current compilation is for host.
virtual llvm::GlobalVariable * getThrowInfo(QualType T)
All available information about a concrete callee.
static CGCallee forDirect(llvm::Constant *functionPtr, const CGCalleeInfo &abstractInfo=CGCalleeInfo())
llvm::DIType * getOrCreateStandaloneType(QualType Ty, SourceLocation Loc)
Emit standalone debug info for a type.
CGFunctionInfo - Class to encapsulate the information about a function definition.
virtual void EmitGCMemmoveCollectable(CodeGen::CodeGenFunction &CGF, Address DestPtr, Address SrcPtr, llvm::Value *Size)=0
EnqueuedBlockInfo emitOpenCLEnqueuedBlock(CodeGenFunction &CGF, const Expr *E)
CallArgList - Type for representing both the value and type of arguments in a call.
void add(RValue rvalue, QualType type)
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
llvm::Value * EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E, llvm::Triple::ArchType Arch)
llvm::Value * EmitFromMemory(llvm::Value *Value, QualType Ty)
EmitFromMemory - Change a scalar value from its memory representation to its value representation.
llvm::Value * EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID, const CallExpr *E)
void FinishFunction(SourceLocation EndLoc=SourceLocation())
FinishFunction - Complete IR generation of the current function.
llvm::Value * EmitLifetimeStart(llvm::TypeSize Size, llvm::Value *Addr)
void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, llvm::Value *V, QualType Type, CharUnits Alignment=CharUnits::Zero(), SanitizerSet SkippedChecks=SanitizerSet(), llvm::Value *ArraySize=nullptr)
Emit a check that V is the address of storage of the appropriate size and alignment for an object of ...
std::pair< RValue, llvm::Value * > EmitAtomicCompareExchange(LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, llvm::AtomicOrdering Success=llvm::AtomicOrdering::SequentiallyConsistent, llvm::AtomicOrdering Failure=llvm::AtomicOrdering::SequentiallyConsistent, bool IsWeak=false, AggValueSlot Slot=AggValueSlot::ignored())
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
llvm::Value * EmitSVEPredicateCast(llvm::Value *Pred, llvm::ScalableVectorType *VTy)
llvm::CallInst * EmitTrapCall(llvm::Intrinsic::ID IntrID)
Emit a call to trap or debugtrap and attach function attribute "trap-func-name" if specified.
SanitizerSet SanOpts
Sanitizers enabled for this function.
RValue EmitBuiltinIsAligned(const CallExpr *E)
Emit IR for __builtin_is_aligned.
LValue EmitAggExprToLValue(const Expr *E)
EmitAggExprToLValue - Emit the computation of the specified expression of aggregate type into a tempo...
void EmitNonNullArgCheck(RValue RV, QualType ArgType, SourceLocation ArgLoc, AbstractCallee AC, unsigned ParmNum)
Create a check for a function parameter that may potentially be declared as non-null.
llvm::Value * EmitHexagonBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
void GetAArch64SVEProcessedOperands(unsigned BuiltinID, const CallExpr *E, SmallVectorImpl< llvm::Value * > &Ops, SVETypeFlags TypeFlags)
llvm::Value * EmitAMDGPUBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
void EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr)
void pushLifetimeExtendedDestroy(CleanupKind kind, Address addr, QualType type, Destroyer *destroyer, bool useEHCleanupForArray)
llvm::Value * EmitCheckedArgForBuiltin(const Expr *E, BuiltinCheckKind Kind)
Emits an argument for a call to a builtin.
llvm::Value * EmitSVEGatherLoad(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
CleanupKind getARCCleanupKind()
Retrieves the default cleanup kind for an ARC cleanup.
llvm::Value * EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitHLSLBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitAArch64SVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
RValue emitBuiltinOSLogFormat(const CallExpr &E)
Emit IR for __builtin_os_log_format.
LValue EmitLValue(const Expr *E, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitLValue - Emit code to compute a designator that specifies the location of the expression.
llvm::Value * EmitVAStartEnd(llvm::Value *ArgValue, bool IsStart)
Emits a call to an LLVM variable-argument intrinsic, either llvm.va_start or llvm....
llvm::Value * EmitSVEMaskedStore(const CallExpr *, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
llvm::Value * EmitSVEReinterpret(llvm::Value *Val, llvm::Type *Ty)
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
llvm::Value * EmitSEHExceptionInfo()
RValue EmitBuiltinAlignTo(const CallExpr *E, bool AlignUp)
Emit IR for __builtin_align_up/__builtin_align_down.
const LangOptions & getLangOpts() const
void ProcessOrderScopeAMDGCN(llvm::Value *Order, llvm::Value *Scope, llvm::AtomicOrdering &AO, llvm::SyncScope::ID &SSID)
llvm::Constant * EmitCheckTypeDescriptor(QualType T)
Emit a description of a type in a format suitable for passing to a runtime sanitizer handler.
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
void EmitTrapCheck(llvm::Value *Checked, SanitizerHandler CheckHandlerID)
Create a basic block that will call the trap intrinsic, and emit a conditional branch to it,...
void EmitUnreachable(SourceLocation Loc)
Emit a reached-unreachable diagnostic if Loc is valid and runtime checking is enabled.
llvm::Value * EmitSVETupleCreate(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType, ArrayRef< llvm::Value * > Ops)
ComplexPairTy EmitComplexExpr(const Expr *E, bool IgnoreReal=false, bool IgnoreImag=false)
EmitComplexExpr - Emit the computation of the specified expression of complex type,...
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
TypeCheckKind
Situations in which we might emit a check for the suitability of a pointer or glvalue.
@ TCK_Store
Checking the destination of a store. Must be suitably sized and aligned.
@ TCK_Load
Checking the operand of a load. Must be suitably sized and aligned.
llvm::Value * EmitSMELdrStr(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee, ReturnValueSlot ReturnValue, const CallArgList &Args, llvm::CallBase **callOrInvoke, bool IsMustTail, SourceLocation Loc)
EmitCall - Generate a call of the given function, expecting the given result type,...
llvm::Value * FormSVEBuiltinResult(llvm::Value *Call)
FormSVEBuiltinResult - Returns the struct of scalable vectors as a wider vector.
llvm::Value * EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx, const llvm::ElementCount &Count)
llvm::Type * ConvertTypeForMem(QualType T)
llvm::Value * EmitSVEMaskedLoad(const CallExpr *, llvm::Type *ReturnTy, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID, bool IsZExtReturn)
bool AlwaysEmitXRayCustomEvents() const
AlwaysEmitXRayCustomEvents - Return true if we must unconditionally emit XRay custom event handling c...
llvm::Value * EmitSVEDupX(llvm::Value *Scalar)
@ Default
! No language constraints on evaluation order.
const TargetInfo & getTarget() const
llvm::Value * vectorWrapScalar16(llvm::Value *Op)
llvm::Function * LookupNeonLLVMIntrinsic(unsigned IntrinsicID, unsigned Modifier, llvm::Type *ArgTy, const CallExpr *E)
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
llvm::Value * EmitSEHExceptionCode()
Address EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitPointerWithAlignment - Given an expression with a pointer type, emit the value and compute our be...
llvm::Value * EmitTargetBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
EmitTargetBuiltinExpr - Emit the given builtin call.
llvm::Value * EmitCountedByFieldExpr(const Expr *Base, const FieldDecl *FAMDecl, const FieldDecl *CountDecl)
Build an expression accessing the "counted_by" field.
RValue EmitAnyExprToTemp(const Expr *E)
EmitAnyExprToTemp - Similarly to EmitAnyExpr(), however, the result will always be accessible even if...
RValue EmitCoroutineIntrinsic(const CallExpr *E, unsigned int IID)
llvm::Value * EmitAArch64SMEBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
RValue EmitAMDGPUDevicePrintfCallExpr(const CallExpr *E)
Address EmitArrayToPointerDecay(const Expr *Array, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
void EmitCheck(ArrayRef< std::pair< llvm::Value *, SanitizerMask > > Checked, SanitizerHandler Check, ArrayRef< llvm::Constant * > StaticArgs, ArrayRef< llvm::Value * > DynamicArgs)
Create a basic block that will either trap or call a handler function in the UBSan runtime with the p...
RValue EmitBuiltinNewDeleteCall(const FunctionProtoType *Type, const CallExpr *TheCallExpr, bool IsDelete)
llvm::Value * EmitSVETupleSetOrGet(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType, ArrayRef< llvm::Value * > Ops)
RValue EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
llvm::Value * EmitSVEPrefetchLoad(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
llvm::Value * EmitAArch64CompareBuiltinExpr(llvm::Value *Op, llvm::Type *Ty, const llvm::CmpInst::Predicate Fp, const llvm::CmpInst::Predicate Ip, const llvm::Twine &Name="")
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
llvm::ScalableVectorType * getSVEType(const SVETypeFlags &TypeFlags)
RValue emitRotate(const CallExpr *E, bool IsRotateRight)
llvm::Constant * EmitCheckSourceLocation(SourceLocation Loc)
Emit a description of a source location in a format suitable for passing to a runtime sanitizer handl...
void ErrorUnsupported(const Stmt *S, const char *Type)
ErrorUnsupported - Print out an error that codegen doesn't support the specified stmt yet.
CGDebugInfo * getDebugInfo()
Address EmitVAListRef(const Expr *E)
llvm::Value * EmitNeonShiftVector(llvm::Value *V, llvm::Type *Ty, bool negateForRightShift)
llvm::Value * EmitSVEMovl(const SVETypeFlags &TypeFlags, llvm::ArrayRef< llvm::Value * > Ops, unsigned BuiltinID)
void emitAlignmentAssumption(llvm::Value *PtrValue, QualType Ty, SourceLocation Loc, SourceLocation AssumptionLoc, llvm::Value *Alignment, llvm::Value *OffsetValue=nullptr)
const TargetCodeGenInfo & getTargetHooks() const
llvm::Value * EmitARMBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
llvm::Type * getEltType(const SVETypeFlags &TypeFlags)
void EmitAggExpr(const Expr *E, AggValueSlot AS)
EmitAggExpr - Emit the computation of the specified expression of aggregate type.
bool ShouldXRayInstrumentFunction() const
ShouldXRayInstrument - Return true if the current function should be instrumented with XRay nop sleds...
const FieldDecl * FindFlexibleArrayMemberField(ASTContext &Ctx, const RecordDecl *RD, StringRef Name, uint64_t &Offset)
llvm::Value * EmitSVEPMull(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
llvm::Value * EmitToMemory(llvm::Value *Value, QualType Ty)
EmitToMemory - Change a scalar value from its value representation to its in-memory representation.
RValue EmitOpenMPDevicePrintfCallExpr(const CallExpr *E)
bool IsInPreservedAIRegion
True if CodeGen currently emits code inside presereved access index region.
llvm::Value * EmitARCRetain(QualType type, llvm::Value *value)
bool AlwaysEmitXRayTypedEvents() const
AlwaysEmitXRayTypedEvents - Return true if clang must unconditionally emit XRay typed event handling ...
void SetSqrtFPAccuracy(llvm::Value *Val)
Set the minimum required accuracy of the given sqrt operation based on CodeGenOpts.
llvm::Value * EmitSVEScatterStore(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::CallInst * EmitNounwindRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
ASTContext & getContext() const
llvm::Function * generateBuiltinOSLogHelperFunction(const analyze_os_log::OSLogBufferLayout &Layout, CharUnits BufferAlignment)
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
CGCallee EmitCallee(const Expr *E)
Address CreateMemTemp(QualType T, const Twine &Name="tmp", Address *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
llvm::Value * EmitScalarOrConstFoldImmArg(unsigned ICEArguments, unsigned Idx, const CallExpr *E)
void checkTargetFeatures(const CallExpr *E, const FunctionDecl *TargetDecl)
llvm::Value * BuildVector(ArrayRef< llvm::Value * > Ops)
llvm::Value * EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitARMCDEBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
llvm::CallInst * EmitRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
llvm::Type * ConvertType(QualType T)
llvm::CallBase * EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee, ArrayRef< llvm::Value * > args, const Twine &name="")
llvm::Value * EmitSystemZBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitSMEReadWrite(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T)
llvm::Value * EmitSMELd1St1(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::Value * EmitPPCBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
static Destroyer destroyARCStrongPrecise
void EmitARCIntrinsicUse(ArrayRef< llvm::Value * > values)
RValue EmitNVPTXDevicePrintfCallExpr(const CallExpr *E)
llvm::Value * EmitSVEStructLoad(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Address EmitMSVAListRef(const Expr *E)
Emit a "reference" to a __builtin_ms_va_list; this is always the value of the expression,...
llvm::Value * EmitCheckedInBoundsGEP(llvm::Type *ElemTy, llvm::Value *Ptr, ArrayRef< llvm::Value * > IdxList, bool SignedIndices, bool IsSubtraction, SourceLocation Loc, const Twine &Name="")
Same as IRBuilder::CreateInBoundsGEP, but additionally emits a check to detect undefined behavior whe...
llvm::Value * EmitNeonRShiftImm(llvm::Value *Vec, llvm::Value *Amt, llvm::Type *Ty, bool usgn, const char *name)
SmallVector< llvm::Type *, 2 > getSVEOverloadTypes(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType, ArrayRef< llvm::Value * > Ops)
static bool hasAggregateEvaluationKind(QualType T)
const FieldDecl * FindCountedByField(const FieldDecl *FD)
Find the FieldDecl specified in a FAM's "counted_by" attribute.
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
llvm::Value * EmitARMMVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
llvm::Value * EmitSVEStructStore(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
llvm::Value * EmitSEHAbnormalTermination()
llvm::Value * EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E)
AggValueSlot CreateAggTemp(QualType T, const Twine &Name="tmp", Address *Alloca=nullptr)
CreateAggTemp - Create a temporary memory object for the given aggregate type.
Address ReturnValue
ReturnValue - The temporary alloca to hold the return value.
llvm::Value * EmitSVEAllTruePred(const SVETypeFlags &TypeFlags)
RValue GetUndefRValue(QualType Ty)
GetUndefRValue - Get an appropriate 'undef' rvalue for the given type.
llvm::Type * SVEBuiltinMemEltTy(const SVETypeFlags &TypeFlags)
SVEBuiltinMemEltTy - Returns the memory element type for this memory access builtin.
llvm::LLVMContext & getLLVMContext()
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
llvm::Value * EmitSMEZero(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::Value * EmitRISCVBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
llvm::Value * EmitCommonNeonBuiltinExpr(unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic, const char *NameHint, unsigned Modifier, const CallExpr *E, SmallVectorImpl< llvm::Value * > &Ops, Address PtrOp0, Address PtrOp1, llvm::Triple::ArchType Arch)
llvm::Value * EmitNeonCall(llvm::Function *F, SmallVectorImpl< llvm::Value * > &O, const char *name, unsigned shift=0, bool rightshift=false)
llvm::Value * EmitAnnotationCall(llvm::Function *AnnotationFn, llvm::Value *AnnotatedVal, StringRef AnnotationStr, SourceLocation Location, const AnnotateAttr *Attr)
Emit an annotation call (intrinsic).
llvm::ScalableVectorType * getSVEPredType(const SVETypeFlags &TypeFlags)
llvm::Value * EmitSVEGatherPrefetch(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
llvm::Value * EmitBPFBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
This class organizes the cross-function state that is used while generating LLVM code.
llvm::Module & getModule() const
llvm::FunctionCallee CreateRuntimeFunction(llvm::FunctionType *Ty, StringRef Name, llvm::AttributeList ExtraAttrs=llvm::AttributeList(), bool Local=false, bool AssumeConvergent=false)
Create or return a runtime function declaration with the specified type and name.
llvm::Constant * getBuiltinLibFunction(const FunctionDecl *FD, unsigned BuiltinID)
Given a builtin id for a function like "__builtin_fabsf", return a Function* for "fabsf".
DiagnosticsEngine & getDiags() const
void ErrorUnsupported(const Stmt *S, const char *Type)
Print out an error that codegen doesn't support the specified stmt yet.
CGCUDARuntime & getCUDARuntime()
Return a reference to the configured CUDA runtime.
CodeGenTypes & getTypes()
CGOpenCLRuntime & getOpenCLRuntime()
Return a reference to the configured OpenCL runtime.
const TargetInfo & getTarget() const
const llvm::DataLayout & getDataLayout() const
void Error(SourceLocation loc, StringRef error)
Emit a general error that something can't be done.
CGCXXABI & getCXXABI() const
llvm::Constant * GetFunctionStart(const ValueDecl *Decl)
const llvm::Triple & getTriple() const
void DecorateInstructionWithTBAA(llvm::Instruction *Inst, TBAAAccessInfo TBAAInfo)
DecorateInstructionWithTBAA - Decorate the instruction with a TBAA tag.
llvm::Constant * CreateRuntimeVariable(llvm::Type *Ty, StringRef Name)
Create a new runtime global variable with the specified type and name.
TBAAAccessInfo getTBAAAccessInfo(QualType AccessType)
getTBAAAccessInfo - Get TBAA information that describes an access to an object of the given type.
ASTContext & getContext() const
const CodeGenOptions & getCodeGenOpts() const
StringRef getMangledName(GlobalDecl GD)
CharUnits getNaturalPointeeTypeAlignment(QualType T, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
llvm::LLVMContext & getLLVMContext()
CGObjCRuntime & getObjCRuntime()
Return a reference to the configured Objective-C runtime.
void SetLLVMFunctionAttributes(GlobalDecl GD, const CGFunctionInfo &Info, llvm::Function *F, bool IsThunk)
Set the LLVM function attributes (sext, zext, etc).
void SetLLVMFunctionAttributesForDefinition(const Decl *D, llvm::Function *F)
Set the LLVM function attributes which only apply to a function definition.
llvm::Function * getIntrinsic(unsigned IID, ArrayRef< llvm::Type * > Tys=std::nullopt)
ConstantAddress GetAddrOfConstantCString(const std::string &Str, const char *GlobalName=nullptr)
Returns a pointer to a character array containing the literal and a terminating '\0' character.
llvm::Type * ConvertType(QualType T)
ConvertType - Convert type T into a llvm::Type.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
const CGFunctionInfo & arrangeBuiltinFunctionCall(QualType resultType, const CallArgList &args)
llvm::Constant * emitAbstract(const Expr *E, QualType T)
Emit the result of the given expression as an abstract constant, asserting that it succeeded.
Information for lazily generating a cleanup.
FunctionArgList - Type for representing both the decl and type of parameters to a function.
LValue - This represents an lvalue references.
llvm::Value * getBitFieldPointer() const
Address getAddress(CodeGenFunction &CGF) const
void setNontemporal(bool Value)
llvm::Value * getPointer(CodeGenFunction &CGF) const
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
static RValue getIgnored()
static RValue get(llvm::Value *V)
static RValue getAggregate(Address addr, bool isVolatile=false)
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
ReturnValueSlot - Contains the address where the return value of a function can be stored,...
virtual llvm::Value * encodeReturnAddress(CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const
Performs the code-generation required to convert the address of an instruction into a return address ...
virtual llvm::Value * decodeReturnAddress(CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const
Performs the code-generation required to convert a return address as stored by the system into the ac...
const T & getABIInfo() const
virtual int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const
Determines the DWARF register number for the stack pointer, for exception-handling purposes.
virtual llvm::Value * testFPKind(llvm::Value *V, unsigned BuiltinID, CGBuilderTy &Builder, CodeGenModule &CGM) const
Performs a target specific test of a floating point value for things like IsNaN, Infinity,...
Complex values, per C99 6.2.5p11.
Represents a concrete matrix type with constant number of rows and columns.
RecordDecl * getOuterLexicalRecordContext()
Retrieve the outermost lexically enclosing record context.
decl_range decls() const
decls_begin/decls_end - Iterate over the declarations stored in this context.
Decl - This represents one declaration (or definition), e.g.
FunctionDecl * getAsFunction() LLVM_READONLY
Returns the function itself, or the templated function if this is a function template.
DeclContext * getDeclContext()
static bool isFlexibleArrayMemberLike(ASTContext &Context, const Decl *D, QualType Ty, LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel, bool IgnoreTemplateOrMacroSubstitution)
Whether it resembles a flexible array member.
Concrete class used by the front-end to report problems and issues.
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
This represents one expression.
bool EvaluateAsInt(EvalResult &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer,...
Expr * IgnoreParenCasts() LLVM_READONLY
Skip past any parentheses and casts which might surround this expression until reaching a fixed point...
llvm::APSInt EvaluateKnownConstInt(const ASTContext &Ctx, SmallVectorImpl< PartialDiagnosticAt > *Diag=nullptr) const
EvaluateKnownConstInt - Call EvaluateAsRValue and return the folded integer.
Expr * IgnoreParenImpCasts() LLVM_READONLY
Skip past any parentheses and implicit casts which might surround this expression until reaching a fi...
bool EvaluateAsFloat(llvm::APFloat &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsFloat - Return true if this is a constant which we can fold and convert to a floating point...
Expr * IgnoreParens() LLVM_READONLY
Skip past any parentheses which might surround this expression until reaching a fixed point.
@ NPC_ValueDependentIsNotNull
Specifies that a value-dependent expression should be considered to never be a null pointer constant.
ExprObjectKind getObjectKind() const
getObjectKind - The object kind that this expression produces.
bool EvaluateAsRValue(EvalResult &Result, const ASTContext &Ctx, bool InConstantContext=false) const
EvaluateAsRValue - Return true if this is a constant which we can fold to an rvalue using any crazy t...
bool HasSideEffects(const ASTContext &Ctx, bool IncludePossibleEffects=true) const
HasSideEffects - This routine returns true for all those expressions which have any effect other than...
Expr * IgnoreImpCasts() LLVM_READONLY
Skip past any implicit casts which might surround this expression until reaching a fixed point.
NullPointerConstantKind isNullPointerConstant(ASTContext &Ctx, NullPointerConstantValueDependence NPC) const
isNullPointerConstant - C99 6.3.2.3p3 - Test if this reduces down to a Null pointer constant.
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
std::optional< llvm::APSInt > getIntegerConstantExpr(const ASTContext &Ctx, SourceLocation *Loc=nullptr) const
isIntegerConstantExpr - Return the value if this expression is a valid integer constant expression.
bool tryEvaluateObjectSize(uint64_t &Result, ASTContext &Ctx, unsigned Type) const
If the current Expr is a pointer, this will try to statically determine the number of bytes available...
const ValueDecl * getAsBuiltinConstantDeclRef(const ASTContext &Context) const
If this expression is an unambiguous reference to a single declaration, in the style of __builtin_fun...
Represents difference between two FPOptions values.
Represents a member of a struct/union/class.
Represents a function declaration or definition.
const ParmVarDecl * getParamDecl(unsigned i) const
Represents a prototype with parameter type info, e.g.
GlobalDecl - represents a global declaration.
const Decl * getDecl() const
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
StrictFlexArraysLevelKind
StringRef getName() const
Get the name of identifier for this declaration as a StringRef.
std::string getNameAsString() const
Get a human-readable name for the declaration, even if it is one of the special kinds of names (C++ c...
Flags to identify the types for overloaded Neon builtins.
EltType getEltType() const
PointerType - C99 6.7.5.1 - Pointer Declarators.
QualType getPointeeType() const
A (possibly-)qualified type.
bool isVolatileQualified() const
Determine whether this type is volatile-qualified.
bool isWebAssemblyFuncrefType() const
Returns true if it is a WebAssembly Funcref Type.
LangAS getAddressSpace() const
Return the address space of this type.
bool isWebAssemblyExternrefType() const
Returns true if it is a WebAssembly Externref Type.
The collection of all-type qualifiers we support.
Represents a struct/union/class.
Flags to identify the types for overloaded SVE builtins.
bool isZExtReturn() const
bool isReverseUSDOT() const
bool isOverloadNone() const
MemEltType getMemEltType() const
bool isGatherLoad() const
bool isOverloadCvt() const
EltType getEltType() const
bool isOverloadDefault() const
bool isOverloadWhileRW() const
bool isReverseMergeAnyAccOp() const
bool isReductionQV() const
bool isInsertOp1SVALL() const
bool isAppendSVALL() const
bool isReverseMergeAnyBinOp() const
bool isStructStore() const
bool isTupleCreate() const
bool isGatherPrefetch() const
bool hasSplatOperand() const
MergeType getMergeType() const
bool isByteIndexed() const
bool isStructLoad() const
bool isOverloadWhileOrMultiVecCvt() const
unsigned getSplatOperand() const
bool isScatterStore() const
bool isReverseCompare() const
Scope - A scope is a transient data structure that is used while parsing the program.
Encodes a location in the source.
SourceLocation getBeginLoc() const LLVM_READONLY
Exposes information about the current target.
TargetOptions & getTargetOpts() const
Retrieve the target options.
virtual bool hasLegalHalfType() const
Determine whether _Float16 is supported on this target.
const llvm::Triple & getTriple() const
Returns the target triple of the primary target.
bool isLittleEndian() const
unsigned getMaxOpenCLWorkGroupSize() const
virtual bool checkArithmeticFenceSupported() const
Controls if __arithmetic_fence is supported in the targeted backend.
unsigned getSuitableAlign() const
Return the alignment that is the largest alignment ever used for any scalar/SIMD data type on the tar...
virtual std::string_view getClobbers() const =0
Returns a string of target-specific clobbers, in LLVM format.
llvm::CodeObjectVersionKind CodeObjectVersion
Code object version for AMDGPU.
The base class of the type hierarchy.
CXXRecordDecl * getAsCXXRecordDecl() const
Retrieves the CXXRecordDecl that this type refers to, either because the type is a RecordType or beca...
bool isBlockPointerType() const
bool isBooleanType() const
bool isSignedIntegerType() const
Return true if this is an integer type that is signed, according to C99 6.2.5p4 [char,...
bool isComplexType() const
isComplexType() does not include complex integers (a GCC extension).
bool isPointerType() const
bool isIntegerType() const
isIntegerType() does not include complex integers (a GCC extension).
const T * castAs() const
Member-template castAs<specific type>.
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
bool isIntegralOrEnumerationType() const
Determine whether this type is an integral or enumeration type.
bool hasUnsignedIntegerRepresentation() const
Determine whether this type has an unsigned integer representation of some sort, e....
bool isBitIntType() const
bool hasSignedIntegerRepresentation() const
Determine whether this type has an signed integer representation of some sort, e.g....
bool isObjCObjectPointerType() const
bool hasFloatingRepresentation() const
Determine whether this type has a floating-point representation of some sort, e.g....
bool isFloatingType() const
bool isUnsignedIntegerType() const
Return true if this is an integer type that is unsigned, according to C99 6.2.5p6 [which returns true...
const T * getAs() const
Member-template getAs<specific type>'.
RecordDecl * getAsRecordDecl() const
Retrieves the RecordDecl this type refers to.
Represent the declaration of a variable (in which case it is an lvalue) a function (in which case it ...
Represents a GCC generic vector type.
unsigned getNumElements() const
SmallVector< OSLogBufferItem, 4 > Items
unsigned char getNumArgsByte() const
unsigned char getSummaryByte() const
Defines the clang::TargetInfo interface.
@ Type
The l-value was considered opaque, so the alignment was determined from a type.
llvm::Constant * initializationPatternFor(CodeGenModule &, llvm::Type *)
TypeEvaluationKind
The kind of evaluation to perform on values of a particular type.
@ EHCleanup
Denotes a cleanup that should run when a scope is exited using exceptional control flow (a throw stat...
constexpr XRayInstrMask Typed
constexpr XRayInstrMask Custom
bool computeOSLogBufferLayout(clang::ASTContext &Ctx, const clang::CallExpr *E, OSLogBufferLayout &layout)
const void * Store
Store - This opaque type encapsulates an immutable mapping from locations to values.
bool Dup(InterpState &S, CodePtr OpPC)
bool Ret(InterpState &S, CodePtr &PC, APValue &Result)
bool Zero(InterpState &S, CodePtr OpPC)
bool Neg(InterpState &S, CodePtr OpPC)
bool Load(InterpState &S, CodePtr OpPC)
bool Cast(InterpState &S, CodePtr OpPC)
The JSON file list parser is used to communicate input to InstallAPI.
@ DType
'dtype' clause, an alias for 'device_type', stored separately for diagnostic purposes.
@ OK_BitField
A bitfield object is a bitfield on a C or C++ record.
bool operator<(DeclarationName LHS, DeclarationName RHS)
Ordering on two declaration names.
@ Asm
Assembly: we accept this only so that we can preprocess it.
@ Result
The result type of a method or function.
LangAS
Defines the address space values used by the address space qualifier of QualType.
@ Success
Template argument deduction was successful.
@ Other
Other implicit parameter.
YAML serialization mapping.
llvm::PointerType * VoidPtrTy
llvm::IntegerType * Int64Ty
llvm::PointerType * ConstGlobalsPtrTy
void* in the address space for constant globals
CharUnits getIntAlign() const
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
llvm::Type * HalfTy
half, bfloat, float, double
llvm::IntegerType * SizeTy
llvm::IntegerType * Int32Ty
llvm::IntegerType * IntPtrTy
llvm::IntegerType * IntTy
int
llvm::IntegerType * Int16Ty
llvm::PointerType * Int8PtrTy
llvm::PointerType * UnqualPtrTy
llvm::PointerType * AllocaInt8PtrTy
LangAS getASTAllocaAddressSpace() const
EvalResult is a struct with detailed info about an evaluated expression.
APValue Val
Val - This is the value the expression can be folded to.
void clear(SanitizerMask K=SanitizerKind::All)
Disable the sanitizers specified in K.
void set(SanitizerMask K, bool Value)
Enable or disable a certain (single) sanitizer.
bool has(SanitizerMask K) const
Check if a certain (single) sanitizer is enabled.
bool has(XRayInstrMask K) const
#define scalbln(__x, __y)
#define copysign(__x, __y)
#define remquo(__x, __y, __z)
#define nextafter(__x, __y)
#define nexttoward(__x, __y)
#define remainder(__x, __y)
#define fma(__x, __y, __z)