35#include "llvm/ADT/APFloat.h"
36#include "llvm/ADT/APInt.h"
37#include "llvm/ADT/FloatingPointMode.h"
38#include "llvm/ADT/SmallPtrSet.h"
39#include "llvm/ADT/StringExtras.h"
40#include "llvm/Analysis/ValueTracking.h"
41#include "llvm/IR/DataLayout.h"
42#include "llvm/IR/InlineAsm.h"
43#include "llvm/IR/Intrinsics.h"
44#include "llvm/IR/IntrinsicsAArch64.h"
45#include "llvm/IR/IntrinsicsAMDGPU.h"
46#include "llvm/IR/IntrinsicsARM.h"
47#include "llvm/IR/IntrinsicsBPF.h"
48#include "llvm/IR/IntrinsicsDirectX.h"
49#include "llvm/IR/IntrinsicsHexagon.h"
50#include "llvm/IR/IntrinsicsNVPTX.h"
51#include "llvm/IR/IntrinsicsPowerPC.h"
52#include "llvm/IR/IntrinsicsR600.h"
53#include "llvm/IR/IntrinsicsRISCV.h"
54#include "llvm/IR/IntrinsicsS390.h"
55#include "llvm/IR/IntrinsicsVE.h"
56#include "llvm/IR/IntrinsicsWebAssembly.h"
57#include "llvm/IR/IntrinsicsX86.h"
58#include "llvm/IR/MDBuilder.h"
59#include "llvm/IR/MatrixBuilder.h"
60#include "llvm/Support/ConvertUTF.h"
61#include "llvm/Support/MathExtras.h"
62#include "llvm/Support/ScopedPrinter.h"
63#include "llvm/TargetParser/AArch64TargetParser.h"
64#include "llvm/TargetParser/X86TargetParser.h"
69using namespace CodeGen;
73 Align AlignmentInBytes) {
75 switch (CGF.
getLangOpts().getTrivialAutoVarInit()) {
76 case LangOptions::TrivialAutoVarInitKind::Uninitialized:
79 case LangOptions::TrivialAutoVarInitKind::Zero:
80 Byte = CGF.
Builder.getInt8(0x00);
82 case LangOptions::TrivialAutoVarInitKind::Pattern: {
84 Byte = llvm::dyn_cast<llvm::ConstantInt>(
92 I->addAnnotationMetadata(
"auto-init");
107 static SmallDenseMap<unsigned, StringRef, 64> F128Builtins{
108 {Builtin::BI__builtin___fprintf_chk,
"__fprintf_chkieee128"},
109 {Builtin::BI__builtin___printf_chk,
"__printf_chkieee128"},
110 {Builtin::BI__builtin___snprintf_chk,
"__snprintf_chkieee128"},
111 {Builtin::BI__builtin___sprintf_chk,
"__sprintf_chkieee128"},
112 {Builtin::BI__builtin___vfprintf_chk,
"__vfprintf_chkieee128"},
113 {Builtin::BI__builtin___vprintf_chk,
"__vprintf_chkieee128"},
114 {Builtin::BI__builtin___vsnprintf_chk,
"__vsnprintf_chkieee128"},
115 {Builtin::BI__builtin___vsprintf_chk,
"__vsprintf_chkieee128"},
116 {Builtin::BI__builtin_fprintf,
"__fprintfieee128"},
117 {Builtin::BI__builtin_printf,
"__printfieee128"},
118 {Builtin::BI__builtin_snprintf,
"__snprintfieee128"},
119 {Builtin::BI__builtin_sprintf,
"__sprintfieee128"},
120 {Builtin::BI__builtin_vfprintf,
"__vfprintfieee128"},
121 {Builtin::BI__builtin_vprintf,
"__vprintfieee128"},
122 {Builtin::BI__builtin_vsnprintf,
"__vsnprintfieee128"},
123 {Builtin::BI__builtin_vsprintf,
"__vsprintfieee128"},
124 {Builtin::BI__builtin_fscanf,
"__fscanfieee128"},
125 {Builtin::BI__builtin_scanf,
"__scanfieee128"},
126 {Builtin::BI__builtin_sscanf,
"__sscanfieee128"},
127 {Builtin::BI__builtin_vfscanf,
"__vfscanfieee128"},
128 {Builtin::BI__builtin_vscanf,
"__vscanfieee128"},
129 {Builtin::BI__builtin_vsscanf,
"__vsscanfieee128"},
130 {Builtin::BI__builtin_nexttowardf128,
"__nexttowardieee128"},
136 static SmallDenseMap<unsigned, StringRef, 4> AIXLongDouble64Builtins{
137 {Builtin::BI__builtin_frexpl,
"frexp"},
138 {Builtin::BI__builtin_ldexpl,
"ldexp"},
139 {Builtin::BI__builtin_modfl,
"modf"},
145 if (FD->
hasAttr<AsmLabelAttr>())
151 &
getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad() &&
152 F128Builtins.contains(BuiltinID))
153 Name = F128Builtins[BuiltinID];
156 &llvm::APFloat::IEEEdouble() &&
157 AIXLongDouble64Builtins.contains(BuiltinID))
158 Name = AIXLongDouble64Builtins[BuiltinID];
163 llvm::FunctionType *Ty =
166 return GetOrCreateLLVMFunction(Name, Ty, D,
false);
172 QualType T, llvm::IntegerType *IntType) {
175 if (
V->getType()->isPointerTy())
176 return CGF.
Builder.CreatePtrToInt(
V, IntType);
178 assert(
V->getType() == IntType);
186 if (ResultType->isPointerTy())
187 return CGF.
Builder.CreateIntToPtr(
V, ResultType);
189 assert(
V->getType() == ResultType);
200 if (Align % Bytes != 0) {
213 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
223 llvm::IntegerType *IntType = llvm::IntegerType::get(
227 llvm::Type *ValueType = Val->getType();
255 llvm::AtomicRMWInst::BinOp Kind,
264 llvm::AtomicRMWInst::BinOp Kind,
266 Instruction::BinaryOps Op,
267 bool Invert =
false) {
276 llvm::IntegerType *IntType = llvm::IntegerType::get(
280 llvm::Type *ValueType = Val->getType();
284 Kind, DestAddr, Val, llvm::AtomicOrdering::SequentiallyConsistent);
289 llvm::ConstantInt::getAllOnesValue(IntType));
313 llvm::IntegerType *IntType = llvm::IntegerType::get(
317 llvm::Type *ValueType = Cmp->getType();
322 DestAddr, Cmp, New, llvm::AtomicOrdering::SequentiallyConsistent,
323 llvm::AtomicOrdering::SequentiallyConsistent);
326 return CGF.
Builder.CreateZExt(CGF.
Builder.CreateExtractValue(Pair, 1),
349 AtomicOrdering SuccessOrdering = AtomicOrdering::SequentiallyConsistent) {
364 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release ?
365 AtomicOrdering::Monotonic :
373 DestAddr, Comparand, Exchange, SuccessOrdering, FailureOrdering);
374 Result->setVolatile(
true);
392 AtomicOrdering SuccessOrdering) {
399 assert(DestPtr->getType()->isPointerTy());
400 assert(!ExchangeHigh->getType()->isPointerTy());
401 assert(!ExchangeLow->getType()->isPointerTy());
404 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release
405 ? AtomicOrdering::Monotonic
410 llvm::Type *Int128Ty = llvm::IntegerType::get(CGF.
getLLVMContext(), 128);
411 Address DestAddr(DestPtr, Int128Ty,
416 ExchangeHigh = CGF.
Builder.CreateZExt(ExchangeHigh, Int128Ty);
417 ExchangeLow = CGF.
Builder.CreateZExt(ExchangeLow, Int128Ty);
419 CGF.
Builder.CreateShl(ExchangeHigh, llvm::ConstantInt::get(Int128Ty, 64));
420 llvm::Value *Exchange = CGF.
Builder.CreateOr(ExchangeHigh, ExchangeLow);
426 SuccessOrdering, FailureOrdering);
432 CXI->setVolatile(
true);
444 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
450 AtomicRMWInst::Add, DestAddr, ConstantInt::get(IntTy, 1), Ordering);
451 return CGF.
Builder.CreateAdd(
Result, ConstantInt::get(IntTy, 1));
456 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
462 AtomicRMWInst::Sub, DestAddr, ConstantInt::get(IntTy, 1), Ordering);
463 return CGF.
Builder.CreateSub(
Result, ConstantInt::get(IntTy, 1));
474 Load->setVolatile(
true);
484 llvm::StoreInst *Store =
486 Store->setVolatile(
true);
494 const CallExpr *E,
unsigned IntrinsicID,
495 unsigned ConstrainedIntrinsicID) {
498 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
499 if (CGF.
Builder.getIsFPConstrained()) {
501 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0 });
504 return CGF.
Builder.CreateCall(F, Src0);
511 const CallExpr *E,
unsigned IntrinsicID,
512 unsigned ConstrainedIntrinsicID) {
516 if (CGF.
Builder.getIsFPConstrained()) {
517 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
519 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0, Src1 });
522 return CGF.
Builder.CreateCall(F, { Src0, Src1 });
529 llvm::Intrinsic::ID ConstrainedIntrinsicID) {
533 if (CGF.
Builder.getIsFPConstrained()) {
534 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
536 {Src0->getType(), Src1->getType()});
537 return CGF.
Builder.CreateConstrainedFPCall(F, {Src0, Src1});
542 return CGF.
Builder.CreateCall(F, {Src0, Src1});
548 const CallExpr *E,
unsigned IntrinsicID,
549 unsigned ConstrainedIntrinsicID) {
554 if (CGF.
Builder.getIsFPConstrained()) {
555 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
557 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0, Src1, Src2 });
560 return CGF.
Builder.CreateCall(F, { Src0, Src1, Src2 });
567 unsigned IntrinsicID,
568 unsigned ConstrainedIntrinsicID,
572 if (CGF.
Builder.getIsFPConstrained())
577 if (CGF.
Builder.getIsFPConstrained())
578 return CGF.
Builder.CreateConstrainedFPCall(F, Args);
580 return CGF.
Builder.CreateCall(F, Args);
586 unsigned IntrinsicID,
587 llvm::StringRef Name =
"") {
591 return CGF.
Builder.CreateCall(F, Src0, Name);
597 unsigned IntrinsicID) {
602 return CGF.
Builder.CreateCall(F, { Src0, Src1 });
608 unsigned IntrinsicID) {
614 return CGF.
Builder.CreateCall(F, { Src0, Src1, Src2 });
620 unsigned IntrinsicID) {
625 return CGF.
Builder.CreateCall(F, {Src0, Src1});
631 unsigned IntrinsicID,
632 unsigned ConstrainedIntrinsicID) {
636 if (CGF.
Builder.getIsFPConstrained()) {
637 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
639 {ResultType, Src0->getType()});
640 return CGF.
Builder.CreateConstrainedFPCall(F, {Src0});
644 return CGF.
Builder.CreateCall(F, Src0);
649 llvm::Intrinsic::ID IntrinsicID) {
657 llvm::Value *
Call = CGF.
Builder.CreateCall(F, Src0);
659 llvm::Value *Exp = CGF.
Builder.CreateExtractValue(
Call, 1);
670 Call->setDoesNotAccessMemory();
679 llvm::Type *Ty =
V->getType();
680 int Width = Ty->getPrimitiveSizeInBits();
681 llvm::Type *IntTy = llvm::IntegerType::get(
C, Width);
683 if (Ty->isPPC_FP128Ty()) {
693 Value *ShiftCst = llvm::ConstantInt::get(IntTy, Width);
698 IntTy = llvm::IntegerType::get(
C, Width);
701 Value *Zero = llvm::Constant::getNullValue(IntTy);
702 return CGF.
Builder.CreateICmpSLT(
V, Zero);
706 const CallExpr *E, llvm::Constant *calleeValue) {
721 const llvm::Intrinsic::ID IntrinsicID,
722 llvm::Value *
X, llvm::Value *Y,
723 llvm::Value *&Carry) {
725 assert(
X->getType() == Y->getType() &&
726 "Arguments must be the same type. (Did you forget to make sure both "
727 "arguments have the same integer width?)");
730 llvm::Value *Tmp = CGF.
Builder.CreateCall(Callee, {
X, Y});
731 Carry = CGF.
Builder.CreateExtractValue(Tmp, 1);
732 return CGF.
Builder.CreateExtractValue(Tmp, 0);
736 unsigned IntrinsicID,
739 llvm::MDNode *RNode = MDHelper.createRange(APInt(32, low), APInt(32, high));
741 llvm::Instruction *
Call = CGF.
Builder.CreateCall(F);
742 Call->setMetadata(llvm::LLVMContext::MD_range, RNode);
743 Call->setMetadata(llvm::LLVMContext::MD_noundef,
749 struct WidthAndSignedness {
755static WidthAndSignedness
769static struct WidthAndSignedness
771 assert(Types.size() > 0 &&
"Empty list of types.");
775 for (
const auto &
Type : Types) {
784 for (
const auto &
Type : Types) {
786 if (Width < MinWidth) {
795 Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend;
806 return From == To || (From == 0 && To == 1) || (From == 3 && To == 2);
811 return ConstantInt::get(ResType, (
Type & 2) ? 0 : -1,
true);
815CodeGenFunction::evaluateOrEmitBuiltinObjectSize(
const Expr *E,
unsigned Type,
816 llvm::IntegerType *ResType,
817 llvm::Value *EmittedE,
821 return emitBuiltinObjectSize(E,
Type, ResType, EmittedE, IsDynamic);
822 return ConstantInt::get(ResType, ObjectSize,
true);
830 uint32_t FieldNo = 0;
836 if ((!FAMDecl || FD == FAMDecl) &&
838 Ctx, FD, FD->getType(), StrictFlexArraysLevel,
866 if (FD->getType()->isCountAttributedType())
878CodeGenFunction::emitFlexibleArrayMemberSize(
const Expr *E,
unsigned Type,
879 llvm::IntegerType *ResType) {
908 const Expr *Idx =
nullptr;
910 if (
const auto *UO = dyn_cast<UnaryOperator>(
Base);
911 UO && UO->getOpcode() == UO_AddrOf) {
913 if (
const auto *ASE = dyn_cast<ArraySubscriptExpr>(SubExpr)) {
914 Base = ASE->getBase()->IgnoreParenImpCasts();
917 if (
const auto *IL = dyn_cast<IntegerLiteral>(Idx)) {
918 int64_t Val = IL->getValue().getSExtValue();
935 if (
const auto *ME = dyn_cast<MemberExpr>(
Base)) {
937 const ValueDecl *VD = ME->getMemberDecl();
939 FAMDecl = dyn_cast<FieldDecl>(VD);
942 }
else if (
const auto *DRE = dyn_cast<DeclRefExpr>(
Base)) {
944 QualType Ty = DRE->getDecl()->getType();
1003 CountedByInst =
Builder.CreateIntCast(CountedByInst, ResType, IsSigned);
1006 Value *IdxInst =
nullptr;
1014 IdxInst =
Builder.CreateIntCast(IdxInst, ResType, IdxSigned);
1019 Builder.CreateSub(CountedByInst, IdxInst,
"", !IsSigned, IsSigned);
1025 llvm::Constant *ElemSize =
1026 llvm::ConstantInt::get(ResType,
Size.getQuantity(), IsSigned);
1028 Builder.CreateMul(CountedByInst, ElemSize,
"", !IsSigned, IsSigned);
1029 FAMSize =
Builder.CreateIntCast(FAMSize, ResType, IsSigned);
1030 Value *Res = FAMSize;
1032 if (isa<DeclRefExpr>(
Base)) {
1037 llvm::Constant *FAMOffset = ConstantInt::get(ResType, Offset, IsSigned);
1038 Value *OffsetAndFAMSize =
1039 Builder.CreateAdd(FAMOffset, Res,
"", !IsSigned, IsSigned);
1042 llvm::Constant *SizeofStruct =
1048 ?
Builder.CreateBinaryIntrinsic(llvm::Intrinsic::smax,
1049 OffsetAndFAMSize, SizeofStruct)
1050 :
Builder.CreateBinaryIntrinsic(llvm::Intrinsic::umax,
1051 OffsetAndFAMSize, SizeofStruct);
1061 return Builder.CreateSelect(Cmp, Res, ConstantInt::get(ResType, 0, IsSigned));
1074CodeGenFunction::emitBuiltinObjectSize(
const Expr *E,
unsigned Type,
1075 llvm::IntegerType *ResType,
1076 llvm::Value *EmittedE,
bool IsDynamic) {
1080 auto *Param = dyn_cast<ParmVarDecl>(D->getDecl());
1081 auto *PS = D->getDecl()->getAttr<PassObjectSizeAttr>();
1082 if (Param !=
nullptr && PS !=
nullptr &&
1084 auto Iter = SizeArguments.find(Param);
1085 assert(
Iter != SizeArguments.end());
1088 auto DIter = LocalDeclMap.find(D);
1089 assert(DIter != LocalDeclMap.end());
1099 if (
Value *
V = emitFlexibleArrayMemberSize(E,
Type, ResType))
1110 assert(Ptr->
getType()->isPointerTy() &&
1111 "Non-pointer passed to __builtin_object_size?");
1127 enum ActionKind : uint8_t { TestOnly, Complement, Reset,
Set };
1128 enum InterlockingKind : uint8_t {
1137 InterlockingKind Interlocking;
1140 static BitTest decodeBitTestBuiltin(
unsigned BuiltinID);
1145llvm::IntrinsicInst *getConvergenceToken(llvm::BasicBlock *BB) {
1146 for (
auto &I : *BB) {
1147 auto *II = dyn_cast<llvm::IntrinsicInst>(&I);
1148 if (II && isConvergenceControlIntrinsic(II->getIntrinsicID()))
1157CodeGenFunction::addConvergenceControlToken(llvm::CallBase *Input,
1158 llvm::Value *ParentToken) {
1159 llvm::Value *bundleArgs[] = {ParentToken};
1160 llvm::OperandBundleDef OB(
"convergencectrl", bundleArgs);
1161 auto Output = llvm::CallBase::addOperandBundle(
1162 Input, llvm::LLVMContext::OB_convergencectrl, OB, Input);
1163 Input->replaceAllUsesWith(Output);
1164 Input->eraseFromParent();
1168llvm::IntrinsicInst *
1169CodeGenFunction::emitConvergenceLoopToken(llvm::BasicBlock *BB,
1170 llvm::Value *ParentToken) {
1171 CGBuilderTy::InsertPoint IP =
Builder.saveIP();
1172 Builder.SetInsertPoint(&BB->front());
1173 auto CB =
Builder.CreateIntrinsic(
1174 llvm::Intrinsic::experimental_convergence_loop, {}, {});
1177 auto I = addConvergenceControlToken(CB, ParentToken);
1178 return cast<llvm::IntrinsicInst>(I);
1181llvm::IntrinsicInst *
1182CodeGenFunction::getOrEmitConvergenceEntryToken(llvm::Function *F) {
1183 auto *BB = &F->getEntryBlock();
1184 auto *token = getConvergenceToken(BB);
1192 CGBuilderTy::InsertPoint IP =
Builder.saveIP();
1193 Builder.SetInsertPoint(&BB->front());
1194 auto I =
Builder.CreateIntrinsic(
1195 llvm::Intrinsic::experimental_convergence_entry, {}, {});
1196 assert(isa<llvm::IntrinsicInst>(I));
1199 return cast<llvm::IntrinsicInst>(I);
1202llvm::IntrinsicInst *
1203CodeGenFunction::getOrEmitConvergenceLoopToken(
const LoopInfo *LI) {
1204 assert(LI !=
nullptr);
1206 auto *token = getConvergenceToken(LI->
getHeader());
1210 llvm::IntrinsicInst *PII =
1212 ? emitConvergenceLoopToken(
1214 : getOrEmitConvergenceEntryToken(LI->getHeader()->getParent());
1216 return emitConvergenceLoopToken(LI->
getHeader(), PII);
1221 llvm::Value *ParentToken =
1224 : getOrEmitConvergenceEntryToken(Input->getFunction());
1225 return addConvergenceControlToken(Input, ParentToken);
1228BitTest BitTest::decodeBitTestBuiltin(
unsigned BuiltinID) {
1229 switch (BuiltinID) {
1231 case Builtin::BI_bittest:
1232 return {TestOnly, Unlocked,
false};
1233 case Builtin::BI_bittestandcomplement:
1234 return {Complement, Unlocked,
false};
1235 case Builtin::BI_bittestandreset:
1236 return {Reset, Unlocked,
false};
1237 case Builtin::BI_bittestandset:
1238 return {
Set, Unlocked,
false};
1239 case Builtin::BI_interlockedbittestandreset:
1240 return {Reset, Sequential,
false};
1241 case Builtin::BI_interlockedbittestandset:
1242 return {
Set, Sequential,
false};
1245 case Builtin::BI_bittest64:
1246 return {TestOnly, Unlocked,
true};
1247 case Builtin::BI_bittestandcomplement64:
1248 return {Complement, Unlocked,
true};
1249 case Builtin::BI_bittestandreset64:
1250 return {Reset, Unlocked,
true};
1251 case Builtin::BI_bittestandset64:
1252 return {
Set, Unlocked,
true};
1253 case Builtin::BI_interlockedbittestandreset64:
1254 return {Reset, Sequential,
true};
1255 case Builtin::BI_interlockedbittestandset64:
1256 return {
Set, Sequential,
true};
1259 case Builtin::BI_interlockedbittestandset_acq:
1260 return {
Set, Acquire,
false};
1261 case Builtin::BI_interlockedbittestandset_rel:
1262 return {
Set, Release,
false};
1263 case Builtin::BI_interlockedbittestandset_nf:
1264 return {
Set, NoFence,
false};
1265 case Builtin::BI_interlockedbittestandreset_acq:
1266 return {Reset, Acquire,
false};
1267 case Builtin::BI_interlockedbittestandreset_rel:
1268 return {Reset, Release,
false};
1269 case Builtin::BI_interlockedbittestandreset_nf:
1270 return {Reset, NoFence,
false};
1272 llvm_unreachable(
"expected only bittest intrinsics");
1277 case BitTest::TestOnly:
return '\0';
1278 case BitTest::Complement:
return 'c';
1279 case BitTest::Reset:
return 'r';
1280 case BitTest::Set:
return 's';
1282 llvm_unreachable(
"invalid action");
1290 char SizeSuffix = BT.Is64Bit ?
'q' :
'l';
1294 raw_svector_ostream AsmOS(
Asm);
1295 if (BT.Interlocking != BitTest::Unlocked)
1300 AsmOS << SizeSuffix <<
" $2, ($1)";
1303 std::string Constraints =
"={@ccc},r,r,~{cc},~{memory}";
1305 if (!MachineClobbers.empty()) {
1307 Constraints += MachineClobbers;
1309 llvm::IntegerType *IntType = llvm::IntegerType::get(
1312 llvm::FunctionType *FTy =
1313 llvm::FunctionType::get(CGF.
Int8Ty, {CGF.UnqualPtrTy, IntType},
false);
1315 llvm::InlineAsm *IA =
1316 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
1317 return CGF.
Builder.CreateCall(IA, {BitBase, BitPos});
1320static llvm::AtomicOrdering
1323 case BitTest::Unlocked:
return llvm::AtomicOrdering::NotAtomic;
1324 case BitTest::Sequential:
return llvm::AtomicOrdering::SequentiallyConsistent;
1325 case BitTest::Acquire:
return llvm::AtomicOrdering::Acquire;
1326 case BitTest::Release:
return llvm::AtomicOrdering::Release;
1327 case BitTest::NoFence:
return llvm::AtomicOrdering::Monotonic;
1329 llvm_unreachable(
"invalid interlocking");
1342 BitTest BT = BitTest::decodeBitTestBuiltin(BuiltinID);
1354 BitPos, llvm::ConstantInt::get(BitPos->
getType(), 3),
"bittest.byteidx");
1357 ByteIndex,
"bittest.byteaddr"),
1361 llvm::ConstantInt::get(CGF.
Int8Ty, 0x7));
1364 Value *Mask =
nullptr;
1365 if (BT.Action != BitTest::TestOnly) {
1366 Mask = CGF.
Builder.CreateShl(llvm::ConstantInt::get(CGF.
Int8Ty, 1), PosLow,
1373 Value *OldByte =
nullptr;
1374 if (Ordering != llvm::AtomicOrdering::NotAtomic) {
1377 llvm::AtomicRMWInst::BinOp RMWOp = llvm::AtomicRMWInst::Or;
1378 if (BT.Action == BitTest::Reset) {
1379 Mask = CGF.
Builder.CreateNot(Mask);
1380 RMWOp = llvm::AtomicRMWInst::And;
1386 Value *NewByte =
nullptr;
1387 switch (BT.Action) {
1388 case BitTest::TestOnly:
1391 case BitTest::Complement:
1392 NewByte = CGF.
Builder.CreateXor(OldByte, Mask);
1394 case BitTest::Reset:
1395 NewByte = CGF.
Builder.CreateAnd(OldByte, CGF.
Builder.CreateNot(Mask));
1398 NewByte = CGF.
Builder.CreateOr(OldByte, Mask);
1407 Value *ShiftedByte = CGF.
Builder.CreateLShr(OldByte, PosLow,
"bittest.shr");
1409 ShiftedByte, llvm::ConstantInt::get(CGF.
Int8Ty, 1),
"bittest.res");
1418 raw_svector_ostream AsmOS(
Asm);
1419 llvm::IntegerType *RetType = CGF.
Int32Ty;
1421 switch (BuiltinID) {
1422 case clang::PPC::BI__builtin_ppc_ldarx:
1426 case clang::PPC::BI__builtin_ppc_lwarx:
1430 case clang::PPC::BI__builtin_ppc_lharx:
1434 case clang::PPC::BI__builtin_ppc_lbarx:
1439 llvm_unreachable(
"Expected only PowerPC load reserve intrinsics");
1442 AsmOS <<
"$0, ${1:y}";
1444 std::string Constraints =
"=r,*Z,~{memory}";
1446 if (!MachineClobbers.empty()) {
1448 Constraints += MachineClobbers;
1452 llvm::FunctionType *FTy = llvm::FunctionType::get(RetType, {PtrType},
false);
1454 llvm::InlineAsm *IA =
1455 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
1456 llvm::CallInst *CI = CGF.
Builder.CreateCall(IA, {Addr});
1458 0, Attribute::get(CGF.
getLLVMContext(), Attribute::ElementType, RetType));
1463enum class MSVCSetJmpKind {
1475 llvm::Value *Arg1 =
nullptr;
1476 llvm::Type *Arg1Ty =
nullptr;
1478 bool IsVarArg =
false;
1479 if (SJKind == MSVCSetJmpKind::_setjmp3) {
1482 Arg1 = llvm::ConstantInt::get(CGF.
IntTy, 0);
1485 Name = SJKind == MSVCSetJmpKind::_setjmp ?
"_setjmp" :
"_setjmpex";
1488 Arg1 = CGF.
Builder.CreateCall(
1491 Arg1 = CGF.
Builder.CreateCall(
1493 llvm::ConstantInt::get(CGF.
Int32Ty, 0));
1497 llvm::Type *ArgTypes[2] = {CGF.
Int8PtrTy, Arg1Ty};
1498 llvm::AttributeList ReturnsTwiceAttr = llvm::AttributeList::get(
1500 llvm::Attribute::ReturnsTwice);
1502 llvm::FunctionType::get(CGF.
IntTy, ArgTypes, IsVarArg), Name,
1503 ReturnsTwiceAttr,
true);
1505 llvm::Value *Buf = CGF.
Builder.CreateBitOrPointerCast(
1507 llvm::Value *Args[] = {Buf, Arg1};
1509 CB->setAttributes(ReturnsTwiceAttr);
1557static std::optional<CodeGenFunction::MSVCIntrin>
1560 switch (BuiltinID) {
1562 return std::nullopt;
1563 case clang::ARM::BI_BitScanForward:
1564 case clang::ARM::BI_BitScanForward64:
1565 return MSVCIntrin::_BitScanForward;
1566 case clang::ARM::BI_BitScanReverse:
1567 case clang::ARM::BI_BitScanReverse64:
1568 return MSVCIntrin::_BitScanReverse;
1569 case clang::ARM::BI_InterlockedAnd64:
1570 return MSVCIntrin::_InterlockedAnd;
1571 case clang::ARM::BI_InterlockedExchange64:
1572 return MSVCIntrin::_InterlockedExchange;
1573 case clang::ARM::BI_InterlockedExchangeAdd64:
1574 return MSVCIntrin::_InterlockedExchangeAdd;
1575 case clang::ARM::BI_InterlockedExchangeSub64:
1576 return MSVCIntrin::_InterlockedExchangeSub;
1577 case clang::ARM::BI_InterlockedOr64:
1578 return MSVCIntrin::_InterlockedOr;
1579 case clang::ARM::BI_InterlockedXor64:
1580 return MSVCIntrin::_InterlockedXor;
1581 case clang::ARM::BI_InterlockedDecrement64:
1582 return MSVCIntrin::_InterlockedDecrement;
1583 case clang::ARM::BI_InterlockedIncrement64:
1584 return MSVCIntrin::_InterlockedIncrement;
1585 case clang::ARM::BI_InterlockedExchangeAdd8_acq:
1586 case clang::ARM::BI_InterlockedExchangeAdd16_acq:
1587 case clang::ARM::BI_InterlockedExchangeAdd_acq:
1588 case clang::ARM::BI_InterlockedExchangeAdd64_acq:
1589 return MSVCIntrin::_InterlockedExchangeAdd_acq;
1590 case clang::ARM::BI_InterlockedExchangeAdd8_rel:
1591 case clang::ARM::BI_InterlockedExchangeAdd16_rel:
1592 case clang::ARM::BI_InterlockedExchangeAdd_rel:
1593 case clang::ARM::BI_InterlockedExchangeAdd64_rel:
1594 return MSVCIntrin::_InterlockedExchangeAdd_rel;
1595 case clang::ARM::BI_InterlockedExchangeAdd8_nf:
1596 case clang::ARM::BI_InterlockedExchangeAdd16_nf:
1597 case clang::ARM::BI_InterlockedExchangeAdd_nf:
1598 case clang::ARM::BI_InterlockedExchangeAdd64_nf:
1599 return MSVCIntrin::_InterlockedExchangeAdd_nf;
1600 case clang::ARM::BI_InterlockedExchange8_acq:
1601 case clang::ARM::BI_InterlockedExchange16_acq:
1602 case clang::ARM::BI_InterlockedExchange_acq:
1603 case clang::ARM::BI_InterlockedExchange64_acq:
1604 return MSVCIntrin::_InterlockedExchange_acq;
1605 case clang::ARM::BI_InterlockedExchange8_rel:
1606 case clang::ARM::BI_InterlockedExchange16_rel:
1607 case clang::ARM::BI_InterlockedExchange_rel:
1608 case clang::ARM::BI_InterlockedExchange64_rel:
1609 return MSVCIntrin::_InterlockedExchange_rel;
1610 case clang::ARM::BI_InterlockedExchange8_nf:
1611 case clang::ARM::BI_InterlockedExchange16_nf:
1612 case clang::ARM::BI_InterlockedExchange_nf:
1613 case clang::ARM::BI_InterlockedExchange64_nf:
1614 return MSVCIntrin::_InterlockedExchange_nf;
1615 case clang::ARM::BI_InterlockedCompareExchange8_acq:
1616 case clang::ARM::BI_InterlockedCompareExchange16_acq:
1617 case clang::ARM::BI_InterlockedCompareExchange_acq:
1618 case clang::ARM::BI_InterlockedCompareExchange64_acq:
1619 return MSVCIntrin::_InterlockedCompareExchange_acq;
1620 case clang::ARM::BI_InterlockedCompareExchange8_rel:
1621 case clang::ARM::BI_InterlockedCompareExchange16_rel:
1622 case clang::ARM::BI_InterlockedCompareExchange_rel:
1623 case clang::ARM::BI_InterlockedCompareExchange64_rel:
1624 return MSVCIntrin::_InterlockedCompareExchange_rel;
1625 case clang::ARM::BI_InterlockedCompareExchange8_nf:
1626 case clang::ARM::BI_InterlockedCompareExchange16_nf:
1627 case clang::ARM::BI_InterlockedCompareExchange_nf:
1628 case clang::ARM::BI_InterlockedCompareExchange64_nf:
1629 return MSVCIntrin::_InterlockedCompareExchange_nf;
1630 case clang::ARM::BI_InterlockedOr8_acq:
1631 case clang::ARM::BI_InterlockedOr16_acq:
1632 case clang::ARM::BI_InterlockedOr_acq:
1633 case clang::ARM::BI_InterlockedOr64_acq:
1634 return MSVCIntrin::_InterlockedOr_acq;
1635 case clang::ARM::BI_InterlockedOr8_rel:
1636 case clang::ARM::BI_InterlockedOr16_rel:
1637 case clang::ARM::BI_InterlockedOr_rel:
1638 case clang::ARM::BI_InterlockedOr64_rel:
1639 return MSVCIntrin::_InterlockedOr_rel;
1640 case clang::ARM::BI_InterlockedOr8_nf:
1641 case clang::ARM::BI_InterlockedOr16_nf:
1642 case clang::ARM::BI_InterlockedOr_nf:
1643 case clang::ARM::BI_InterlockedOr64_nf:
1644 return MSVCIntrin::_InterlockedOr_nf;
1645 case clang::ARM::BI_InterlockedXor8_acq:
1646 case clang::ARM::BI_InterlockedXor16_acq:
1647 case clang::ARM::BI_InterlockedXor_acq:
1648 case clang::ARM::BI_InterlockedXor64_acq:
1649 return MSVCIntrin::_InterlockedXor_acq;
1650 case clang::ARM::BI_InterlockedXor8_rel:
1651 case clang::ARM::BI_InterlockedXor16_rel:
1652 case clang::ARM::BI_InterlockedXor_rel:
1653 case clang::ARM::BI_InterlockedXor64_rel:
1654 return MSVCIntrin::_InterlockedXor_rel;
1655 case clang::ARM::BI_InterlockedXor8_nf:
1656 case clang::ARM::BI_InterlockedXor16_nf:
1657 case clang::ARM::BI_InterlockedXor_nf:
1658 case clang::ARM::BI_InterlockedXor64_nf:
1659 return MSVCIntrin::_InterlockedXor_nf;
1660 case clang::ARM::BI_InterlockedAnd8_acq:
1661 case clang::ARM::BI_InterlockedAnd16_acq:
1662 case clang::ARM::BI_InterlockedAnd_acq:
1663 case clang::ARM::BI_InterlockedAnd64_acq:
1664 return MSVCIntrin::_InterlockedAnd_acq;
1665 case clang::ARM::BI_InterlockedAnd8_rel:
1666 case clang::ARM::BI_InterlockedAnd16_rel:
1667 case clang::ARM::BI_InterlockedAnd_rel:
1668 case clang::ARM::BI_InterlockedAnd64_rel:
1669 return MSVCIntrin::_InterlockedAnd_rel;
1670 case clang::ARM::BI_InterlockedAnd8_nf:
1671 case clang::ARM::BI_InterlockedAnd16_nf:
1672 case clang::ARM::BI_InterlockedAnd_nf:
1673 case clang::ARM::BI_InterlockedAnd64_nf:
1674 return MSVCIntrin::_InterlockedAnd_nf;
1675 case clang::ARM::BI_InterlockedIncrement16_acq:
1676 case clang::ARM::BI_InterlockedIncrement_acq:
1677 case clang::ARM::BI_InterlockedIncrement64_acq:
1678 return MSVCIntrin::_InterlockedIncrement_acq;
1679 case clang::ARM::BI_InterlockedIncrement16_rel:
1680 case clang::ARM::BI_InterlockedIncrement_rel:
1681 case clang::ARM::BI_InterlockedIncrement64_rel:
1682 return MSVCIntrin::_InterlockedIncrement_rel;
1683 case clang::ARM::BI_InterlockedIncrement16_nf:
1684 case clang::ARM::BI_InterlockedIncrement_nf:
1685 case clang::ARM::BI_InterlockedIncrement64_nf:
1686 return MSVCIntrin::_InterlockedIncrement_nf;
1687 case clang::ARM::BI_InterlockedDecrement16_acq:
1688 case clang::ARM::BI_InterlockedDecrement_acq:
1689 case clang::ARM::BI_InterlockedDecrement64_acq:
1690 return MSVCIntrin::_InterlockedDecrement_acq;
1691 case clang::ARM::BI_InterlockedDecrement16_rel:
1692 case clang::ARM::BI_InterlockedDecrement_rel:
1693 case clang::ARM::BI_InterlockedDecrement64_rel:
1694 return MSVCIntrin::_InterlockedDecrement_rel;
1695 case clang::ARM::BI_InterlockedDecrement16_nf:
1696 case clang::ARM::BI_InterlockedDecrement_nf:
1697 case clang::ARM::BI_InterlockedDecrement64_nf:
1698 return MSVCIntrin::_InterlockedDecrement_nf;
1700 llvm_unreachable(
"must return from switch");
1703static std::optional<CodeGenFunction::MSVCIntrin>
1706 switch (BuiltinID) {
1708 return std::nullopt;
1709 case clang::AArch64::BI_BitScanForward:
1710 case clang::AArch64::BI_BitScanForward64:
1711 return MSVCIntrin::_BitScanForward;
1712 case clang::AArch64::BI_BitScanReverse:
1713 case clang::AArch64::BI_BitScanReverse64:
1714 return MSVCIntrin::_BitScanReverse;
1715 case clang::AArch64::BI_InterlockedAnd64:
1716 return MSVCIntrin::_InterlockedAnd;
1717 case clang::AArch64::BI_InterlockedExchange64:
1718 return MSVCIntrin::_InterlockedExchange;
1719 case clang::AArch64::BI_InterlockedExchangeAdd64:
1720 return MSVCIntrin::_InterlockedExchangeAdd;
1721 case clang::AArch64::BI_InterlockedExchangeSub64:
1722 return MSVCIntrin::_InterlockedExchangeSub;
1723 case clang::AArch64::BI_InterlockedOr64:
1724 return MSVCIntrin::_InterlockedOr;
1725 case clang::AArch64::BI_InterlockedXor64:
1726 return MSVCIntrin::_InterlockedXor;
1727 case clang::AArch64::BI_InterlockedDecrement64:
1728 return MSVCIntrin::_InterlockedDecrement;
1729 case clang::AArch64::BI_InterlockedIncrement64:
1730 return MSVCIntrin::_InterlockedIncrement;
1731 case clang::AArch64::BI_InterlockedExchangeAdd8_acq:
1732 case clang::AArch64::BI_InterlockedExchangeAdd16_acq:
1733 case clang::AArch64::BI_InterlockedExchangeAdd_acq:
1734 case clang::AArch64::BI_InterlockedExchangeAdd64_acq:
1735 return MSVCIntrin::_InterlockedExchangeAdd_acq;
1736 case clang::AArch64::BI_InterlockedExchangeAdd8_rel:
1737 case clang::AArch64::BI_InterlockedExchangeAdd16_rel:
1738 case clang::AArch64::BI_InterlockedExchangeAdd_rel:
1739 case clang::AArch64::BI_InterlockedExchangeAdd64_rel:
1740 return MSVCIntrin::_InterlockedExchangeAdd_rel;
1741 case clang::AArch64::BI_InterlockedExchangeAdd8_nf:
1742 case clang::AArch64::BI_InterlockedExchangeAdd16_nf:
1743 case clang::AArch64::BI_InterlockedExchangeAdd_nf:
1744 case clang::AArch64::BI_InterlockedExchangeAdd64_nf:
1745 return MSVCIntrin::_InterlockedExchangeAdd_nf;
1746 case clang::AArch64::BI_InterlockedExchange8_acq:
1747 case clang::AArch64::BI_InterlockedExchange16_acq:
1748 case clang::AArch64::BI_InterlockedExchange_acq:
1749 case clang::AArch64::BI_InterlockedExchange64_acq:
1750 return MSVCIntrin::_InterlockedExchange_acq;
1751 case clang::AArch64::BI_InterlockedExchange8_rel:
1752 case clang::AArch64::BI_InterlockedExchange16_rel:
1753 case clang::AArch64::BI_InterlockedExchange_rel:
1754 case clang::AArch64::BI_InterlockedExchange64_rel:
1755 return MSVCIntrin::_InterlockedExchange_rel;
1756 case clang::AArch64::BI_InterlockedExchange8_nf:
1757 case clang::AArch64::BI_InterlockedExchange16_nf:
1758 case clang::AArch64::BI_InterlockedExchange_nf:
1759 case clang::AArch64::BI_InterlockedExchange64_nf:
1760 return MSVCIntrin::_InterlockedExchange_nf;
1761 case clang::AArch64::BI_InterlockedCompareExchange8_acq:
1762 case clang::AArch64::BI_InterlockedCompareExchange16_acq:
1763 case clang::AArch64::BI_InterlockedCompareExchange_acq:
1764 case clang::AArch64::BI_InterlockedCompareExchange64_acq:
1765 return MSVCIntrin::_InterlockedCompareExchange_acq;
1766 case clang::AArch64::BI_InterlockedCompareExchange8_rel:
1767 case clang::AArch64::BI_InterlockedCompareExchange16_rel:
1768 case clang::AArch64::BI_InterlockedCompareExchange_rel:
1769 case clang::AArch64::BI_InterlockedCompareExchange64_rel:
1770 return MSVCIntrin::_InterlockedCompareExchange_rel;
1771 case clang::AArch64::BI_InterlockedCompareExchange8_nf:
1772 case clang::AArch64::BI_InterlockedCompareExchange16_nf:
1773 case clang::AArch64::BI_InterlockedCompareExchange_nf:
1774 case clang::AArch64::BI_InterlockedCompareExchange64_nf:
1775 return MSVCIntrin::_InterlockedCompareExchange_nf;
1776 case clang::AArch64::BI_InterlockedCompareExchange128:
1777 return MSVCIntrin::_InterlockedCompareExchange128;
1778 case clang::AArch64::BI_InterlockedCompareExchange128_acq:
1779 return MSVCIntrin::_InterlockedCompareExchange128_acq;
1780 case clang::AArch64::BI_InterlockedCompareExchange128_nf:
1781 return MSVCIntrin::_InterlockedCompareExchange128_nf;
1782 case clang::AArch64::BI_InterlockedCompareExchange128_rel:
1783 return MSVCIntrin::_InterlockedCompareExchange128_rel;
1784 case clang::AArch64::BI_InterlockedOr8_acq:
1785 case clang::AArch64::BI_InterlockedOr16_acq:
1786 case clang::AArch64::BI_InterlockedOr_acq:
1787 case clang::AArch64::BI_InterlockedOr64_acq:
1788 return MSVCIntrin::_InterlockedOr_acq;
1789 case clang::AArch64::BI_InterlockedOr8_rel:
1790 case clang::AArch64::BI_InterlockedOr16_rel:
1791 case clang::AArch64::BI_InterlockedOr_rel:
1792 case clang::AArch64::BI_InterlockedOr64_rel:
1793 return MSVCIntrin::_InterlockedOr_rel;
1794 case clang::AArch64::BI_InterlockedOr8_nf:
1795 case clang::AArch64::BI_InterlockedOr16_nf:
1796 case clang::AArch64::BI_InterlockedOr_nf:
1797 case clang::AArch64::BI_InterlockedOr64_nf:
1798 return MSVCIntrin::_InterlockedOr_nf;
1799 case clang::AArch64::BI_InterlockedXor8_acq:
1800 case clang::AArch64::BI_InterlockedXor16_acq:
1801 case clang::AArch64::BI_InterlockedXor_acq:
1802 case clang::AArch64::BI_InterlockedXor64_acq:
1803 return MSVCIntrin::_InterlockedXor_acq;
1804 case clang::AArch64::BI_InterlockedXor8_rel:
1805 case clang::AArch64::BI_InterlockedXor16_rel:
1806 case clang::AArch64::BI_InterlockedXor_rel:
1807 case clang::AArch64::BI_InterlockedXor64_rel:
1808 return MSVCIntrin::_InterlockedXor_rel;
1809 case clang::AArch64::BI_InterlockedXor8_nf:
1810 case clang::AArch64::BI_InterlockedXor16_nf:
1811 case clang::AArch64::BI_InterlockedXor_nf:
1812 case clang::AArch64::BI_InterlockedXor64_nf:
1813 return MSVCIntrin::_InterlockedXor_nf;
1814 case clang::AArch64::BI_InterlockedAnd8_acq:
1815 case clang::AArch64::BI_InterlockedAnd16_acq:
1816 case clang::AArch64::BI_InterlockedAnd_acq:
1817 case clang::AArch64::BI_InterlockedAnd64_acq:
1818 return MSVCIntrin::_InterlockedAnd_acq;
1819 case clang::AArch64::BI_InterlockedAnd8_rel:
1820 case clang::AArch64::BI_InterlockedAnd16_rel:
1821 case clang::AArch64::BI_InterlockedAnd_rel:
1822 case clang::AArch64::BI_InterlockedAnd64_rel:
1823 return MSVCIntrin::_InterlockedAnd_rel;
1824 case clang::AArch64::BI_InterlockedAnd8_nf:
1825 case clang::AArch64::BI_InterlockedAnd16_nf:
1826 case clang::AArch64::BI_InterlockedAnd_nf:
1827 case clang::AArch64::BI_InterlockedAnd64_nf:
1828 return MSVCIntrin::_InterlockedAnd_nf;
1829 case clang::AArch64::BI_InterlockedIncrement16_acq:
1830 case clang::AArch64::BI_InterlockedIncrement_acq:
1831 case clang::AArch64::BI_InterlockedIncrement64_acq:
1832 return MSVCIntrin::_InterlockedIncrement_acq;
1833 case clang::AArch64::BI_InterlockedIncrement16_rel:
1834 case clang::AArch64::BI_InterlockedIncrement_rel:
1835 case clang::AArch64::BI_InterlockedIncrement64_rel:
1836 return MSVCIntrin::_InterlockedIncrement_rel;
1837 case clang::AArch64::BI_InterlockedIncrement16_nf:
1838 case clang::AArch64::BI_InterlockedIncrement_nf:
1839 case clang::AArch64::BI_InterlockedIncrement64_nf:
1840 return MSVCIntrin::_InterlockedIncrement_nf;
1841 case clang::AArch64::BI_InterlockedDecrement16_acq:
1842 case clang::AArch64::BI_InterlockedDecrement_acq:
1843 case clang::AArch64::BI_InterlockedDecrement64_acq:
1844 return MSVCIntrin::_InterlockedDecrement_acq;
1845 case clang::AArch64::BI_InterlockedDecrement16_rel:
1846 case clang::AArch64::BI_InterlockedDecrement_rel:
1847 case clang::AArch64::BI_InterlockedDecrement64_rel:
1848 return MSVCIntrin::_InterlockedDecrement_rel;
1849 case clang::AArch64::BI_InterlockedDecrement16_nf:
1850 case clang::AArch64::BI_InterlockedDecrement_nf:
1851 case clang::AArch64::BI_InterlockedDecrement64_nf:
1852 return MSVCIntrin::_InterlockedDecrement_nf;
1854 llvm_unreachable(
"must return from switch");
1857static std::optional<CodeGenFunction::MSVCIntrin>
1860 switch (BuiltinID) {
1862 return std::nullopt;
1863 case clang::X86::BI_BitScanForward:
1864 case clang::X86::BI_BitScanForward64:
1865 return MSVCIntrin::_BitScanForward;
1866 case clang::X86::BI_BitScanReverse:
1867 case clang::X86::BI_BitScanReverse64:
1868 return MSVCIntrin::_BitScanReverse;
1869 case clang::X86::BI_InterlockedAnd64:
1870 return MSVCIntrin::_InterlockedAnd;
1871 case clang::X86::BI_InterlockedCompareExchange128:
1872 return MSVCIntrin::_InterlockedCompareExchange128;
1873 case clang::X86::BI_InterlockedExchange64:
1874 return MSVCIntrin::_InterlockedExchange;
1875 case clang::X86::BI_InterlockedExchangeAdd64:
1876 return MSVCIntrin::_InterlockedExchangeAdd;
1877 case clang::X86::BI_InterlockedExchangeSub64:
1878 return MSVCIntrin::_InterlockedExchangeSub;
1879 case clang::X86::BI_InterlockedOr64:
1880 return MSVCIntrin::_InterlockedOr;
1881 case clang::X86::BI_InterlockedXor64:
1882 return MSVCIntrin::_InterlockedXor;
1883 case clang::X86::BI_InterlockedDecrement64:
1884 return MSVCIntrin::_InterlockedDecrement;
1885 case clang::X86::BI_InterlockedIncrement64:
1886 return MSVCIntrin::_InterlockedIncrement;
1888 llvm_unreachable(
"must return from switch");
1894 switch (BuiltinID) {
1895 case MSVCIntrin::_BitScanForward:
1896 case MSVCIntrin::_BitScanReverse: {
1900 llvm::Type *ArgType = ArgValue->
getType();
1901 llvm::Type *IndexType = IndexAddress.getElementType();
1904 Value *ArgZero = llvm::Constant::getNullValue(ArgType);
1905 Value *ResZero = llvm::Constant::getNullValue(ResultType);
1906 Value *ResOne = llvm::ConstantInt::get(ResultType, 1);
1911 PHINode *
Result =
Builder.CreatePHI(ResultType, 2,
"bitscan_result");
1914 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, ArgZero);
1916 Builder.CreateCondBr(IsZero, End, NotZero);
1919 Builder.SetInsertPoint(NotZero);
1921 if (BuiltinID == MSVCIntrin::_BitScanForward) {
1924 ZeroCount =
Builder.CreateIntCast(ZeroCount, IndexType,
false);
1927 unsigned ArgWidth = cast<llvm::IntegerType>(ArgType)->getBitWidth();
1928 Value *ArgTypeLastIndex = llvm::ConstantInt::get(IndexType, ArgWidth - 1);
1932 ZeroCount =
Builder.CreateIntCast(ZeroCount, IndexType,
false);
1933 Value *Index =
Builder.CreateNSWSub(ArgTypeLastIndex, ZeroCount);
1937 Result->addIncoming(ResOne, NotZero);
1942 case MSVCIntrin::_InterlockedAnd:
1944 case MSVCIntrin::_InterlockedExchange:
1946 case MSVCIntrin::_InterlockedExchangeAdd:
1948 case MSVCIntrin::_InterlockedExchangeSub:
1950 case MSVCIntrin::_InterlockedOr:
1952 case MSVCIntrin::_InterlockedXor:
1954 case MSVCIntrin::_InterlockedExchangeAdd_acq:
1956 AtomicOrdering::Acquire);
1957 case MSVCIntrin::_InterlockedExchangeAdd_rel:
1959 AtomicOrdering::Release);
1960 case MSVCIntrin::_InterlockedExchangeAdd_nf:
1962 AtomicOrdering::Monotonic);
1963 case MSVCIntrin::_InterlockedExchange_acq:
1965 AtomicOrdering::Acquire);
1966 case MSVCIntrin::_InterlockedExchange_rel:
1968 AtomicOrdering::Release);
1969 case MSVCIntrin::_InterlockedExchange_nf:
1971 AtomicOrdering::Monotonic);
1972 case MSVCIntrin::_InterlockedCompareExchange_acq:
1974 case MSVCIntrin::_InterlockedCompareExchange_rel:
1976 case MSVCIntrin::_InterlockedCompareExchange_nf:
1978 case MSVCIntrin::_InterlockedCompareExchange128:
1980 *
this, E, AtomicOrdering::SequentiallyConsistent);
1981 case MSVCIntrin::_InterlockedCompareExchange128_acq:
1983 case MSVCIntrin::_InterlockedCompareExchange128_rel:
1985 case MSVCIntrin::_InterlockedCompareExchange128_nf:
1987 case MSVCIntrin::_InterlockedOr_acq:
1989 AtomicOrdering::Acquire);
1990 case MSVCIntrin::_InterlockedOr_rel:
1992 AtomicOrdering::Release);
1993 case MSVCIntrin::_InterlockedOr_nf:
1995 AtomicOrdering::Monotonic);
1996 case MSVCIntrin::_InterlockedXor_acq:
1998 AtomicOrdering::Acquire);
1999 case MSVCIntrin::_InterlockedXor_rel:
2001 AtomicOrdering::Release);
2002 case MSVCIntrin::_InterlockedXor_nf:
2004 AtomicOrdering::Monotonic);
2005 case MSVCIntrin::_InterlockedAnd_acq:
2007 AtomicOrdering::Acquire);
2008 case MSVCIntrin::_InterlockedAnd_rel:
2010 AtomicOrdering::Release);
2011 case MSVCIntrin::_InterlockedAnd_nf:
2013 AtomicOrdering::Monotonic);
2014 case MSVCIntrin::_InterlockedIncrement_acq:
2016 case MSVCIntrin::_InterlockedIncrement_rel:
2018 case MSVCIntrin::_InterlockedIncrement_nf:
2020 case MSVCIntrin::_InterlockedDecrement_acq:
2022 case MSVCIntrin::_InterlockedDecrement_rel:
2024 case MSVCIntrin::_InterlockedDecrement_nf:
2027 case MSVCIntrin::_InterlockedDecrement:
2029 case MSVCIntrin::_InterlockedIncrement:
2032 case MSVCIntrin::__fastfail: {
2037 StringRef
Asm, Constraints;
2042 case llvm::Triple::x86:
2043 case llvm::Triple::x86_64:
2045 Constraints =
"{cx}";
2047 case llvm::Triple::thumb:
2049 Constraints =
"{r0}";
2051 case llvm::Triple::aarch64:
2052 Asm =
"brk #0xF003";
2053 Constraints =
"{w0}";
2055 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy, {
Int32Ty},
false);
2056 llvm::InlineAsm *IA =
2057 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
2058 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
2060 llvm::Attribute::NoReturn);
2062 CI->setAttributes(NoReturnAttr);
2066 llvm_unreachable(
"Incorrect MSVC intrinsic!");
2072 CallObjCArcUse(llvm::Value *
object) :
object(
object) {}
2082 BuiltinCheckKind Kind) {
2084 &&
"Unsupported builtin check kind");
2090 SanitizerScope SanScope(
this);
2092 ArgValue, llvm::Constant::getNullValue(ArgValue->
getType()));
2093 EmitCheck(std::make_pair(Cond, SanitizerKind::Builtin),
2094 SanitizerHandler::InvalidBuiltin,
2096 llvm::ConstantInt::get(
Builder.getInt8Ty(), Kind)},
2102 return CGF.
Builder.CreateBinaryIntrinsic(
2103 Intrinsic::abs, ArgValue,
2104 ConstantInt::get(CGF.
Builder.getInt1Ty(), HasNSW));
2108 bool SanitizeOverflow) {
2112 if (
const auto *VCI = dyn_cast<llvm::ConstantInt>(ArgValue)) {
2113 if (!VCI->isMinSignedValue())
2114 return EmitAbs(CGF, ArgValue,
true);
2117 CodeGenFunction::SanitizerScope SanScope(&CGF);
2119 Constant *Zero = Constant::getNullValue(ArgValue->
getType());
2120 Value *ResultAndOverflow = CGF.
Builder.CreateBinaryIntrinsic(
2121 Intrinsic::ssub_with_overflow, Zero, ArgValue);
2124 CGF.
Builder.CreateExtractValue(ResultAndOverflow, 1));
2127 if (SanitizeOverflow) {
2128 CGF.
EmitCheck({{NotOverflow, SanitizerKind::SignedIntegerOverflow}},
2129 SanitizerHandler::NegateOverflow,
2134 CGF.
EmitTrapCheck(NotOverflow, SanitizerHandler::SubOverflow);
2136 Value *CmpResult = CGF.
Builder.CreateICmpSLT(ArgValue, Zero,
"abscond");
2137 return CGF.
Builder.CreateSelect(CmpResult,
Result, ArgValue,
"abs");
2142 QualType UnsignedTy =
C.getIntTypeForBitwidth(Size * 8,
false);
2143 return C.getCanonicalType(UnsignedTy);
2153 raw_svector_ostream OS(Name);
2154 OS <<
"__os_log_helper";
2158 for (
const auto &Item : Layout.
Items)
2159 OS <<
"_" <<
int(Item.getSizeByte()) <<
"_"
2160 <<
int(Item.getDescriptorByte());
2163 if (llvm::Function *F =
CGM.
getModule().getFunction(Name))
2173 for (
unsigned int I = 0, E = Layout.
Items.size(); I < E; ++I) {
2174 char Size = Layout.
Items[I].getSizeByte();
2181 &Ctx.
Idents.
get(std::string(
"arg") + llvm::to_string(I)), ArgTy,
2183 ArgTys.emplace_back(ArgTy);
2194 llvm::Function *Fn = llvm::Function::Create(
2195 FuncTy, llvm::GlobalValue::LinkOnceODRLinkage, Name, &
CGM.
getModule());
2196 Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);
2199 Fn->setDoesNotThrow();
2203 Fn->addFnAttr(llvm::Attribute::NoInline);
2221 for (
const auto &Item : Layout.
Items) {
2223 Builder.getInt8(Item.getDescriptorByte()),
2226 Builder.getInt8(Item.getSizeByte()),
2230 if (!
Size.getQuantity())
2248 "__builtin_os_log_format takes at least 2 arguments");
2259 for (
const auto &Item : Layout.
Items) {
2260 int Size = Item.getSizeByte();
2264 llvm::Value *ArgVal;
2268 for (
unsigned I = 0, E = Item.getMaskType().size(); I < E; ++I)
2269 Val |= ((
uint64_t)Item.getMaskType()[I]) << I * 8;
2270 ArgVal = llvm::Constant::getIntegerValue(
Int64Ty, llvm::APInt(64, Val));
2271 }
else if (
const Expr *TheExpr = Item.getExpr()) {
2277 auto LifetimeExtendObject = [&](
const Expr *E) {
2285 if (isa<CallExpr>(E) || isa<ObjCMessageExpr>(E))
2290 if (TheExpr->getType()->isObjCRetainableType() &&
2291 getLangOpts().ObjCAutoRefCount && LifetimeExtendObject(TheExpr)) {
2293 "Only scalar can be a ObjC retainable type");
2294 if (!isa<Constant>(ArgVal)) {
2308 pushCleanupAfterFullExpr<CallObjCArcUse>(Cleanup, ArgVal);
2312 ArgVal =
Builder.getInt32(Item.getConstValue().getQuantity());
2315 unsigned ArgValSize =
2319 ArgVal =
Builder.CreateBitOrPointerCast(ArgVal,
IntTy);
2335 unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info,
2336 WidthAndSignedness ResultInfo) {
2337 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2338 Op1Info.Width == Op2Info.Width && Op2Info.Width == ResultInfo.Width &&
2339 !Op1Info.Signed && !Op2Info.Signed && ResultInfo.Signed;
2344 const clang::Expr *Op2, WidthAndSignedness Op2Info,
2346 WidthAndSignedness ResultInfo) {
2348 Builtin::BI__builtin_mul_overflow, Op1Info, Op2Info, ResultInfo) &&
2349 "Cannot specialize this multiply");
2354 llvm::Value *HasOverflow;
2356 CGF, llvm::Intrinsic::umul_with_overflow, V1, V2, HasOverflow);
2361 auto IntMax = llvm::APInt::getSignedMaxValue(ResultInfo.Width);
2362 llvm::Value *IntMaxValue = llvm::ConstantInt::get(
Result->getType(), IntMax);
2364 llvm::Value *IntMaxOverflow = CGF.
Builder.CreateICmpUGT(
Result, IntMaxValue);
2365 HasOverflow = CGF.
Builder.CreateOr(HasOverflow, IntMaxOverflow);
2377 WidthAndSignedness Op1Info,
2378 WidthAndSignedness Op2Info,
2379 WidthAndSignedness ResultInfo) {
2380 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2381 std::max(Op1Info.Width, Op2Info.Width) >= ResultInfo.Width &&
2382 Op1Info.Signed != Op2Info.Signed;
2389 WidthAndSignedness Op1Info,
const clang::Expr *Op2,
2390 WidthAndSignedness Op2Info,
2392 WidthAndSignedness ResultInfo) {
2394 Op2Info, ResultInfo) &&
2395 "Not a mixed-sign multipliction we can specialize");
2398 const clang::Expr *SignedOp = Op1Info.Signed ? Op1 : Op2;
2399 const clang::Expr *UnsignedOp = Op1Info.Signed ? Op2 : Op1;
2402 unsigned SignedOpWidth = Op1Info.Signed ? Op1Info.Width : Op2Info.Width;
2403 unsigned UnsignedOpWidth = Op1Info.Signed ? Op2Info.Width : Op1Info.Width;
2406 if (SignedOpWidth < UnsignedOpWidth)
2408 if (UnsignedOpWidth < SignedOpWidth)
2411 llvm::Type *OpTy =
Signed->getType();
2412 llvm::Value *Zero = llvm::Constant::getNullValue(OpTy);
2415 unsigned OpWidth = std::max(Op1Info.Width, Op2Info.Width);
2418 llvm::Value *IsNegative = CGF.
Builder.CreateICmpSLT(
Signed, Zero);
2419 llvm::Value *AbsOfNegative = CGF.
Builder.CreateSub(Zero,
Signed);
2420 llvm::Value *AbsSigned =
2421 CGF.
Builder.CreateSelect(IsNegative, AbsOfNegative,
Signed);
2424 llvm::Value *UnsignedOverflow;
2425 llvm::Value *UnsignedResult =
2429 llvm::Value *Overflow, *
Result;
2430 if (ResultInfo.Signed) {
2434 llvm::APInt::getSignedMaxValue(ResultInfo.Width).zext(OpWidth);
2435 llvm::Value *MaxResult =
2436 CGF.
Builder.CreateAdd(llvm::ConstantInt::get(OpTy, IntMax),
2437 CGF.
Builder.CreateZExt(IsNegative, OpTy));
2438 llvm::Value *SignedOverflow =
2439 CGF.
Builder.CreateICmpUGT(UnsignedResult, MaxResult);
2440 Overflow = CGF.
Builder.CreateOr(UnsignedOverflow, SignedOverflow);
2443 llvm::Value *NegativeResult = CGF.
Builder.CreateNeg(UnsignedResult);
2444 llvm::Value *SignedResult =
2445 CGF.
Builder.CreateSelect(IsNegative, NegativeResult, UnsignedResult);
2449 llvm::Value *Underflow = CGF.
Builder.CreateAnd(
2450 IsNegative, CGF.
Builder.CreateIsNotNull(UnsignedResult));
2451 Overflow = CGF.
Builder.CreateOr(UnsignedOverflow, Underflow);
2452 if (ResultInfo.Width < OpWidth) {
2454 llvm::APInt::getMaxValue(ResultInfo.Width).zext(OpWidth);
2455 llvm::Value *TruncOverflow = CGF.
Builder.CreateICmpUGT(
2456 UnsignedResult, llvm::ConstantInt::get(OpTy, IntMax));
2457 Overflow = CGF.
Builder.CreateOr(Overflow, TruncOverflow);
2462 IsNegative, CGF.
Builder.CreateNeg(UnsignedResult), UnsignedResult);
2466 assert(Overflow &&
Result &&
"Missing overflow or result");
2477 llvm::SmallPtrSetImpl<const Decl *> &Seen) {
2486 if (!Seen.insert(
Record).second)
2489 assert(
Record->hasDefinition() &&
2490 "Incomplete types should already be diagnosed");
2492 if (
Record->isDynamicClass())
2517 llvm::Type *Ty = Src->getType();
2518 ShiftAmt =
Builder.CreateIntCast(ShiftAmt, Ty,
false);
2521 unsigned IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2528 switch (BuiltinID) {
2529#define MUTATE_LDBL(func) \
2530 case Builtin::BI__builtin_##func##l: \
2531 return Builtin::BI__builtin_##func##f128;
2600 if (CGF.
Builder.getIsFPConstrained() &&
2601 CGF.
Builder.getDefaultConstrainedExcept() != fp::ebIgnore) {
2613 auto UBF = CGF->
CGM.
getModule().getOrInsertFunction(Name, FnTy);
2616 for (
auto &&FormalTy : FnTy->params())
2617 Args.push_back(llvm::PoisonValue::get(FormalTy));
2630 !
Result.hasSideEffects()) {
2634 if (
Result.Val.isFloat())
2643 if (
getTarget().getTriple().isPPC64() &&
2644 &
getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad())
2651 const unsigned BuiltinIDIfNoAsmLabel =
2652 FD->
hasAttr<AsmLabelAttr>() ? 0 : BuiltinID;
2654 std::optional<bool> ErrnoOverriden;
2660 if (OP.hasMathErrnoOverride())
2661 ErrnoOverriden = OP.getMathErrnoOverride();
2670 bool ErrnoOverridenToFalseWithOpt =
2671 ErrnoOverriden.has_value() && !ErrnoOverriden.value() && !OptNone &&
2689 switch (BuiltinID) {
2690 case Builtin::BI__builtin_fma:
2691 case Builtin::BI__builtin_fmaf:
2692 case Builtin::BI__builtin_fmal:
2693 case Builtin::BIfma:
2694 case Builtin::BIfmaf:
2695 case Builtin::BIfmal: {
2697 if (Trip.isGNUEnvironment() || Trip.isOSMSVCRT())
2705 bool ConstWithoutErrnoAndExceptions =
2707 bool ConstWithoutExceptions =
2725 bool ConstWithoutErrnoOrExceptions =
2726 ConstWithoutErrnoAndExceptions || ConstWithoutExceptions;
2727 bool GenerateIntrinsics =
2728 (ConstAlways && !OptNone) ||
2730 !(ErrnoOverriden.has_value() && ErrnoOverriden.value()) && !OptNone);
2731 if (!GenerateIntrinsics) {
2732 GenerateIntrinsics =
2733 ConstWithoutErrnoOrExceptions && !ConstWithoutErrnoAndExceptions;
2734 if (!GenerateIntrinsics)
2735 GenerateIntrinsics =
2736 ConstWithoutErrnoOrExceptions &&
2738 !(ErrnoOverriden.has_value() && ErrnoOverriden.value()) && !OptNone);
2739 if (!GenerateIntrinsics)
2740 GenerateIntrinsics =
2741 ConstWithoutErrnoOrExceptions && ErrnoOverridenToFalseWithOpt;
2743 if (GenerateIntrinsics) {
2744 switch (BuiltinIDIfNoAsmLabel) {
2745 case Builtin::BIceil:
2746 case Builtin::BIceilf:
2747 case Builtin::BIceill:
2748 case Builtin::BI__builtin_ceil:
2749 case Builtin::BI__builtin_ceilf:
2750 case Builtin::BI__builtin_ceilf16:
2751 case Builtin::BI__builtin_ceill:
2752 case Builtin::BI__builtin_ceilf128:
2755 Intrinsic::experimental_constrained_ceil));
2757 case Builtin::BIcopysign:
2758 case Builtin::BIcopysignf:
2759 case Builtin::BIcopysignl:
2760 case Builtin::BI__builtin_copysign:
2761 case Builtin::BI__builtin_copysignf:
2762 case Builtin::BI__builtin_copysignf16:
2763 case Builtin::BI__builtin_copysignl:
2764 case Builtin::BI__builtin_copysignf128:
2767 case Builtin::BIcos:
2768 case Builtin::BIcosf:
2769 case Builtin::BIcosl:
2770 case Builtin::BI__builtin_cos:
2771 case Builtin::BI__builtin_cosf:
2772 case Builtin::BI__builtin_cosf16:
2773 case Builtin::BI__builtin_cosl:
2774 case Builtin::BI__builtin_cosf128:
2777 Intrinsic::experimental_constrained_cos));
2779 case Builtin::BIexp:
2780 case Builtin::BIexpf:
2781 case Builtin::BIexpl:
2782 case Builtin::BI__builtin_exp:
2783 case Builtin::BI__builtin_expf:
2784 case Builtin::BI__builtin_expf16:
2785 case Builtin::BI__builtin_expl:
2786 case Builtin::BI__builtin_expf128:
2789 Intrinsic::experimental_constrained_exp));
2791 case Builtin::BIexp2:
2792 case Builtin::BIexp2f:
2793 case Builtin::BIexp2l:
2794 case Builtin::BI__builtin_exp2:
2795 case Builtin::BI__builtin_exp2f:
2796 case Builtin::BI__builtin_exp2f16:
2797 case Builtin::BI__builtin_exp2l:
2798 case Builtin::BI__builtin_exp2f128:
2801 Intrinsic::experimental_constrained_exp2));
2802 case Builtin::BI__builtin_exp10:
2803 case Builtin::BI__builtin_exp10f:
2804 case Builtin::BI__builtin_exp10f16:
2805 case Builtin::BI__builtin_exp10l:
2806 case Builtin::BI__builtin_exp10f128: {
2808 if (
Builder.getIsFPConstrained())
2812 case Builtin::BIfabs:
2813 case Builtin::BIfabsf:
2814 case Builtin::BIfabsl:
2815 case Builtin::BI__builtin_fabs:
2816 case Builtin::BI__builtin_fabsf:
2817 case Builtin::BI__builtin_fabsf16:
2818 case Builtin::BI__builtin_fabsl:
2819 case Builtin::BI__builtin_fabsf128:
2822 case Builtin::BIfloor:
2823 case Builtin::BIfloorf:
2824 case Builtin::BIfloorl:
2825 case Builtin::BI__builtin_floor:
2826 case Builtin::BI__builtin_floorf:
2827 case Builtin::BI__builtin_floorf16:
2828 case Builtin::BI__builtin_floorl:
2829 case Builtin::BI__builtin_floorf128:
2832 Intrinsic::experimental_constrained_floor));
2834 case Builtin::BIfma:
2835 case Builtin::BIfmaf:
2836 case Builtin::BIfmal:
2837 case Builtin::BI__builtin_fma:
2838 case Builtin::BI__builtin_fmaf:
2839 case Builtin::BI__builtin_fmaf16:
2840 case Builtin::BI__builtin_fmal:
2841 case Builtin::BI__builtin_fmaf128:
2844 Intrinsic::experimental_constrained_fma));
2846 case Builtin::BIfmax:
2847 case Builtin::BIfmaxf:
2848 case Builtin::BIfmaxl:
2849 case Builtin::BI__builtin_fmax:
2850 case Builtin::BI__builtin_fmaxf:
2851 case Builtin::BI__builtin_fmaxf16:
2852 case Builtin::BI__builtin_fmaxl:
2853 case Builtin::BI__builtin_fmaxf128:
2856 Intrinsic::experimental_constrained_maxnum));
2858 case Builtin::BIfmin:
2859 case Builtin::BIfminf:
2860 case Builtin::BIfminl:
2861 case Builtin::BI__builtin_fmin:
2862 case Builtin::BI__builtin_fminf:
2863 case Builtin::BI__builtin_fminf16:
2864 case Builtin::BI__builtin_fminl:
2865 case Builtin::BI__builtin_fminf128:
2868 Intrinsic::experimental_constrained_minnum));
2872 case Builtin::BIfmod:
2873 case Builtin::BIfmodf:
2874 case Builtin::BIfmodl:
2875 case Builtin::BI__builtin_fmod:
2876 case Builtin::BI__builtin_fmodf:
2877 case Builtin::BI__builtin_fmodf16:
2878 case Builtin::BI__builtin_fmodl:
2879 case Builtin::BI__builtin_fmodf128: {
2880 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
2886 case Builtin::BIlog:
2887 case Builtin::BIlogf:
2888 case Builtin::BIlogl:
2889 case Builtin::BI__builtin_log:
2890 case Builtin::BI__builtin_logf:
2891 case Builtin::BI__builtin_logf16:
2892 case Builtin::BI__builtin_logl:
2893 case Builtin::BI__builtin_logf128:
2896 Intrinsic::experimental_constrained_log));
2898 case Builtin::BIlog10:
2899 case Builtin::BIlog10f:
2900 case Builtin::BIlog10l:
2901 case Builtin::BI__builtin_log10:
2902 case Builtin::BI__builtin_log10f:
2903 case Builtin::BI__builtin_log10f16:
2904 case Builtin::BI__builtin_log10l:
2905 case Builtin::BI__builtin_log10f128:
2908 Intrinsic::experimental_constrained_log10));
2910 case Builtin::BIlog2:
2911 case Builtin::BIlog2f:
2912 case Builtin::BIlog2l:
2913 case Builtin::BI__builtin_log2:
2914 case Builtin::BI__builtin_log2f:
2915 case Builtin::BI__builtin_log2f16:
2916 case Builtin::BI__builtin_log2l:
2917 case Builtin::BI__builtin_log2f128:
2920 Intrinsic::experimental_constrained_log2));
2922 case Builtin::BInearbyint:
2923 case Builtin::BInearbyintf:
2924 case Builtin::BInearbyintl:
2925 case Builtin::BI__builtin_nearbyint:
2926 case Builtin::BI__builtin_nearbyintf:
2927 case Builtin::BI__builtin_nearbyintl:
2928 case Builtin::BI__builtin_nearbyintf128:
2930 Intrinsic::nearbyint,
2931 Intrinsic::experimental_constrained_nearbyint));
2933 case Builtin::BIpow:
2934 case Builtin::BIpowf:
2935 case Builtin::BIpowl:
2936 case Builtin::BI__builtin_pow:
2937 case Builtin::BI__builtin_powf:
2938 case Builtin::BI__builtin_powf16:
2939 case Builtin::BI__builtin_powl:
2940 case Builtin::BI__builtin_powf128:
2943 Intrinsic::experimental_constrained_pow));
2945 case Builtin::BIrint:
2946 case Builtin::BIrintf:
2947 case Builtin::BIrintl:
2948 case Builtin::BI__builtin_rint:
2949 case Builtin::BI__builtin_rintf:
2950 case Builtin::BI__builtin_rintf16:
2951 case Builtin::BI__builtin_rintl:
2952 case Builtin::BI__builtin_rintf128:
2955 Intrinsic::experimental_constrained_rint));
2957 case Builtin::BIround:
2958 case Builtin::BIroundf:
2959 case Builtin::BIroundl:
2960 case Builtin::BI__builtin_round:
2961 case Builtin::BI__builtin_roundf:
2962 case Builtin::BI__builtin_roundf16:
2963 case Builtin::BI__builtin_roundl:
2964 case Builtin::BI__builtin_roundf128:
2967 Intrinsic::experimental_constrained_round));
2969 case Builtin::BIroundeven:
2970 case Builtin::BIroundevenf:
2971 case Builtin::BIroundevenl:
2972 case Builtin::BI__builtin_roundeven:
2973 case Builtin::BI__builtin_roundevenf:
2974 case Builtin::BI__builtin_roundevenf16:
2975 case Builtin::BI__builtin_roundevenl:
2976 case Builtin::BI__builtin_roundevenf128:
2978 Intrinsic::roundeven,
2979 Intrinsic::experimental_constrained_roundeven));
2981 case Builtin::BIsin:
2982 case Builtin::BIsinf:
2983 case Builtin::BIsinl:
2984 case Builtin::BI__builtin_sin:
2985 case Builtin::BI__builtin_sinf:
2986 case Builtin::BI__builtin_sinf16:
2987 case Builtin::BI__builtin_sinl:
2988 case Builtin::BI__builtin_sinf128:
2991 Intrinsic::experimental_constrained_sin));
2993 case Builtin::BIsqrt:
2994 case Builtin::BIsqrtf:
2995 case Builtin::BIsqrtl:
2996 case Builtin::BI__builtin_sqrt:
2997 case Builtin::BI__builtin_sqrtf:
2998 case Builtin::BI__builtin_sqrtf16:
2999 case Builtin::BI__builtin_sqrtl:
3000 case Builtin::BI__builtin_sqrtf128:
3001 case Builtin::BI__builtin_elementwise_sqrt: {
3003 *
this, E, Intrinsic::sqrt, Intrinsic::experimental_constrained_sqrt);
3007 case Builtin::BItrunc:
3008 case Builtin::BItruncf:
3009 case Builtin::BItruncl:
3010 case Builtin::BI__builtin_trunc:
3011 case Builtin::BI__builtin_truncf:
3012 case Builtin::BI__builtin_truncf16:
3013 case Builtin::BI__builtin_truncl:
3014 case Builtin::BI__builtin_truncf128:
3017 Intrinsic::experimental_constrained_trunc));
3019 case Builtin::BIlround:
3020 case Builtin::BIlroundf:
3021 case Builtin::BIlroundl:
3022 case Builtin::BI__builtin_lround:
3023 case Builtin::BI__builtin_lroundf:
3024 case Builtin::BI__builtin_lroundl:
3025 case Builtin::BI__builtin_lroundf128:
3027 *
this, E, Intrinsic::lround,
3028 Intrinsic::experimental_constrained_lround));
3030 case Builtin::BIllround:
3031 case Builtin::BIllroundf:
3032 case Builtin::BIllroundl:
3033 case Builtin::BI__builtin_llround:
3034 case Builtin::BI__builtin_llroundf:
3035 case Builtin::BI__builtin_llroundl:
3036 case Builtin::BI__builtin_llroundf128:
3038 *
this, E, Intrinsic::llround,
3039 Intrinsic::experimental_constrained_llround));
3041 case Builtin::BIlrint:
3042 case Builtin::BIlrintf:
3043 case Builtin::BIlrintl:
3044 case Builtin::BI__builtin_lrint:
3045 case Builtin::BI__builtin_lrintf:
3046 case Builtin::BI__builtin_lrintl:
3047 case Builtin::BI__builtin_lrintf128:
3049 *
this, E, Intrinsic::lrint,
3050 Intrinsic::experimental_constrained_lrint));
3052 case Builtin::BIllrint:
3053 case Builtin::BIllrintf:
3054 case Builtin::BIllrintl:
3055 case Builtin::BI__builtin_llrint:
3056 case Builtin::BI__builtin_llrintf:
3057 case Builtin::BI__builtin_llrintl:
3058 case Builtin::BI__builtin_llrintf128:
3060 *
this, E, Intrinsic::llrint,
3061 Intrinsic::experimental_constrained_llrint));
3062 case Builtin::BI__builtin_ldexp:
3063 case Builtin::BI__builtin_ldexpf:
3064 case Builtin::BI__builtin_ldexpl:
3065 case Builtin::BI__builtin_ldexpf16:
3066 case Builtin::BI__builtin_ldexpf128: {
3068 *
this, E, Intrinsic::ldexp,
3069 Intrinsic::experimental_constrained_ldexp));
3079 Value *Val = A.emitRawPointer(*
this);
3085 SkippedChecks.
set(SanitizerKind::All);
3086 SkippedChecks.
clear(SanitizerKind::Alignment);
3089 if (
auto *CE = dyn_cast<ImplicitCastExpr>(Arg))
3090 if (CE->getCastKind() == CK_BitCast)
3091 Arg = CE->getSubExpr();
3092 EmitTypeCheck(Kind, Loc, Val, Arg->getType(), A.getAlignment(),
3097 switch (BuiltinIDIfNoAsmLabel) {
3099 case Builtin::BI__builtin___CFStringMakeConstantString:
3100 case Builtin::BI__builtin___NSStringMakeConstantString:
3102 case Builtin::BI__builtin_stdarg_start:
3103 case Builtin::BI__builtin_va_start:
3104 case Builtin::BI__va_start:
3105 case Builtin::BI__builtin_va_end:
3109 BuiltinID != Builtin::BI__builtin_va_end);
3111 case Builtin::BI__builtin_va_copy: {
3118 case Builtin::BIabs:
3119 case Builtin::BIlabs:
3120 case Builtin::BIllabs:
3121 case Builtin::BI__builtin_abs:
3122 case Builtin::BI__builtin_labs:
3123 case Builtin::BI__builtin_llabs: {
3124 bool SanitizeOverflow =
SanOpts.
has(SanitizerKind::SignedIntegerOverflow);
3127 switch (
getLangOpts().getSignedOverflowBehavior()) {
3132 if (!SanitizeOverflow) {
3144 case Builtin::BI__builtin_complex: {
3149 case Builtin::BI__builtin_conj:
3150 case Builtin::BI__builtin_conjf:
3151 case Builtin::BI__builtin_conjl:
3152 case Builtin::BIconj:
3153 case Builtin::BIconjf:
3154 case Builtin::BIconjl: {
3156 Value *Real = ComplexVal.first;
3157 Value *Imag = ComplexVal.second;
3158 Imag =
Builder.CreateFNeg(Imag,
"neg");
3161 case Builtin::BI__builtin_creal:
3162 case Builtin::BI__builtin_crealf:
3163 case Builtin::BI__builtin_creall:
3164 case Builtin::BIcreal:
3165 case Builtin::BIcrealf:
3166 case Builtin::BIcreall: {
3171 case Builtin::BI__builtin_preserve_access_index: {
3192 case Builtin::BI__builtin_cimag:
3193 case Builtin::BI__builtin_cimagf:
3194 case Builtin::BI__builtin_cimagl:
3195 case Builtin::BIcimag:
3196 case Builtin::BIcimagf:
3197 case Builtin::BIcimagl: {
3202 case Builtin::BI__builtin_clrsb:
3203 case Builtin::BI__builtin_clrsbl:
3204 case Builtin::BI__builtin_clrsbll: {
3208 llvm::Type *ArgType = ArgValue->
getType();
3212 Value *
Zero = llvm::Constant::getNullValue(ArgType);
3213 Value *IsNeg =
Builder.CreateICmpSLT(ArgValue, Zero,
"isneg");
3215 Value *Tmp =
Builder.CreateSelect(IsNeg, Inverse, ArgValue);
3222 case Builtin::BI__builtin_ctzs:
3223 case Builtin::BI__builtin_ctz:
3224 case Builtin::BI__builtin_ctzl:
3225 case Builtin::BI__builtin_ctzll:
3226 case Builtin::BI__builtin_ctzg: {
3227 bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_ctzg &&
3234 llvm::Type *ArgType = ArgValue->
getType();
3241 if (
Result->getType() != ResultType)
3247 Value *
Zero = Constant::getNullValue(ArgType);
3248 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, Zero,
"iszero");
3250 Value *ResultOrFallback =
3251 Builder.CreateSelect(IsZero, FallbackValue,
Result,
"ctzg");
3254 case Builtin::BI__builtin_clzs:
3255 case Builtin::BI__builtin_clz:
3256 case Builtin::BI__builtin_clzl:
3257 case Builtin::BI__builtin_clzll:
3258 case Builtin::BI__builtin_clzg: {
3259 bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_clzg &&
3266 llvm::Type *ArgType = ArgValue->
getType();
3273 if (
Result->getType() != ResultType)
3279 Value *
Zero = Constant::getNullValue(ArgType);
3280 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, Zero,
"iszero");
3282 Value *ResultOrFallback =
3283 Builder.CreateSelect(IsZero, FallbackValue,
Result,
"clzg");
3286 case Builtin::BI__builtin_ffs:
3287 case Builtin::BI__builtin_ffsl:
3288 case Builtin::BI__builtin_ffsll: {
3292 llvm::Type *ArgType = ArgValue->
getType();
3297 Builder.CreateAdd(
Builder.CreateCall(F, {ArgValue, Builder.getTrue()}),
3298 llvm::ConstantInt::get(ArgType, 1));
3299 Value *
Zero = llvm::Constant::getNullValue(ArgType);
3300 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, Zero,
"iszero");
3302 if (
Result->getType() != ResultType)
3307 case Builtin::BI__builtin_parity:
3308 case Builtin::BI__builtin_parityl:
3309 case Builtin::BI__builtin_parityll: {
3313 llvm::Type *ArgType = ArgValue->
getType();
3319 if (
Result->getType() != ResultType)
3324 case Builtin::BI__lzcnt16:
3325 case Builtin::BI__lzcnt:
3326 case Builtin::BI__lzcnt64: {
3329 llvm::Type *ArgType = ArgValue->
getType();
3334 if (
Result->getType() != ResultType)
3339 case Builtin::BI__popcnt16:
3340 case Builtin::BI__popcnt:
3341 case Builtin::BI__popcnt64:
3342 case Builtin::BI__builtin_popcount:
3343 case Builtin::BI__builtin_popcountl:
3344 case Builtin::BI__builtin_popcountll:
3345 case Builtin::BI__builtin_popcountg: {
3348 llvm::Type *ArgType = ArgValue->
getType();
3353 if (
Result->getType() != ResultType)
3358 case Builtin::BI__builtin_unpredictable: {
3364 case Builtin::BI__builtin_expect: {
3366 llvm::Type *ArgType = ArgValue->
getType();
3377 Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue},
"expval");
3380 case Builtin::BI__builtin_expect_with_probability: {
3382 llvm::Type *ArgType = ArgValue->
getType();
3385 llvm::APFloat Probability(0.0);
3388 assert(EvalSucceed &&
"probability should be able to evaluate as float");
3390 bool LoseInfo =
false;
3391 Probability.convert(llvm::APFloat::IEEEdouble(),
3392 llvm::RoundingMode::Dynamic, &LoseInfo);
3394 Constant *Confidence = ConstantFP::get(Ty, Probability);
3404 FnExpect, {ArgValue, ExpectedValue, Confidence},
"expval");
3407 case Builtin::BI__builtin_assume_aligned: {
3410 Value *OffsetValue =
3414 ConstantInt *AlignmentCI = cast<ConstantInt>(AlignmentValue);
3415 if (AlignmentCI->getValue().ugt(llvm::Value::MaximumAlignment))
3416 AlignmentCI = ConstantInt::get(AlignmentCI->getIntegerType(),
3417 llvm::Value::MaximumAlignment);
3421 AlignmentCI, OffsetValue);
3424 case Builtin::BI__assume:
3425 case Builtin::BI__builtin_assume: {
3431 Builder.CreateCall(FnAssume, ArgValue);
3434 case Builtin::BI__builtin_assume_separate_storage: {
3441 Value *Values[] = {Value0, Value1};
3442 OperandBundleDefT<Value *> OBD(
"separate_storage", Values);
3446 case Builtin::BI__builtin_allow_runtime_check: {
3450 llvm::Value *Allow =
Builder.CreateCall(
3452 llvm::MetadataAsValue::get(Ctx, llvm::MDString::get(Ctx, Kind)));
3455 case Builtin::BI__arithmetic_fence: {
3458 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3459 llvm::FastMathFlags FMF =
Builder.getFastMathFlags();
3460 bool isArithmeticFenceEnabled =
3461 FMF.allowReassoc() &&
3465 if (isArithmeticFenceEnabled) {
3468 Value *Real =
Builder.CreateArithmeticFence(ComplexVal.first,
3470 Value *Imag =
Builder.CreateArithmeticFence(ComplexVal.second,
3475 Value *Real = ComplexVal.first;
3476 Value *Imag = ComplexVal.second;
3480 if (isArithmeticFenceEnabled)
3485 case Builtin::BI__builtin_bswap16:
3486 case Builtin::BI__builtin_bswap32:
3487 case Builtin::BI__builtin_bswap64:
3488 case Builtin::BI_byteswap_ushort:
3489 case Builtin::BI_byteswap_ulong:
3490 case Builtin::BI_byteswap_uint64: {
3493 case Builtin::BI__builtin_bitreverse8:
3494 case Builtin::BI__builtin_bitreverse16:
3495 case Builtin::BI__builtin_bitreverse32:
3496 case Builtin::BI__builtin_bitreverse64: {
3499 case Builtin::BI__builtin_rotateleft8:
3500 case Builtin::BI__builtin_rotateleft16:
3501 case Builtin::BI__builtin_rotateleft32:
3502 case Builtin::BI__builtin_rotateleft64:
3503 case Builtin::BI_rotl8:
3504 case Builtin::BI_rotl16:
3505 case Builtin::BI_rotl:
3506 case Builtin::BI_lrotl:
3507 case Builtin::BI_rotl64:
3510 case Builtin::BI__builtin_rotateright8:
3511 case Builtin::BI__builtin_rotateright16:
3512 case Builtin::BI__builtin_rotateright32:
3513 case Builtin::BI__builtin_rotateright64:
3514 case Builtin::BI_rotr8:
3515 case Builtin::BI_rotr16:
3516 case Builtin::BI_rotr:
3517 case Builtin::BI_lrotr:
3518 case Builtin::BI_rotr64:
3521 case Builtin::BI__builtin_constant_p: {
3532 return RValue::get(ConstantInt::get(ResultType, 0));
3537 return RValue::get(ConstantInt::get(ResultType, 0));
3549 if (
Result->getType() != ResultType)
3553 case Builtin::BI__builtin_dynamic_object_size:
3554 case Builtin::BI__builtin_object_size: {
3561 bool IsDynamic = BuiltinID == Builtin::BI__builtin_dynamic_object_size;
3563 nullptr, IsDynamic));
3565 case Builtin::BI__builtin_prefetch: {
3569 llvm::ConstantInt::get(
Int32Ty, 0);
3571 llvm::ConstantInt::get(
Int32Ty, 3);
3577 case Builtin::BI__builtin_readcyclecounter: {
3581 case Builtin::BI__builtin_readsteadycounter: {
3585 case Builtin::BI__builtin___clear_cache: {
3591 case Builtin::BI__builtin_trap:
3594 case Builtin::BI__debugbreak:
3597 case Builtin::BI__builtin_unreachable: {
3606 case Builtin::BI__builtin_powi:
3607 case Builtin::BI__builtin_powif:
3608 case Builtin::BI__builtin_powil: {
3612 if (
Builder.getIsFPConstrained()) {
3615 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3622 { Src0->getType(), Src1->getType() });
3625 case Builtin::BI__builtin_frexpl: {
3629 if (&
getTarget().getLongDoubleFormat() == &llvm::APFloat::PPCDoubleDouble())
3633 case Builtin::BI__builtin_frexp:
3634 case Builtin::BI__builtin_frexpf:
3635 case Builtin::BI__builtin_frexpf128:
3636 case Builtin::BI__builtin_frexpf16:
3638 case Builtin::BI__builtin_isgreater:
3639 case Builtin::BI__builtin_isgreaterequal:
3640 case Builtin::BI__builtin_isless:
3641 case Builtin::BI__builtin_islessequal:
3642 case Builtin::BI__builtin_islessgreater:
3643 case Builtin::BI__builtin_isunordered: {
3646 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3650 switch (BuiltinID) {
3651 default: llvm_unreachable(
"Unknown ordered comparison");
3652 case Builtin::BI__builtin_isgreater:
3653 LHS =
Builder.CreateFCmpOGT(LHS, RHS,
"cmp");
3655 case Builtin::BI__builtin_isgreaterequal:
3656 LHS =
Builder.CreateFCmpOGE(LHS, RHS,
"cmp");
3658 case Builtin::BI__builtin_isless:
3659 LHS =
Builder.CreateFCmpOLT(LHS, RHS,
"cmp");
3661 case Builtin::BI__builtin_islessequal:
3662 LHS =
Builder.CreateFCmpOLE(LHS, RHS,
"cmp");
3664 case Builtin::BI__builtin_islessgreater:
3665 LHS =
Builder.CreateFCmpONE(LHS, RHS,
"cmp");
3667 case Builtin::BI__builtin_isunordered:
3668 LHS =
Builder.CreateFCmpUNO(LHS, RHS,
"cmp");
3675 case Builtin::BI__builtin_isnan: {
3676 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3685 case Builtin::BI__builtin_issignaling: {
3686 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3693 case Builtin::BI__builtin_isinf: {
3694 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3703 case Builtin::BIfinite:
3704 case Builtin::BI__finite:
3705 case Builtin::BIfinitef:
3706 case Builtin::BI__finitef:
3707 case Builtin::BIfinitel:
3708 case Builtin::BI__finitel:
3709 case Builtin::BI__builtin_isfinite: {
3710 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3719 case Builtin::BI__builtin_isnormal: {
3720 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3727 case Builtin::BI__builtin_issubnormal: {
3728 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3731 Builder.CreateZExt(
Builder.createIsFPClass(
V, FPClassTest::fcSubnormal),
3735 case Builtin::BI__builtin_iszero: {
3736 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3743 case Builtin::BI__builtin_isfpclass: {
3748 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3754 case Builtin::BI__builtin_nondeterministic_value: {
3763 case Builtin::BI__builtin_elementwise_abs: {
3768 QT = VecTy->getElementType();
3772 Builder.getFalse(),
nullptr,
"elt.abs");
3779 case Builtin::BI__builtin_elementwise_ceil:
3782 case Builtin::BI__builtin_elementwise_exp:
3785 case Builtin::BI__builtin_elementwise_exp2:
3788 case Builtin::BI__builtin_elementwise_log:
3791 case Builtin::BI__builtin_elementwise_log2:
3794 case Builtin::BI__builtin_elementwise_log10:
3797 case Builtin::BI__builtin_elementwise_pow: {
3800 case Builtin::BI__builtin_elementwise_bitreverse:
3803 case Builtin::BI__builtin_elementwise_cos:
3806 case Builtin::BI__builtin_elementwise_floor:
3809 case Builtin::BI__builtin_elementwise_roundeven:
3812 case Builtin::BI__builtin_elementwise_round:
3815 case Builtin::BI__builtin_elementwise_rint:
3818 case Builtin::BI__builtin_elementwise_nearbyint:
3821 case Builtin::BI__builtin_elementwise_sin:
3825 case Builtin::BI__builtin_elementwise_trunc:
3828 case Builtin::BI__builtin_elementwise_canonicalize:
3830 emitUnaryBuiltin(*
this, E, llvm::Intrinsic::canonicalize,
"elt.canonicalize"));
3831 case Builtin::BI__builtin_elementwise_copysign:
3833 case Builtin::BI__builtin_elementwise_fma:
3835 case Builtin::BI__builtin_elementwise_add_sat:
3836 case Builtin::BI__builtin_elementwise_sub_sat: {
3840 assert(Op0->
getType()->isIntOrIntVectorTy() &&
"integer type expected");
3843 Ty = VecTy->getElementType();
3846 if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_add_sat)
3847 Opc = IsSigned ? llvm::Intrinsic::sadd_sat : llvm::Intrinsic::uadd_sat;
3849 Opc = IsSigned ? llvm::Intrinsic::ssub_sat : llvm::Intrinsic::usub_sat;
3850 Result =
Builder.CreateBinaryIntrinsic(Opc, Op0, Op1,
nullptr,
"elt.sat");
3854 case Builtin::BI__builtin_elementwise_max: {
3858 if (Op0->
getType()->isIntOrIntVectorTy()) {
3861 Ty = VecTy->getElementType();
3863 ? llvm::Intrinsic::smax
3864 : llvm::Intrinsic::umax,
3865 Op0, Op1,
nullptr,
"elt.max");
3870 case Builtin::BI__builtin_elementwise_min: {
3874 if (Op0->
getType()->isIntOrIntVectorTy()) {
3877 Ty = VecTy->getElementType();
3879 ? llvm::Intrinsic::smin
3880 : llvm::Intrinsic::umin,
3881 Op0, Op1,
nullptr,
"elt.min");
3887 case Builtin::BI__builtin_reduce_max: {
3888 auto GetIntrinsicID = [](
QualType QT) {
3890 QT = VecTy->getElementType();
3892 return llvm::Intrinsic::vector_reduce_smax;
3894 return llvm::Intrinsic::vector_reduce_umax;
3896 return llvm::Intrinsic::vector_reduce_fmax;
3899 *
this, E, GetIntrinsicID(E->
getArg(0)->
getType()),
"rdx.min"));
3902 case Builtin::BI__builtin_reduce_min: {
3903 auto GetIntrinsicID = [](
QualType QT) {
3905 QT = VecTy->getElementType();
3907 return llvm::Intrinsic::vector_reduce_smin;
3909 return llvm::Intrinsic::vector_reduce_umin;
3911 return llvm::Intrinsic::vector_reduce_fmin;
3915 *
this, E, GetIntrinsicID(E->
getArg(0)->
getType()),
"rdx.min"));
3918 case Builtin::BI__builtin_reduce_add:
3920 *
this, E, llvm::Intrinsic::vector_reduce_add,
"rdx.add"));
3921 case Builtin::BI__builtin_reduce_mul:
3923 *
this, E, llvm::Intrinsic::vector_reduce_mul,
"rdx.mul"));
3924 case Builtin::BI__builtin_reduce_xor:
3926 *
this, E, llvm::Intrinsic::vector_reduce_xor,
"rdx.xor"));
3927 case Builtin::BI__builtin_reduce_or:
3929 *
this, E, llvm::Intrinsic::vector_reduce_or,
"rdx.or"));
3930 case Builtin::BI__builtin_reduce_and:
3932 *
this, E, llvm::Intrinsic::vector_reduce_and,
"rdx.and"));
3934 case Builtin::BI__builtin_matrix_transpose: {
3938 Value *
Result = MB.CreateMatrixTranspose(MatValue, MatrixTy->getNumRows(),
3939 MatrixTy->getNumColumns());
3943 case Builtin::BI__builtin_matrix_column_major_load: {
3949 assert(PtrTy &&
"arg0 must be of pointer type");
3959 ResultTy->getNumRows(), ResultTy->getNumColumns(),
"matrix");
3963 case Builtin::BI__builtin_matrix_column_major_store: {
3971 assert(PtrTy &&
"arg1 must be of pointer type");
3980 MatrixTy->getNumRows(), MatrixTy->getNumColumns());
3984 case Builtin::BI__builtin_isinf_sign: {
3986 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3991 AbsArg, ConstantFP::getInfinity(Arg->
getType()),
"isinf");
3997 Value *NegativeOne = ConstantInt::get(
IntTy, -1);
3998 Value *SignResult =
Builder.CreateSelect(IsNeg, NegativeOne, One);
4003 case Builtin::BI__builtin_flt_rounds: {
4008 if (
Result->getType() != ResultType)
4014 case Builtin::BI__builtin_set_flt_rounds: {
4022 case Builtin::BI__builtin_fpclassify: {
4023 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
4034 "fpclassify_result");
4038 Value *IsZero =
Builder.CreateFCmpOEQ(
V, Constant::getNullValue(Ty),
4042 Builder.CreateCondBr(IsZero, End, NotZero);
4046 Builder.SetInsertPoint(NotZero);
4050 Builder.CreateCondBr(IsNan, End, NotNan);
4051 Result->addIncoming(NanLiteral, NotZero);
4054 Builder.SetInsertPoint(NotNan);
4057 Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(
V->getType()),
4061 Builder.CreateCondBr(IsInf, End, NotInf);
4062 Result->addIncoming(InfLiteral, NotNan);
4065 Builder.SetInsertPoint(NotInf);
4066 APFloat Smallest = APFloat::getSmallestNormalized(
4069 Builder.CreateFCmpUGE(VAbs, ConstantFP::get(
V->getContext(), Smallest),
4071 Value *NormalResult =
4075 Result->addIncoming(NormalResult, NotInf);
4088 case Builtin::BIalloca:
4089 case Builtin::BI_alloca:
4090 case Builtin::BI__builtin_alloca_uninitialized:
4091 case Builtin::BI__builtin_alloca: {
4095 const Align SuitableAlignmentInBytes =
4099 AllocaInst *AI =
Builder.CreateAlloca(
Builder.getInt8Ty(), Size);
4100 AI->setAlignment(SuitableAlignmentInBytes);
4101 if (BuiltinID != Builtin::BI__builtin_alloca_uninitialized)
4113 case Builtin::BI__builtin_alloca_with_align_uninitialized:
4114 case Builtin::BI__builtin_alloca_with_align: {
4117 auto *AlignmentInBitsCI = cast<ConstantInt>(AlignmentInBitsValue);
4118 unsigned AlignmentInBits = AlignmentInBitsCI->getZExtValue();
4119 const Align AlignmentInBytes =
4121 AllocaInst *AI =
Builder.CreateAlloca(
Builder.getInt8Ty(), Size);
4122 AI->setAlignment(AlignmentInBytes);
4123 if (BuiltinID != Builtin::BI__builtin_alloca_with_align_uninitialized)
4135 case Builtin::BIbzero:
4136 case Builtin::BI__builtin_bzero: {
4145 case Builtin::BIbcopy:
4146 case Builtin::BI__builtin_bcopy: {
4160 case Builtin::BImemcpy:
4161 case Builtin::BI__builtin_memcpy:
4162 case Builtin::BImempcpy:
4163 case Builtin::BI__builtin_mempcpy: {
4170 if (BuiltinID == Builtin::BImempcpy ||
4171 BuiltinID == Builtin::BI__builtin_mempcpy)
4178 case Builtin::BI__builtin_memcpy_inline: {
4189 case Builtin::BI__builtin_char_memchr:
4190 BuiltinID = Builtin::BI__builtin_memchr;
4193 case Builtin::BI__builtin___memcpy_chk: {
4200 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
4201 if (
Size.ugt(DstSize))
4205 Value *SizeVal = llvm::ConstantInt::get(
Builder.getContext(), Size);
4210 case Builtin::BI__builtin_objc_memmove_collectable: {
4215 DestAddr, SrcAddr, SizeVal);
4219 case Builtin::BI__builtin___memmove_chk: {
4226 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
4227 if (
Size.ugt(DstSize))
4231 Value *SizeVal = llvm::ConstantInt::get(
Builder.getContext(), Size);
4236 case Builtin::BImemmove:
4237 case Builtin::BI__builtin_memmove: {
4246 case Builtin::BImemset:
4247 case Builtin::BI__builtin_memset: {
4257 case Builtin::BI__builtin_memset_inline: {
4269 case Builtin::BI__builtin___memset_chk: {
4276 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
4277 if (
Size.ugt(DstSize))
4282 Value *SizeVal = llvm::ConstantInt::get(
Builder.getContext(), Size);
4286 case Builtin::BI__builtin_wmemchr: {
4289 if (!
getTarget().getTriple().isOSMSVCRT())
4297 BasicBlock *Entry =
Builder.GetInsertBlock();
4302 Builder.CreateCondBr(SizeEq0, Exit, CmpEq);
4306 StrPhi->addIncoming(Str, Entry);
4308 SizePhi->addIncoming(Size, Entry);
4312 Value *FoundChr =
Builder.CreateConstInBoundsGEP1_32(WCharTy, StrPhi, 0);
4314 Builder.CreateCondBr(StrEqChr, Exit, Next);
4317 Value *NextStr =
Builder.CreateConstInBoundsGEP1_32(WCharTy, StrPhi, 1);
4319 Value *NextSizeEq0 =
4320 Builder.CreateICmpEQ(NextSize, ConstantInt::get(
SizeTy, 0));
4321 Builder.CreateCondBr(NextSizeEq0, Exit, CmpEq);
4322 StrPhi->addIncoming(NextStr, Next);
4323 SizePhi->addIncoming(NextSize, Next);
4327 Ret->addIncoming(llvm::Constant::getNullValue(Str->
getType()), Entry);
4328 Ret->addIncoming(llvm::Constant::getNullValue(Str->
getType()), Next);
4329 Ret->addIncoming(FoundChr, CmpEq);
4332 case Builtin::BI__builtin_wmemcmp: {
4335 if (!
getTarget().getTriple().isOSMSVCRT())
4344 BasicBlock *Entry =
Builder.GetInsertBlock();
4350 Builder.CreateCondBr(SizeEq0, Exit, CmpGT);
4354 DstPhi->addIncoming(Dst, Entry);
4356 SrcPhi->addIncoming(Src, Entry);
4358 SizePhi->addIncoming(Size, Entry);
4364 Builder.CreateCondBr(DstGtSrc, Exit, CmpLT);
4368 Builder.CreateCondBr(DstLtSrc, Exit, Next);
4371 Value *NextDst =
Builder.CreateConstInBoundsGEP1_32(WCharTy, DstPhi, 1);
4372 Value *NextSrc =
Builder.CreateConstInBoundsGEP1_32(WCharTy, SrcPhi, 1);
4374 Value *NextSizeEq0 =
4375 Builder.CreateICmpEQ(NextSize, ConstantInt::get(
SizeTy, 0));
4376 Builder.CreateCondBr(NextSizeEq0, Exit, CmpGT);
4377 DstPhi->addIncoming(NextDst, Next);
4378 SrcPhi->addIncoming(NextSrc, Next);
4379 SizePhi->addIncoming(NextSize, Next);
4383 Ret->addIncoming(ConstantInt::get(
IntTy, 0), Entry);
4384 Ret->addIncoming(ConstantInt::get(
IntTy, 1), CmpGT);
4385 Ret->addIncoming(ConstantInt::get(
IntTy, -1), CmpLT);
4386 Ret->addIncoming(ConstantInt::get(
IntTy, 0), Next);
4389 case Builtin::BI__builtin_dwarf_cfa: {
4402 llvm::ConstantInt::get(
Int32Ty, Offset)));
4404 case Builtin::BI__builtin_return_address: {
4410 case Builtin::BI_ReturnAddress: {
4414 case Builtin::BI__builtin_frame_address: {
4420 case Builtin::BI__builtin_extract_return_addr: {
4425 case Builtin::BI__builtin_frob_return_addr: {
4430 case Builtin::BI__builtin_dwarf_sp_column: {
4431 llvm::IntegerType *Ty
4440 case Builtin::BI__builtin_init_dwarf_reg_size_table: {
4446 case Builtin::BI__builtin_eh_return: {
4450 llvm::IntegerType *
IntTy = cast<llvm::IntegerType>(
Int->getType());
4451 assert((
IntTy->getBitWidth() == 32 ||
IntTy->getBitWidth() == 64) &&
4452 "LLVM's __builtin_eh_return only supports 32- and 64-bit variants");
4455 : Intrinsic::eh_return_i64);
4464 case Builtin::BI__builtin_unwind_init: {
4469 case Builtin::BI__builtin_extend_pointer: {
4494 case Builtin::BI__builtin_setjmp: {
4501 ConstantInt::get(
Int32Ty, 0));
4515 case Builtin::BI__builtin_longjmp: {
4529 case Builtin::BI__builtin_launder: {
4538 case Builtin::BI__sync_fetch_and_add:
4539 case Builtin::BI__sync_fetch_and_sub:
4540 case Builtin::BI__sync_fetch_and_or:
4541 case Builtin::BI__sync_fetch_and_and:
4542 case Builtin::BI__sync_fetch_and_xor:
4543 case Builtin::BI__sync_fetch_and_nand:
4544 case Builtin::BI__sync_add_and_fetch:
4545 case Builtin::BI__sync_sub_and_fetch:
4546 case Builtin::BI__sync_and_and_fetch:
4547 case Builtin::BI__sync_or_and_fetch:
4548 case Builtin::BI__sync_xor_and_fetch:
4549 case Builtin::BI__sync_nand_and_fetch:
4550 case Builtin::BI__sync_val_compare_and_swap:
4551 case Builtin::BI__sync_bool_compare_and_swap:
4552 case Builtin::BI__sync_lock_test_and_set:
4553 case Builtin::BI__sync_lock_release:
4554 case Builtin::BI__sync_swap:
4555 llvm_unreachable(
"Shouldn't make it through sema");
4556 case Builtin::BI__sync_fetch_and_add_1:
4557 case Builtin::BI__sync_fetch_and_add_2:
4558 case Builtin::BI__sync_fetch_and_add_4:
4559 case Builtin::BI__sync_fetch_and_add_8:
4560 case Builtin::BI__sync_fetch_and_add_16:
4562 case Builtin::BI__sync_fetch_and_sub_1:
4563 case Builtin::BI__sync_fetch_and_sub_2:
4564 case Builtin::BI__sync_fetch_and_sub_4:
4565 case Builtin::BI__sync_fetch_and_sub_8:
4566 case Builtin::BI__sync_fetch_and_sub_16:
4568 case Builtin::BI__sync_fetch_and_or_1:
4569 case Builtin::BI__sync_fetch_and_or_2:
4570 case Builtin::BI__sync_fetch_and_or_4:
4571 case Builtin::BI__sync_fetch_and_or_8:
4572 case Builtin::BI__sync_fetch_and_or_16:
4574 case Builtin::BI__sync_fetch_and_and_1:
4575 case Builtin::BI__sync_fetch_and_and_2:
4576 case Builtin::BI__sync_fetch_and_and_4:
4577 case Builtin::BI__sync_fetch_and_and_8:
4578 case Builtin::BI__sync_fetch_and_and_16:
4580 case Builtin::BI__sync_fetch_and_xor_1:
4581 case Builtin::BI__sync_fetch_and_xor_2:
4582 case Builtin::BI__sync_fetch_and_xor_4:
4583 case Builtin::BI__sync_fetch_and_xor_8:
4584 case Builtin::BI__sync_fetch_and_xor_16:
4586 case Builtin::BI__sync_fetch_and_nand_1:
4587 case Builtin::BI__sync_fetch_and_nand_2:
4588 case Builtin::BI__sync_fetch_and_nand_4:
4589 case Builtin::BI__sync_fetch_and_nand_8:
4590 case Builtin::BI__sync_fetch_and_nand_16:
4594 case Builtin::BI__sync_fetch_and_min:
4596 case Builtin::BI__sync_fetch_and_max:
4598 case Builtin::BI__sync_fetch_and_umin:
4600 case Builtin::BI__sync_fetch_and_umax:
4603 case Builtin::BI__sync_add_and_fetch_1:
4604 case Builtin::BI__sync_add_and_fetch_2:
4605 case Builtin::BI__sync_add_and_fetch_4:
4606 case Builtin::BI__sync_add_and_fetch_8:
4607 case Builtin::BI__sync_add_and_fetch_16:
4609 llvm::Instruction::Add);
4610 case Builtin::BI__sync_sub_and_fetch_1:
4611 case Builtin::BI__sync_sub_and_fetch_2:
4612 case Builtin::BI__sync_sub_and_fetch_4:
4613 case Builtin::BI__sync_sub_and_fetch_8:
4614 case Builtin::BI__sync_sub_and_fetch_16:
4616 llvm::Instruction::Sub);
4617 case Builtin::BI__sync_and_and_fetch_1:
4618 case Builtin::BI__sync_and_and_fetch_2:
4619 case Builtin::BI__sync_and_and_fetch_4:
4620 case Builtin::BI__sync_and_and_fetch_8:
4621 case Builtin::BI__sync_and_and_fetch_16:
4623 llvm::Instruction::And);
4624 case Builtin::BI__sync_or_and_fetch_1:
4625 case Builtin::BI__sync_or_and_fetch_2:
4626 case Builtin::BI__sync_or_and_fetch_4:
4627 case Builtin::BI__sync_or_and_fetch_8:
4628 case Builtin::BI__sync_or_and_fetch_16:
4630 llvm::Instruction::Or);
4631 case Builtin::BI__sync_xor_and_fetch_1:
4632 case Builtin::BI__sync_xor_and_fetch_2:
4633 case Builtin::BI__sync_xor_and_fetch_4:
4634 case Builtin::BI__sync_xor_and_fetch_8:
4635 case Builtin::BI__sync_xor_and_fetch_16:
4637 llvm::Instruction::Xor);
4638 case Builtin::BI__sync_nand_and_fetch_1:
4639 case Builtin::BI__sync_nand_and_fetch_2:
4640 case Builtin::BI__sync_nand_and_fetch_4:
4641 case Builtin::BI__sync_nand_and_fetch_8:
4642 case Builtin::BI__sync_nand_and_fetch_16:
4644 llvm::Instruction::And,
true);
4646 case Builtin::BI__sync_val_compare_and_swap_1:
4647 case Builtin::BI__sync_val_compare_and_swap_2:
4648 case Builtin::BI__sync_val_compare_and_swap_4:
4649 case Builtin::BI__sync_val_compare_and_swap_8:
4650 case Builtin::BI__sync_val_compare_and_swap_16:
4653 case Builtin::BI__sync_bool_compare_and_swap_1:
4654 case Builtin::BI__sync_bool_compare_and_swap_2:
4655 case Builtin::BI__sync_bool_compare_and_swap_4:
4656 case Builtin::BI__sync_bool_compare_and_swap_8:
4657 case Builtin::BI__sync_bool_compare_and_swap_16:
4660 case Builtin::BI__sync_swap_1:
4661 case Builtin::BI__sync_swap_2:
4662 case Builtin::BI__sync_swap_4:
4663 case Builtin::BI__sync_swap_8:
4664 case Builtin::BI__sync_swap_16:
4667 case Builtin::BI__sync_lock_test_and_set_1:
4668 case Builtin::BI__sync_lock_test_and_set_2:
4669 case Builtin::BI__sync_lock_test_and_set_4:
4670 case Builtin::BI__sync_lock_test_and_set_8:
4671 case Builtin::BI__sync_lock_test_and_set_16:
4674 case Builtin::BI__sync_lock_release_1:
4675 case Builtin::BI__sync_lock_release_2:
4676 case Builtin::BI__sync_lock_release_4:
4677 case Builtin::BI__sync_lock_release_8:
4678 case Builtin::BI__sync_lock_release_16: {
4684 llvm::StoreInst *
Store =
4686 Store->setAtomic(llvm::AtomicOrdering::Release);
4690 case Builtin::BI__sync_synchronize: {
4698 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent);
4702 case Builtin::BI__builtin_nontemporal_load:
4704 case Builtin::BI__builtin_nontemporal_store:
4706 case Builtin::BI__c11_atomic_is_lock_free:
4707 case Builtin::BI__atomic_is_lock_free: {
4711 const char *LibCallName =
"__atomic_is_lock_free";
4715 if (BuiltinID == Builtin::BI__atomic_is_lock_free)
4729 case Builtin::BI__atomic_test_and_set: {
4741 if (isa<llvm::ConstantInt>(Order)) {
4742 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
4743 AtomicRMWInst *
Result =
nullptr;
4748 llvm::AtomicOrdering::Monotonic);
4753 llvm::AtomicOrdering::Acquire);
4757 llvm::AtomicOrdering::Release);
4762 llvm::AtomicOrdering::AcquireRelease);
4766 llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
4767 llvm::AtomicOrdering::SequentiallyConsistent);
4770 Result->setVolatile(Volatile);
4776 llvm::BasicBlock *BBs[5] = {
4783 llvm::AtomicOrdering Orders[5] = {
4784 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Acquire,
4785 llvm::AtomicOrdering::Release, llvm::AtomicOrdering::AcquireRelease,
4786 llvm::AtomicOrdering::SequentiallyConsistent};
4788 Order =
Builder.CreateIntCast(Order,
Builder.getInt32Ty(),
false);
4789 llvm::SwitchInst *SI =
Builder.CreateSwitch(Order, BBs[0]);
4791 Builder.SetInsertPoint(ContBB);
4794 for (
unsigned i = 0; i < 5; ++i) {
4795 Builder.SetInsertPoint(BBs[i]);
4797 Ptr, NewVal, Orders[i]);
4798 RMW->setVolatile(Volatile);
4799 Result->addIncoming(RMW, BBs[i]);
4803 SI->addCase(
Builder.getInt32(0), BBs[0]);
4804 SI->addCase(
Builder.getInt32(1), BBs[1]);
4805 SI->addCase(
Builder.getInt32(2), BBs[1]);
4806 SI->addCase(
Builder.getInt32(3), BBs[2]);
4807 SI->addCase(
Builder.getInt32(4), BBs[3]);
4808 SI->addCase(
Builder.getInt32(5), BBs[4]);
4810 Builder.SetInsertPoint(ContBB);
4814 case Builtin::BI__atomic_clear: {
4823 if (isa<llvm::ConstantInt>(Order)) {
4824 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
4829 Store->setOrdering(llvm::AtomicOrdering::Monotonic);
4832 Store->setOrdering(llvm::AtomicOrdering::Release);
4835 Store->setOrdering(llvm::AtomicOrdering::SequentiallyConsistent);
4843 llvm::BasicBlock *BBs[3] = {
4848 llvm::AtomicOrdering Orders[3] = {
4849 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Release,
4850 llvm::AtomicOrdering::SequentiallyConsistent};
4852 Order =
Builder.CreateIntCast(Order,
Builder.getInt32Ty(),
false);
4853 llvm::SwitchInst *SI =
Builder.CreateSwitch(Order, BBs[0]);
4855 for (
unsigned i = 0; i < 3; ++i) {
4856 Builder.SetInsertPoint(BBs[i]);
4858 Store->setOrdering(Orders[i]);
4862 SI->addCase(
Builder.getInt32(0), BBs[0]);
4863 SI->addCase(
Builder.getInt32(3), BBs[1]);
4864 SI->addCase(
Builder.getInt32(5), BBs[2]);
4866 Builder.SetInsertPoint(ContBB);
4870 case Builtin::BI__atomic_thread_fence:
4871 case Builtin::BI__atomic_signal_fence:
4872 case Builtin::BI__c11_atomic_thread_fence:
4873 case Builtin::BI__c11_atomic_signal_fence: {
4874 llvm::SyncScope::ID SSID;
4875 if (BuiltinID == Builtin::BI__atomic_signal_fence ||
4876 BuiltinID == Builtin::BI__c11_atomic_signal_fence)
4877 SSID = llvm::SyncScope::SingleThread;
4879 SSID = llvm::SyncScope::System;
4881 if (isa<llvm::ConstantInt>(Order)) {
4882 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
4889 Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
4892 Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
4895 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
4898 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
4904 llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB;
4911 Order =
Builder.CreateIntCast(Order,
Builder.getInt32Ty(),
false);
4912 llvm::SwitchInst *SI =
Builder.CreateSwitch(Order, ContBB);
4914 Builder.SetInsertPoint(AcquireBB);
4915 Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
4917 SI->addCase(
Builder.getInt32(1), AcquireBB);
4918 SI->addCase(
Builder.getInt32(2), AcquireBB);
4920 Builder.SetInsertPoint(ReleaseBB);
4921 Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
4923 SI->addCase(
Builder.getInt32(3), ReleaseBB);
4925 Builder.SetInsertPoint(AcqRelBB);
4926 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
4928 SI->addCase(
Builder.getInt32(4), AcqRelBB);
4930 Builder.SetInsertPoint(SeqCstBB);
4931 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
4933 SI->addCase(
Builder.getInt32(5), SeqCstBB);
4935 Builder.SetInsertPoint(ContBB);
4939 case Builtin::BI__builtin_signbit:
4940 case Builtin::BI__builtin_signbitf:
4941 case Builtin::BI__builtin_signbitl: {
4946 case Builtin::BI__warn_memset_zero_len:
4948 case Builtin::BI__annotation: {
4953 assert(Str->getCharByteWidth() == 2);
4954 StringRef WideBytes = Str->getBytes();
4955 std::string StrUtf8;
4956 if (!convertUTF16ToUTF8String(
4957 ArrayRef(WideBytes.data(), WideBytes.size()), StrUtf8)) {
4961 Strings.push_back(llvm::MDString::get(
getLLVMContext(), StrUtf8));
4971 case Builtin::BI__builtin_annotation: {
4980 StringRef Str = cast<StringLiteral>(AnnotationStrExpr)->getString();
4984 case Builtin::BI__builtin_addcb:
4985 case Builtin::BI__builtin_addcs:
4986 case Builtin::BI__builtin_addc:
4987 case Builtin::BI__builtin_addcl:
4988 case Builtin::BI__builtin_addcll:
4989 case Builtin::BI__builtin_subcb:
4990 case Builtin::BI__builtin_subcs:
4991 case Builtin::BI__builtin_subc:
4992 case Builtin::BI__builtin_subcl:
4993 case Builtin::BI__builtin_subcll: {
5019 llvm::Intrinsic::ID IntrinsicId;
5020 switch (BuiltinID) {
5021 default: llvm_unreachable(
"Unknown multiprecision builtin id.");
5022 case Builtin::BI__builtin_addcb:
5023 case Builtin::BI__builtin_addcs:
5024 case Builtin::BI__builtin_addc:
5025 case Builtin::BI__builtin_addcl:
5026 case Builtin::BI__builtin_addcll:
5027 IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
5029 case Builtin::BI__builtin_subcb:
5030 case Builtin::BI__builtin_subcs:
5031 case Builtin::BI__builtin_subc:
5032 case Builtin::BI__builtin_subcl:
5033 case Builtin::BI__builtin_subcll:
5034 IntrinsicId = llvm::Intrinsic::usub_with_overflow;
5039 llvm::Value *Carry1;
5042 llvm::Value *Carry2;
5044 Sum1, Carryin, Carry2);
5045 llvm::Value *CarryOut =
Builder.CreateZExt(
Builder.CreateOr(Carry1, Carry2),
5051 case Builtin::BI__builtin_add_overflow:
5052 case Builtin::BI__builtin_sub_overflow:
5053 case Builtin::BI__builtin_mul_overflow: {
5061 WidthAndSignedness LeftInfo =
5063 WidthAndSignedness RightInfo =
5065 WidthAndSignedness ResultInfo =
5072 RightInfo, ResultArg, ResultQTy,
5078 *
this, LeftArg, LeftInfo, RightArg, RightInfo, ResultArg, ResultQTy,
5081 WidthAndSignedness EncompassingInfo =
5084 llvm::Type *EncompassingLLVMTy =
5089 llvm::Intrinsic::ID IntrinsicId;
5090 switch (BuiltinID) {
5092 llvm_unreachable(
"Unknown overflow builtin id.");
5093 case Builtin::BI__builtin_add_overflow:
5094 IntrinsicId = EncompassingInfo.Signed
5095 ? llvm::Intrinsic::sadd_with_overflow
5096 : llvm::Intrinsic::uadd_with_overflow;
5098 case Builtin::BI__builtin_sub_overflow:
5099 IntrinsicId = EncompassingInfo.Signed
5100 ? llvm::Intrinsic::ssub_with_overflow
5101 : llvm::Intrinsic::usub_with_overflow;
5103 case Builtin::BI__builtin_mul_overflow:
5104 IntrinsicId = EncompassingInfo.Signed
5105 ? llvm::Intrinsic::smul_with_overflow
5106 : llvm::Intrinsic::umul_with_overflow;
5115 Left =
Builder.CreateIntCast(Left, EncompassingLLVMTy, LeftInfo.Signed);
5116 Right =
Builder.CreateIntCast(Right, EncompassingLLVMTy, RightInfo.Signed);
5119 llvm::Value *Overflow, *
Result;
5122 if (EncompassingInfo.Width > ResultInfo.Width) {
5125 llvm::Value *ResultTrunc =
Builder.CreateTrunc(
Result, ResultLLVMTy);
5129 llvm::Value *ResultTruncExt =
Builder.CreateIntCast(
5130 ResultTrunc, EncompassingLLVMTy, ResultInfo.Signed);
5131 llvm::Value *TruncationOverflow =
5134 Overflow =
Builder.CreateOr(Overflow, TruncationOverflow);
5146 case Builtin::BI__builtin_uadd_overflow:
5147 case Builtin::BI__builtin_uaddl_overflow:
5148 case Builtin::BI__builtin_uaddll_overflow:
5149 case Builtin::BI__builtin_usub_overflow:
5150 case Builtin::BI__builtin_usubl_overflow:
5151 case Builtin::BI__builtin_usubll_overflow:
5152 case Builtin::BI__builtin_umul_overflow:
5153 case Builtin::BI__builtin_umull_overflow:
5154 case Builtin::BI__builtin_umulll_overflow:
5155 case Builtin::BI__builtin_sadd_overflow:
5156 case Builtin::BI__builtin_saddl_overflow:
5157 case Builtin::BI__builtin_saddll_overflow:
5158 case Builtin::BI__builtin_ssub_overflow:
5159 case Builtin::BI__builtin_ssubl_overflow:
5160 case Builtin::BI__builtin_ssubll_overflow:
5161 case Builtin::BI__builtin_smul_overflow:
5162 case Builtin::BI__builtin_smull_overflow:
5163 case Builtin::BI__builtin_smulll_overflow: {
5173 llvm::Intrinsic::ID IntrinsicId;
5174 switch (BuiltinID) {
5175 default: llvm_unreachable(
"Unknown overflow builtin id.");
5176 case Builtin::BI__builtin_uadd_overflow:
5177 case Builtin::BI__builtin_uaddl_overflow:
5178 case Builtin::BI__builtin_uaddll_overflow:
5179 IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
5181 case Builtin::BI__builtin_usub_overflow:
5182 case Builtin::BI__builtin_usubl_overflow:
5183 case Builtin::BI__builtin_usubll_overflow:
5184 IntrinsicId = llvm::Intrinsic::usub_with_overflow;
5186 case Builtin::BI__builtin_umul_overflow:
5187 case Builtin::BI__builtin_umull_overflow:
5188 case Builtin::BI__builtin_umulll_overflow:
5189 IntrinsicId = llvm::Intrinsic::umul_with_overflow;
5191 case Builtin::BI__builtin_sadd_overflow:
5192 case Builtin::BI__builtin_saddl_overflow:
5193 case Builtin::BI__builtin_saddll_overflow:
5194 IntrinsicId = llvm::Intrinsic::sadd_with_overflow;
5196 case Builtin::BI__builtin_ssub_overflow:
5197 case Builtin::BI__builtin_ssubl_overflow:
5198 case Builtin::BI__builtin_ssubll_overflow:
5199 IntrinsicId = llvm::Intrinsic::ssub_with_overflow;
5201 case Builtin::BI__builtin_smul_overflow:
5202 case Builtin::BI__builtin_smull_overflow:
5203 case Builtin::BI__builtin_smulll_overflow:
5204 IntrinsicId = llvm::Intrinsic::smul_with_overflow;
5215 case Builtin::BIaddressof:
5216 case Builtin::BI__addressof:
5217 case Builtin::BI__builtin_addressof:
5219 case Builtin::BI__builtin_function_start:
5222 case Builtin::BI__builtin_operator_new:
5225 case Builtin::BI__builtin_operator_delete:
5230 case Builtin::BI__builtin_is_aligned:
5232 case Builtin::BI__builtin_align_up:
5234 case Builtin::BI__builtin_align_down:
5237 case Builtin::BI__noop:
5240 case Builtin::BI__builtin_call_with_static_chain: {
5247 case Builtin::BI_InterlockedExchange8:
5248 case Builtin::BI_InterlockedExchange16:
5249 case Builtin::BI_InterlockedExchange:
5250 case Builtin::BI_InterlockedExchangePointer:
5253 case Builtin::BI_InterlockedCompareExchangePointer:
5254 case Builtin::BI_InterlockedCompareExchangePointer_nf: {
5256 llvm::IntegerType *IntType = IntegerType::get(
5262 RTy = Exchange->getType();
5263 Exchange =
Builder.CreatePtrToInt(Exchange, IntType);
5265 llvm::Value *Comparand =
5269 BuiltinID == Builtin::BI_InterlockedCompareExchangePointer_nf ?
5270 AtomicOrdering::Monotonic : AtomicOrdering::SequentiallyConsistent;
5273 Ordering, Ordering);
5274 Result->setVolatile(
true);
5280 case Builtin::BI_InterlockedCompareExchange8:
5281 case Builtin::BI_InterlockedCompareExchange16:
5282 case Builtin::BI_InterlockedCompareExchange:
5283 case Builtin::BI_InterlockedCompareExchange64:
5285 case Builtin::BI_InterlockedIncrement16:
5286 case Builtin::BI_InterlockedIncrement:
5289 case Builtin::BI_InterlockedDecrement16:
5290 case Builtin::BI_InterlockedDecrement:
5293 case Builtin::BI_InterlockedAnd8:
5294 case Builtin::BI_InterlockedAnd16:
5295 case Builtin::BI_InterlockedAnd:
5297 case Builtin::BI_InterlockedExchangeAdd8:
5298 case Builtin::BI_InterlockedExchangeAdd16:
5299 case Builtin::BI_InterlockedExchangeAdd:
5302 case Builtin::BI_InterlockedExchangeSub8:
5303 case Builtin::BI_InterlockedExchangeSub16:
5304 case Builtin::BI_InterlockedExchangeSub:
5307 case Builtin::BI_InterlockedOr8:
5308 case Builtin::BI_InterlockedOr16:
5309 case Builtin::BI_InterlockedOr:
5311 case Builtin::BI_InterlockedXor8:
5312 case Builtin::BI_InterlockedXor16:
5313 case Builtin::BI_InterlockedXor:
5316 case Builtin::BI_bittest64:
5317 case Builtin::BI_bittest:
5318 case Builtin::BI_bittestandcomplement64:
5319 case Builtin::BI_bittestandcomplement:
5320 case Builtin::BI_bittestandreset64:
5321 case Builtin::BI_bittestandreset:
5322 case Builtin::BI_bittestandset64:
5323 case Builtin::BI_bittestandset:
5324 case Builtin::BI_interlockedbittestandreset:
5325 case Builtin::BI_interlockedbittestandreset64:
5326 case Builtin::BI_interlockedbittestandset64:
5327 case Builtin::BI_interlockedbittestandset:
5328 case Builtin::BI_interlockedbittestandset_acq:
5329 case Builtin::BI_interlockedbittestandset_rel:
5330 case Builtin::BI_interlockedbittestandset_nf:
5331 case Builtin::BI_interlockedbittestandreset_acq:
5332 case Builtin::BI_interlockedbittestandreset_rel:
5333 case Builtin::BI_interlockedbittestandreset_nf:
5338 case Builtin::BI__iso_volatile_load8:
5339 case Builtin::BI__iso_volatile_load16:
5340 case Builtin::BI__iso_volatile_load32:
5341 case Builtin::BI__iso_volatile_load64:
5343 case Builtin::BI__iso_volatile_store8:
5344 case Builtin::BI__iso_volatile_store16:
5345 case Builtin::BI__iso_volatile_store32:
5346 case Builtin::BI__iso_volatile_store64:
5349 case Builtin::BI__builtin_ptrauth_auth:
5350 case Builtin::BI__builtin_ptrauth_auth_and_resign:
5351 case Builtin::BI__builtin_ptrauth_blend_discriminator:
5352 case Builtin::BI__builtin_ptrauth_sign_generic_data:
5353 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
5354 case Builtin::BI__builtin_ptrauth_strip: {
5361 llvm::Type *OrigValueType = Args[0]->getType();
5362 if (OrigValueType->isPointerTy())
5365 switch (BuiltinID) {
5366 case Builtin::BI__builtin_ptrauth_auth_and_resign:
5367 if (Args[4]->getType()->isPointerTy())
5371 case Builtin::BI__builtin_ptrauth_auth:
5372 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
5373 if (Args[2]->getType()->isPointerTy())
5377 case Builtin::BI__builtin_ptrauth_sign_generic_data:
5378 if (Args[1]->getType()->isPointerTy())
5382 case Builtin::BI__builtin_ptrauth_blend_discriminator:
5383 case Builtin::BI__builtin_ptrauth_strip:
5388 auto IntrinsicID = [&]() ->
unsigned {
5389 switch (BuiltinID) {
5390 case Builtin::BI__builtin_ptrauth_auth:
5391 return llvm::Intrinsic::ptrauth_auth;
5392 case Builtin::BI__builtin_ptrauth_auth_and_resign:
5393 return llvm::Intrinsic::ptrauth_resign;
5394 case Builtin::BI__builtin_ptrauth_blend_discriminator:
5395 return llvm::Intrinsic::ptrauth_blend;
5396 case Builtin::BI__builtin_ptrauth_sign_generic_data:
5397 return llvm::Intrinsic::ptrauth_sign_generic;
5398 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
5399 return llvm::Intrinsic::ptrauth_sign;
5400 case Builtin::BI__builtin_ptrauth_strip:
5401 return llvm::Intrinsic::ptrauth_strip;
5403 llvm_unreachable(
"bad ptrauth intrinsic");
5408 if (BuiltinID != Builtin::BI__builtin_ptrauth_sign_generic_data &&
5409 BuiltinID != Builtin::BI__builtin_ptrauth_blend_discriminator &&
5410 OrigValueType->isPointerTy()) {
5416 case Builtin::BI__exception_code:
5417 case Builtin::BI_exception_code:
5419 case Builtin::BI__exception_info:
5420 case Builtin::BI_exception_info:
5422 case Builtin::BI__abnormal_termination:
5423 case Builtin::BI_abnormal_termination:
5425 case Builtin::BI_setjmpex:
5430 case Builtin::BI_setjmp:
5433 if (
getTarget().getTriple().getArch() == llvm::Triple::x86)
5435 else if (
getTarget().getTriple().getArch() == llvm::Triple::aarch64)
5442 case Builtin::BImove:
5443 case Builtin::BImove_if_noexcept:
5444 case Builtin::BIforward:
5445 case Builtin::BIforward_like:
5446 case Builtin::BIas_const:
5448 case Builtin::BI__GetExceptionInfo: {
5449 if (llvm::GlobalVariable *GV =
5455 case Builtin::BI__fastfail:
5458 case Builtin::BI__builtin_coro_id:
5460 case Builtin::BI__builtin_coro_promise:
5462 case Builtin::BI__builtin_coro_resume:
5465 case Builtin::BI__builtin_coro_frame:
5467 case Builtin::BI__builtin_coro_noop:
5469 case Builtin::BI__builtin_coro_free:
5471 case Builtin::BI__builtin_coro_destroy:
5474 case Builtin::BI__builtin_coro_done:
5476 case Builtin::BI__builtin_coro_alloc:
5478 case Builtin::BI__builtin_coro_begin:
5480 case Builtin::BI__builtin_coro_end:
5482 case Builtin::BI__builtin_coro_suspend:
5484 case Builtin::BI__builtin_coro_size:
5486 case Builtin::BI__builtin_coro_align:
5490 case Builtin::BIread_pipe:
5491 case Builtin::BIwrite_pipe: {
5495 Value *PacketSize = OpenCLRT.getPipeElemSize(E->
getArg(0));
5496 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->
getArg(0));
5499 unsigned GenericAS =
5501 llvm::Type *I8PTy = llvm::PointerType::get(
getLLVMContext(), GenericAS);
5505 const char *Name = (BuiltinID == Builtin::BIread_pipe) ?
"__read_pipe_2"
5510 llvm::FunctionType *FTy = llvm::FunctionType::get(
5515 {Arg0, BCast, PacketSize, PacketAlign}));
5518 "Illegal number of parameters to pipe function");
5519 const char *Name = (BuiltinID == Builtin::BIread_pipe) ?
"__read_pipe_4"
5526 llvm::FunctionType *FTy = llvm::FunctionType::get(
5535 {Arg0, Arg1, Arg2, BCast, PacketSize, PacketAlign}));
5540 case Builtin::BIreserve_read_pipe:
5541 case Builtin::BIreserve_write_pipe:
5542 case Builtin::BIwork_group_reserve_read_pipe:
5543 case Builtin::BIwork_group_reserve_write_pipe:
5544 case Builtin::BIsub_group_reserve_read_pipe:
5545 case Builtin::BIsub_group_reserve_write_pipe: {
5548 if (BuiltinID == Builtin::BIreserve_read_pipe)
5549 Name =
"__reserve_read_pipe";
5550 else if (BuiltinID == Builtin::BIreserve_write_pipe)
5551 Name =
"__reserve_write_pipe";
5552 else if (BuiltinID == Builtin::BIwork_group_reserve_read_pipe)
5553 Name =
"__work_group_reserve_read_pipe";
5554 else if (BuiltinID == Builtin::BIwork_group_reserve_write_pipe)
5555 Name =
"__work_group_reserve_write_pipe";
5556 else if (BuiltinID == Builtin::BIsub_group_reserve_read_pipe)
5557 Name =
"__sub_group_reserve_read_pipe";
5559 Name =
"__sub_group_reserve_write_pipe";
5565 Value *PacketSize = OpenCLRT.getPipeElemSize(E->
getArg(0));
5566 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->
getArg(0));
5570 llvm::FunctionType *FTy = llvm::FunctionType::get(
5577 {Arg0, Arg1, PacketSize, PacketAlign}));
5581 case Builtin::BIcommit_read_pipe:
5582 case Builtin::BIcommit_write_pipe:
5583 case Builtin::BIwork_group_commit_read_pipe:
5584 case Builtin::BIwork_group_commit_write_pipe:
5585 case Builtin::BIsub_group_commit_read_pipe:
5586 case Builtin::BIsub_group_commit_write_pipe: {
5588 if (BuiltinID == Builtin::BIcommit_read_pipe)
5589 Name =
"__commit_read_pipe";
5590 else if (BuiltinID == Builtin::BIcommit_write_pipe)
5591 Name =
"__commit_write_pipe";
5592 else if (BuiltinID == Builtin::BIwork_group_commit_read_pipe)
5593 Name =
"__work_group_commit_read_pipe";
5594 else if (BuiltinID == Builtin::BIwork_group_commit_write_pipe)
5595 Name =
"__work_group_commit_write_pipe";
5596 else if (BuiltinID == Builtin::BIsub_group_commit_read_pipe)
5597 Name =
"__sub_group_commit_read_pipe";
5599 Name =
"__sub_group_commit_write_pipe";
5604 Value *PacketSize = OpenCLRT.getPipeElemSize(E->
getArg(0));
5605 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->
getArg(0));
5609 llvm::FunctionType *FTy =
5614 {Arg0, Arg1, PacketSize, PacketAlign}));
5617 case Builtin::BIget_pipe_num_packets:
5618 case Builtin::BIget_pipe_max_packets: {
5619 const char *BaseName;
5621 if (BuiltinID == Builtin::BIget_pipe_num_packets)
5622 BaseName =
"__get_pipe_num_packets";
5624 BaseName =
"__get_pipe_max_packets";
5625 std::string Name = std::string(BaseName) +
5626 std::string(PipeTy->isReadOnly() ?
"_ro" :
"_wo");
5631 Value *PacketSize = OpenCLRT.getPipeElemSize(E->
getArg(0));
5632 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->
getArg(0));
5634 llvm::FunctionType *FTy = llvm::FunctionType::get(
5638 {Arg0, PacketSize, PacketAlign}));
5642 case Builtin::BIto_global:
5643 case Builtin::BIto_local:
5644 case Builtin::BIto_private: {
5646 auto NewArgT = llvm::PointerType::get(
5649 auto NewRetT = llvm::PointerType::get(
5653 auto FTy = llvm::FunctionType::get(NewRetT, {NewArgT},
false);
5654 llvm::Value *NewArg;
5655 if (Arg0->
getType()->getPointerAddressSpace() !=
5656 NewArgT->getPointerAddressSpace())
5659 NewArg =
Builder.CreateBitOrPointerCast(Arg0, NewArgT);
5675 case Builtin::BIenqueue_kernel: {
5680 llvm::Type *GenericVoidPtrTy =
Builder.getPtrTy(
5692 Name =
"__enqueue_kernel_basic";
5693 llvm::Type *ArgTys[] = {QueueTy,
Int32Ty, RangeTy, GenericVoidPtrTy,
5695 llvm::FunctionType *FTy = llvm::FunctionType::get(
5701 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5702 llvm::Value *
Block =
5703 Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5705 AttrBuilder B(
Builder.getContext());
5707 llvm::AttributeList ByValAttrSet =
5708 llvm::AttributeList::get(
CGM.
getModule().getContext(), 3U, B);
5712 {Queue, Flags, Range, Kernel, Block});
5713 RTCall->setAttributes(ByValAttrSet);
5716 assert(NumArgs >= 5 &&
"Invalid enqueue_kernel signature");
5720 auto CreateArrayForSizeVar = [=](
unsigned First)
5721 -> std::tuple<llvm::Value *, llvm::Value *, llvm::Value *> {
5722 llvm::APInt ArraySize(32, NumArgs -
First);
5724 getContext().getSizeType(), ArraySize,
nullptr,
5728 llvm::Value *TmpPtr = Tmp.getPointer();
5731 llvm::Value *ElemPtr;
5734 auto *
Zero = llvm::ConstantInt::get(
IntTy, 0);
5735 for (
unsigned I =
First; I < NumArgs; ++I) {
5736 auto *Index = llvm::ConstantInt::get(
IntTy, I -
First);
5746 return std::tie(ElemPtr, TmpSize, TmpPtr);
5752 Name =
"__enqueue_kernel_varargs";
5756 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5757 auto *
Block =
Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5758 llvm::Value *ElemPtr, *TmpSize, *TmpPtr;
5759 std::tie(ElemPtr, TmpSize, TmpPtr) = CreateArrayForSizeVar(4);
5763 llvm::Value *
const Args[] = {Queue, Flags,
5767 llvm::Type *
const ArgTys[] = {
5768 QueueTy,
IntTy, RangeTy, GenericVoidPtrTy,
5769 GenericVoidPtrTy,
IntTy, ElemPtr->getType()};
5771 llvm::FunctionType *FTy = llvm::FunctionType::get(
Int32Ty, ArgTys,
false);
5780 llvm::PointerType *PtrTy = llvm::PointerType::get(
5784 llvm::Value *NumEvents =
5790 llvm::Value *EventWaitList =
nullptr;
5793 EventWaitList = llvm::ConstantPointerNull::get(PtrTy);
5800 EventWaitList =
Builder.CreatePointerCast(EventWaitList, PtrTy);
5802 llvm::Value *EventRet =
nullptr;
5805 EventRet = llvm::ConstantPointerNull::get(PtrTy);
5814 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5815 llvm::Value *
Block =
5816 Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5818 std::vector<llvm::Type *> ArgTys = {
5820 PtrTy, PtrTy, GenericVoidPtrTy, GenericVoidPtrTy};
5822 std::vector<llvm::Value *> Args = {Queue, Flags,
Range,
5823 NumEvents, EventWaitList, EventRet,
5828 Name =
"__enqueue_kernel_basic_events";
5829 llvm::FunctionType *FTy = llvm::FunctionType::get(
5837 Args.push_back(ConstantInt::get(
Int32Ty, NumArgs - 7));
5839 Name =
"__enqueue_kernel_events_varargs";
5841 llvm::Value *ElemPtr, *TmpSize, *TmpPtr;
5842 std::tie(ElemPtr, TmpSize, TmpPtr) = CreateArrayForSizeVar(7);
5843 Args.push_back(ElemPtr);
5844 ArgTys.push_back(ElemPtr->getType());
5846 llvm::FunctionType *FTy = llvm::FunctionType::get(
5855 llvm_unreachable(
"Unexpected enqueue_kernel signature");
5859 case Builtin::BIget_kernel_work_group_size: {
5860 llvm::Type *GenericVoidPtrTy =
Builder.getPtrTy(
5865 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5866 Value *Arg =
Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5869 llvm::FunctionType::get(
IntTy, {GenericVoidPtrTy, GenericVoidPtrTy},
5871 "__get_kernel_work_group_size_impl"),
5874 case Builtin::BIget_kernel_preferred_work_group_size_multiple: {
5875 llvm::Type *GenericVoidPtrTy =
Builder.getPtrTy(
5880 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5881 Value *Arg =
Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5884 llvm::FunctionType::get(
IntTy, {GenericVoidPtrTy, GenericVoidPtrTy},
5886 "__get_kernel_preferred_work_group_size_multiple_impl"),
5889 case Builtin::BIget_kernel_max_sub_group_size_for_ndrange:
5890 case Builtin::BIget_kernel_sub_group_count_for_ndrange: {
5891 llvm::Type *GenericVoidPtrTy =
Builder.getPtrTy(
5898 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5901 BuiltinID == Builtin::BIget_kernel_max_sub_group_size_for_ndrange
5902 ?
"__get_kernel_max_sub_group_size_for_ndrange_impl"
5903 :
"__get_kernel_sub_group_count_for_ndrange_impl";
5906 llvm::FunctionType::get(
5907 IntTy, {NDRange->getType(), GenericVoidPtrTy, GenericVoidPtrTy},
5910 {NDRange, Kernel, Block}));
5912 case Builtin::BI__builtin_store_half:
5913 case Builtin::BI__builtin_store_halff: {
5920 case Builtin::BI__builtin_load_half: {
5925 case Builtin::BI__builtin_load_halff: {
5930 case Builtin::BI__builtin_printf:
5931 case Builtin::BIprintf:
5932 if (
getTarget().getTriple().isNVPTX() ||
5943 case Builtin::BI__builtin_canonicalize:
5944 case Builtin::BI__builtin_canonicalizef:
5945 case Builtin::BI__builtin_canonicalizef16:
5946 case Builtin::BI__builtin_canonicalizel:
5949 case Builtin::BI__builtin_thread_pointer: {
5950 if (!
getContext().getTargetInfo().isTLSSupported())
5955 case Builtin::BI__builtin_os_log_format:
5958 case Builtin::BI__xray_customevent: {
5971 auto FTy = F->getFunctionType();
5972 auto Arg0 = E->
getArg(0);
5974 auto Arg0Ty = Arg0->
getType();
5975 auto PTy0 = FTy->getParamType(0);
5976 if (PTy0 != Arg0Val->getType()) {
5977 if (Arg0Ty->isArrayType())
5980 Arg0Val =
Builder.CreatePointerCast(Arg0Val, PTy0);
5983 auto PTy1 = FTy->getParamType(1);
5985 Arg1 =
Builder.CreateTruncOrBitCast(Arg1, PTy1);
5989 case Builtin::BI__xray_typedevent: {
6005 auto FTy = F->getFunctionType();
6007 auto PTy0 = FTy->getParamType(0);
6009 Arg0 =
Builder.CreateTruncOrBitCast(Arg0, PTy0);
6010 auto Arg1 = E->
getArg(1);
6012 auto Arg1Ty = Arg1->
getType();
6013 auto PTy1 = FTy->getParamType(1);
6014 if (PTy1 != Arg1Val->getType()) {
6015 if (Arg1Ty->isArrayType())
6018 Arg1Val =
Builder.CreatePointerCast(Arg1Val, PTy1);
6021 auto PTy2 = FTy->getParamType(2);
6023 Arg2 =
Builder.CreateTruncOrBitCast(Arg2, PTy2);
6027 case Builtin::BI__builtin_ms_va_start:
6028 case Builtin::BI__builtin_ms_va_end:
6031 BuiltinID == Builtin::BI__builtin_ms_va_start));
6033 case Builtin::BI__builtin_ms_va_copy: {
6050 case Builtin::BI__builtin_get_device_side_mangled_name: {
6054 llvm::Constant *Zeros[] = {llvm::ConstantInt::get(
SizeTy, 0),
6055 llvm::ConstantInt::get(
SizeTy, 0)};
6056 auto *Ptr = llvm::ConstantExpr::getGetElementPtr(Str.getElementType(),
6057 Str.getPointer(), Zeros);
6083 LargestVectorWidth = std::max(LargestVectorWidth, VectorWidth);
6087 Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic;
6089 llvm::Triple::getArchTypePrefix(
getTarget().getTriple().getArch());
6090 if (!Prefix.empty()) {
6091 IntrinsicID = Intrinsic::getIntrinsicForClangBuiltin(Prefix.data(), Name);
6095 if (IntrinsicID == Intrinsic::not_intrinsic)
6096 IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(Prefix.data(), Name);
6099 if (IntrinsicID != Intrinsic::not_intrinsic) {
6104 unsigned ICEArguments = 0;
6110 llvm::FunctionType *FTy = F->getFunctionType();
6112 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; ++i) {
6116 llvm::Type *PTy = FTy->getParamType(i);
6117 if (PTy != ArgValue->
getType()) {
6119 if (
auto *PtrTy = dyn_cast<llvm::PointerType>(PTy)) {
6120 if (PtrTy->getAddressSpace() !=
6121 ArgValue->
getType()->getPointerAddressSpace()) {
6124 PtrTy->getAddressSpace()));
6130 if (PTy->isX86_AMXTy())
6131 ArgValue =
Builder.CreateIntrinsic(Intrinsic::x86_cast_vector_to_tile,
6132 {ArgValue->
getType()}, {ArgValue});
6134 ArgValue =
Builder.CreateBitCast(ArgValue, PTy);
6137 Args.push_back(ArgValue);
6143 llvm::Type *RetTy =
VoidTy;
6147 if (RetTy !=
V->getType()) {
6149 if (
auto *PtrTy = dyn_cast<llvm::PointerType>(RetTy)) {
6150 if (PtrTy->getAddressSpace() !=
V->getType()->getPointerAddressSpace()) {
6153 PtrTy->getAddressSpace()));
6159 if (
V->getType()->isX86_AMXTy())
6160 V =
Builder.CreateIntrinsic(Intrinsic::x86_cast_tile_to_vector, {RetTy},
6166 if (RetTy->isVoidTy())
6186 if (
V->getType()->isVoidTy())
6193 llvm_unreachable(
"No current target builtin returns complex");
6195 llvm_unreachable(
"Bad evaluation kind in EmitBuiltinExpr");
6212 unsigned BuiltinID,
const CallExpr *E,
6214 llvm::Triple::ArchType Arch) {
6226 case llvm::Triple::arm:
6227 case llvm::Triple::armeb:
6228 case llvm::Triple::thumb:
6229 case llvm::Triple::thumbeb:
6231 case llvm::Triple::aarch64:
6232 case llvm::Triple::aarch64_32:
6233 case llvm::Triple::aarch64_be:
6235 case llvm::Triple::bpfeb:
6236 case llvm::Triple::bpfel:
6238 case llvm::Triple::x86:
6239 case llvm::Triple::x86_64:
6241 case llvm::Triple::ppc:
6242 case llvm::Triple::ppcle:
6243 case llvm::Triple::ppc64:
6244 case llvm::Triple::ppc64le:
6246 case llvm::Triple::r600:
6247 case llvm::Triple::amdgcn:
6249 case llvm::Triple::systemz:
6251 case llvm::Triple::nvptx:
6252 case llvm::Triple::nvptx64:
6254 case llvm::Triple::wasm32:
6255 case llvm::Triple::wasm64:
6257 case llvm::Triple::hexagon:
6259 case llvm::Triple::riscv32:
6260 case llvm::Triple::riscv64:
6271 assert(
getContext().getAuxTargetInfo() &&
"Missing aux target info");
6283 bool HasLegalHalfType =
true,
6285 bool AllowBFloatArgsAndRet =
true) {
6286 int IsQuad = TypeFlags.
isQuad();
6290 return llvm::FixedVectorType::get(CGF->
Int8Ty, V1Ty ? 1 : (8 << IsQuad));
6293 return llvm::FixedVectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
6295 if (AllowBFloatArgsAndRet)
6296 return llvm::FixedVectorType::get(CGF->
BFloatTy, V1Ty ? 1 : (4 << IsQuad));
6298 return llvm::FixedVectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
6300 if (HasLegalHalfType)
6301 return llvm::FixedVectorType::get(CGF->
HalfTy, V1Ty ? 1 : (4 << IsQuad));
6303 return llvm::FixedVectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
6305 return llvm::FixedVectorType::get(CGF->
Int32Ty, V1Ty ? 1 : (2 << IsQuad));
6308 return llvm::FixedVectorType::get(CGF->
Int64Ty, V1Ty ? 1 : (1 << IsQuad));
6313 return llvm::FixedVectorType::get(CGF->
Int8Ty, 16);
6315 return llvm::FixedVectorType::get(CGF->
FloatTy, V1Ty ? 1 : (2 << IsQuad));
6317 return llvm::FixedVectorType::get(CGF->
DoubleTy, V1Ty ? 1 : (1 << IsQuad));
6319 llvm_unreachable(
"Unknown vector element type!");
6324 int IsQuad = IntTypeFlags.
isQuad();
6327 return llvm::FixedVectorType::get(CGF->
HalfTy, (4 << IsQuad));
6329 return llvm::FixedVectorType::get(CGF->
FloatTy, (2 << IsQuad));
6331 return llvm::FixedVectorType::get(CGF->
DoubleTy, (1 << IsQuad));
6333 llvm_unreachable(
"Type can't be converted to floating-point!");
6338 const ElementCount &Count) {
6339 Value *SV = llvm::ConstantVector::getSplat(Count,
C);
6340 return Builder.CreateShuffleVector(
V,
V, SV,
"lane");
6344 ElementCount EC = cast<llvm::VectorType>(
V->getType())->getElementCount();
6350 unsigned shift,
bool rightshift) {
6352 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
6353 ai != ae; ++ai, ++j) {
6354 if (F->isConstrainedFPIntrinsic())
6355 if (ai->getType()->isMetadataTy())
6357 if (shift > 0 && shift == j)
6360 Ops[j] =
Builder.CreateBitCast(Ops[j], ai->getType(), name);
6363 if (F->isConstrainedFPIntrinsic())
6364 return Builder.CreateConstrainedFPCall(F, Ops, name);
6366 return Builder.CreateCall(F, Ops, name);
6371 int SV = cast<ConstantInt>(
V)->getSExtValue();
6372 return ConstantInt::get(Ty, neg ? -SV : SV);
6377 llvm::Type *Ty,
bool usgn,
6379 llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
6381 int ShiftAmt = cast<ConstantInt>(Shift)->getSExtValue();
6382 int EltSize = VTy->getScalarSizeInBits();
6384 Vec =
Builder.CreateBitCast(Vec, Ty);
6388 if (ShiftAmt == EltSize) {
6391 return llvm::ConstantAggregateZero::get(VTy);
6396 Shift = ConstantInt::get(VTy->getElementType(), ShiftAmt);
6402 return Builder.CreateLShr(Vec, Shift, name);
6404 return Builder.CreateAShr(Vec, Shift, name);
6430struct ARMVectorIntrinsicInfo {
6431 const char *NameHint;
6433 unsigned LLVMIntrinsic;
6434 unsigned AltLLVMIntrinsic;
6437 bool operator<(
unsigned RHSBuiltinID)
const {
6438 return BuiltinID < RHSBuiltinID;
6440 bool operator<(
const ARMVectorIntrinsicInfo &TE)
const {
6441 return BuiltinID < TE.BuiltinID;
6446#define NEONMAP0(NameBase) \
6447 { #NameBase, NEON::BI__builtin_neon_ ## NameBase, 0, 0, 0 }
6449#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
6450 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
6451 Intrinsic::LLVMIntrinsic, 0, TypeModifier }
6453#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier) \
6454 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
6455 Intrinsic::LLVMIntrinsic, Intrinsic::AltLLVMIntrinsic, \
6459 NEONMAP1(__a32_vcvt_bf16_f32, arm_neon_vcvtfp2bf, 0),
6466 NEONMAP1(vabs_v, arm_neon_vabs, 0),
6467 NEONMAP1(vabsq_v, arm_neon_vabs, 0),
6471 NEONMAP1(vaesdq_u8, arm_neon_aesd, 0),
6472 NEONMAP1(vaeseq_u8, arm_neon_aese, 0),
6473 NEONMAP1(vaesimcq_u8, arm_neon_aesimc, 0),
6474 NEONMAP1(vaesmcq_u8, arm_neon_aesmc, 0),
6475 NEONMAP1(vbfdot_f32, arm_neon_bfdot, 0),
6476 NEONMAP1(vbfdotq_f32, arm_neon_bfdot, 0),
6477 NEONMAP1(vbfmlalbq_f32, arm_neon_bfmlalb, 0),
6478 NEONMAP1(vbfmlaltq_f32, arm_neon_bfmlalt, 0),
6479 NEONMAP1(vbfmmlaq_f32, arm_neon_bfmmla, 0),
6492 NEONMAP1(vcage_v, arm_neon_vacge, 0),
6493 NEONMAP1(vcageq_v, arm_neon_vacge, 0),
6494 NEONMAP1(vcagt_v, arm_neon_vacgt, 0),
6495 NEONMAP1(vcagtq_v, arm_neon_vacgt, 0),
6496 NEONMAP1(vcale_v, arm_neon_vacge, 0),
6497 NEONMAP1(vcaleq_v, arm_neon_vacge, 0),
6498 NEONMAP1(vcalt_v, arm_neon_vacgt, 0),
6499 NEONMAP1(vcaltq_v, arm_neon_vacgt, 0),
6516 NEONMAP1(vcvt_f16_f32, arm_neon_vcvtfp2hf, 0),
6519 NEONMAP1(vcvt_f32_f16, arm_neon_vcvthf2fp, 0),
6521 NEONMAP1(vcvt_n_f16_s16, arm_neon_vcvtfxs2fp, 0),
6522 NEONMAP1(vcvt_n_f16_u16, arm_neon_vcvtfxu2fp, 0),
6523 NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
6524 NEONMAP1(vcvt_n_s16_f16, arm_neon_vcvtfp2fxs, 0),
6525 NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0),
6526 NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0),
6527 NEONMAP1(vcvt_n_u16_f16, arm_neon_vcvtfp2fxu, 0),
6528 NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0),
6529 NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0),
6536 NEONMAP1(vcvta_s16_f16, arm_neon_vcvtas, 0),
6537 NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0),
6538 NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0),
6539 NEONMAP1(vcvta_u16_f16, arm_neon_vcvtau, 0),
6540 NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0),
6541 NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0),
6542 NEONMAP1(vcvtaq_s16_f16, arm_neon_vcvtas, 0),
6543 NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0),
6544 NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0),
6545 NEONMAP1(vcvtaq_u16_f16, arm_neon_vcvtau, 0),
6546 NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0),
6547 NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0),
6548 NEONMAP1(vcvth_bf16_f32, arm_neon_vcvtbfp2bf, 0),
6549 NEONMAP1(vcvtm_s16_f16, arm_neon_vcvtms, 0),
6550 NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0),
6551 NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0),
6552 NEONMAP1(vcvtm_u16_f16, arm_neon_vcvtmu, 0),
6553 NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0),
6554 NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0),
6555 NEONMAP1(vcvtmq_s16_f16, arm_neon_vcvtms, 0),
6556 NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0),
6557 NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0),
6558 NEONMAP1(vcvtmq_u16_f16, arm_neon_vcvtmu, 0),
6559 NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0),
6560 NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0),
6561 NEONMAP1(vcvtn_s16_f16, arm_neon_vcvtns, 0),
6562 NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0),
6563 NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0),
6564 NEONMAP1(vcvtn_u16_f16, arm_neon_vcvtnu, 0),
6565 NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0),
6566 NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0),
6567 NEONMAP1(vcvtnq_s16_f16, arm_neon_vcvtns, 0),
6568 NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0),
6569 NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0),
6570 NEONMAP1(vcvtnq_u16_f16, arm_neon_vcvtnu, 0),
6571 NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0),
6572 NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0),
6573 NEONMAP1(vcvtp_s16_f16, arm_neon_vcvtps, 0),
6574 NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0),
6575 NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0),
6576 NEONMAP1(vcvtp_u16_f16, arm_neon_vcvtpu, 0),
6577 NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0),
6578 NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0),
6579 NEONMAP1(vcvtpq_s16_f16, arm_neon_vcvtps, 0),
6580 NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0),
6581 NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0),
6582 NEONMAP1(vcvtpq_u16_f16, arm_neon_vcvtpu, 0),
6583 NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0),
6584 NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0),
6588 NEONMAP1(vcvtq_n_f16_s16, arm_neon_vcvtfxs2fp, 0),
6589 NEONMAP1(vcvtq_n_f16_u16, arm_neon_vcvtfxu2fp, 0),
6590 NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
6591 NEONMAP1(vcvtq_n_s16_f16, arm_neon_vcvtfp2fxs, 0),
6592 NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0),
6593 NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0),
6594 NEONMAP1(vcvtq_n_u16_f16, arm_neon_vcvtfp2fxu, 0),
6595 NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0),
6596 NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0),
6603 NEONMAP1(vdot_s32, arm_neon_sdot, 0),
6604 NEONMAP1(vdot_u32, arm_neon_udot, 0),
6605 NEONMAP1(vdotq_s32, arm_neon_sdot, 0),
6606 NEONMAP1(vdotq_u32, arm_neon_udot, 0),
6616 NEONMAP1(vld1_v, arm_neon_vld1, 0),
6617 NEONMAP1(vld1_x2_v, arm_neon_vld1x2, 0),
6618 NEONMAP1(vld1_x3_v, arm_neon_vld1x3, 0),
6619 NEONMAP1(vld1_x4_v, arm_neon_vld1x4, 0),
6621 NEONMAP1(vld1q_v, arm_neon_vld1, 0),
6622 NEONMAP1(vld1q_x2_v, arm_neon_vld1x2, 0),
6623 NEONMAP1(vld1q_x3_v, arm_neon_vld1x3, 0),
6624 NEONMAP1(vld1q_x4_v, arm_neon_vld1x4, 0),
6625 NEONMAP1(vld2_dup_v, arm_neon_vld2dup, 0),
6626 NEONMAP1(vld2_lane_v, arm_neon_vld2lane, 0),
6627 NEONMAP1(vld2_v, arm_neon_vld2, 0),
6628 NEONMAP1(vld2q_dup_v, arm_neon_vld2dup, 0),
6629 NEONMAP1(vld2q_lane_v, arm_neon_vld2lane, 0),
6630 NEONMAP1(vld2q_v, arm_neon_vld2, 0),
6631 NEONMAP1(vld3_dup_v, arm_neon_vld3dup, 0),
6632 NEONMAP1(vld3_lane_v, arm_neon_vld3lane, 0),
6633 NEONMAP1(vld3_v, arm_neon_vld3, 0),
6634 NEONMAP1(vld3q_dup_v, arm_neon_vld3dup, 0),
6635 NEONMAP1(vld3q_lane_v, arm_neon_vld3lane, 0),
6636 NEONMAP1(vld3q_v, arm_neon_vld3, 0),
6637 NEONMAP1(vld4_dup_v, arm_neon_vld4dup, 0),
6638 NEONMAP1(vld4_lane_v, arm_neon_vld4lane, 0),
6639 NEONMAP1(vld4_v, arm_neon_vld4, 0),
6640 NEONMAP1(vld4q_dup_v, arm_neon_vld4dup, 0),
6641 NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0),
6642 NEONMAP1(vld4q_v, arm_neon_vld4, 0),
6651 NEONMAP1(vmmlaq_s32, arm_neon_smmla, 0),
6652 NEONMAP1(vmmlaq_u32, arm_neon_ummla, 0),
6670 NEONMAP2(vqdmlal_v, arm_neon_vqdmull, sadd_sat, 0),
6671 NEONMAP2(vqdmlsl_v, arm_neon_vqdmull, ssub_sat, 0),
6695 NEONMAP1(vqshlu_n_v, arm_neon_vqshiftsu, 0),
6696 NEONMAP1(vqshluq_n_v, arm_neon_vqshiftsu, 0),
6700 NEONMAP2(vrecpe_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
6701 NEONMAP2(vrecpeq_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
6724 NEONMAP2(vrsqrte_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
6725 NEONMAP2(vrsqrteq_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
6729 NEONMAP1(vsha1su0q_u32, arm_neon_sha1su0, 0),
6730 NEONMAP1(vsha1su1q_u32, arm_neon_sha1su1, 0),
6731 NEONMAP1(vsha256h2q_u32, arm_neon_sha256h2, 0),
6732 NEONMAP1(vsha256hq_u32, arm_neon_sha256h, 0),
6733 NEONMAP1(vsha256su0q_u32, arm_neon_sha256su0, 0),
6734 NEONMAP1(vsha256su1q_u32, arm_neon_sha256su1, 0),
6743 NEONMAP1(vst1_v, arm_neon_vst1, 0),
6744 NEONMAP1(vst1_x2_v, arm_neon_vst1x2, 0),
6745 NEONMAP1(vst1_x3_v, arm_neon_vst1x3, 0),
6746 NEONMAP1(vst1_x4_v, arm_neon_vst1x4, 0),
6747 NEONMAP1(vst1q_v, arm_neon_vst1, 0),
6748 NEONMAP1(vst1q_x2_v, arm_neon_vst1x2, 0),
6749 NEONMAP1(vst1q_x3_v, arm_neon_vst1x3, 0),
6750 NEONMAP1(vst1q_x4_v, arm_neon_vst1x4, 0),
6751 NEONMAP1(vst2_lane_v, arm_neon_vst2lane, 0),
6752 NEONMAP1(vst2_v, arm_neon_vst2, 0),
6753 NEONMAP1(vst2q_lane_v, arm_neon_vst2lane, 0),
6754 NEONMAP1(vst2q_v, arm_neon_vst2, 0),
6755 NEONMAP1(vst3_lane_v, arm_neon_vst3lane, 0),
6756 NEONMAP1(vst3_v, arm_neon_vst3, 0),
6757 NEONMAP1(vst3q_lane_v, arm_neon_vst3lane, 0),
6758 NEONMAP1(vst3q_v, arm_neon_vst3, 0),
6759 NEONMAP1(vst4_lane_v, arm_neon_vst4lane, 0),
6760 NEONMAP1(vst4_v, arm_neon_vst4, 0),
6761 NEONMAP1(vst4q_lane_v, arm_neon_vst4lane, 0),
6762 NEONMAP1(vst4q_v, arm_neon_vst4, 0),
6768 NEONMAP1(vusdot_s32, arm_neon_usdot, 0),
6769 NEONMAP1(vusdotq_s32, arm_neon_usdot, 0),
6770 NEONMAP1(vusmmlaq_s32, arm_neon_usmmla, 0),
6778 NEONMAP1(__a64_vcvtq_low_bf16_f32, aarch64_neon_bfcvtn, 0),
6783 NEONMAP1(vabs_v, aarch64_neon_abs, 0),
6784 NEONMAP1(vabsq_v, aarch64_neon_abs, 0),
6789 NEONMAP1(vaesdq_u8, aarch64_crypto_aesd, 0),
6790 NEONMAP1(vaeseq_u8, aarch64_crypto_aese, 0),
6791 NEONMAP1(vaesimcq_u8, aarch64_crypto_aesimc, 0),
6792 NEONMAP1(vaesmcq_u8, aarch64_crypto_aesmc, 0),
6801 NEONMAP1(vbfdot_f32, aarch64_neon_bfdot, 0),
6802 NEONMAP1(vbfdotq_f32, aarch64_neon_bfdot, 0),
6803 NEONMAP1(vbfmlalbq_f32, aarch64_neon_bfmlalb, 0),
6804 NEONMAP1(vbfmlaltq_f32, aarch64_neon_bfmlalt, 0),
6805 NEONMAP1(vbfmmlaq_f32, aarch64_neon_bfmmla, 0),
6816 NEONMAP1(vcage_v, aarch64_neon_facge, 0),
6817 NEONMAP1(vcageq_v, aarch64_neon_facge, 0),
6818 NEONMAP1(vcagt_v, aarch64_neon_facgt, 0),
6819 NEONMAP1(vcagtq_v, aarch64_neon_facgt, 0),
6820 NEONMAP1(vcale_v, aarch64_neon_facge, 0),
6821 NEONMAP1(vcaleq_v, aarch64_neon_facge, 0),
6822 NEONMAP1(vcalt_v, aarch64_neon_facgt, 0),
6823 NEONMAP1(vcaltq_v, aarch64_neon_facgt, 0),
6860 NEONMAP1(vcvt_f16_f32, aarch64_neon_vcvtfp2hf, 0),
6863 NEONMAP1(vcvt_f32_f16, aarch64_neon_vcvthf2fp, 0),
6865 NEONMAP1(vcvt_n_f16_s16, aarch64_neon_vcvtfxs2fp, 0),
6866 NEONMAP1(vcvt_n_f16_u16, aarch64_neon_vcvtfxu2fp, 0),
6867 NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
6868 NEONMAP2(vcvt_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
6869 NEONMAP1(vcvt_n_s16_f16, aarch64_neon_vcvtfp2fxs, 0),
6870 NEONMAP1(vcvt_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
6871 NEONMAP1(vcvt_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
6872 NEONMAP1(vcvt_n_u16_f16, aarch64_neon_vcvtfp2fxu, 0),
6873 NEONMAP1(vcvt_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
6874 NEONMAP1(vcvt_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
6878 NEONMAP1(vcvtq_high_bf16_f32, aarch64_neon_bfcvtn2, 0),
6879 NEONMAP1(vcvtq_n_f16_s16, aarch64_neon_vcvtfxs2fp, 0),
6880 NEONMAP1(vcvtq_n_f16_u16, aarch64_neon_vcvtfxu2fp, 0),
6881 NEONMAP2(vcvtq_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
6882 NEONMAP2(vcvtq_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
6883 NEONMAP1(vcvtq_n_s16_f16, aarch64_neon_vcvtfp2fxs, 0),
6884 NEONMAP1(vcvtq_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
6885 NEONMAP1(vcvtq_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
6886 NEONMAP1(vcvtq_n_u16_f16, aarch64_neon_vcvtfp2fxu, 0),
6887 NEONMAP1(vcvtq_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
6888 NEONMAP1(vcvtq_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
6890 NEONMAP1(vdot_s32, aarch64_neon_sdot, 0),
6891 NEONMAP1(vdot_u32, aarch64_neon_udot, 0),
6892 NEONMAP1(vdotq_s32, aarch64_neon_sdot, 0),
6893 NEONMAP1(vdotq_u32, aarch64_neon_udot, 0),
6906 NEONMAP1(vfmlal_high_f16, aarch64_neon_fmlal2, 0),
6907 NEONMAP1(vfmlal_low_f16, aarch64_neon_fmlal, 0),
6908 NEONMAP1(vfmlalq_high_f16, aarch64_neon_fmlal2, 0),
6909 NEONMAP1(vfmlalq_low_f16, aarch64_neon_fmlal, 0),
6910 NEONMAP1(vfmlsl_high_f16, aarch64_neon_fmlsl2, 0),
6911 NEONMAP1(vfmlsl_low_f16, aarch64_neon_fmlsl, 0),
6912 NEONMAP1(vfmlslq_high_f16, aarch64_neon_fmlsl2, 0),
6913 NEONMAP1(vfmlslq_low_f16, aarch64_neon_fmlsl, 0),
6918 NEONMAP1(vld1_x2_v, aarch64_neon_ld1x2, 0),
6919 NEONMAP1(vld1_x3_v, aarch64_neon_ld1x3, 0),
6920 NEONMAP1(vld1_x4_v, aarch64_neon_ld1x4, 0),
6921 NEONMAP1(vld1q_x2_v, aarch64_neon_ld1x2, 0),
6922 NEONMAP1(vld1q_x3_v, aarch64_neon_ld1x3, 0),
6923 NEONMAP1(vld1q_x4_v, aarch64_neon_ld1x4, 0),
6924 NEONMAP1(vmmlaq_s32, aarch64_neon_smmla, 0),
6925 NEONMAP1(vmmlaq_u32, aarch64_neon_ummla, 0),
6938 NEONMAP2(vqdmlal_v, aarch64_neon_sqdmull, aarch64_neon_sqadd, 0),
6939 NEONMAP2(vqdmlsl_v, aarch64_neon_sqdmull, aarch64_neon_sqsub, 0),
6940 NEONMAP1(vqdmulh_lane_v, aarch64_neon_sqdmulh_lane, 0),
6941 NEONMAP1(vqdmulh_laneq_v, aarch64_neon_sqdmulh_laneq, 0),
6943 NEONMAP1(vqdmulhq_lane_v, aarch64_neon_sqdmulh_lane, 0),
6944 NEONMAP1(vqdmulhq_laneq_v, aarch64_neon_sqdmulh_laneq, 0),
6959 NEONMAP1(vqrdmulh_lane_v, aarch64_neon_sqrdmulh_lane, 0),
6960 NEONMAP1(vqrdmulh_laneq_v, aarch64_neon_sqrdmulh_laneq, 0),
6962 NEONMAP1(vqrdmulhq_lane_v, aarch64_neon_sqrdmulh_lane, 0),
6963 NEONMAP1(vqrdmulhq_laneq_v, aarch64_neon_sqrdmulh_laneq, 0),
6971 NEONMAP1(vqshlu_n_v, aarch64_neon_sqshlu, 0),
6972 NEONMAP1(vqshluq_n_v, aarch64_neon_sqshlu, 0),
6976 NEONMAP1(vrax1q_u64, aarch64_crypto_rax1, 0),
6977 NEONMAP2(vrecpe_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
6978 NEONMAP2(vrecpeq_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
7005 NEONMAP2(vrsqrte_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
7006 NEONMAP2(vrsqrteq_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
7010 NEONMAP1(vsha1su0q_u32, aarch64_crypto_sha1su0, 0),
7011 NEONMAP1(vsha1su1q_u32, aarch64_crypto_sha1su1, 0),
7012 NEONMAP1(vsha256h2q_u32, aarch64_crypto_sha256h2, 0),
7013 NEONMAP1(vsha256hq_u32, aarch64_crypto_sha256h, 0),
7014 NEONMAP1(vsha256su0q_u32, aarch64_crypto_sha256su0, 0),
7015 NEONMAP1(vsha256su1q_u32, aarch64_crypto_sha256su1, 0),
7016 NEONMAP1(vsha512h2q_u64, aarch64_crypto_sha512h2, 0),
7017 NEONMAP1(vsha512hq_u64, aarch64_crypto_sha512h, 0),
7018 NEONMAP1(vsha512su0q_u64, aarch64_crypto_sha512su0, 0),
7019 NEONMAP1(vsha512su1q_u64, aarch64_crypto_sha512su1, 0),
7028 NEONMAP1(vsm3partw1q_u32, aarch64_crypto_sm3partw1, 0),
7029 NEONMAP1(vsm3partw2q_u32, aarch64_crypto_sm3partw2, 0),
7030 NEONMAP1(vsm3ss1q_u32, aarch64_crypto_sm3ss1, 0),
7031 NEONMAP1(vsm3tt1aq_u32, aarch64_crypto_sm3tt1a, 0),
7032 NEONMAP1(vsm3tt1bq_u32, aarch64_crypto_sm3tt1b, 0),
7033 NEONMAP1(vsm3tt2aq_u32, aarch64_crypto_sm3tt2a, 0),
7034 NEONMAP1(vsm3tt2bq_u32, aarch64_crypto_sm3tt2b, 0),
7035 NEONMAP1(vsm4ekeyq_u32, aarch64_crypto_sm4ekey, 0),
7036 NEONMAP1(vsm4eq_u32, aarch64_crypto_sm4e, 0),
7037 NEONMAP1(vst1_x2_v, aarch64_neon_st1x2, 0),
7038 NEONMAP1(vst1_x3_v, aarch64_neon_st1x3, 0),
7039 NEONMAP1(vst1_x4_v, aarch64_neon_st1x4, 0),
7040 NEONMAP1(vst1q_x2_v, aarch64_neon_st1x2, 0),
7041 NEONMAP1(vst1q_x3_v, aarch64_neon_st1x3, 0),
7042 NEONMAP1(vst1q_x4_v, aarch64_neon_st1x4, 0),
7046 NEONMAP1(vusdot_s32, aarch64_neon_usdot, 0),
7047 NEONMAP1(vusdotq_s32, aarch64_neon_usdot, 0),
7048 NEONMAP1(vusmmlaq_s32, aarch64_neon_usmmla, 0),
7049 NEONMAP1(vxarq_u64, aarch64_crypto_xar, 0),
7087 NEONMAP1(vcvth_bf16_f32, aarch64_neon_bfcvt, 0),
7106 NEONMAP1(vcvtxd_f32_f64, aarch64_sisd_fcvtxn, 0),
7127 NEONMAP1(vmull_p64, aarch64_neon_pmull64, 0),
7155 NEONMAP1(vqdmulls_s32, aarch64_neon_sqdmulls_scalar, 0),
7236 NEONMAP1(vsha1cq_u32, aarch64_crypto_sha1c, 0),
7237 NEONMAP1(vsha1h_u32, aarch64_crypto_sha1h, 0),
7238 NEONMAP1(vsha1mq_u32, aarch64_crypto_sha1m, 0),
7239 NEONMAP1(vsha1pq_u32, aarch64_crypto_sha1p, 0),
7293 { NEON::BI__builtin_neon_splat_lane_bf16, NEON::BI__builtin_neon_splat_lane_v, },
7294 { NEON::BI__builtin_neon_splat_laneq_bf16, NEON::BI__builtin_neon_splat_laneq_v, },
7295 { NEON::BI__builtin_neon_splatq_lane_bf16, NEON::BI__builtin_neon_splatq_lane_v, },
7296 { NEON::BI__builtin_neon_splatq_laneq_bf16, NEON::BI__builtin_neon_splatq_laneq_v, },
7297 { NEON::BI__builtin_neon_vabd_f16, NEON::BI__builtin_neon_vabd_v, },
7298 { NEON::BI__builtin_neon_vabdq_f16, NEON::BI__builtin_neon_vabdq_v, },
7299 { NEON::BI__builtin_neon_vabs_f16, NEON::BI__builtin_neon_vabs_v, },
7300 { NEON::BI__builtin_neon_vabsq_f16, NEON::BI__builtin_neon_vabsq_v, },
7301 { NEON::BI__builtin_neon_vcage_f16, NEON::BI__builtin_neon_vcage_v, },
7302 { NEON::BI__builtin_neon_vcageq_f16, NEON::BI__builtin_neon_vcageq_v, },
7303 { NEON::BI__builtin_neon_vcagt_f16, NEON::BI__builtin_neon_vcagt_v, },
7304 { NEON::BI__builtin_neon_vcagtq_f16, NEON::BI__builtin_neon_vcagtq_v, },
7305 { NEON::BI__builtin_neon_vcale_f16, NEON::BI__builtin_neon_vcale_v, },
7306 { NEON::BI__builtin_neon_vcaleq_f16, NEON::BI__builtin_neon_vcaleq_v, },
7307 { NEON::BI__builtin_neon_vcalt_f16, NEON::BI__builtin_neon_vcalt_v, },
7308 { NEON::BI__builtin_neon_vcaltq_f16, NEON::BI__builtin_neon_vcaltq_v, },
7309 { NEON::BI__builtin_neon_vceqz_f16, NEON::BI__builtin_neon_vceqz_v, },
7310 { NEON::BI__builtin_neon_vceqzq_f16, NEON::BI__builtin_neon_vceqzq_v, },
7311 { NEON::BI__builtin_neon_vcgez_f16, NEON::BI__builtin_neon_vcgez_v, },
7312 { NEON::BI__builtin_neon_vcgezq_f16, NEON::BI__builtin_neon_vcgezq_v, },
7313 { NEON::BI__builtin_neon_vcgtz_f16, NEON::BI__builtin_neon_vcgtz_v, },
7314 { NEON::BI__builtin_neon_vcgtzq_f16, NEON::BI__builtin_neon_vcgtzq_v, },
7315 { NEON::BI__builtin_neon_vclez_f16, NEON::BI__builtin_neon_vclez_v, },
7316 { NEON::BI__builtin_neon_vclezq_f16, NEON::BI__builtin_neon_vclezq_v, },
7317 { NEON::BI__builtin_neon_vcltz_f16, NEON::BI__builtin_neon_vcltz_v, },
7318 { NEON::BI__builtin_neon_vcltzq_f16, NEON::BI__builtin_neon_vcltzq_v, },
7319 { NEON::BI__builtin_neon_vfma_f16, NEON::BI__builtin_neon_vfma_v, },
7320 { NEON::BI__builtin_neon_vfma_lane_f16, NEON::BI__builtin_neon_vfma_lane_v, },
7321 { NEON::BI__builtin_neon_vfma_laneq_f16, NEON::BI__builtin_neon_vfma_laneq_v, },
7322 { NEON::BI__builtin_neon_vfmaq_f16, NEON::BI__builtin_neon_vfmaq_v, },
7323 { NEON::BI__builtin_neon_vfmaq_lane_f16, NEON::BI__builtin_neon_vfmaq_lane_v, },
7324 { NEON::BI__builtin_neon_vfmaq_laneq_f16, NEON::BI__builtin_neon_vfmaq_laneq_v, },
7325 { NEON::BI__builtin_neon_vld1_bf16_x2, NEON::BI__builtin_neon_vld1_x2_v },
7326 { NEON::BI__builtin_neon_vld1_bf16_x3, NEON::BI__builtin_neon_vld1_x3_v },
7327 { NEON::BI__builtin_neon_vld1_bf16_x4, NEON::BI__builtin_neon_vld1_x4_v },
7328 { NEON::BI__builtin_neon_vld1_bf16, NEON::BI__builtin_neon_vld1_v },
7329 { NEON::BI__builtin_neon_vld1_dup_bf16, NEON::BI__builtin_neon_vld1_dup_v },
7330 { NEON::BI__builtin_neon_vld1_lane_bf16, NEON::BI__builtin_neon_vld1_lane_v },
7331 { NEON::BI__builtin_neon_vld1q_bf16_x2, NEON::BI__builtin_neon_vld1q_x2_v },
7332 { NEON::BI__builtin_neon_vld1q_bf16_x3, NEON::BI__builtin_neon_vld1q_x3_v },
7333 { NEON::BI__builtin_neon_vld1q_bf16_x4, NEON::BI__builtin_neon_vld1q_x4_v },
7334 { NEON::BI__builtin_neon_vld1q_bf16, NEON::BI__builtin_neon_vld1q_v },
7335 { NEON::BI__builtin_neon_vld1q_dup_bf16, NEON::BI__builtin_neon_vld1q_dup_v },
7336 { NEON::BI__builtin_neon_vld1q_lane_bf16, NEON::BI__builtin_neon_vld1q_lane_v },
7337 { NEON::BI__builtin_neon_vld2_bf16, NEON::BI__builtin_neon_vld2_v },
7338 { NEON::BI__builtin_neon_vld2_dup_bf16, NEON::BI__builtin_neon_vld2_dup_v },
7339 { NEON::BI__builtin_neon_vld2_lane_bf16, NEON::BI__builtin_neon_vld2_lane_v },
7340 { NEON::BI__builtin_neon_vld2q_bf16, NEON::BI__builtin_neon_vld2q_v },
7341 { NEON::BI__builtin_neon_vld2q_dup_bf16, NEON::BI__builtin_neon_vld2q_dup_v },
7342 { NEON::BI__builtin_neon_vld2q_lane_bf16, NEON::BI__builtin_neon_vld2q_lane_v },
7343 { NEON::BI__builtin_neon_vld3_bf16, NEON::BI__builtin_neon_vld3_v },
7344 { NEON::BI__builtin_neon_vld3_dup_bf16, NEON::BI__builtin_neon_vld3_dup_v },
7345 { NEON::BI__builtin_neon_vld3_lane_bf16, NEON::BI__builtin_neon_vld3_lane_v },
7346 { NEON::BI__builtin_neon_vld3q_bf16, NEON::BI__builtin_neon_vld3q_v },
7347 { NEON::BI__builtin_neon_vld3q_dup_bf16, NEON::BI__builtin_neon_vld3q_dup_v },
7348 { NEON::BI__builtin_neon_vld3q_lane_bf16, NEON::BI__builtin_neon_vld3q_lane_v },
7349 { NEON::BI__builtin_neon_vld4_bf16, NEON::BI__builtin_neon_vld4_v },
7350 { NEON::BI__builtin_neon_vld4_dup_bf16, NEON::BI__builtin_neon_vld4_dup_v },
7351 { NEON::BI__builtin_neon_vld4_lane_bf16, NEON::BI__builtin_neon_vld4_lane_v },
7352 { NEON::BI__builtin_neon_vld4q_bf16, NEON::BI__builtin_neon_vld4q_v },
7353 { NEON::BI__builtin_neon_vld4q_dup_bf16, NEON::BI__builtin_neon_vld4q_dup_v },
7354 { NEON::BI__builtin_neon_vld4q_lane_bf16, NEON::BI__builtin_neon_vld4q_lane_v },
7355 { NEON::BI__builtin_neon_vmax_f16, NEON::BI__builtin_neon_vmax_v, },
7356 { NEON::BI__builtin_neon_vmaxnm_f16, NEON::BI__builtin_neon_vmaxnm_v, },
7357 { NEON::BI__builtin_neon_vmaxnmq_f16, NEON::BI__builtin_neon_vmaxnmq_v, },
7358 { NEON::BI__builtin_neon_vmaxq_f16, NEON::BI__builtin_neon_vmaxq_v, },
7359 { NEON::BI__builtin_neon_vmin_f16, NEON::BI__builtin_neon_vmin_v, },
7360 { NEON::BI__builtin_neon_vminnm_f16, NEON::BI__builtin_neon_vminnm_v, },
7361 { NEON::BI__builtin_neon_vminnmq_f16, NEON::BI__builtin_neon_vminnmq_v, },
7362 { NEON::BI__builtin_neon_vminq_f16, NEON::BI__builtin_neon_vminq_v, },
7363 { NEON::BI__builtin_neon_vmulx_f16, NEON::BI__builtin_neon_vmulx_v, },
7364 { NEON::BI__builtin_neon_vmulxq_f16, NEON::BI__builtin_neon_vmulxq_v, },
7365 { NEON::BI__builtin_neon_vpadd_f16, NEON::BI__builtin_neon_vpadd_v, },
7366 { NEON::BI__builtin_neon_vpaddq_f16, NEON::BI__builtin_neon_vpaddq_v, },
7367 { NEON::BI__builtin_neon_vpmax_f16, NEON::BI__builtin_neon_vpmax_v, },
7368 { NEON::BI__builtin_neon_vpmaxnm_f16, NEON::BI__builtin_neon_vpmaxnm_v, },
7369 { NEON::BI__builtin_neon_vpmaxnmq_f16, NEON::BI__builtin_neon_vpmaxnmq_v, },
7370 { NEON::BI__builtin_neon_vpmaxq_f16, NEON::BI__builtin_neon_vpmaxq_v, },
7371 { NEON::BI__builtin_neon_vpmin_f16, NEON::BI__builtin_neon_vpmin_v, },
7372 { NEON::BI__builtin_neon_vpminnm_f16, NEON::BI__builtin_neon_vpminnm_v, },
7373 { NEON::BI__builtin_neon_vpminnmq_f16, NEON::BI__builtin_neon_vpminnmq_v, },
7374 { NEON::BI__builtin_neon_vpminq_f16, NEON::BI__builtin_neon_vpminq_v, },
7375 { NEON::BI__builtin_neon_vrecpe_f16, NEON::BI__builtin_neon_vrecpe_v, },
7376 { NEON::BI__builtin_neon_vrecpeq_f16, NEON::BI__builtin_neon_vrecpeq_v, },
7377 { NEON::BI__builtin_neon_vrecps_f16, NEON::BI__builtin_neon_vrecps_v, },
7378 { NEON::BI__builtin_neon_vrecpsq_f16, NEON::BI__builtin_neon_vrecpsq_v, },
7379 { NEON::BI__builtin_neon_vrnd_f16, NEON::BI__builtin_neon_vrnd_v, },
7380 { NEON::BI__builtin_neon_vrnda_f16, NEON::BI__builtin_neon_vrnda_v, },
7381 { NEON::BI__builtin_neon_vrndaq_f16, NEON::BI__builtin_neon_vrndaq_v, },
7382 { NEON::BI__builtin_neon_vrndi_f16, NEON::BI__builtin_neon_vrndi_v, },
7383 { NEON::BI__builtin_neon_vrndiq_f16, NEON::BI__builtin_neon_vrndiq_v, },
7384 { NEON::BI__builtin_neon_vrndm_f16, NEON::BI__builtin_neon_vrndm_v, },
7385 { NEON::BI__builtin_neon_vrndmq_f16, NEON::BI__builtin_neon_vrndmq_v, },
7386 { NEON::BI__builtin_neon_vrndn_f16, NEON::BI__builtin_neon_vrndn_v, },
7387 { NEON::BI__builtin_neon_vrndnq_f16, NEON::BI__builtin_neon_vrndnq_v, },
7388 { NEON::BI__builtin_neon_vrndp_f16, NEON::BI__builtin_neon_vrndp_v, },
7389 { NEON::BI__builtin_neon_vrndpq_f16, NEON::BI__builtin_neon_vrndpq_v, },
7390 { NEON::BI__builtin_neon_vrndq_f16, NEON::BI__builtin_neon_vrndq_v, },
7391 { NEON::BI__builtin_neon_vrndx_f16, NEON::BI__builtin_neon_vrndx_v, },
7392 { NEON::BI__builtin_neon_vrndxq_f16, NEON::BI__builtin_neon_vrndxq_v, },
7393 { NEON::BI__builtin_neon_vrsqrte_f16, NEON::BI__builtin_neon_vrsqrte_v, },
7394 { NEON::BI__builtin_neon_vrsqrteq_f16, NEON::BI__builtin_neon_vrsqrteq_v, },
7395 { NEON::BI__builtin_neon_vrsqrts_f16, NEON::BI__builtin_neon_vrsqrts_v, },
7396 { NEON::BI__builtin_neon_vrsqrtsq_f16, NEON::BI__builtin_neon_vrsqrtsq_v, },
7397 { NEON::BI__builtin_neon_vsqrt_f16, NEON::BI__builtin_neon_vsqrt_v, },
7398 { NEON::BI__builtin_neon_vsqrtq_f16, NEON::BI__builtin_neon_vsqrtq_v, },
7399 { NEON::BI__builtin_neon_vst1_bf16_x2, NEON::BI__builtin_neon_vst1_x2_v },
7400 { NEON::BI__builtin_neon_vst1_bf16_x3, NEON::BI__builtin_neon_vst1_x3_v },
7401 { NEON::BI__builtin_neon_vst1_bf16_x4, NEON::BI__builtin_neon_vst1_x4_v },
7402 { NEON::BI__builtin_neon_vst1_bf16, NEON::BI__builtin_neon_vst1_v },
7403 { NEON::BI__builtin_neon_vst1_lane_bf16, NEON::BI__builtin_neon_vst1_lane_v },
7404 { NEON::BI__builtin_neon_vst1q_bf16_x2, NEON::BI__builtin_neon_vst1q_x2_v },
7405 { NEON::BI__builtin_neon_vst1q_bf16_x3, NEON::BI__builtin_neon_vst1q_x3_v },
7406 { NEON::BI__builtin_neon_vst1q_bf16_x4, NEON::BI__builtin_neon_vst1q_x4_v },
7407 { NEON::BI__builtin_neon_vst1q_bf16, NEON::BI__builtin_neon_vst1q_v },
7408 { NEON::BI__builtin_neon_vst1q_lane_bf16, NEON::BI__builtin_neon_vst1q_lane_v },
7409 { NEON::BI__builtin_neon_vst2_bf16, NEON::BI__builtin_neon_vst2_v },
7410 { NEON::BI__builtin_neon_vst2_lane_bf16, NEON::BI__builtin_neon_vst2_lane_v },
7411 { NEON::BI__builtin_neon_vst2q_bf16, NEON::BI__builtin_neon_vst2q_v },
7412 { NEON::BI__builtin_neon_vst2q_lane_bf16, NEON::BI__builtin_neon_vst2q_lane_v },
7413 { NEON::BI__builtin_neon_vst3_bf16, NEON::BI__builtin_neon_vst3_v },
7414 { NEON::BI__builtin_neon_vst3_lane_bf16, NEON::BI__builtin_neon_vst3_lane_v },
7415 { NEON::BI__builtin_neon_vst3q_bf16, NEON::BI__builtin_neon_vst3q_v },
7416 { NEON::BI__builtin_neon_vst3q_lane_bf16, NEON::BI__builtin_neon_vst3q_lane_v },
7417 { NEON::BI__builtin_neon_vst4_bf16, NEON::BI__builtin_neon_vst4_v },
7418 { NEON::BI__builtin_neon_vst4_lane_bf16, NEON::BI__builtin_neon_vst4_lane_v },
7419 { NEON::BI__builtin_neon_vst4q_bf16, NEON::BI__builtin_neon_vst4q_v },
7420 { NEON::BI__builtin_neon_vst4q_lane_bf16, NEON::BI__builtin_neon_vst4q_lane_v },
7424 { NEON::BI__builtin_neon_vldap1_lane_u64, NEON::BI__builtin_neon_vldap1_lane_s64 },
7425 { NEON::BI__builtin_neon_vldap1_lane_f64, NEON::BI__builtin_neon_vldap1_lane_s64 },
7426 { NEON::BI__builtin_neon_vldap1_lane_p64, NEON::BI__builtin_neon_vldap1_lane_s64 },
7427 { NEON::BI__builtin_neon_vldap1q_lane_u64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
7428 { NEON::BI__builtin_neon_vldap1q_lane_f64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
7429 { NEON::BI__builtin_neon_vldap1q_lane_p64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
7430 { NEON::BI__builtin_neon_vstl1_lane_u64, NEON::BI__builtin_neon_vstl1_lane_s64 },
7431 { NEON::BI__builtin_neon_vstl1_lane_f64, NEON::BI__builtin_neon_vstl1_lane_s64 },
7432 { NEON::BI__builtin_neon_vstl1_lane_p64, NEON::BI__builtin_neon_vstl1_lane_s64 },
7433 { NEON::BI__builtin_neon_vstl1q_lane_u64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
7434 { NEON::BI__builtin_neon_vstl1q_lane_f64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
7435 { NEON::BI__builtin_neon_vstl1q_lane_p64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
7442#define SVEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
7444 #NameBase, SVE::BI__builtin_sve_##NameBase, Intrinsic::LLVMIntrinsic, 0, \
7448#define SVEMAP2(NameBase, TypeModifier) \
7449 { #NameBase, SVE::BI__builtin_sve_##NameBase, 0, 0, TypeModifier }
7451#define GET_SVE_LLVM_INTRINSIC_MAP
7452#include "clang/Basic/arm_sve_builtin_cg.inc"
7453#include "clang/Basic/BuiltinsAArch64NeonSVEBridge_cg.def"
7454#undef GET_SVE_LLVM_INTRINSIC_MAP
7460#define SMEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
7462 #NameBase, SME::BI__builtin_sme_##NameBase, Intrinsic::LLVMIntrinsic, 0, \
7466#define SMEMAP2(NameBase, TypeModifier) \
7467 { #NameBase, SME::BI__builtin_sme_##NameBase, 0, 0, TypeModifier }
7469#define GET_SME_LLVM_INTRINSIC_MAP
7470#include "clang/Basic/arm_sme_builtin_cg.inc"
7471#undef GET_SME_LLVM_INTRINSIC_MAP
7484static const ARMVectorIntrinsicInfo *
7486 unsigned BuiltinID,
bool &MapProvenSorted) {
7489 if (!MapProvenSorted) {
7490 assert(llvm::is_sorted(IntrinsicMap));
7491 MapProvenSorted =
true;
7495 const ARMVectorIntrinsicInfo *Builtin =
7496 llvm::lower_bound(IntrinsicMap, BuiltinID);
7498 if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID)
7506 llvm::Type *ArgType,
7519 Ty = llvm::FixedVectorType::get(
7520 Ty, VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1);
7527 int Elts = VectorSize ? VectorSize / ArgType->getPrimitiveSizeInBits() : 1;
7528 ArgType = llvm::FixedVectorType::get(ArgType, Elts);
7532 Tys.push_back(ArgType);
7535 Tys.push_back(ArgType);
7546 unsigned BuiltinID = SISDInfo.BuiltinID;
7547 unsigned int Int = SISDInfo.LLVMIntrinsic;
7548 unsigned Modifier = SISDInfo.TypeModifier;
7549 const char *
s = SISDInfo.NameHint;
7551 switch (BuiltinID) {
7552 case NEON::BI__builtin_neon_vcled_s64:
7553 case NEON::BI__builtin_neon_vcled_u64:
7554 case NEON::BI__builtin_neon_vcles_f32:
7555 case NEON::BI__builtin_neon_vcled_f64:
7556 case NEON::BI__builtin_neon_vcltd_s64:
7557 case NEON::BI__builtin_neon_vcltd_u64:
7558 case NEON::BI__builtin_neon_vclts_f32:
7559 case NEON::BI__builtin_neon_vcltd_f64:
7560 case NEON::BI__builtin_neon_vcales_f32:
7561 case NEON::BI__builtin_neon_vcaled_f64:
7562 case NEON::BI__builtin_neon_vcalts_f32:
7563 case NEON::BI__builtin_neon_vcaltd_f64:
7567 std::swap(Ops[0], Ops[1]);
7571 assert(Int &&
"Generic code assumes a valid intrinsic");
7579 ConstantInt *C0 = ConstantInt::get(CGF.
SizeTy, 0);
7580 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
7581 ai != ae; ++ai, ++j) {
7582 llvm::Type *ArgTy = ai->getType();
7583 if (Ops[j]->getType()->getPrimitiveSizeInBits() ==
7584 ArgTy->getPrimitiveSizeInBits())
7587 assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy());
7590 Ops[j] = CGF.
Builder.CreateTruncOrBitCast(
7591 Ops[j], cast<llvm::VectorType>(ArgTy)->getElementType());
7593 CGF.
Builder.CreateInsertElement(PoisonValue::get(ArgTy), Ops[j], C0);
7598 if (ResultType->getPrimitiveSizeInBits().getFixedValue() <
7599 Result->getType()->getPrimitiveSizeInBits().getFixedValue())
7606 unsigned BuiltinID,
unsigned LLVMIntrinsic,
unsigned AltLLVMIntrinsic,
7607 const char *NameHint,
unsigned Modifier,
const CallExpr *E,
7609 llvm::Triple::ArchType Arch) {
7612 std::optional<llvm::APSInt> NeonTypeConst =
7619 bool Usgn =
Type.isUnsigned();
7620 bool Quad =
Type.isQuad();
7622 const bool AllowBFloatArgsAndRet =
7625 llvm::FixedVectorType *VTy =
7626 GetNeonType(
this,
Type, HasLegalHalfType,
false, AllowBFloatArgsAndRet);
7627 llvm::Type *Ty = VTy;
7631 auto getAlignmentValue32 = [&](
Address addr) ->
Value* {
7632 return Builder.getInt32(addr.getAlignment().getQuantity());
7635 unsigned Int = LLVMIntrinsic;
7637 Int = AltLLVMIntrinsic;
7639 switch (BuiltinID) {
7641 case NEON::BI__builtin_neon_splat_lane_v:
7642 case NEON::BI__builtin_neon_splat_laneq_v:
7643 case NEON::BI__builtin_neon_splatq_lane_v:
7644 case NEON::BI__builtin_neon_splatq_laneq_v: {
7645 auto NumElements = VTy->getElementCount();
7646 if (BuiltinID == NEON::BI__builtin_neon_splatq_lane_v)
7647 NumElements = NumElements * 2;
7648 if (BuiltinID == NEON::BI__builtin_neon_splat_laneq_v)
7649 NumElements = NumElements.divideCoefficientBy(2);
7651 Ops[0] =
Builder.CreateBitCast(Ops[0], VTy);
7652 return EmitNeonSplat(Ops[0], cast<ConstantInt>(Ops[1]), NumElements);
7654 case NEON::BI__builtin_neon_vpadd_v:
7655 case NEON::BI__builtin_neon_vpaddq_v:
7657 if (VTy->getElementType()->isFloatingPointTy() &&
7658 Int == Intrinsic::aarch64_neon_addp)
7659 Int = Intrinsic::aarch64_neon_faddp;
7661 case NEON::BI__builtin_neon_vabs_v:
7662 case NEON::BI__builtin_neon_vabsq_v:
7663 if (VTy->getElementType()->isFloatingPointTy())
7666 case NEON::BI__builtin_neon_vadd_v:
7667 case NEON::BI__builtin_neon_vaddq_v: {
7668 llvm::Type *VTy = llvm::FixedVectorType::get(
Int8Ty, Quad ? 16 : 8);
7669 Ops[0] =
Builder.CreateBitCast(Ops[0], VTy);
7670 Ops[1] =
Builder.CreateBitCast(Ops[1], VTy);
7671 Ops[0] =
Builder.CreateXor(Ops[0], Ops[1]);
7672 return Builder.CreateBitCast(Ops[0], Ty);
7674 case NEON::BI__builtin_neon_vaddhn_v: {
7675 llvm::FixedVectorType *SrcTy =
7676 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
7679 Ops[0] =
Builder.CreateBitCast(Ops[0], SrcTy);
7680 Ops[1] =
Builder.CreateBitCast(Ops[1], SrcTy);
7681 Ops[0] =
Builder.CreateAdd(Ops[0], Ops[1],
"vaddhn");
7684 Constant *ShiftAmt =
7685 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
7686 Ops[0] =
Builder.CreateLShr(Ops[0], ShiftAmt,
"vaddhn");
7689 return Builder.CreateTrunc(Ops[0], VTy,
"vaddhn");
7691 case NEON::BI__builtin_neon_vcale_v:
7692 case NEON::BI__builtin_neon_vcaleq_v:
7693 case NEON::BI__builtin_neon_vcalt_v:
7694 case NEON::BI__builtin_neon_vcaltq_v:
7695 std::swap(Ops[0], Ops[1]);
7697 case NEON::BI__builtin_neon_vcage_v:
7698 case NEON::BI__builtin_neon_vcageq_v:
7699 case NEON::BI__builtin_neon_vcagt_v:
7700 case NEON::BI__builtin_neon_vcagtq_v: {
7702 switch (VTy->getScalarSizeInBits()) {
7703 default: llvm_unreachable(
"unexpected type");
7714 auto *VecFlt = llvm::FixedVectorType::get(Ty, VTy->getNumElements());
7715 llvm::Type *Tys[] = { VTy, VecFlt };
7719 case NEON::BI__builtin_neon_vceqz_v:
7720 case NEON::BI__builtin_neon_vceqzq_v:
7722 ICmpInst::ICMP_EQ,
"vceqz");
7723 case NEON::BI__builtin_neon_vcgez_v:
7724 case NEON::BI__builtin_neon_vcgezq_v:
7726 ICmpInst::ICMP_SGE,
"vcgez");
7727 case NEON::BI__builtin_neon_vclez_v:
7728 case NEON::BI__builtin_neon_vclezq_v:
7730 ICmpInst::ICMP_SLE,
"vclez");
7731 case NEON::BI__builtin_neon_vcgtz_v:
7732 case NEON::BI__builtin_neon_vcgtzq_v:
7734 ICmpInst::ICMP_SGT,
"vcgtz");
7735 case NEON::BI__builtin_neon_vcltz_v:
7736 case NEON::BI__builtin_neon_vcltzq_v:
7738 ICmpInst::ICMP_SLT,
"vcltz");
7739 case NEON::BI__builtin_neon_vclz_v:
7740 case NEON::BI__builtin_neon_vclzq_v:
7745 case NEON::BI__builtin_neon_vcvt_f32_v:
7746 case NEON::BI__builtin_neon_vcvtq_f32_v:
7747 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
7750 return Usgn ?
Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
7751 :
Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
7752 case NEON::BI__builtin_neon_vcvt_f16_s16:
7753 case NEON::BI__builtin_neon_vcvt_f16_u16:
7754 case NEON::BI__builtin_neon_vcvtq_f16_s16:
7755 case NEON::BI__builtin_neon_vcvtq_f16_u16:
7756 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
7759 return Usgn ?
Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
7760 :
Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
7761 case NEON::BI__builtin_neon_vcvt_n_f16_s16:
7762 case NEON::BI__builtin_neon_vcvt_n_f16_u16:
7763 case NEON::BI__builtin_neon_vcvtq_n_f16_s16:
7764 case NEON::BI__builtin_neon_vcvtq_n_f16_u16: {
7769 case NEON::BI__builtin_neon_vcvt_n_f32_v:
7770 case NEON::BI__builtin_neon_vcvt_n_f64_v:
7771 case NEON::BI__builtin_neon_vcvtq_n_f32_v:
7772 case NEON::BI__builtin_neon_vcvtq_n_f64_v: {
7774 Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
7778 case NEON::BI__builtin_neon_vcvt_n_s16_f16:
7779 case NEON::BI__builtin_neon_vcvt_n_s32_v:
7780 case NEON::BI__builtin_neon_vcvt_n_u16_f16:
7781 case NEON::BI__builtin_neon_vcvt_n_u32_v:
7782 case NEON::BI__builtin_neon_vcvt_n_s64_v:
7783 case NEON::BI__builtin_neon_vcvt_n_u64_v:
7784 case NEON::BI__builtin_neon_vcvtq_n_s16_f16:
7785 case NEON::BI__builtin_neon_vcvtq_n_s32_v:
7786 case NEON::BI__builtin_neon_vcvtq_n_u16_f16:
7787 case NEON::BI__builtin_neon_vcvtq_n_u32_v:
7788 case NEON::BI__builtin_neon_vcvtq_n_s64_v:
7789 case NEON::BI__builtin_neon_vcvtq_n_u64_v: {
7794 case NEON::BI__builtin_neon_vcvt_s32_v:
7795 case NEON::BI__builtin_neon_vcvt_u32_v:
7796 case NEON::BI__builtin_neon_vcvt_s64_v:
7797 case NEON::BI__builtin_neon_vcvt_u64_v:
7798 case NEON::BI__builtin_neon_vcvt_s16_f16:
7799 case NEON::BI__builtin_neon_vcvt_u16_f16:
7800 case NEON::BI__builtin_neon_vcvtq_s32_v:
7801 case NEON::BI__builtin_neon_vcvtq_u32_v:
7802 case NEON::BI__builtin_neon_vcvtq_s64_v:
7803 case NEON::BI__builtin_neon_vcvtq_u64_v:
7804 case NEON::BI__builtin_neon_vcvtq_s16_f16:
7805 case NEON::BI__builtin_neon_vcvtq_u16_f16: {
7807 return Usgn ?
Builder.CreateFPToUI(Ops[0], Ty,
"vcvt")
7808 :
Builder.CreateFPToSI(Ops[0], Ty,
"vcvt");
7810 case NEON::BI__builtin_neon_vcvta_s16_f16:
7811 case NEON::BI__builtin_neon_vcvta_s32_v:
7812 case NEON::BI__builtin_neon_vcvta_s64_v:
7813 case NEON::BI__builtin_neon_vcvta_u16_f16:
7814 case NEON::BI__builtin_neon_vcvta_u32_v:
7815 case NEON::BI__builtin_neon_vcvta_u64_v:
7816 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
7817 case NEON::BI__builtin_neon_vcvtaq_s32_v:
7818 case NEON::BI__builtin_neon_vcvtaq_s64_v:
7819 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
7820 case NEON::BI__builtin_neon_vcvtaq_u32_v:
7821 case NEON::BI__builtin_neon_vcvtaq_u64_v:
7822 case NEON::BI__builtin_neon_vcvtn_s16_f16:
7823 case NEON::BI__builtin_neon_vcvtn_s32_v:
7824 case NEON::BI__builtin_neon_vcvtn_s64_v:
7825 case NEON::BI__builtin_neon_vcvtn_u16_f16:
7826 case NEON::BI__builtin_neon_vcvtn_u32_v:
7827 case NEON::BI__builtin_neon_vcvtn_u64_v:
7828 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
7829 case NEON::BI__builtin_neon_vcvtnq_s32_v:
7830 case NEON::BI__builtin_neon_vcvtnq_s64_v:
7831 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
7832 case NEON::BI__builtin_neon_vcvtnq_u32_v:
7833 case NEON::BI__builtin_neon_vcvtnq_u64_v:
7834 case NEON::BI__builtin_neon_vcvtp_s16_f16:
7835 case NEON::BI__builtin_neon_vcvtp_s32_v:
7836 case NEON::BI__builtin_neon_vcvtp_s64_v:
7837 case NEON::BI__builtin_neon_vcvtp_u16_f16:
7838 case NEON::BI__builtin_neon_vcvtp_u32_v:
7839 case NEON::BI__builtin_neon_vcvtp_u64_v:
7840 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
7841 case NEON::BI__builtin_neon_vcvtpq_s32_v:
7842 case NEON::BI__builtin_neon_vcvtpq_s64_v:
7843 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
7844 case NEON::BI__builtin_neon_vcvtpq_u32_v:
7845 case NEON::BI__builtin_neon_vcvtpq_u64_v:
7846 case NEON::BI__builtin_neon_vcvtm_s16_f16:
7847 case NEON::BI__builtin_neon_vcvtm_s32_v:
7848 case NEON::BI__builtin_neon_vcvtm_s64_v:
7849 case NEON::BI__builtin_neon_vcvtm_u16_f16:
7850 case NEON::BI__builtin_neon_vcvtm_u32_v:
7851 case NEON::BI__builtin_neon_vcvtm_u64_v:
7852 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
7853 case NEON::BI__builtin_neon_vcvtmq_s32_v:
7854 case NEON::BI__builtin_neon_vcvtmq_s64_v:
7855 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
7856 case NEON::BI__builtin_neon_vcvtmq_u32_v:
7857 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
7861 case NEON::BI__builtin_neon_vcvtx_f32_v: {
7862 llvm::Type *Tys[2] = { VTy->getTruncatedElementVectorType(VTy), Ty};
7866 case NEON::BI__builtin_neon_vext_v:
7867 case NEON::BI__builtin_neon_vextq_v: {
7868 int CV = cast<ConstantInt>(Ops[2])->getSExtValue();
7870 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
7871 Indices.push_back(i+CV);
7873 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
7874 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
7875 return Builder.CreateShuffleVector(Ops[0], Ops[1], Indices,
"vext");
7877 case NEON::BI__builtin_neon_vfma_v:
7878 case NEON::BI__builtin_neon_vfmaq_v: {
7879 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
7880 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
7881 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
7885 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
7886 {Ops[1], Ops[2], Ops[0]});
7888 case NEON::BI__builtin_neon_vld1_v:
7889 case NEON::BI__builtin_neon_vld1q_v: {
7891 Ops.push_back(getAlignmentValue32(PtrOp0));
7894 case NEON::BI__builtin_neon_vld1_x2_v:
7895 case NEON::BI__builtin_neon_vld1q_x2_v:
7896 case NEON::BI__builtin_neon_vld1_x3_v:
7897 case NEON::BI__builtin_neon_vld1q_x3_v:
7898 case NEON::BI__builtin_neon_vld1_x4_v:
7899 case NEON::BI__builtin_neon_vld1q_x4_v: {
7902 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld1xN");
7905 case NEON::BI__builtin_neon_vld2_v:
7906 case NEON::BI__builtin_neon_vld2q_v:
7907 case NEON::BI__builtin_neon_vld3_v:
7908 case NEON::BI__builtin_neon_vld3q_v:
7909 case NEON::BI__builtin_neon_vld4_v:
7910 case NEON::BI__builtin_neon_vld4q_v:
7911 case NEON::BI__builtin_neon_vld2_dup_v:
7912 case NEON::BI__builtin_neon_vld2q_dup_v:
7913 case NEON::BI__builtin_neon_vld3_dup_v:
7914 case NEON::BI__builtin_neon_vld3q_dup_v:
7915 case NEON::BI__builtin_neon_vld4_dup_v:
7916 case NEON::BI__builtin_neon_vld4q_dup_v: {
7919 Value *Align = getAlignmentValue32(PtrOp1);
7920 Ops[1] =
Builder.CreateCall(F, {Ops[1], Align}, NameHint);
7923 case NEON::BI__builtin_neon_vld1_dup_v:
7924 case NEON::BI__builtin_neon_vld1q_dup_v: {
7925 Value *
V = PoisonValue::get(Ty);
7928 llvm::Constant *CI = ConstantInt::get(
SizeTy, 0);
7929 Ops[0] =
Builder.CreateInsertElement(
V, Ld, CI);
7932 case NEON::BI__builtin_neon_vld2_lane_v:
7933 case NEON::BI__builtin_neon_vld2q_lane_v:
7934 case NEON::BI__builtin_neon_vld3_lane_v:
7935 case NEON::BI__builtin_neon_vld3q_lane_v:
7936 case NEON::BI__builtin_neon_vld4_lane_v:
7937 case NEON::BI__builtin_neon_vld4q_lane_v: {
7940 for (
unsigned I = 2; I < Ops.size() - 1; ++I)
7941 Ops[I] =
Builder.CreateBitCast(Ops[I], Ty);
7942 Ops.push_back(getAlignmentValue32(PtrOp1));
7946 case NEON::BI__builtin_neon_vmovl_v: {
7947 llvm::FixedVectorType *DTy =
7948 llvm::FixedVectorType::getTruncatedElementVectorType(VTy);
7949 Ops[0] =
Builder.CreateBitCast(Ops[0], DTy);
7951 return Builder.CreateZExt(Ops[0], Ty,
"vmovl");
7952 return Builder.CreateSExt(Ops[0], Ty,
"vmovl");
7954 case NEON::BI__builtin_neon_vmovn_v: {
7955 llvm::FixedVectorType *QTy =
7956 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
7957 Ops[0] =
Builder.CreateBitCast(Ops[0], QTy);
7958 return Builder.CreateTrunc(Ops[0], Ty,
"vmovn");
7960 case NEON::BI__builtin_neon_vmull_v:
7966 Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls;
7969 case NEON::BI__builtin_neon_vpadal_v:
7970 case NEON::BI__builtin_neon_vpadalq_v: {
7972 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
7976 llvm::FixedVectorType::get(EltTy, VTy->getNumElements() * 2);
7977 llvm::Type *Tys[2] = { Ty, NarrowTy };
7980 case NEON::BI__builtin_neon_vpaddl_v:
7981 case NEON::BI__builtin_neon_vpaddlq_v: {
7983 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
7984 llvm::Type *EltTy = llvm::IntegerType::get(
getLLVMContext(), EltBits / 2);
7986 llvm::FixedVectorType::get(EltTy, VTy->getNumElements() * 2);
7987 llvm::Type *Tys[2] = { Ty, NarrowTy };
7990 case NEON::BI__builtin_neon_vqdmlal_v:
7991 case NEON::BI__builtin_neon_vqdmlsl_v: {
7998 case NEON::BI__builtin_neon_vqdmulhq_lane_v:
7999 case NEON::BI__builtin_neon_vqdmulh_lane_v:
8000 case NEON::BI__builtin_neon_vqrdmulhq_lane_v:
8001 case NEON::BI__builtin_neon_vqrdmulh_lane_v: {
8002 auto *RTy = cast<llvm::FixedVectorType>(Ty);
8003 if (BuiltinID == NEON::BI__builtin_neon_vqdmulhq_lane_v ||
8004 BuiltinID == NEON::BI__builtin_neon_vqrdmulhq_lane_v)
8005 RTy = llvm::FixedVectorType::get(RTy->getElementType(),
8006 RTy->getNumElements() * 2);
8007 llvm::Type *Tys[2] = {
8012 case NEON::BI__builtin_neon_vqdmulhq_laneq_v:
8013 case NEON::BI__builtin_neon_vqdmulh_laneq_v:
8014 case NEON::BI__builtin_neon_vqrdmulhq_laneq_v:
8015 case NEON::BI__builtin_neon_vqrdmulh_laneq_v: {
8016 llvm::Type *Tys[2] = {
8021 case NEON::BI__builtin_neon_vqshl_n_v:
8022 case NEON::BI__builtin_neon_vqshlq_n_v:
8025 case NEON::BI__builtin_neon_vqshlu_n_v:
8026 case NEON::BI__builtin_neon_vqshluq_n_v:
8029 case NEON::BI__builtin_neon_vrecpe_v:
8030 case NEON::BI__builtin_neon_vrecpeq_v:
8031 case NEON::BI__builtin_neon_vrsqrte_v:
8032 case NEON::BI__builtin_neon_vrsqrteq_v:
8033 Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic;
8035 case NEON::BI__builtin_neon_vrndi_v:
8036 case NEON::BI__builtin_neon_vrndiq_v:
8038 ? Intrinsic::experimental_constrained_nearbyint
8039 : Intrinsic::nearbyint;
8041 case NEON::BI__builtin_neon_vrshr_n_v:
8042 case NEON::BI__builtin_neon_vrshrq_n_v:
8045 case NEON::BI__builtin_neon_vsha512hq_u64:
8046 case NEON::BI__builtin_neon_vsha512h2q_u64:
8047 case NEON::BI__builtin_neon_vsha512su0q_u64:
8048 case NEON::BI__builtin_neon_vsha512su1q_u64: {
8052 case NEON::BI__builtin_neon_vshl_n_v:
8053 case NEON::BI__builtin_neon_vshlq_n_v:
8055 return Builder.CreateShl(
Builder.CreateBitCast(Ops[0],Ty), Ops[1],
8057 case NEON::BI__builtin_neon_vshll_n_v: {
8058 llvm::FixedVectorType *SrcTy =
8059 llvm::FixedVectorType::getTruncatedElementVectorType(VTy);
8060 Ops[0] =
Builder.CreateBitCast(Ops[0], SrcTy);
8062 Ops[0] =
Builder.CreateZExt(Ops[0], VTy);
8064 Ops[0] =
Builder.CreateSExt(Ops[0], VTy);
8066 return Builder.CreateShl(Ops[0], Ops[1],
"vshll_n");
8068 case NEON::BI__builtin_neon_vshrn_n_v: {
8069 llvm::FixedVectorType *SrcTy =
8070 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
8071 Ops[0] =
Builder.CreateBitCast(Ops[0], SrcTy);
8074 Ops[0] =
Builder.CreateLShr(Ops[0], Ops[1]);
8076 Ops[0] =
Builder.CreateAShr(Ops[0], Ops[1]);
8077 return Builder.CreateTrunc(Ops[0], Ty,
"vshrn_n");
8079 case NEON::BI__builtin_neon_vshr_n_v:
8080 case NEON::BI__builtin_neon_vshrq_n_v:
8082 case NEON::BI__builtin_neon_vst1_v:
8083 case NEON::BI__builtin_neon_vst1q_v:
8084 case NEON::BI__builtin_neon_vst2_v:
8085 case NEON::BI__builtin_neon_vst2q_v:
8086 case NEON::BI__builtin_neon_vst3_v:
8087 case NEON::BI__builtin_neon_vst3q_v:
8088 case NEON::BI__builtin_neon_vst4_v:
8089 case NEON::BI__builtin_neon_vst4q_v:
8090 case NEON::BI__builtin_neon_vst2_lane_v:
8091 case NEON::BI__builtin_neon_vst2q_lane_v:
8092 case NEON::BI__builtin_neon_vst3_lane_v:
8093 case NEON::BI__builtin_neon_vst3q_lane_v:
8094 case NEON::BI__builtin_neon_vst4_lane_v:
8095 case NEON::BI__builtin_neon_vst4q_lane_v: {
8097 Ops.push_back(getAlignmentValue32(PtrOp0));
8100 case NEON::BI__builtin_neon_vsm3partw1q_u32:
8101 case NEON::BI__builtin_neon_vsm3partw2q_u32:
8102 case NEON::BI__builtin_neon_vsm3ss1q_u32:
8103 case NEON::BI__builtin_neon_vsm4ekeyq_u32:
8104 case NEON::BI__builtin_neon_vsm4eq_u32: {
8108 case NEON::BI__builtin_neon_vsm3tt1aq_u32:
8109 case NEON::BI__builtin_neon_vsm3tt1bq_u32:
8110 case NEON::BI__builtin_neon_vsm3tt2aq_u32:
8111 case NEON::BI__builtin_neon_vsm3tt2bq_u32: {
8116 case NEON::BI__builtin_neon_vst1_x2_v:
8117 case NEON::BI__builtin_neon_vst1q_x2_v:
8118 case NEON::BI__builtin_neon_vst1_x3_v:
8119 case NEON::BI__builtin_neon_vst1q_x3_v:
8120 case NEON::BI__builtin_neon_vst1_x4_v:
8121 case NEON::BI__builtin_neon_vst1q_x4_v: {
8124 if (Arch == llvm::Triple::aarch64 || Arch == llvm::Triple::aarch64_be ||
8125 Arch == llvm::Triple::aarch64_32) {
8127 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
8133 case NEON::BI__builtin_neon_vsubhn_v: {
8134 llvm::FixedVectorType *SrcTy =
8135 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
8138 Ops[0] =
Builder.CreateBitCast(Ops[0], SrcTy);
8139 Ops[1] =
Builder.CreateBitCast(Ops[1], SrcTy);
8140 Ops[0] =
Builder.CreateSub(Ops[0], Ops[1],
"vsubhn");
8143 Constant *ShiftAmt =
8144 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
8145 Ops[0] =
Builder.CreateLShr(Ops[0], ShiftAmt,
"vsubhn");
8148 return Builder.CreateTrunc(Ops[0], VTy,
"vsubhn");
8150 case NEON::BI__builtin_neon_vtrn_v:
8151 case NEON::BI__builtin_neon_vtrnq_v: {
8152 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
8153 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
8154 Value *SV =
nullptr;
8156 for (
unsigned vi = 0; vi != 2; ++vi) {
8158 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
8159 Indices.push_back(i+vi);
8160 Indices.push_back(i+e+vi);
8162 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8163 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vtrn");
8168 case NEON::BI__builtin_neon_vtst_v:
8169 case NEON::BI__builtin_neon_vtstq_v: {
8170 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
8171 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
8172 Ops[0] =
Builder.CreateAnd(Ops[0], Ops[1]);
8173 Ops[0] =
Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
8174 ConstantAggregateZero::get(Ty));
8175 return Builder.CreateSExt(Ops[0], Ty,
"vtst");
8177 case NEON::BI__builtin_neon_vuzp_v:
8178 case NEON::BI__builtin_neon_vuzpq_v: {
8179 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
8180 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
8181 Value *SV =
nullptr;
8183 for (
unsigned vi = 0; vi != 2; ++vi) {
8185 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
8186 Indices.push_back(2*i+vi);
8188 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8189 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vuzp");
8194 case NEON::BI__builtin_neon_vxarq_u64: {
8199 case NEON::BI__builtin_neon_vzip_v:
8200 case NEON::BI__builtin_neon_vzipq_v: {
8201 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
8202 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
8203 Value *SV =
nullptr;
8205 for (
unsigned vi = 0; vi != 2; ++vi) {
8207 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
8208 Indices.push_back((i + vi*e) >> 1);
8209 Indices.push_back(((i + vi*e) >> 1)+e);
8211 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8212 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vzip");
8217 case NEON::BI__builtin_neon_vdot_s32:
8218 case NEON::BI__builtin_neon_vdot_u32:
8219 case NEON::BI__builtin_neon_vdotq_s32:
8220 case NEON::BI__builtin_neon_vdotq_u32: {
8222 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8223 llvm::Type *Tys[2] = { Ty, InputTy };
8226 case NEON::BI__builtin_neon_vfmlal_low_f16:
8227 case NEON::BI__builtin_neon_vfmlalq_low_f16: {
8229 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8230 llvm::Type *Tys[2] = { Ty, InputTy };
8233 case NEON::BI__builtin_neon_vfmlsl_low_f16:
8234 case NEON::BI__builtin_neon_vfmlslq_low_f16: {
8236 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8237 llvm::Type *Tys[2] = { Ty, InputTy };
8240 case NEON::BI__builtin_neon_vfmlal_high_f16:
8241 case NEON::BI__builtin_neon_vfmlalq_high_f16: {
8243 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8244 llvm::Type *Tys[2] = { Ty, InputTy };
8247 case NEON::BI__builtin_neon_vfmlsl_high_f16:
8248 case NEON::BI__builtin_neon_vfmlslq_high_f16: {
8250 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8251 llvm::Type *Tys[2] = { Ty, InputTy };
8254 case NEON::BI__builtin_neon_vmmlaq_s32:
8255 case NEON::BI__builtin_neon_vmmlaq_u32: {
8257 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8258 llvm::Type *Tys[2] = { Ty, InputTy };
8261 case NEON::BI__builtin_neon_vusmmlaq_s32: {
8263 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8264 llvm::Type *Tys[2] = { Ty, InputTy };
8267 case NEON::BI__builtin_neon_vusdot_s32:
8268 case NEON::BI__builtin_neon_vusdotq_s32: {
8270 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8271 llvm::Type *Tys[2] = { Ty, InputTy };
8274 case NEON::BI__builtin_neon_vbfdot_f32:
8275 case NEON::BI__builtin_neon_vbfdotq_f32: {
8276 llvm::Type *InputTy =
8277 llvm::FixedVectorType::get(
BFloatTy, Ty->getPrimitiveSizeInBits() / 16);
8278 llvm::Type *Tys[2] = { Ty, InputTy };
8281 case NEON::BI__builtin_neon___a32_vcvt_bf16_f32: {
8282 llvm::Type *Tys[1] = { Ty };
8289 assert(Int &&
"Expected valid intrinsic number");
8302 Value *Op, llvm::Type *Ty,
const CmpInst::Predicate Fp,
8303 const CmpInst::Predicate Ip,
const Twine &Name) {
8304 llvm::Type *OTy = Op->
getType();
8310 if (BitCastInst *BI = dyn_cast<BitCastInst>(Op))
8311 OTy = BI->getOperand(0)->getType();
8313 Op =
Builder.CreateBitCast(Op, OTy);
8314 if (OTy->getScalarType()->isFloatingPointTy()) {
8315 if (Fp == CmpInst::FCMP_OEQ)
8316 Op =
Builder.CreateFCmp(Fp, Op, Constant::getNullValue(OTy));
8318 Op =
Builder.CreateFCmpS(Fp, Op, Constant::getNullValue(OTy));
8320 Op =
Builder.CreateICmp(Ip, Op, Constant::getNullValue(OTy));
8322 return Builder.CreateSExt(Op, Ty, Name);
8327 llvm::Type *ResTy,
unsigned IntID,
8331 TblOps.push_back(ExtOp);
8335 auto *TblTy = cast<llvm::FixedVectorType>(Ops[0]->getType());
8336 for (
unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) {
8337 Indices.push_back(2*i);
8338 Indices.push_back(2*i+1);
8341 int PairPos = 0, End = Ops.size() - 1;
8342 while (PairPos < End) {
8343 TblOps.push_back(CGF.
Builder.CreateShuffleVector(Ops[PairPos],
8344 Ops[PairPos+1], Indices,
8351 if (PairPos == End) {
8352 Value *ZeroTbl = ConstantAggregateZero::get(TblTy);
8353 TblOps.push_back(CGF.
Builder.CreateShuffleVector(Ops[PairPos],
8354 ZeroTbl, Indices, Name));
8358 TblOps.push_back(IndexOp);
8364Value *CodeGenFunction::GetValueForARMHint(
unsigned BuiltinID) {
8366 switch (BuiltinID) {
8369 case clang::ARM::BI__builtin_arm_nop:
8372 case clang::ARM::BI__builtin_arm_yield:
8373 case clang::ARM::BI__yield:
8376 case clang::ARM::BI__builtin_arm_wfe:
8377 case clang::ARM::BI__wfe:
8380 case clang::ARM::BI__builtin_arm_wfi:
8381 case clang::ARM::BI__wfi:
8384 case clang::ARM::BI__builtin_arm_sev:
8385 case clang::ARM::BI__sev:
8388 case clang::ARM::BI__builtin_arm_sevl:
8389 case clang::ARM::BI__sevl:
8407 llvm::Type *RegisterType,
8408 llvm::Type *ValueType,
bool isExecHi) {
8413 llvm::Value *
Call = Builder.CreateCall(F, {Builder.getInt1(
true)});
8416 Value *Rt2 = Builder.CreateLShr(
Call, 32);
8417 Rt2 = Builder.CreateTrunc(Rt2, CGF.
Int32Ty);
8429 llvm::Type *RegisterType,
8430 llvm::Type *ValueType,
8432 StringRef SysReg =
"") {
8434 assert((RegisterType->isIntegerTy(32) || RegisterType->isIntegerTy(64) ||
8435 RegisterType->isIntegerTy(128)) &&
8436 "Unsupported size for register.");
8442 if (SysReg.empty()) {
8444 SysReg = cast<clang::StringLiteral>(SysRegStrExpr)->getString();
8447 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) };
8448 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
8449 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
8451 llvm::Type *Types[] = { RegisterType };
8453 bool MixedTypes = RegisterType->isIntegerTy(64) && ValueType->isIntegerTy(32);
8454 assert(!(RegisterType->isIntegerTy(32) && ValueType->isIntegerTy(64))
8455 &&
"Can't fit 64-bit value in 32-bit register");
8457 if (AccessKind !=
Write) {
8460 AccessKind ==
VolatileRead ? llvm::Intrinsic::read_volatile_register
8461 : llvm::Intrinsic::read_register,
8463 llvm::Value *
Call = Builder.CreateCall(F, Metadata);
8467 return Builder.CreateTrunc(
Call, ValueType);
8469 if (ValueType->isPointerTy())
8471 return Builder.CreateIntToPtr(
Call, ValueType);
8476 llvm::Function *F = CGM.
getIntrinsic(llvm::Intrinsic::write_register, Types);
8480 ArgValue = Builder.CreateZExt(ArgValue, RegisterType);
8481 return Builder.CreateCall(F, { Metadata, ArgValue });
8484 if (ValueType->isPointerTy()) {
8486 ArgValue = Builder.CreatePtrToInt(ArgValue, RegisterType);
8487 return Builder.CreateCall(F, { Metadata, ArgValue });
8490 return Builder.CreateCall(F, { Metadata, ArgValue });
8496 switch (BuiltinID) {
8498 case NEON::BI__builtin_neon_vget_lane_i8:
8499 case NEON::BI__builtin_neon_vget_lane_i16:
8500 case NEON::BI__builtin_neon_vget_lane_bf16:
8501 case NEON::BI__builtin_neon_vget_lane_i32:
8502 case NEON::BI__builtin_neon_vget_lane_i64:
8503 case NEON::BI__builtin_neon_vget_lane_f32:
8504 case NEON::BI__builtin_neon_vgetq_lane_i8:
8505 case NEON::BI__builtin_neon_vgetq_lane_i16:
8506 case NEON::BI__builtin_neon_vgetq_lane_bf16:
8507 case NEON::BI__builtin_neon_vgetq_lane_i32:
8508 case NEON::BI__builtin_neon_vgetq_lane_i64:
8509 case NEON::BI__builtin_neon_vgetq_lane_f32:
8510 case NEON::BI__builtin_neon_vduph_lane_bf16:
8511 case NEON::BI__builtin_neon_vduph_laneq_bf16:
8512 case NEON::BI__builtin_neon_vset_lane_i8:
8513 case NEON::BI__builtin_neon_vset_lane_i16:
8514 case NEON::BI__builtin_neon_vset_lane_bf16:
8515 case NEON::BI__builtin_neon_vset_lane_i32:
8516 case NEON::BI__builtin_neon_vset_lane_i64:
8517 case NEON::BI__builtin_neon_vset_lane_f32:
8518 case NEON::BI__builtin_neon_vsetq_lane_i8:
8519 case NEON::BI__builtin_neon_vsetq_lane_i16:
8520 case NEON::BI__builtin_neon_vsetq_lane_bf16:
8521 case NEON::BI__builtin_neon_vsetq_lane_i32:
8522 case NEON::BI__builtin_neon_vsetq_lane_i64:
8523 case NEON::BI__builtin_neon_vsetq_lane_f32:
8524 case NEON::BI__builtin_neon_vsha1h_u32:
8525 case NEON::BI__builtin_neon_vsha1cq_u32:
8526 case NEON::BI__builtin_neon_vsha1pq_u32:
8527 case NEON::BI__builtin_neon_vsha1mq_u32:
8528 case NEON::BI__builtin_neon_vcvth_bf16_f32:
8529 case clang::ARM::BI_MoveToCoprocessor:
8530 case clang::ARM::BI_MoveToCoprocessor2:
8539 llvm::Triple::ArchType Arch) {
8540 if (
auto Hint = GetValueForARMHint(BuiltinID))
8543 if (BuiltinID == clang::ARM::BI__emit) {
8545 llvm::FunctionType *FTy =
8546 llvm::FunctionType::get(
VoidTy,
false);
8550 llvm_unreachable(
"Sema will ensure that the parameter is constant");
8553 uint64_t ZExtValue =
Value.zextOrTrunc(IsThumb ? 16 : 32).getZExtValue();
8555 llvm::InlineAsm *Emit =
8556 IsThumb ? InlineAsm::get(FTy,
".inst.n 0x" + utohexstr(ZExtValue),
"",
8558 : InlineAsm::get(FTy,
".inst 0x" + utohexstr(ZExtValue),
"",
8561 return Builder.CreateCall(Emit);
8564 if (BuiltinID == clang::ARM::BI__builtin_arm_dbg) {
8569 if (BuiltinID == clang::ARM::BI__builtin_arm_prefetch) {
8581 if (BuiltinID == clang::ARM::BI__builtin_arm_rbit) {
8584 CGM.
getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
8587 if (BuiltinID == clang::ARM::BI__builtin_arm_clz ||
8588 BuiltinID == clang::ARM::BI__builtin_arm_clz64) {
8592 if (BuiltinID == clang::ARM::BI__builtin_arm_clz64)
8598 if (BuiltinID == clang::ARM::BI__builtin_arm_cls) {
8602 if (BuiltinID == clang::ARM::BI__builtin_arm_cls64) {
8608 if (BuiltinID == clang::ARM::BI__clear_cache) {
8609 assert(E->
getNumArgs() == 2 &&
"__clear_cache takes 2 arguments");
8612 for (
unsigned i = 0; i < 2; i++)
8615 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
8616 StringRef Name = FD->
getName();
8620 if (BuiltinID == clang::ARM::BI__builtin_arm_mcrr ||
8621 BuiltinID == clang::ARM::BI__builtin_arm_mcrr2) {
8624 switch (BuiltinID) {
8625 default: llvm_unreachable(
"unexpected builtin");
8626 case clang::ARM::BI__builtin_arm_mcrr:
8629 case clang::ARM::BI__builtin_arm_mcrr2:
8651 return Builder.CreateCall(F, {Coproc, Opc1, Rt, Rt2, CRm});
8654 if (BuiltinID == clang::ARM::BI__builtin_arm_mrrc ||
8655 BuiltinID == clang::ARM::BI__builtin_arm_mrrc2) {
8658 switch (BuiltinID) {
8659 default: llvm_unreachable(
"unexpected builtin");
8660 case clang::ARM::BI__builtin_arm_mrrc:
8663 case clang::ARM::BI__builtin_arm_mrrc2:
8671 Value *RtAndRt2 =
Builder.CreateCall(F, {Coproc, Opc1, CRm});
8681 Value *ShiftCast = llvm::ConstantInt::get(
Int64Ty, 32);
8682 RtAndRt2 =
Builder.CreateShl(Rt, ShiftCast,
"shl",
true);
8683 RtAndRt2 =
Builder.CreateOr(RtAndRt2, Rt1);
8688 if (BuiltinID == clang::ARM::BI__builtin_arm_ldrexd ||
8689 ((BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
8690 BuiltinID == clang::ARM::BI__builtin_arm_ldaex) &&
8692 BuiltinID == clang::ARM::BI__ldrexd) {
8695 switch (BuiltinID) {
8696 default: llvm_unreachable(
"unexpected builtin");
8697 case clang::ARM::BI__builtin_arm_ldaex:
8700 case clang::ARM::BI__builtin_arm_ldrexd:
8701 case clang::ARM::BI__builtin_arm_ldrex:
8702 case clang::ARM::BI__ldrexd:
8716 Val =
Builder.CreateShl(Val0, ShiftCst,
"shl",
true );
8717 Val =
Builder.CreateOr(Val, Val1);
8721 if (BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
8722 BuiltinID == clang::ARM::BI__builtin_arm_ldaex) {
8731 BuiltinID == clang::ARM::BI__builtin_arm_ldaex ? Intrinsic::arm_ldaex
8732 : Intrinsic::arm_ldrex,
8734 CallInst *Val =
Builder.CreateCall(F, LoadAddr,
"ldrex");
8738 if (RealResTy->isPointerTy())
8739 return Builder.CreateIntToPtr(Val, RealResTy);
8741 llvm::Type *IntResTy = llvm::IntegerType::get(
8743 return Builder.CreateBitCast(
Builder.CreateTruncOrBitCast(Val, IntResTy),
8748 if (BuiltinID == clang::ARM::BI__builtin_arm_strexd ||
8749 ((BuiltinID == clang::ARM::BI__builtin_arm_stlex ||
8750 BuiltinID == clang::ARM::BI__builtin_arm_strex) &&
8753 BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlexd
8754 : Intrinsic::arm_strexd);
8767 return Builder.CreateCall(F, {Arg0, Arg1, StPtr},
"strexd");
8770 if (BuiltinID == clang::ARM::BI__builtin_arm_strex ||
8771 BuiltinID == clang::ARM::BI__builtin_arm_stlex) {
8776 llvm::Type *StoreTy =
8779 if (StoreVal->
getType()->isPointerTy())
8782 llvm::Type *
IntTy = llvm::IntegerType::get(
8790 BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlex
8791 : Intrinsic::arm_strex,
8794 CallInst *CI =
Builder.CreateCall(F, {StoreVal, StoreAddr},
"strex");
8796 1, Attribute::get(
getLLVMContext(), Attribute::ElementType, StoreTy));
8800 if (BuiltinID == clang::ARM::BI__builtin_arm_clrex) {
8806 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
8807 switch (BuiltinID) {
8808 case clang::ARM::BI__builtin_arm_crc32b:
8809 CRCIntrinsicID = Intrinsic::arm_crc32b;
break;
8810 case clang::ARM::BI__builtin_arm_crc32cb:
8811 CRCIntrinsicID = Intrinsic::arm_crc32cb;
break;
8812 case clang::ARM::BI__builtin_arm_crc32h:
8813 CRCIntrinsicID = Intrinsic::arm_crc32h;
break;
8814 case clang::ARM::BI__builtin_arm_crc32ch:
8815 CRCIntrinsicID = Intrinsic::arm_crc32ch;
break;
8816 case clang::ARM::BI__builtin_arm_crc32w:
8817 case clang::ARM::BI__builtin_arm_crc32d:
8818 CRCIntrinsicID = Intrinsic::arm_crc32w;
break;
8819 case clang::ARM::BI__builtin_arm_crc32cw:
8820 case clang::ARM::BI__builtin_arm_crc32cd:
8821 CRCIntrinsicID = Intrinsic::arm_crc32cw;
break;
8824 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
8830 if (BuiltinID == clang::ARM::BI__builtin_arm_crc32d ||
8831 BuiltinID == clang::ARM::BI__builtin_arm_crc32cd) {
8839 return Builder.CreateCall(F, {Res, Arg1b});
8844 return Builder.CreateCall(F, {Arg0, Arg1});
8848 if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
8849 BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
8850 BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
8851 BuiltinID == clang::ARM::BI__builtin_arm_wsr ||
8852 BuiltinID == clang::ARM::BI__builtin_arm_wsr64 ||
8853 BuiltinID == clang::ARM::BI__builtin_arm_wsrp) {
8856 if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
8857 BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
8858 BuiltinID == clang::ARM::BI__builtin_arm_rsrp)
8861 bool IsPointerBuiltin = BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
8862 BuiltinID == clang::ARM::BI__builtin_arm_wsrp;
8864 bool Is64Bit = BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
8865 BuiltinID == clang::ARM::BI__builtin_arm_wsr64;
8867 llvm::Type *ValueType;
8868 llvm::Type *RegisterType;
8869 if (IsPointerBuiltin) {
8872 }
else if (Is64Bit) {
8873 ValueType = RegisterType =
Int64Ty;
8875 ValueType = RegisterType =
Int32Ty;
8882 if (BuiltinID == ARM::BI__builtin_sponentry) {
8901 return P.first == BuiltinID;
8904 BuiltinID = It->second;
8908 unsigned ICEArguments = 0;
8913 auto getAlignmentValue32 = [&](
Address addr) ->
Value* {
8914 return Builder.getInt32(addr.getAlignment().getQuantity());
8921 unsigned NumArgs = E->
getNumArgs() - (HasExtraArg ? 1 : 0);
8922 for (
unsigned i = 0, e = NumArgs; i != e; i++) {
8924 switch (BuiltinID) {
8925 case NEON::BI__builtin_neon_vld1_v:
8926 case NEON::BI__builtin_neon_vld1q_v:
8927 case NEON::BI__builtin_neon_vld1q_lane_v:
8928 case NEON::BI__builtin_neon_vld1_lane_v:
8929 case NEON::BI__builtin_neon_vld1_dup_v:
8930 case NEON::BI__builtin_neon_vld1q_dup_v:
8931 case NEON::BI__builtin_neon_vst1_v:
8932 case NEON::BI__builtin_neon_vst1q_v:
8933 case NEON::BI__builtin_neon_vst1q_lane_v:
8934 case NEON::BI__builtin_neon_vst1_lane_v:
8935 case NEON::BI__builtin_neon_vst2_v:
8936 case NEON::BI__builtin_neon_vst2q_v:
8937 case NEON::BI__builtin_neon_vst2_lane_v:
8938 case NEON::BI__builtin_neon_vst2q_lane_v:
8939 case NEON::BI__builtin_neon_vst3_v:
8940 case NEON::BI__builtin_neon_vst3q_v:
8941 case NEON::BI__builtin_neon_vst3_lane_v:
8942 case NEON::BI__builtin_neon_vst3q_lane_v:
8943 case NEON::BI__builtin_neon_vst4_v:
8944 case NEON::BI__builtin_neon_vst4q_v:
8945 case NEON::BI__builtin_neon_vst4_lane_v:
8946 case NEON::BI__builtin_neon_vst4q_lane_v:
8955 switch (BuiltinID) {
8956 case NEON::BI__builtin_neon_vld2_v:
8957 case NEON::BI__builtin_neon_vld2q_v:
8958 case NEON::BI__builtin_neon_vld3_v:
8959 case NEON::BI__builtin_neon_vld3q_v:
8960 case NEON::BI__builtin_neon_vld4_v:
8961 case NEON::BI__builtin_neon_vld4q_v:
8962 case NEON::BI__builtin_neon_vld2_lane_v:
8963 case NEON::BI__builtin_neon_vld2q_lane_v:
8964 case NEON::BI__builtin_neon_vld3_lane_v:
8965 case NEON::BI__builtin_neon_vld3q_lane_v:
8966 case NEON::BI__builtin_neon_vld4_lane_v:
8967 case NEON::BI__builtin_neon_vld4q_lane_v:
8968 case NEON::BI__builtin_neon_vld2_dup_v:
8969 case NEON::BI__builtin_neon_vld2q_dup_v:
8970 case NEON::BI__builtin_neon_vld3_dup_v:
8971 case NEON::BI__builtin_neon_vld3q_dup_v:
8972 case NEON::BI__builtin_neon_vld4_dup_v:
8973 case NEON::BI__builtin_neon_vld4q_dup_v:
8985 switch (BuiltinID) {
8988 case NEON::BI__builtin_neon_vget_lane_i8:
8989 case NEON::BI__builtin_neon_vget_lane_i16:
8990 case NEON::BI__builtin_neon_vget_lane_i32:
8991 case NEON::BI__builtin_neon_vget_lane_i64:
8992 case NEON::BI__builtin_neon_vget_lane_bf16:
8993 case NEON::BI__builtin_neon_vget_lane_f32:
8994 case NEON::BI__builtin_neon_vgetq_lane_i8:
8995 case NEON::BI__builtin_neon_vgetq_lane_i16:
8996 case NEON::BI__builtin_neon_vgetq_lane_i32:
8997 case NEON::BI__builtin_neon_vgetq_lane_i64:
8998 case NEON::BI__builtin_neon_vgetq_lane_bf16:
8999 case NEON::BI__builtin_neon_vgetq_lane_f32:
9000 case NEON::BI__builtin_neon_vduph_lane_bf16:
9001 case NEON::BI__builtin_neon_vduph_laneq_bf16:
9002 return Builder.CreateExtractElement(Ops[0], Ops[1],
"vget_lane");
9004 case NEON::BI__builtin_neon_vrndns_f32: {
9006 llvm::Type *Tys[] = {Arg->
getType()};
9008 return Builder.CreateCall(F, {Arg},
"vrndn"); }
9010 case NEON::BI__builtin_neon_vset_lane_i8:
9011 case NEON::BI__builtin_neon_vset_lane_i16:
9012 case NEON::BI__builtin_neon_vset_lane_i32:
9013 case NEON::BI__builtin_neon_vset_lane_i64:
9014 case NEON::BI__builtin_neon_vset_lane_bf16:
9015 case NEON::BI__builtin_neon_vset_lane_f32:
9016 case NEON::BI__builtin_neon_vsetq_lane_i8:
9017 case NEON::BI__builtin_neon_vsetq_lane_i16:
9018 case NEON::BI__builtin_neon_vsetq_lane_i32:
9019 case NEON::BI__builtin_neon_vsetq_lane_i64:
9020 case NEON::BI__builtin_neon_vsetq_lane_bf16:
9021 case NEON::BI__builtin_neon_vsetq_lane_f32:
9022 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
9024 case NEON::BI__builtin_neon_vsha1h_u32:
9027 case NEON::BI__builtin_neon_vsha1cq_u32:
9030 case NEON::BI__builtin_neon_vsha1pq_u32:
9033 case NEON::BI__builtin_neon_vsha1mq_u32:
9037 case NEON::BI__builtin_neon_vcvth_bf16_f32: {
9044 case clang::ARM::BI_MoveToCoprocessor:
9045 case clang::ARM::BI_MoveToCoprocessor2: {
9047 ? Intrinsic::arm_mcr
9048 : Intrinsic::arm_mcr2);
9049 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0],
9050 Ops[3], Ops[4], Ops[5]});
9055 assert(HasExtraArg);
9057 std::optional<llvm::APSInt>
Result =
9062 if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f ||
9063 BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_d) {
9066 if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f)
9072 bool usgn =
Result->getZExtValue() == 1;
9073 unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr;
9077 return Builder.CreateCall(F, Ops,
"vcvtr");
9082 bool usgn =
Type.isUnsigned();
9083 bool rightShift =
false;
9085 llvm::FixedVectorType *VTy =
9088 llvm::Type *Ty = VTy;
9099 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
9100 Builtin->NameHint, Builtin->TypeModifier, E, Ops, PtrOp0, PtrOp1, Arch);
9103 switch (BuiltinID) {
9104 default:
return nullptr;
9105 case NEON::BI__builtin_neon_vld1q_lane_v:
9108 if (VTy->getElementType()->isIntegerTy(64)) {
9110 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
9111 int Lane = cast<ConstantInt>(Ops[2])->getZExtValue();
9112 Value *SV = llvm::ConstantVector::get(ConstantInt::get(
Int32Ty, 1-Lane));
9113 Ops[1] =
Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
9115 Ty = llvm::FixedVectorType::get(VTy->getElementType(), 1);
9118 Value *Align = getAlignmentValue32(PtrOp0);
9121 int Indices[] = {1 - Lane, Lane};
9122 return Builder.CreateShuffleVector(Ops[1], Ld, Indices,
"vld1q_lane");
9125 case NEON::BI__builtin_neon_vld1_lane_v: {
9126 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
9129 return Builder.CreateInsertElement(Ops[1], Ld, Ops[2],
"vld1_lane");
9131 case NEON::BI__builtin_neon_vqrshrn_n_v:
9133 usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns;
9136 case NEON::BI__builtin_neon_vqrshrun_n_v:
9138 Ops,
"vqrshrun_n", 1,
true);
9139 case NEON::BI__builtin_neon_vqshrn_n_v:
9140 Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns;
9143 case NEON::BI__builtin_neon_vqshrun_n_v:
9145 Ops,
"vqshrun_n", 1,
true);
9146 case NEON::BI__builtin_neon_vrecpe_v:
9147 case NEON::BI__builtin_neon_vrecpeq_v:
9150 case NEON::BI__builtin_neon_vrshrn_n_v:
9152 Ops,
"vrshrn_n", 1,
true);
9153 case NEON::BI__builtin_neon_vrsra_n_v:
9154 case NEON::BI__builtin_neon_vrsraq_n_v:
9155 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
9156 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
9158 Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
9160 return Builder.CreateAdd(Ops[0], Ops[1],
"vrsra_n");
9161 case NEON::BI__builtin_neon_vsri_n_v:
9162 case NEON::BI__builtin_neon_vsriq_n_v:
9165 case NEON::BI__builtin_neon_vsli_n_v:
9166 case NEON::BI__builtin_neon_vsliq_n_v:
9170 case NEON::BI__builtin_neon_vsra_n_v:
9171 case NEON::BI__builtin_neon_vsraq_n_v:
9172 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
9174 return Builder.CreateAdd(Ops[0], Ops[1]);
9175 case NEON::BI__builtin_neon_vst1q_lane_v:
9178 if (VTy->getElementType()->isIntegerTy(64)) {
9179 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
9180 Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2]));
9181 Ops[1] =
Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
9182 Ops[2] = getAlignmentValue32(PtrOp0);
9183 llvm::Type *Tys[] = {
Int8PtrTy, Ops[1]->getType()};
9188 case NEON::BI__builtin_neon_vst1_lane_v: {
9189 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
9190 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2]);
9194 case NEON::BI__builtin_neon_vtbl1_v:
9197 case NEON::BI__builtin_neon_vtbl2_v:
9200 case NEON::BI__builtin_neon_vtbl3_v:
9203 case NEON::BI__builtin_neon_vtbl4_v:
9206 case NEON::BI__builtin_neon_vtbx1_v:
9209 case NEON::BI__builtin_neon_vtbx2_v:
9212 case NEON::BI__builtin_neon_vtbx3_v:
9215 case NEON::BI__builtin_neon_vtbx4_v:
9221template<
typename Integer>
9230 return Unsigned ? Builder.CreateZExt(
V,
T) : Builder.CreateSExt(
V,
T);
9240 unsigned LaneBits = cast<llvm::VectorType>(
V->getType())
9242 ->getPrimitiveSizeInBits();
9243 if (Shift == LaneBits) {
9248 return llvm::Constant::getNullValue(
V->getType());
9252 return Unsigned ? Builder.CreateLShr(
V, Shift) : Builder.CreateAShr(
V, Shift);
9259 unsigned Elements = 128 /
V->getType()->getPrimitiveSizeInBits();
9260 return Builder.CreateVectorSplat(Elements,
V);
9266 llvm::Type *DestType) {
9279 V->getType()->getScalarSizeInBits() != DestType->getScalarSizeInBits()) {
9280 return Builder.CreateCall(
9282 {DestType, V->getType()}),
9285 return Builder.CreateBitCast(
V, DestType);
9293 unsigned InputElements =
9294 cast<llvm::FixedVectorType>(
V->getType())->getNumElements();
9295 for (
unsigned i = 0; i < InputElements; i += 2)
9296 Indices.push_back(i + Odd);
9297 return Builder.CreateShuffleVector(
V, Indices);
9303 assert(V0->getType() == V1->getType() &&
"Can't zip different vector types");
9305 unsigned InputElements =
9306 cast<llvm::FixedVectorType>(V0->getType())->getNumElements();
9307 for (
unsigned i = 0; i < InputElements; i++) {
9308 Indices.push_back(i);
9309 Indices.push_back(i + InputElements);
9311 return Builder.CreateShuffleVector(V0, V1, Indices);
9314template<
unsigned HighBit,
unsigned OtherBits>
9318 llvm::Type *
T = cast<llvm::VectorType>(VT)->getElementType();
9319 unsigned LaneBits =
T->getPrimitiveSizeInBits();
9320 uint32_t
Value = HighBit << (LaneBits - 1);
9322 Value |= (1UL << (LaneBits - 1)) - 1;
9323 llvm::Value *Lane = llvm::ConstantInt::get(
T,
Value);
9329 unsigned ReverseWidth) {
9333 unsigned LaneSize =
V->getType()->getScalarSizeInBits();
9334 unsigned Elements = 128 / LaneSize;
9335 unsigned Mask = ReverseWidth / LaneSize - 1;
9336 for (
unsigned i = 0; i < Elements; i++)
9337 Indices.push_back(i ^ Mask);
9338 return Builder.CreateShuffleVector(
V, Indices);
9344 llvm::Triple::ArchType Arch) {
9345 enum class CustomCodeGen { VLD24, VST24 } CustomCodeGenType;
9346 Intrinsic::ID IRIntr;
9347 unsigned NumVectors;
9350 switch (BuiltinID) {
9351 #include "clang/Basic/arm_mve_builtin_cg.inc"
9362 switch (CustomCodeGenType) {
9364 case CustomCodeGen::VLD24: {
9368 auto MvecCType = E->
getType();
9370 assert(MvecLType->isStructTy() &&
9371 "Return type for vld[24]q should be a struct");
9372 assert(MvecLType->getStructNumElements() == 1 &&
9373 "Return-type struct for vld[24]q should have one element");
9374 auto MvecLTypeInner = MvecLType->getStructElementType(0);
9375 assert(MvecLTypeInner->isArrayTy() &&
9376 "Return-type struct for vld[24]q should contain an array");
9377 assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
9378 "Array member of return-type struct vld[24]q has wrong length");
9379 auto VecLType = MvecLTypeInner->getArrayElementType();
9381 Tys.push_back(VecLType);
9383 auto Addr = E->
getArg(0);
9389 Value *MvecOut = PoisonValue::get(MvecLType);
9390 for (
unsigned i = 0; i < NumVectors; ++i) {
9391 Value *Vec =
Builder.CreateExtractValue(LoadResult, i);
9392 MvecOut =
Builder.CreateInsertValue(MvecOut, Vec, {0, i});
9401 case CustomCodeGen::VST24: {
9405 auto Addr = E->
getArg(0);
9411 assert(MvecLType->isStructTy() &&
"Data type for vst2q should be a struct");
9412 assert(MvecLType->getStructNumElements() == 1 &&
9413 "Data-type struct for vst2q should have one element");
9414 auto MvecLTypeInner = MvecLType->getStructElementType(0);
9415 assert(MvecLTypeInner->isArrayTy() &&
9416 "Data-type struct for vst2q should contain an array");
9417 assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
9418 "Array member of return-type struct vld[24]q has wrong length");
9419 auto VecLType = MvecLTypeInner->getArrayElementType();
9421 Tys.push_back(VecLType);
9426 for (
unsigned i = 0; i < NumVectors; i++)
9427 Ops.push_back(
Builder.CreateExtractValue(Mvec, {0, i}));
9430 Value *ToReturn =
nullptr;
9431 for (
unsigned i = 0; i < NumVectors; i++) {
9432 Ops.push_back(llvm::ConstantInt::get(
Int32Ty, i));
9433 ToReturn =
Builder.CreateCall(F, Ops);
9439 llvm_unreachable(
"unknown custom codegen type.");
9445 llvm::Triple::ArchType Arch) {
9446 switch (BuiltinID) {
9449#include "clang/Basic/arm_cde_builtin_cg.inc"
9456 llvm::Triple::ArchType Arch) {
9457 unsigned int Int = 0;
9458 const char *
s =
nullptr;
9460 switch (BuiltinID) {
9463 case NEON::BI__builtin_neon_vtbl1_v:
9464 case NEON::BI__builtin_neon_vqtbl1_v:
9465 case NEON::BI__builtin_neon_vqtbl1q_v:
9466 case NEON::BI__builtin_neon_vtbl2_v:
9467 case NEON::BI__builtin_neon_vqtbl2_v:
9468 case NEON::BI__builtin_neon_vqtbl2q_v:
9469 case NEON::BI__builtin_neon_vtbl3_v:
9470 case NEON::BI__builtin_neon_vqtbl3_v:
9471 case NEON::BI__builtin_neon_vqtbl3q_v:
9472 case NEON::BI__builtin_neon_vtbl4_v:
9473 case NEON::BI__builtin_neon_vqtbl4_v:
9474 case NEON::BI__builtin_neon_vqtbl4q_v:
9476 case NEON::BI__builtin_neon_vtbx1_v:
9477 case NEON::BI__builtin_neon_vqtbx1_v:
9478 case NEON::BI__builtin_neon_vqtbx1q_v:
9479 case NEON::BI__builtin_neon_vtbx2_v:
9480 case NEON::BI__builtin_neon_vqtbx2_v:
9481 case NEON::BI__builtin_neon_vqtbx2q_v:
9482 case NEON::BI__builtin_neon_vtbx3_v:
9483 case NEON::BI__builtin_neon_vqtbx3_v:
9484 case NEON::BI__builtin_neon_vqtbx3q_v:
9485 case NEON::BI__builtin_neon_vtbx4_v:
9486 case NEON::BI__builtin_neon_vqtbx4_v:
9487 case NEON::BI__builtin_neon_vqtbx4q_v:
9495 std::optional<llvm::APSInt>
Result =
9510 switch (BuiltinID) {
9511 case NEON::BI__builtin_neon_vtbl1_v: {
9513 Ty, Intrinsic::aarch64_neon_tbl1,
"vtbl1");
9515 case NEON::BI__builtin_neon_vtbl2_v: {
9517 Ty, Intrinsic::aarch64_neon_tbl1,
"vtbl1");
9519 case NEON::BI__builtin_neon_vtbl3_v: {
9521 Ty, Intrinsic::aarch64_neon_tbl2,
"vtbl2");
9523 case NEON::BI__builtin_neon_vtbl4_v: {
9525 Ty, Intrinsic::aarch64_neon_tbl2,
"vtbl2");
9527 case NEON::BI__builtin_neon_vtbx1_v: {
9530 Intrinsic::aarch64_neon_tbl1,
"vtbl1");
9532 llvm::Constant *EightV = ConstantInt::get(Ty, 8);
9533 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV);
9534 CmpRes = Builder.CreateSExt(CmpRes, Ty);
9536 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
9537 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
9538 return Builder.CreateOr(EltsFromInput, EltsFromTbl,
"vtbx");
9540 case NEON::BI__builtin_neon_vtbx2_v: {
9542 Ty, Intrinsic::aarch64_neon_tbx1,
"vtbx1");
9544 case NEON::BI__builtin_neon_vtbx3_v: {
9547 Intrinsic::aarch64_neon_tbl2,
"vtbl2");
9549 llvm::Constant *TwentyFourV = ConstantInt::get(Ty, 24);
9550 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4],
9552 CmpRes = Builder.CreateSExt(CmpRes, Ty);
9554 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
9555 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
9556 return Builder.CreateOr(EltsFromInput, EltsFromTbl,
"vtbx");
9558 case NEON::BI__builtin_neon_vtbx4_v: {
9560 Ty, Intrinsic::aarch64_neon_tbx2,
"vtbx2");
9562 case NEON::BI__builtin_neon_vqtbl1_v:
9563 case NEON::BI__builtin_neon_vqtbl1q_v:
9564 Int = Intrinsic::aarch64_neon_tbl1;
s =
"vtbl1";
break;
9565 case NEON::BI__builtin_neon_vqtbl2_v:
9566 case NEON::BI__builtin_neon_vqtbl2q_v: {
9567 Int = Intrinsic::aarch64_neon_tbl2;
s =
"vtbl2";
break;
9568 case NEON::BI__builtin_neon_vqtbl3_v:
9569 case NEON::BI__builtin_neon_vqtbl3q_v:
9570 Int = Intrinsic::aarch64_neon_tbl3;
s =
"vtbl3";
break;
9571 case NEON::BI__builtin_neon_vqtbl4_v:
9572 case NEON::BI__builtin_neon_vqtbl4q_v:
9573 Int = Intrinsic::aarch64_neon_tbl4;
s =
"vtbl4";
break;
9574 case NEON::BI__builtin_neon_vqtbx1_v:
9575 case NEON::BI__builtin_neon_vqtbx1q_v:
9576 Int = Intrinsic::aarch64_neon_tbx1;
s =
"vtbx1";
break;
9577 case NEON::BI__builtin_neon_vqtbx2_v:
9578 case NEON::BI__builtin_neon_vqtbx2q_v:
9579 Int = Intrinsic::aarch64_neon_tbx2;
s =
"vtbx2";
break;
9580 case NEON::BI__builtin_neon_vqtbx3_v:
9581 case NEON::BI__builtin_neon_vqtbx3q_v:
9582 Int = Intrinsic::aarch64_neon_tbx3;
s =
"vtbx3";
break;
9583 case NEON::BI__builtin_neon_vqtbx4_v:
9584 case NEON::BI__builtin_neon_vqtbx4q_v:
9585 Int = Intrinsic::aarch64_neon_tbx4;
s =
"vtbx4";
break;
9597 auto *VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
9599 Value *
V = PoisonValue::get(VTy);
9600 llvm::Constant *CI = ConstantInt::get(
SizeTy, 0);
9601 Op =
Builder.CreateInsertElement(
V, Op, CI);
9610 case SVETypeFlags::MemEltTyDefault:
9612 case SVETypeFlags::MemEltTyInt8:
9614 case SVETypeFlags::MemEltTyInt16:
9616 case SVETypeFlags::MemEltTyInt32:
9618 case SVETypeFlags::MemEltTyInt64:
9621 llvm_unreachable(
"Unknown MemEltType");
9627 llvm_unreachable(
"Invalid SVETypeFlag!");
9629 case SVETypeFlags::EltTyInt8:
9631 case SVETypeFlags::EltTyInt16:
9633 case SVETypeFlags::EltTyInt32:
9635 case SVETypeFlags::EltTyInt64:
9637 case SVETypeFlags::EltTyInt128:
9640 case SVETypeFlags::EltTyFloat16:
9642 case SVETypeFlags::EltTyFloat32:
9644 case SVETypeFlags::EltTyFloat64:
9647 case SVETypeFlags::EltTyBFloat16:
9650 case SVETypeFlags::EltTyBool8:
9651 case SVETypeFlags::EltTyBool16:
9652 case SVETypeFlags::EltTyBool32:
9653 case SVETypeFlags::EltTyBool64:
9660llvm::ScalableVectorType *
9663 default: llvm_unreachable(
"Unhandled SVETypeFlag!");
9665 case SVETypeFlags::EltTyInt8:
9666 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 16);
9667 case SVETypeFlags::EltTyInt16:
9668 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 8);
9669 case SVETypeFlags::EltTyInt32:
9670 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 4);
9671 case SVETypeFlags::EltTyInt64:
9672 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 2);
9674 case SVETypeFlags::EltTyBFloat16:
9675 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 8);
9676 case SVETypeFlags::EltTyFloat16:
9677 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 8);
9678 case SVETypeFlags::EltTyFloat32:
9679 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 4);
9680 case SVETypeFlags::EltTyFloat64:
9681 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 2);
9683 case SVETypeFlags::EltTyBool8:
9684 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 16);
9685 case SVETypeFlags::EltTyBool16:
9686 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 8);
9687 case SVETypeFlags::EltTyBool32:
9688 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 4);
9689 case SVETypeFlags::EltTyBool64:
9690 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 2);
9695llvm::ScalableVectorType *
9699 llvm_unreachable(
"Invalid SVETypeFlag!");
9701 case SVETypeFlags::EltTyInt8:
9702 return llvm::ScalableVectorType::get(
Builder.getInt8Ty(), 16);
9703 case SVETypeFlags::EltTyInt16:
9704 return llvm::ScalableVectorType::get(
Builder.getInt16Ty(), 8);
9705 case SVETypeFlags::EltTyInt32:
9706 return llvm::ScalableVectorType::get(
Builder.getInt32Ty(), 4);
9707 case SVETypeFlags::EltTyInt64:
9708 return llvm::ScalableVectorType::get(
Builder.getInt64Ty(), 2);
9710 case SVETypeFlags::EltTyFloat16:
9711 return llvm::ScalableVectorType::get(
Builder.getHalfTy(), 8);
9712 case SVETypeFlags::EltTyBFloat16:
9713 return llvm::ScalableVectorType::get(
Builder.getBFloatTy(), 8);
9714 case SVETypeFlags::EltTyFloat32:
9715 return llvm::ScalableVectorType::get(
Builder.getFloatTy(), 4);
9716 case SVETypeFlags::EltTyFloat64:
9717 return llvm::ScalableVectorType::get(
Builder.getDoubleTy(), 2);
9719 case SVETypeFlags::EltTyBool8:
9720 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 16);
9721 case SVETypeFlags::EltTyBool16:
9722 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 8);
9723 case SVETypeFlags::EltTyBool32:
9724 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 4);
9725 case SVETypeFlags::EltTyBool64:
9726 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 2);
9741 return llvm::ScalableVectorType::get(EltTy, NumElts);
9747 llvm::ScalableVectorType *VTy) {
9749 if (isa<TargetExtType>(Pred->
getType()) &&
9750 cast<TargetExtType>(Pred->
getType())->getName() ==
"aarch64.svcount")
9753 auto *RTy = llvm::VectorType::get(IntegerType::get(
getLLVMContext(), 1), VTy);
9758 llvm::Type *IntrinsicTy;
9759 switch (VTy->getMinNumElements()) {
9761 llvm_unreachable(
"unsupported element count!");
9766 IntID = Intrinsic::aarch64_sve_convert_from_svbool;
9770 IntID = Intrinsic::aarch64_sve_convert_to_svbool;
9771 IntrinsicTy = Pred->
getType();
9777 assert(
C->getType() == RTy &&
"Unexpected return type!");
9785 auto *OverloadedTy =
9789 if (Ops[1]->getType()->isVectorTy())
9809 Ops[0], cast<llvm::ScalableVectorType>(F->getArg(0)->getType()));
9814 if (Ops.size() == 2) {
9815 assert(Ops[1]->getType()->isVectorTy() &&
"Scalar base requires an offset");
9816 Ops.push_back(ConstantInt::get(
Int64Ty, 0));
9821 if (!TypeFlags.
isByteIndexed() && Ops[1]->getType()->isVectorTy()) {
9822 unsigned BytesPerElt =
9823 OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
9824 Ops[2] =
Builder.CreateShl(Ops[2], Log2_32(BytesPerElt));
9839 auto *OverloadedTy =
9844 Ops.insert(Ops.begin(), Ops.pop_back_val());
9847 if (Ops[2]->getType()->isVectorTy())
9862 if (Ops.size() == 3) {
9863 assert(Ops[1]->getType()->isVectorTy() &&
"Scalar base requires an offset");
9864 Ops.push_back(ConstantInt::get(
Int64Ty, 0));
9869 Ops[0] =
Builder.CreateTrunc(Ops[0], OverloadedTy);
9879 Ops[1], cast<llvm::ScalableVectorType>(F->getArg(1)->getType()));
9883 if (!TypeFlags.
isByteIndexed() && Ops[2]->getType()->isVectorTy()) {
9884 unsigned BytesPerElt =
9885 OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
9886 Ops[3] =
Builder.CreateShl(Ops[3], Log2_32(BytesPerElt));
9889 return Builder.CreateCall(F, Ops);
9897 auto *OverloadedTy = dyn_cast<llvm::ScalableVectorType>(Ops[1]->getType());
9899 OverloadedTy = cast<llvm::ScalableVectorType>(Ops[2]->getType());
9905 if (Ops[1]->getType()->isVectorTy()) {
9906 if (Ops.size() == 3) {
9908 Ops.push_back(ConstantInt::get(
Int64Ty, 0));
9911 std::swap(Ops[2], Ops[3]);
9915 unsigned BytesPerElt = MemEltTy->getPrimitiveSizeInBits() / 8;
9916 if (BytesPerElt > 1)
9917 Ops[2] =
Builder.CreateShl(Ops[2], Log2_32(BytesPerElt));
9922 return Builder.CreateCall(F, Ops);
9928 llvm::ScalableVectorType *VTy =
getSVEType(TypeFlags);
9932 case Intrinsic::aarch64_sve_ld2_sret:
9933 case Intrinsic::aarch64_sve_ld1_pn_x2:
9934 case Intrinsic::aarch64_sve_ldnt1_pn_x2:
9935 case Intrinsic::aarch64_sve_ld2q_sret:
9938 case Intrinsic::aarch64_sve_ld3_sret:
9939 case Intrinsic::aarch64_sve_ld3q_sret:
9942 case Intrinsic::aarch64_sve_ld4_sret:
9943 case Intrinsic::aarch64_sve_ld1_pn_x4:
9944 case Intrinsic::aarch64_sve_ldnt1_pn_x4:
9945 case Intrinsic::aarch64_sve_ld4q_sret:
9949 llvm_unreachable(
"unknown intrinsic!");
9951 auto RetTy = llvm::VectorType::get(VTy->getElementType(),
9952 VTy->getElementCount() * N);
9955 Value *BasePtr = Ops[1];
9963 unsigned MinElts = VTy->getMinNumElements();
9964 Value *
Ret = llvm::PoisonValue::get(RetTy);
9965 for (
unsigned I = 0; I < N; I++) {
9968 Ret =
Builder.CreateInsertVector(RetTy, Ret, SRet, Idx);
9976 llvm::ScalableVectorType *VTy =
getSVEType(TypeFlags);
9980 case Intrinsic::aarch64_sve_st2:
9981 case Intrinsic::aarch64_sve_st1_pn_x2:
9982 case Intrinsic::aarch64_sve_stnt1_pn_x2:
9983 case Intrinsic::aarch64_sve_st2q:
9986 case Intrinsic::aarch64_sve_st3:
9987 case Intrinsic::aarch64_sve_st3q:
9990 case Intrinsic::aarch64_sve_st4:
9991 case Intrinsic::aarch64_sve_st1_pn_x4:
9992 case Intrinsic::aarch64_sve_stnt1_pn_x4:
9993 case Intrinsic::aarch64_sve_st4q:
9997 llvm_unreachable(
"unknown intrinsic!");
10001 Value *BasePtr = Ops[1];
10004 if (Ops.size() > (2 + N))
10010 for (
unsigned I = Ops.size() - N; I < Ops.size(); ++I)
10011 Operands.push_back(Ops[I]);
10012 Operands.append({Predicate, BasePtr});
10015 return Builder.CreateCall(F, Operands);
10023 unsigned BuiltinID) {
10035 llvm::ScalableVectorType *Ty =
getSVEType(TypeFlags);
10041 llvm::Type *OverloadedTy =
getSVEType(TypeFlags);
10048 unsigned BuiltinID) {
10051 auto *MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
10054 Value *BasePtr = Ops[1];
10057 if (Ops.size() > 3)
10060 Value *PrfOp = Ops.back();
10063 return Builder.CreateCall(F, {Predicate, BasePtr, PrfOp});
10067 llvm::Type *ReturnTy,
10069 unsigned IntrinsicID,
10070 bool IsZExtReturn) {
10077 auto VectorTy = cast<llvm::ScalableVectorType>(ReturnTy);
10078 llvm::ScalableVectorType *MemoryTy =
nullptr;
10079 llvm::ScalableVectorType *PredTy =
nullptr;
10080 bool IsQuadLoad =
false;
10081 switch (IntrinsicID) {
10082 case Intrinsic::aarch64_sve_ld1uwq:
10083 case Intrinsic::aarch64_sve_ld1udq:
10084 MemoryTy = llvm::ScalableVectorType::get(MemEltTy, 1);
10085 PredTy = llvm::ScalableVectorType::get(
10090 MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
10096 Value *BasePtr = Ops[1];
10099 if (Ops.size() > 2)
10104 cast<llvm::Instruction>(
Builder.CreateCall(F, {Predicate, BasePtr}));
10111 return IsZExtReturn ?
Builder.CreateZExt(Load, VectorTy)
10112 :
Builder.CreateSExt(Load, VectorTy);
10117 unsigned IntrinsicID) {
10124 auto VectorTy = cast<llvm::ScalableVectorType>(Ops.back()->getType());
10125 auto MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
10127 auto PredTy = MemoryTy;
10128 auto AddrMemoryTy = MemoryTy;
10129 bool IsQuadStore =
false;
10131 switch (IntrinsicID) {
10132 case Intrinsic::aarch64_sve_st1wq:
10133 case Intrinsic::aarch64_sve_st1dq:
10134 AddrMemoryTy = llvm::ScalableVectorType::get(MemEltTy, 1);
10136 llvm::ScalableVectorType::get(IntegerType::get(
getLLVMContext(), 1), 1);
10137 IsQuadStore =
true;
10143 Value *BasePtr = Ops[1];
10146 if (Ops.size() == 4)
10151 IsQuadStore ? Ops.back() :
Builder.CreateTrunc(Ops.back(), MemoryTy);
10156 cast<llvm::Instruction>(
Builder.CreateCall(F, {Val, Predicate, BasePtr}));
10169 NewOps.push_back(Ops[2]);
10171 llvm::Value *BasePtr = Ops[3];
10175 if (Ops.size() == 5) {
10178 llvm::Value *StreamingVectorLengthCall =
10179 Builder.CreateCall(StreamingVectorLength);
10180 llvm::Value *Mulvl =
10181 Builder.CreateMul(StreamingVectorLengthCall, Ops[4],
"mulvl");
10185 NewOps.push_back(BasePtr);
10186 NewOps.push_back(Ops[0]);
10187 NewOps.push_back(Ops[1]);
10189 return Builder.CreateCall(F, NewOps);
10201 return Builder.CreateCall(F, Ops);
10208 if (Ops.size() == 0)
10209 Ops.push_back(llvm::ConstantInt::get(
Int32Ty, 255));
10211 return Builder.CreateCall(F, Ops);
10217 if (Ops.size() == 2)
10218 Ops.push_back(
Builder.getInt32(0));
10222 return Builder.CreateCall(F, Ops);
10228 return Builder.CreateVectorSplat(
10229 cast<llvm::VectorType>(Ty)->getElementCount(), Scalar);
10243 return Builder.CreateBitCast(Val, Ty);
10248 auto *SplatZero = Constant::getNullValue(Ty);
10249 Ops.insert(Ops.begin(), SplatZero);
10254 auto *SplatUndef = UndefValue::get(Ty);
10255 Ops.insert(Ops.begin(), SplatUndef);
10260 llvm::Type *ResultType,
10265 llvm::Type *DefaultType =
getSVEType(TypeFlags);
10268 return {DefaultType, Ops[1]->getType()};
10274 return {Ops[0]->getType(), Ops.back()->getType()};
10276 if (TypeFlags.
isReductionQV() && !ResultType->isScalableTy() &&
10277 ResultType->isVectorTy())
10278 return {ResultType, Ops[1]->getType()};
10281 return {DefaultType};
10288 "Expects TypleFlag isTupleSet or TypeFlags.isTupleSet()");
10290 unsigned I = cast<ConstantInt>(Ops[1])->getSExtValue();
10291 auto *SingleVecTy = dyn_cast<llvm::ScalableVectorType>(
10292 TypeFlags.
isTupleSet() ? Ops[2]->getType() : Ty);
10294 I * SingleVecTy->getMinNumElements());
10297 return Builder.CreateInsertVector(Ty, Ops[0], Ops[2], Idx);
10298 return Builder.CreateExtractVector(Ty, Ops[0], Idx);
10304 assert(TypeFlags.
isTupleCreate() &&
"Expects TypleFlag isTupleCreate");
10306 auto *SrcTy = dyn_cast<llvm::ScalableVectorType>(Ops[0]->getType());
10307 unsigned MinElts = SrcTy->getMinNumElements();
10308 Value *
Call = llvm::PoisonValue::get(Ty);
10309 for (
unsigned I = 0; I < Ops.size(); I++) {
10320 auto *StructTy = dyn_cast<StructType>(
Call->getType());
10324 auto *VTy = dyn_cast<ScalableVectorType>(StructTy->getTypeAtIndex(0
U));
10327 unsigned N = StructTy->getNumElements();
10330 bool IsPredTy = VTy->getElementType()->isIntegerTy(1);
10331 unsigned MinElts = IsPredTy ? 16 : VTy->getMinNumElements();
10333 ScalableVectorType *WideVTy =
10334 ScalableVectorType::get(VTy->getElementType(), MinElts * N);
10335 Value *
Ret = llvm::PoisonValue::get(WideVTy);
10336 for (
unsigned I = 0; I < N; ++I) {
10338 assert(SRet->
getType() == VTy &&
"Unexpected type for result value");
10343 SRet, ScalableVectorType::get(
Builder.getInt1Ty(), 16));
10345 Ret =
Builder.CreateInsertVector(WideVTy, Ret, SRet, Idx);
10356 unsigned ICEArguments = 0;
10365 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; i++) {
10366 bool IsICE = ICEArguments & (1 << i);
10372 std::optional<llvm::APSInt>
Result =
10374 assert(
Result &&
"Expected argument to be a constant");
10384 if (IsTupleGetOrSet || !isa<ScalableVectorType>(Arg->getType())) {
10385 Ops.push_back(Arg);
10389 auto *VTy = cast<ScalableVectorType>(Arg->getType());
10390 unsigned MinElts = VTy->getMinNumElements();
10391 bool IsPred = VTy->getElementType()->isIntegerTy(1);
10392 unsigned N = (MinElts * VTy->getScalarSizeInBits()) / (IsPred ? 16 : 128);
10395 Ops.push_back(Arg);
10399 for (
unsigned I = 0; I < N; ++I) {
10402 ScalableVectorType::get(VTy->getElementType(), MinElts / N);
10403 Ops.push_back(
Builder.CreateExtractVector(NewVTy, Arg, Idx));
10411 if (BuiltinID >= SVE::BI__builtin_sve_reinterpret_s8_s8 &&
10412 BuiltinID <= SVE::BI__builtin_sve_reinterpret_f64_f64_x4) {
10427 else if (TypeFlags.
isStore())
10445 else if (TypeFlags.
isUndef())
10446 return UndefValue::get(Ty);
10447 else if (Builtin->LLVMIntrinsic != 0) {
10448 if (TypeFlags.
getMergeType() == SVETypeFlags::MergeZeroExp)
10451 if (TypeFlags.
getMergeType() == SVETypeFlags::MergeAnyExp)
10457 Ops.push_back(
Builder.getInt32( 31));
10459 Ops.insert(&Ops[1],
Builder.getInt32( 31));
10462 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
10463 if (
auto PredTy = dyn_cast<llvm::VectorType>(Ops[i]->getType()))
10464 if (PredTy->getElementType()->isIntegerTy(1))
10474 std::swap(Ops[1], Ops[2]);
10476 std::swap(Ops[1], Ops[2]);
10479 std::swap(Ops[1], Ops[2]);
10482 std::swap(Ops[1], Ops[3]);
10485 if (TypeFlags.
getMergeType() == SVETypeFlags::MergeZero) {
10486 llvm::Type *OpndTy = Ops[1]->getType();
10487 auto *SplatZero = Constant::getNullValue(OpndTy);
10488 Ops[1] =
Builder.CreateSelect(Ops[0], Ops[1], SplatZero);
10496 if (
auto PredTy = dyn_cast<llvm::VectorType>(
Call->getType()))
10497 if (PredTy->getScalarType()->isIntegerTy(1))
10503 switch (BuiltinID) {
10507 case SVE::BI__builtin_sve_svreinterpret_b: {
10511 CGM.
getIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool, SVCountTy);
10512 return Builder.CreateCall(CastFromSVCountF, Ops[0]);
10514 case SVE::BI__builtin_sve_svreinterpret_c: {
10518 CGM.
getIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool, SVCountTy);
10519 return Builder.CreateCall(CastToSVCountF, Ops[0]);
10522 case SVE::BI__builtin_sve_svpsel_lane_b8:
10523 case SVE::BI__builtin_sve_svpsel_lane_b16:
10524 case SVE::BI__builtin_sve_svpsel_lane_b32:
10525 case SVE::BI__builtin_sve_svpsel_lane_b64:
10526 case SVE::BI__builtin_sve_svpsel_lane_c8:
10527 case SVE::BI__builtin_sve_svpsel_lane_c16:
10528 case SVE::BI__builtin_sve_svpsel_lane_c32:
10529 case SVE::BI__builtin_sve_svpsel_lane_c64: {
10530 bool IsSVCount = isa<TargetExtType>(Ops[0]->getType());
10531 assert(((!IsSVCount || cast<TargetExtType>(Ops[0]->getType())->
getName() ==
10532 "aarch64.svcount")) &&
10533 "Unexpected TargetExtType");
10537 CGM.
getIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool, SVCountTy);
10539 CGM.
getIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool, SVCountTy);
10543 llvm::Value *Ops0 =
10544 IsSVCount ?
Builder.CreateCall(CastFromSVCountF, Ops[0]) : Ops[0];
10546 llvm::Value *PSel =
Builder.CreateCall(F, {Ops0, Ops1, Ops[2]});
10547 return IsSVCount ?
Builder.CreateCall(CastToSVCountF, PSel) : PSel;
10549 case SVE::BI__builtin_sve_svmov_b_z: {
10552 llvm::Type* OverloadedTy =
getSVEType(TypeFlags);
10554 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[1]});
10557 case SVE::BI__builtin_sve_svnot_b_z: {
10560 llvm::Type* OverloadedTy =
getSVEType(TypeFlags);
10562 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[0]});
10565 case SVE::BI__builtin_sve_svmovlb_u16:
10566 case SVE::BI__builtin_sve_svmovlb_u32:
10567 case SVE::BI__builtin_sve_svmovlb_u64:
10568 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllb);
10570 case SVE::BI__builtin_sve_svmovlb_s16:
10571 case SVE::BI__builtin_sve_svmovlb_s32:
10572 case SVE::BI__builtin_sve_svmovlb_s64:
10573 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllb);
10575 case SVE::BI__builtin_sve_svmovlt_u16:
10576 case SVE::BI__builtin_sve_svmovlt_u32:
10577 case SVE::BI__builtin_sve_svmovlt_u64:
10578 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllt);
10580 case SVE::BI__builtin_sve_svmovlt_s16:
10581 case SVE::BI__builtin_sve_svmovlt_s32:
10582 case SVE::BI__builtin_sve_svmovlt_s64:
10583 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllt);
10585 case SVE::BI__builtin_sve_svpmullt_u16:
10586 case SVE::BI__builtin_sve_svpmullt_u64:
10587 case SVE::BI__builtin_sve_svpmullt_n_u16:
10588 case SVE::BI__builtin_sve_svpmullt_n_u64:
10589 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullt_pair);
10591 case SVE::BI__builtin_sve_svpmullb_u16:
10592 case SVE::BI__builtin_sve_svpmullb_u64:
10593 case SVE::BI__builtin_sve_svpmullb_n_u16:
10594 case SVE::BI__builtin_sve_svpmullb_n_u64:
10595 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullb_pair);
10597 case SVE::BI__builtin_sve_svdup_n_b8:
10598 case SVE::BI__builtin_sve_svdup_n_b16:
10599 case SVE::BI__builtin_sve_svdup_n_b32:
10600 case SVE::BI__builtin_sve_svdup_n_b64: {
10602 Builder.CreateICmpNE(Ops[0], Constant::getNullValue(Ops[0]->getType()));
10603 llvm::ScalableVectorType *OverloadedTy =
getSVEType(TypeFlags);
10608 case SVE::BI__builtin_sve_svdupq_n_b8:
10609 case SVE::BI__builtin_sve_svdupq_n_b16:
10610 case SVE::BI__builtin_sve_svdupq_n_b32:
10611 case SVE::BI__builtin_sve_svdupq_n_b64:
10612 case SVE::BI__builtin_sve_svdupq_n_u8:
10613 case SVE::BI__builtin_sve_svdupq_n_s8:
10614 case SVE::BI__builtin_sve_svdupq_n_u64:
10615 case SVE::BI__builtin_sve_svdupq_n_f64:
10616 case SVE::BI__builtin_sve_svdupq_n_s64:
10617 case SVE::BI__builtin_sve_svdupq_n_u16:
10618 case SVE::BI__builtin_sve_svdupq_n_f16:
10619 case SVE::BI__builtin_sve_svdupq_n_bf16:
10620 case SVE::BI__builtin_sve_svdupq_n_s16:
10621 case SVE::BI__builtin_sve_svdupq_n_u32:
10622 case SVE::BI__builtin_sve_svdupq_n_f32:
10623 case SVE::BI__builtin_sve_svdupq_n_s32: {
10626 unsigned NumOpnds = Ops.size();
10629 cast<llvm::VectorType>(Ty)->getElementType()->isIntegerTy(1);
10634 llvm::Type *EltTy = Ops[0]->getType();
10639 for (
unsigned I = 0; I < NumOpnds; ++I)
10640 VecOps.push_back(
Builder.CreateZExt(Ops[I], EltTy));
10645 OverloadedTy, PoisonValue::get(OverloadedTy), Vec,
Builder.getInt64(0));
10660 : Intrinsic::aarch64_sve_cmpne_wide,
10667 case SVE::BI__builtin_sve_svpfalse_b:
10668 return ConstantInt::getFalse(Ty);
10670 case SVE::BI__builtin_sve_svpfalse_c: {
10671 auto SVBoolTy = ScalableVectorType::get(
Builder.getInt1Ty(), 16);
10674 return Builder.CreateCall(CastToSVCountF, ConstantInt::getFalse(SVBoolTy));
10677 case SVE::BI__builtin_sve_svlen_bf16:
10678 case SVE::BI__builtin_sve_svlen_f16:
10679 case SVE::BI__builtin_sve_svlen_f32:
10680 case SVE::BI__builtin_sve_svlen_f64:
10681 case SVE::BI__builtin_sve_svlen_s8:
10682 case SVE::BI__builtin_sve_svlen_s16:
10683 case SVE::BI__builtin_sve_svlen_s32:
10684 case SVE::BI__builtin_sve_svlen_s64:
10685 case SVE::BI__builtin_sve_svlen_u8:
10686 case SVE::BI__builtin_sve_svlen_u16:
10687 case SVE::BI__builtin_sve_svlen_u32:
10688 case SVE::BI__builtin_sve_svlen_u64: {
10690 auto VTy = cast<llvm::VectorType>(
getSVEType(TF));
10692 llvm::ConstantInt::get(Ty, VTy->getElementCount().getKnownMinValue());
10698 case SVE::BI__builtin_sve_svtbl2_u8:
10699 case SVE::BI__builtin_sve_svtbl2_s8:
10700 case SVE::BI__builtin_sve_svtbl2_u16:
10701 case SVE::BI__builtin_sve_svtbl2_s16:
10702 case SVE::BI__builtin_sve_svtbl2_u32:
10703 case SVE::BI__builtin_sve_svtbl2_s32:
10704 case SVE::BI__builtin_sve_svtbl2_u64:
10705 case SVE::BI__builtin_sve_svtbl2_s64:
10706 case SVE::BI__builtin_sve_svtbl2_f16:
10707 case SVE::BI__builtin_sve_svtbl2_bf16:
10708 case SVE::BI__builtin_sve_svtbl2_f32:
10709 case SVE::BI__builtin_sve_svtbl2_f64: {
10711 auto VTy = cast<llvm::ScalableVectorType>(
getSVEType(TF));
10713 return Builder.CreateCall(F, Ops);
10716 case SVE::BI__builtin_sve_svset_neonq_s8:
10717 case SVE::BI__builtin_sve_svset_neonq_s16:
10718 case SVE::BI__builtin_sve_svset_neonq_s32:
10719 case SVE::BI__builtin_sve_svset_neonq_s64:
10720 case SVE::BI__builtin_sve_svset_neonq_u8:
10721 case SVE::BI__builtin_sve_svset_neonq_u16:
10722 case SVE::BI__builtin_sve_svset_neonq_u32:
10723 case SVE::BI__builtin_sve_svset_neonq_u64:
10724 case SVE::BI__builtin_sve_svset_neonq_f16:
10725 case SVE::BI__builtin_sve_svset_neonq_f32:
10726 case SVE::BI__builtin_sve_svset_neonq_f64:
10727 case SVE::BI__builtin_sve_svset_neonq_bf16: {
10728 return Builder.CreateInsertVector(Ty, Ops[0], Ops[1],
Builder.getInt64(0));
10731 case SVE::BI__builtin_sve_svget_neonq_s8:
10732 case SVE::BI__builtin_sve_svget_neonq_s16:
10733 case SVE::BI__builtin_sve_svget_neonq_s32:
10734 case SVE::BI__builtin_sve_svget_neonq_s64:
10735 case SVE::BI__builtin_sve_svget_neonq_u8:
10736 case SVE::BI__builtin_sve_svget_neonq_u16:
10737 case SVE::BI__builtin_sve_svget_neonq_u32:
10738 case SVE::BI__builtin_sve_svget_neonq_u64:
10739 case SVE::BI__builtin_sve_svget_neonq_f16:
10740 case SVE::BI__builtin_sve_svget_neonq_f32:
10741 case SVE::BI__builtin_sve_svget_neonq_f64:
10742 case SVE::BI__builtin_sve_svget_neonq_bf16: {
10743 return Builder.CreateExtractVector(Ty, Ops[0],
Builder.getInt64(0));
10746 case SVE::BI__builtin_sve_svdup_neonq_s8:
10747 case SVE::BI__builtin_sve_svdup_neonq_s16:
10748 case SVE::BI__builtin_sve_svdup_neonq_s32:
10749 case SVE::BI__builtin_sve_svdup_neonq_s64:
10750 case SVE::BI__builtin_sve_svdup_neonq_u8:
10751 case SVE::BI__builtin_sve_svdup_neonq_u16:
10752 case SVE::BI__builtin_sve_svdup_neonq_u32:
10753 case SVE::BI__builtin_sve_svdup_neonq_u64:
10754 case SVE::BI__builtin_sve_svdup_neonq_f16:
10755 case SVE::BI__builtin_sve_svdup_neonq_f32:
10756 case SVE::BI__builtin_sve_svdup_neonq_f64:
10757 case SVE::BI__builtin_sve_svdup_neonq_bf16: {
10760 return Builder.CreateIntrinsic(Intrinsic::aarch64_sve_dupq_lane, {Ty},
10772 switch (BuiltinID) {
10775 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x1:
10778 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x2:
10779 case SME::BI__builtin_sme_svsudot_za32_s8_vg1x2:
10782 case SME::BI__builtin_sme_svsudot_za32_s8_vg1x4:
10783 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x4:
10789 for (
unsigned I = 0; I < MultiVec; ++I)
10790 std::swap(Ops[I + 1], Ops[I + 1 + MultiVec]);
10803 return EmitSMELd1St1(TypeFlags, Ops, Builtin->LLVMIntrinsic);
10806 else if (BuiltinID == SME::BI__builtin_sme_svzero_mask_za ||
10807 BuiltinID == SME::BI__builtin_sme_svzero_za)
10808 return EmitSMEZero(TypeFlags, Ops, Builtin->LLVMIntrinsic);
10809 else if (BuiltinID == SME::BI__builtin_sme_svldr_vnum_za ||
10810 BuiltinID == SME::BI__builtin_sme_svstr_vnum_za ||
10811 BuiltinID == SME::BI__builtin_sme_svldr_za ||
10812 BuiltinID == SME::BI__builtin_sme_svstr_za)
10813 return EmitSMELdrStr(TypeFlags, Ops, Builtin->LLVMIntrinsic);
10819 if (Builtin->LLVMIntrinsic == 0)
10823 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
10824 if (
auto PredTy = dyn_cast<llvm::VectorType>(Ops[i]->getType()))
10825 if (PredTy->getElementType()->isIntegerTy(1))
10839 llvm::Triple::ArchType Arch) {
10848 if (BuiltinID == Builtin::BI__builtin_cpu_supports)
10849 return EmitAArch64CpuSupports(E);
10851 unsigned HintID =
static_cast<unsigned>(-1);
10852 switch (BuiltinID) {
10854 case clang::AArch64::BI__builtin_arm_nop:
10857 case clang::AArch64::BI__builtin_arm_yield:
10858 case clang::AArch64::BI__yield:
10861 case clang::AArch64::BI__builtin_arm_wfe:
10862 case clang::AArch64::BI__wfe:
10865 case clang::AArch64::BI__builtin_arm_wfi:
10866 case clang::AArch64::BI__wfi:
10869 case clang::AArch64::BI__builtin_arm_sev:
10870 case clang::AArch64::BI__sev:
10873 case clang::AArch64::BI__builtin_arm_sevl:
10874 case clang::AArch64::BI__sevl:
10879 if (HintID !=
static_cast<unsigned>(-1)) {
10881 return Builder.CreateCall(F, llvm::ConstantInt::get(
Int32Ty, HintID));
10884 if (BuiltinID == clang::AArch64::BI__builtin_arm_trap) {
10890 if (BuiltinID == clang::AArch64::BI__builtin_arm_get_sme_state) {
10895 "__arm_sme_state"));
10897 "aarch64_pstate_sm_compatible");
10898 CI->setAttributes(Attrs);
10899 CI->setCallingConv(
10900 llvm::CallingConv::
10901 AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2);
10908 if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit) {
10910 "rbit of unusual size!");
10913 CGM.
getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
10915 if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit64) {
10917 "rbit of unusual size!");
10920 CGM.
getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
10923 if (BuiltinID == clang::AArch64::BI__builtin_arm_clz ||
10924 BuiltinID == clang::AArch64::BI__builtin_arm_clz64) {
10928 if (BuiltinID == clang::AArch64::BI__builtin_arm_clz64)
10933 if (BuiltinID == clang::AArch64::BI__builtin_arm_cls) {
10938 if (BuiltinID == clang::AArch64::BI__builtin_arm_cls64) {
10944 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint32zf ||
10945 BuiltinID == clang::AArch64::BI__builtin_arm_rint32z) {
10947 llvm::Type *Ty = Arg->getType();
10952 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint64zf ||
10953 BuiltinID == clang::AArch64::BI__builtin_arm_rint64z) {
10955 llvm::Type *Ty = Arg->getType();
10960 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint32xf ||
10961 BuiltinID == clang::AArch64::BI__builtin_arm_rint32x) {
10963 llvm::Type *Ty = Arg->getType();
10968 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint64xf ||
10969 BuiltinID == clang::AArch64::BI__builtin_arm_rint64x) {
10971 llvm::Type *Ty = Arg->getType();
10976 if (BuiltinID == clang::AArch64::BI__builtin_arm_jcvt) {
10978 "__jcvt of unusual size!");
10984 if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b ||
10985 BuiltinID == clang::AArch64::BI__builtin_arm_st64b ||
10986 BuiltinID == clang::AArch64::BI__builtin_arm_st64bv ||
10987 BuiltinID == clang::AArch64::BI__builtin_arm_st64bv0) {
10991 if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b) {
10995 llvm::Value *Val =
Builder.CreateCall(F, MemAddr);
10996 llvm::Value *ToRet;
10997 for (
size_t i = 0; i < 8; i++) {
10998 llvm::Value *ValOffsetPtr =
11009 Args.push_back(MemAddr);
11010 for (
size_t i = 0; i < 8; i++) {
11011 llvm::Value *ValOffsetPtr =
11018 auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_st64b
11019 ? Intrinsic::aarch64_st64b
11020 : BuiltinID == clang::AArch64::BI__builtin_arm_st64bv
11021 ? Intrinsic::aarch64_st64bv
11022 : Intrinsic::aarch64_st64bv0);
11024 return Builder.CreateCall(F, Args);
11028 if (BuiltinID == clang::AArch64::BI__builtin_arm_rndr ||
11029 BuiltinID == clang::AArch64::BI__builtin_arm_rndrrs) {
11031 auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_rndr
11032 ? Intrinsic::aarch64_rndr
11033 : Intrinsic::aarch64_rndrrs);
11035 llvm::Value *Val =
Builder.CreateCall(F);
11036 Value *RandomValue =
Builder.CreateExtractValue(Val, 0);
11045 if (BuiltinID == clang::AArch64::BI__clear_cache) {
11046 assert(E->
getNumArgs() == 2 &&
"__clear_cache takes 2 arguments");
11049 for (
unsigned i = 0; i < 2; i++)
11052 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
11053 StringRef Name = FD->
getName();
11057 if ((BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
11058 BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) &&
11062 ? Intrinsic::aarch64_ldaxp
11063 : Intrinsic::aarch64_ldxp);
11070 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
11071 Val0 =
Builder.CreateZExt(Val0, Int128Ty);
11072 Val1 =
Builder.CreateZExt(Val1, Int128Ty);
11074 Value *ShiftCst = llvm::ConstantInt::get(Int128Ty, 64);
11075 Val =
Builder.CreateShl(Val0, ShiftCst,
"shl",
true );
11076 Val =
Builder.CreateOr(Val, Val1);
11078 }
else if (BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
11079 BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) {
11084 llvm::Type *
IntTy =
11089 ? Intrinsic::aarch64_ldaxr
11090 : Intrinsic::aarch64_ldxr,
11092 CallInst *Val =
Builder.CreateCall(F, LoadAddr,
"ldxr");
11096 if (RealResTy->isPointerTy())
11097 return Builder.CreateIntToPtr(Val, RealResTy);
11099 llvm::Type *IntResTy = llvm::IntegerType::get(
11101 return Builder.CreateBitCast(
Builder.CreateTruncOrBitCast(Val, IntResTy),
11105 if ((BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
11106 BuiltinID == clang::AArch64::BI__builtin_arm_stlex) &&
11110 ? Intrinsic::aarch64_stlxp
11111 : Intrinsic::aarch64_stxp);
11123 return Builder.CreateCall(F, {Arg0, Arg1, StPtr},
"stxp");
11126 if (BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
11127 BuiltinID == clang::AArch64::BI__builtin_arm_stlex) {
11132 llvm::Type *StoreTy =
11135 if (StoreVal->
getType()->isPointerTy())
11138 llvm::Type *
IntTy = llvm::IntegerType::get(
11147 ? Intrinsic::aarch64_stlxr
11148 : Intrinsic::aarch64_stxr,
11150 CallInst *CI =
Builder.CreateCall(F, {StoreVal, StoreAddr},
"stxr");
11152 1, Attribute::get(
getLLVMContext(), Attribute::ElementType, StoreTy));
11156 if (BuiltinID == clang::AArch64::BI__getReg) {
11159 llvm_unreachable(
"Sema will ensure that the parameter is constant");
11165 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, Reg)};
11166 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
11167 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
11169 llvm::Function *F =
11171 return Builder.CreateCall(F, Metadata);
11174 if (BuiltinID == clang::AArch64::BI__break) {
11177 llvm_unreachable(
"Sema will ensure that the parameter is constant");
11179 llvm::Function *F =
CGM.
getIntrinsic(llvm::Intrinsic::aarch64_break);
11183 if (BuiltinID == clang::AArch64::BI__builtin_arm_clrex) {
11185 return Builder.CreateCall(F);
11188 if (BuiltinID == clang::AArch64::BI_ReadWriteBarrier)
11189 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
11190 llvm::SyncScope::SingleThread);
11193 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
11194 switch (BuiltinID) {
11195 case clang::AArch64::BI__builtin_arm_crc32b:
11196 CRCIntrinsicID = Intrinsic::aarch64_crc32b;
break;
11197 case clang::AArch64::BI__builtin_arm_crc32cb:
11198 CRCIntrinsicID = Intrinsic::aarch64_crc32cb;
break;
11199 case clang::AArch64::BI__builtin_arm_crc32h:
11200 CRCIntrinsicID = Intrinsic::aarch64_crc32h;
break;
11201 case clang::AArch64::BI__builtin_arm_crc32ch:
11202 CRCIntrinsicID = Intrinsic::aarch64_crc32ch;
break;
11203 case clang::AArch64::BI__builtin_arm_crc32w:
11204 CRCIntrinsicID = Intrinsic::aarch64_crc32w;
break;
11205 case clang::AArch64::BI__builtin_arm_crc32cw:
11206 CRCIntrinsicID = Intrinsic::aarch64_crc32cw;
break;
11207 case clang::AArch64::BI__builtin_arm_crc32d:
11208 CRCIntrinsicID = Intrinsic::aarch64_crc32x;
break;
11209 case clang::AArch64::BI__builtin_arm_crc32cd:
11210 CRCIntrinsicID = Intrinsic::aarch64_crc32cx;
break;
11213 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
11218 llvm::Type *DataTy = F->getFunctionType()->getParamType(1);
11219 Arg1 =
Builder.CreateZExtOrBitCast(Arg1, DataTy);
11221 return Builder.CreateCall(F, {Arg0, Arg1});
11225 if (BuiltinID == AArch64::BI__builtin_arm_mops_memset_tag) {
11233 CGM.
getIntrinsic(Intrinsic::aarch64_mops_memset_tag), {Dst, Val, Size});
11237 Intrinsic::ID MTEIntrinsicID = Intrinsic::not_intrinsic;
11238 switch (BuiltinID) {
11239 case clang::AArch64::BI__builtin_arm_irg:
11240 MTEIntrinsicID = Intrinsic::aarch64_irg;
break;
11241 case clang::AArch64::BI__builtin_arm_addg:
11242 MTEIntrinsicID = Intrinsic::aarch64_addg;
break;
11243 case clang::AArch64::BI__builtin_arm_gmi:
11244 MTEIntrinsicID = Intrinsic::aarch64_gmi;
break;
11245 case clang::AArch64::BI__builtin_arm_ldg:
11246 MTEIntrinsicID = Intrinsic::aarch64_ldg;
break;
11247 case clang::AArch64::BI__builtin_arm_stg:
11248 MTEIntrinsicID = Intrinsic::aarch64_stg;
break;
11249 case clang::AArch64::BI__builtin_arm_subp:
11250 MTEIntrinsicID = Intrinsic::aarch64_subp;
break;
11253 if (MTEIntrinsicID != Intrinsic::not_intrinsic) {
11256 if (MTEIntrinsicID == Intrinsic::aarch64_irg) {
11264 return Builder.CreatePointerCast(RV,
T);
11266 if (MTEIntrinsicID == Intrinsic::aarch64_addg) {
11274 return Builder.CreatePointerCast(RV,
T);
11276 if (MTEIntrinsicID == Intrinsic::aarch64_gmi) {
11288 if (MTEIntrinsicID == Intrinsic::aarch64_ldg) {
11293 return Builder.CreatePointerCast(RV,
T);
11298 if (MTEIntrinsicID == Intrinsic::aarch64_stg) {
11304 if (MTEIntrinsicID == Intrinsic::aarch64_subp) {
11314 if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
11315 BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
11316 BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
11317 BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
11318 BuiltinID == clang::AArch64::BI__builtin_arm_wsr ||
11319 BuiltinID == clang::AArch64::BI__builtin_arm_wsr64 ||
11320 BuiltinID == clang::AArch64::BI__builtin_arm_wsr128 ||
11321 BuiltinID == clang::AArch64::BI__builtin_arm_wsrp) {
11324 if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
11325 BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
11326 BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
11327 BuiltinID == clang::AArch64::BI__builtin_arm_rsrp)
11330 bool IsPointerBuiltin = BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
11331 BuiltinID == clang::AArch64::BI__builtin_arm_wsrp;
11333 bool Is32Bit = BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
11334 BuiltinID == clang::AArch64::BI__builtin_arm_wsr;
11336 bool Is128Bit = BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
11337 BuiltinID == clang::AArch64::BI__builtin_arm_wsr128;
11339 llvm::Type *ValueType;
11340 llvm::Type *RegisterType =
Int64Ty;
11343 }
else if (Is128Bit) {
11344 llvm::Type *Int128Ty =
11346 ValueType = Int128Ty;
11347 RegisterType = Int128Ty;
11348 }
else if (IsPointerBuiltin) {
11358 if (BuiltinID == clang::AArch64::BI_ReadStatusReg ||
11359 BuiltinID == clang::AArch64::BI_WriteStatusReg) {
11365 std::string SysRegStr;
11366 llvm::raw_string_ostream(SysRegStr) <<
11367 ((1 << 1) | ((SysReg >> 14) & 1)) <<
":" <<
11368 ((SysReg >> 11) & 7) <<
":" <<
11369 ((SysReg >> 7) & 15) <<
":" <<
11370 ((SysReg >> 3) & 15) <<
":" <<
11373 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysRegStr) };
11374 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
11375 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
11377 llvm::Type *RegisterType =
Int64Ty;
11378 llvm::Type *Types[] = { RegisterType };
11380 if (BuiltinID == clang::AArch64::BI_ReadStatusReg) {
11381 llvm::Function *F =
CGM.
getIntrinsic(llvm::Intrinsic::read_register, Types);
11383 return Builder.CreateCall(F, Metadata);
11386 llvm::Function *F =
CGM.
getIntrinsic(llvm::Intrinsic::write_register, Types);
11389 return Builder.CreateCall(F, { Metadata, ArgValue });
11392 if (BuiltinID == clang::AArch64::BI_AddressOfReturnAddress) {
11393 llvm::Function *F =
11395 return Builder.CreateCall(F);
11398 if (BuiltinID == clang::AArch64::BI__builtin_sponentry) {
11400 return Builder.CreateCall(F);
11403 if (BuiltinID == clang::AArch64::BI__mulh ||
11404 BuiltinID == clang::AArch64::BI__umulh) {
11406 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
11408 bool IsSigned = BuiltinID == clang::AArch64::BI__mulh;
11414 Value *MulResult, *HigherBits;
11416 MulResult =
Builder.CreateNSWMul(LHS, RHS);
11417 HigherBits =
Builder.CreateAShr(MulResult, 64);
11419 MulResult =
Builder.CreateNUWMul(LHS, RHS);
11420 HigherBits =
Builder.CreateLShr(MulResult, 64);
11422 HigherBits =
Builder.CreateIntCast(HigherBits, ResType, IsSigned);
11427 if (BuiltinID == AArch64::BI__writex18byte ||
11428 BuiltinID == AArch64::BI__writex18word ||
11429 BuiltinID == AArch64::BI__writex18dword ||
11430 BuiltinID == AArch64::BI__writex18qword) {
11433 llvm::Metadata *Ops[] = {llvm::MDString::get(Context,
"x18")};
11434 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
11435 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
11436 llvm::Function *F =
11438 llvm::Value *X18 =
Builder.CreateCall(F, Metadata);
11449 if (BuiltinID == AArch64::BI__readx18byte ||
11450 BuiltinID == AArch64::BI__readx18word ||
11451 BuiltinID == AArch64::BI__readx18dword ||
11452 BuiltinID == AArch64::BI__readx18qword) {
11457 llvm::Metadata *Ops[] = {llvm::MDString::get(Context,
"x18")};
11458 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
11459 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
11460 llvm::Function *F =
11462 llvm::Value *X18 =
Builder.CreateCall(F, Metadata);
11472 if (BuiltinID == AArch64::BI_CopyDoubleFromInt64 ||
11473 BuiltinID == AArch64::BI_CopyFloatFromInt32 ||
11474 BuiltinID == AArch64::BI_CopyInt32FromFloat ||
11475 BuiltinID == AArch64::BI_CopyInt64FromDouble) {
11478 return Builder.CreateBitCast(Arg, RetTy);
11481 if (BuiltinID == AArch64::BI_CountLeadingOnes ||
11482 BuiltinID == AArch64::BI_CountLeadingOnes64 ||
11483 BuiltinID == AArch64::BI_CountLeadingZeros ||
11484 BuiltinID == AArch64::BI_CountLeadingZeros64) {
11486 llvm::Type *ArgType = Arg->
getType();
11488 if (BuiltinID == AArch64::BI_CountLeadingOnes ||
11489 BuiltinID == AArch64::BI_CountLeadingOnes64)
11490 Arg =
Builder.CreateXor(Arg, Constant::getAllOnesValue(ArgType));
11495 if (BuiltinID == AArch64::BI_CountLeadingOnes64 ||
11496 BuiltinID == AArch64::BI_CountLeadingZeros64)
11501 if (BuiltinID == AArch64::BI_CountLeadingSigns ||
11502 BuiltinID == AArch64::BI_CountLeadingSigns64) {
11505 Function *F = (BuiltinID == AArch64::BI_CountLeadingSigns)
11510 if (BuiltinID == AArch64::BI_CountLeadingSigns64)
11515 if (BuiltinID == AArch64::BI_CountOneBits ||
11516 BuiltinID == AArch64::BI_CountOneBits64) {
11518 llvm::Type *ArgType = ArgValue->
getType();
11522 if (BuiltinID == AArch64::BI_CountOneBits64)
11527 if (BuiltinID == AArch64::BI__prefetch) {
11538 if (std::optional<MSVCIntrin> MsvcIntId =
11544 return P.first == BuiltinID;
11547 BuiltinID = It->second;
11551 unsigned ICEArguments = 0;
11558 for (
unsigned i = 0, e = E->
getNumArgs() - 1; i != e; i++) {
11560 switch (BuiltinID) {
11561 case NEON::BI__builtin_neon_vld1_v:
11562 case NEON::BI__builtin_neon_vld1q_v:
11563 case NEON::BI__builtin_neon_vld1_dup_v:
11564 case NEON::BI__builtin_neon_vld1q_dup_v:
11565 case NEON::BI__builtin_neon_vld1_lane_v:
11566 case NEON::BI__builtin_neon_vld1q_lane_v:
11567 case NEON::BI__builtin_neon_vst1_v:
11568 case NEON::BI__builtin_neon_vst1q_v:
11569 case NEON::BI__builtin_neon_vst1_lane_v:
11570 case NEON::BI__builtin_neon_vst1q_lane_v:
11571 case NEON::BI__builtin_neon_vldap1_lane_s64:
11572 case NEON::BI__builtin_neon_vldap1q_lane_s64:
11573 case NEON::BI__builtin_neon_vstl1_lane_s64:
11574 case NEON::BI__builtin_neon_vstl1q_lane_s64:
11592 assert(
Result &&
"SISD intrinsic should have been handled");
11598 if (std::optional<llvm::APSInt>
Result =
11603 bool usgn =
Type.isUnsigned();
11604 bool quad =
Type.isQuad();
11607 switch (BuiltinID) {
11609 case NEON::BI__builtin_neon_vabsh_f16:
11612 case NEON::BI__builtin_neon_vaddq_p128: {
11615 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
11616 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
11617 Ops[0] =
Builder.CreateXor(Ops[0], Ops[1]);
11618 llvm::Type *Int128Ty = llvm::Type::getIntNTy(
getLLVMContext(), 128);
11619 return Builder.CreateBitCast(Ops[0], Int128Ty);
11621 case NEON::BI__builtin_neon_vldrq_p128: {
11622 llvm::Type *Int128Ty = llvm::Type::getIntNTy(
getLLVMContext(), 128);
11627 case NEON::BI__builtin_neon_vstrq_p128: {
11628 Value *Ptr = Ops[0];
11631 case NEON::BI__builtin_neon_vcvts_f32_u32:
11632 case NEON::BI__builtin_neon_vcvtd_f64_u64:
11635 case NEON::BI__builtin_neon_vcvts_f32_s32:
11636 case NEON::BI__builtin_neon_vcvtd_f64_s64: {
11638 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
11641 Ops[0] =
Builder.CreateBitCast(Ops[0], InTy);
11643 return Builder.CreateUIToFP(Ops[0], FTy);
11644 return Builder.CreateSIToFP(Ops[0], FTy);
11646 case NEON::BI__builtin_neon_vcvth_f16_u16:
11647 case NEON::BI__builtin_neon_vcvth_f16_u32:
11648 case NEON::BI__builtin_neon_vcvth_f16_u64:
11651 case NEON::BI__builtin_neon_vcvth_f16_s16:
11652 case NEON::BI__builtin_neon_vcvth_f16_s32:
11653 case NEON::BI__builtin_neon_vcvth_f16_s64: {
11655 llvm::Type *FTy =
HalfTy;
11657 if (Ops[0]->getType()->getPrimitiveSizeInBits() == 64)
11659 else if (Ops[0]->getType()->getPrimitiveSizeInBits() == 32)
11663 Ops[0] =
Builder.CreateBitCast(Ops[0], InTy);
11665 return Builder.CreateUIToFP(Ops[0], FTy);
11666 return Builder.CreateSIToFP(Ops[0], FTy);
11668 case NEON::BI__builtin_neon_vcvtah_u16_f16:
11669 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
11670 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
11671 case NEON::BI__builtin_neon_vcvtph_u16_f16:
11672 case NEON::BI__builtin_neon_vcvth_u16_f16:
11673 case NEON::BI__builtin_neon_vcvtah_s16_f16:
11674 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
11675 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
11676 case NEON::BI__builtin_neon_vcvtph_s16_f16:
11677 case NEON::BI__builtin_neon_vcvth_s16_f16: {
11680 llvm::Type* FTy =
HalfTy;
11681 llvm::Type *Tys[2] = {InTy, FTy};
11683 switch (BuiltinID) {
11684 default: llvm_unreachable(
"missing builtin ID in switch!");
11685 case NEON::BI__builtin_neon_vcvtah_u16_f16:
11686 Int = Intrinsic::aarch64_neon_fcvtau;
break;
11687 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
11688 Int = Intrinsic::aarch64_neon_fcvtmu;
break;
11689 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
11690 Int = Intrinsic::aarch64_neon_fcvtnu;
break;
11691 case NEON::BI__builtin_neon_vcvtph_u16_f16:
11692 Int = Intrinsic::aarch64_neon_fcvtpu;
break;
11693 case NEON::BI__builtin_neon_vcvth_u16_f16:
11694 Int = Intrinsic::aarch64_neon_fcvtzu;
break;
11695 case NEON::BI__builtin_neon_vcvtah_s16_f16:
11696 Int = Intrinsic::aarch64_neon_fcvtas;
break;
11697 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
11698 Int = Intrinsic::aarch64_neon_fcvtms;
break;
11699 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
11700 Int = Intrinsic::aarch64_neon_fcvtns;
break;
11701 case NEON::BI__builtin_neon_vcvtph_s16_f16:
11702 Int = Intrinsic::aarch64_neon_fcvtps;
break;
11703 case NEON::BI__builtin_neon_vcvth_s16_f16:
11704 Int = Intrinsic::aarch64_neon_fcvtzs;
break;
11709 case NEON::BI__builtin_neon_vcaleh_f16:
11710 case NEON::BI__builtin_neon_vcalth_f16:
11711 case NEON::BI__builtin_neon_vcageh_f16:
11712 case NEON::BI__builtin_neon_vcagth_f16: {
11715 llvm::Type* FTy =
HalfTy;
11716 llvm::Type *Tys[2] = {InTy, FTy};
11718 switch (BuiltinID) {
11719 default: llvm_unreachable(
"missing builtin ID in switch!");
11720 case NEON::BI__builtin_neon_vcageh_f16:
11721 Int = Intrinsic::aarch64_neon_facge;
break;
11722 case NEON::BI__builtin_neon_vcagth_f16:
11723 Int = Intrinsic::aarch64_neon_facgt;
break;
11724 case NEON::BI__builtin_neon_vcaleh_f16:
11725 Int = Intrinsic::aarch64_neon_facge; std::swap(Ops[0], Ops[1]);
break;
11726 case NEON::BI__builtin_neon_vcalth_f16:
11727 Int = Intrinsic::aarch64_neon_facgt; std::swap(Ops[0], Ops[1]);
break;
11732 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
11733 case NEON::BI__builtin_neon_vcvth_n_u16_f16: {
11736 llvm::Type* FTy =
HalfTy;
11737 llvm::Type *Tys[2] = {InTy, FTy};
11739 switch (BuiltinID) {
11740 default: llvm_unreachable(
"missing builtin ID in switch!");
11741 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
11742 Int = Intrinsic::aarch64_neon_vcvtfp2fxs;
break;
11743 case NEON::BI__builtin_neon_vcvth_n_u16_f16:
11744 Int = Intrinsic::aarch64_neon_vcvtfp2fxu;
break;
11749 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
11750 case NEON::BI__builtin_neon_vcvth_n_f16_u16: {
11752 llvm::Type* FTy =
HalfTy;
11754 llvm::Type *Tys[2] = {FTy, InTy};
11756 switch (BuiltinID) {
11757 default: llvm_unreachable(
"missing builtin ID in switch!");
11758 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
11759 Int = Intrinsic::aarch64_neon_vcvtfxs2fp;
11760 Ops[0] =
Builder.CreateSExt(Ops[0], InTy,
"sext");
11762 case NEON::BI__builtin_neon_vcvth_n_f16_u16:
11763 Int = Intrinsic::aarch64_neon_vcvtfxu2fp;
11764 Ops[0] =
Builder.CreateZExt(Ops[0], InTy);
11769 case NEON::BI__builtin_neon_vpaddd_s64: {
11770 auto *Ty = llvm::FixedVectorType::get(
Int64Ty, 2);
11773 Vec =
Builder.CreateBitCast(Vec, Ty,
"v2i64");
11774 llvm::Value *Idx0 = llvm::ConstantInt::get(
SizeTy, 0);
11775 llvm::Value *Idx1 = llvm::ConstantInt::get(
SizeTy, 1);
11776 Value *Op0 =
Builder.CreateExtractElement(Vec, Idx0,
"lane0");
11777 Value *Op1 =
Builder.CreateExtractElement(Vec, Idx1,
"lane1");
11779 return Builder.CreateAdd(Op0, Op1,
"vpaddd");
11781 case NEON::BI__builtin_neon_vpaddd_f64: {
11782 auto *Ty = llvm::FixedVectorType::get(
DoubleTy, 2);
11785 Vec =
Builder.CreateBitCast(Vec, Ty,
"v2f64");
11786 llvm::Value *Idx0 = llvm::ConstantInt::get(
SizeTy, 0);
11787 llvm::Value *Idx1 = llvm::ConstantInt::get(
SizeTy, 1);
11788 Value *Op0 =
Builder.CreateExtractElement(Vec, Idx0,
"lane0");
11789 Value *Op1 =
Builder.CreateExtractElement(Vec, Idx1,
"lane1");
11791 return Builder.CreateFAdd(Op0, Op1,
"vpaddd");
11793 case NEON::BI__builtin_neon_vpadds_f32: {
11794 auto *Ty = llvm::FixedVectorType::get(
FloatTy, 2);
11797 Vec =
Builder.CreateBitCast(Vec, Ty,
"v2f32");
11798 llvm::Value *Idx0 = llvm::ConstantInt::get(
SizeTy, 0);
11799 llvm::Value *Idx1 = llvm::ConstantInt::get(
SizeTy, 1);
11800 Value *Op0 =
Builder.CreateExtractElement(Vec, Idx0,
"lane0");
11801 Value *Op1 =
Builder.CreateExtractElement(Vec, Idx1,
"lane1");
11803 return Builder.CreateFAdd(Op0, Op1,
"vpaddd");
11805 case NEON::BI__builtin_neon_vceqzd_s64:
11806 case NEON::BI__builtin_neon_vceqzd_f64:
11807 case NEON::BI__builtin_neon_vceqzs_f32:
11808 case NEON::BI__builtin_neon_vceqzh_f16:
11812 ICmpInst::FCMP_OEQ, ICmpInst::ICMP_EQ,
"vceqz");
11813 case NEON::BI__builtin_neon_vcgezd_s64:
11814 case NEON::BI__builtin_neon_vcgezd_f64:
11815 case NEON::BI__builtin_neon_vcgezs_f32:
11816 case NEON::BI__builtin_neon_vcgezh_f16:
11820 ICmpInst::FCMP_OGE, ICmpInst::ICMP_SGE,
"vcgez");
11821 case NEON::BI__builtin_neon_vclezd_s64:
11822 case NEON::BI__builtin_neon_vclezd_f64:
11823 case NEON::BI__builtin_neon_vclezs_f32:
11824 case NEON::BI__builtin_neon_vclezh_f16:
11828 ICmpInst::FCMP_OLE, ICmpInst::ICMP_SLE,
"vclez");
11829 case NEON::BI__builtin_neon_vcgtzd_s64:
11830 case NEON::BI__builtin_neon_vcgtzd_f64:
11831 case NEON::BI__builtin_neon_vcgtzs_f32:
11832 case NEON::BI__builtin_neon_vcgtzh_f16:
11836 ICmpInst::FCMP_OGT, ICmpInst::ICMP_SGT,
"vcgtz");
11837 case NEON::BI__builtin_neon_vcltzd_s64:
11838 case NEON::BI__builtin_neon_vcltzd_f64:
11839 case NEON::BI__builtin_neon_vcltzs_f32:
11840 case NEON::BI__builtin_neon_vcltzh_f16:
11844 ICmpInst::FCMP_OLT, ICmpInst::ICMP_SLT,
"vcltz");
11846 case NEON::BI__builtin_neon_vceqzd_u64: {
11850 Builder.CreateICmpEQ(Ops[0], llvm::Constant::getNullValue(
Int64Ty));
11853 case NEON::BI__builtin_neon_vceqd_f64:
11854 case NEON::BI__builtin_neon_vcled_f64:
11855 case NEON::BI__builtin_neon_vcltd_f64:
11856 case NEON::BI__builtin_neon_vcged_f64:
11857 case NEON::BI__builtin_neon_vcgtd_f64: {
11858 llvm::CmpInst::Predicate
P;
11859 switch (BuiltinID) {
11860 default: llvm_unreachable(
"missing builtin ID in switch!");
11861 case NEON::BI__builtin_neon_vceqd_f64:
P = llvm::FCmpInst::FCMP_OEQ;
break;
11862 case NEON::BI__builtin_neon_vcled_f64:
P = llvm::FCmpInst::FCMP_OLE;
break;
11863 case NEON::BI__builtin_neon_vcltd_f64:
P = llvm::FCmpInst::FCMP_OLT;
break;
11864 case NEON::BI__builtin_neon_vcged_f64:
P = llvm::FCmpInst::FCMP_OGE;
break;
11865 case NEON::BI__builtin_neon_vcgtd_f64:
P = llvm::FCmpInst::FCMP_OGT;
break;
11870 if (
P == llvm::FCmpInst::FCMP_OEQ)
11871 Ops[0] =
Builder.CreateFCmp(
P, Ops[0], Ops[1]);
11873 Ops[0] =
Builder.CreateFCmpS(
P, Ops[0], Ops[1]);
11876 case NEON::BI__builtin_neon_vceqs_f32:
11877 case NEON::BI__builtin_neon_vcles_f32:
11878 case NEON::BI__builtin_neon_vclts_f32:
11879 case NEON::BI__builtin_neon_vcges_f32:
11880 case NEON::BI__builtin_neon_vcgts_f32: {
11881 llvm::CmpInst::Predicate
P;
11882 switch (BuiltinID) {
11883 default: llvm_unreachable(
"missing builtin ID in switch!");
11884 case NEON::BI__builtin_neon_vceqs_f32:
P = llvm::FCmpInst::FCMP_OEQ;
break;
11885 case NEON::BI__builtin_neon_vcles_f32:
P = llvm::FCmpInst::FCMP_OLE;
break;
11886 case NEON::BI__builtin_neon_vclts_f32:
P = llvm::FCmpInst::FCMP_OLT;
break;
11887 case NEON::BI__builtin_neon_vcges_f32:
P = llvm::FCmpInst::FCMP_OGE;
break;
11888 case NEON::BI__builtin_neon_vcgts_f32:
P = llvm::FCmpInst::FCMP_OGT;
break;
11893 if (
P == llvm::FCmpInst::FCMP_OEQ)
11894 Ops[0] =
Builder.CreateFCmp(
P, Ops[0], Ops[1]);
11896 Ops[0] =
Builder.CreateFCmpS(
P, Ops[0], Ops[1]);
11899 case NEON::BI__builtin_neon_vceqh_f16:
11900 case NEON::BI__builtin_neon_vcleh_f16:
11901 case NEON::BI__builtin_neon_vclth_f16:
11902 case NEON::BI__builtin_neon_vcgeh_f16:
11903 case NEON::BI__builtin_neon_vcgth_f16: {
11904 llvm::CmpInst::Predicate
P;
11905 switch (BuiltinID) {
11906 default: llvm_unreachable(
"missing builtin ID in switch!");
11907 case NEON::BI__builtin_neon_vceqh_f16:
P = llvm::FCmpInst::FCMP_OEQ;
break;
11908 case NEON::BI__builtin_neon_vcleh_f16:
P = llvm::FCmpInst::FCMP_OLE;
break;
11909 case NEON::BI__builtin_neon_vclth_f16:
P = llvm::FCmpInst::FCMP_OLT;
break;
11910 case NEON::BI__builtin_neon_vcgeh_f16:
P = llvm::FCmpInst::FCMP_OGE;
break;
11911 case NEON::BI__builtin_neon_vcgth_f16:
P = llvm::FCmpInst::FCMP_OGT;
break;
11916 if (
P == llvm::FCmpInst::FCMP_OEQ)
11917 Ops[0] =
Builder.CreateFCmp(
P, Ops[0], Ops[1]);
11919 Ops[0] =
Builder.CreateFCmpS(
P, Ops[0], Ops[1]);
11922 case NEON::BI__builtin_neon_vceqd_s64:
11923 case NEON::BI__builtin_neon_vceqd_u64:
11924 case NEON::BI__builtin_neon_vcgtd_s64:
11925 case NEON::BI__builtin_neon_vcgtd_u64:
11926 case NEON::BI__builtin_neon_vcltd_s64:
11927 case NEON::BI__builtin_neon_vcltd_u64:
11928 case NEON::BI__builtin_neon_vcged_u64:
11929 case NEON::BI__builtin_neon_vcged_s64:
11930 case NEON::BI__builtin_neon_vcled_u64:
11931 case NEON::BI__builtin_neon_vcled_s64: {
11932 llvm::CmpInst::Predicate
P;
11933 switch (BuiltinID) {
11934 default: llvm_unreachable(
"missing builtin ID in switch!");
11935 case NEON::BI__builtin_neon_vceqd_s64:
11936 case NEON::BI__builtin_neon_vceqd_u64:
P = llvm::ICmpInst::ICMP_EQ;
break;
11937 case NEON::BI__builtin_neon_vcgtd_s64:
P = llvm::ICmpInst::ICMP_SGT;
break;
11938 case NEON::BI__builtin_neon_vcgtd_u64:
P = llvm::ICmpInst::ICMP_UGT;
break;
11939 case NEON::BI__builtin_neon_vcltd_s64:
P = llvm::ICmpInst::ICMP_SLT;
break;
11940 case NEON::BI__builtin_neon_vcltd_u64:
P = llvm::ICmpInst::ICMP_ULT;
break;
11941 case NEON::BI__builtin_neon_vcged_u64:
P = llvm::ICmpInst::ICMP_UGE;
break;
11942 case NEON::BI__builtin_neon_vcged_s64:
P = llvm::ICmpInst::ICMP_SGE;
break;
11943 case NEON::BI__builtin_neon_vcled_u64:
P = llvm::ICmpInst::ICMP_ULE;
break;
11944 case NEON::BI__builtin_neon_vcled_s64:
P = llvm::ICmpInst::ICMP_SLE;
break;
11949 Ops[0] =
Builder.CreateICmp(
P, Ops[0], Ops[1]);
11952 case NEON::BI__builtin_neon_vtstd_s64:
11953 case NEON::BI__builtin_neon_vtstd_u64: {
11957 Ops[0] =
Builder.CreateAnd(Ops[0], Ops[1]);
11958 Ops[0] =
Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
11959 llvm::Constant::getNullValue(
Int64Ty));
11962 case NEON::BI__builtin_neon_vset_lane_i8:
11963 case NEON::BI__builtin_neon_vset_lane_i16:
11964 case NEON::BI__builtin_neon_vset_lane_i32:
11965 case NEON::BI__builtin_neon_vset_lane_i64:
11966 case NEON::BI__builtin_neon_vset_lane_bf16:
11967 case NEON::BI__builtin_neon_vset_lane_f32:
11968 case NEON::BI__builtin_neon_vsetq_lane_i8:
11969 case NEON::BI__builtin_neon_vsetq_lane_i16:
11970 case NEON::BI__builtin_neon_vsetq_lane_i32:
11971 case NEON::BI__builtin_neon_vsetq_lane_i64:
11972 case NEON::BI__builtin_neon_vsetq_lane_bf16:
11973 case NEON::BI__builtin_neon_vsetq_lane_f32:
11975 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
11976 case NEON::BI__builtin_neon_vset_lane_f64:
11979 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(
DoubleTy, 1));
11981 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
11982 case NEON::BI__builtin_neon_vsetq_lane_f64:
11985 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(
DoubleTy, 2));
11987 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
11989 case NEON::BI__builtin_neon_vget_lane_i8:
11990 case NEON::BI__builtin_neon_vdupb_lane_i8:
11992 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int8Ty, 8));
11995 case NEON::BI__builtin_neon_vgetq_lane_i8:
11996 case NEON::BI__builtin_neon_vdupb_laneq_i8:
11998 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int8Ty, 16));
12001 case NEON::BI__builtin_neon_vget_lane_i16:
12002 case NEON::BI__builtin_neon_vduph_lane_i16:
12004 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int16Ty, 4));
12007 case NEON::BI__builtin_neon_vgetq_lane_i16:
12008 case NEON::BI__builtin_neon_vduph_laneq_i16:
12010 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int16Ty, 8));
12013 case NEON::BI__builtin_neon_vget_lane_i32:
12014 case NEON::BI__builtin_neon_vdups_lane_i32:
12016 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int32Ty, 2));
12019 case NEON::BI__builtin_neon_vdups_lane_f32:
12021 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
FloatTy, 2));
12024 case NEON::BI__builtin_neon_vgetq_lane_i32:
12025 case NEON::BI__builtin_neon_vdups_laneq_i32:
12027 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int32Ty, 4));
12030 case NEON::BI__builtin_neon_vget_lane_i64:
12031 case NEON::BI__builtin_neon_vdupd_lane_i64:
12033 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int64Ty, 1));
12036 case NEON::BI__builtin_neon_vdupd_lane_f64:
12038 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
DoubleTy, 1));
12041 case NEON::BI__builtin_neon_vgetq_lane_i64:
12042 case NEON::BI__builtin_neon_vdupd_laneq_i64:
12044 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int64Ty, 2));
12047 case NEON::BI__builtin_neon_vget_lane_f32:
12049 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
FloatTy, 2));
12052 case NEON::BI__builtin_neon_vget_lane_f64:
12054 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
DoubleTy, 1));
12057 case NEON::BI__builtin_neon_vgetq_lane_f32:
12058 case NEON::BI__builtin_neon_vdups_laneq_f32:
12060 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
FloatTy, 4));
12063 case NEON::BI__builtin_neon_vgetq_lane_f64:
12064 case NEON::BI__builtin_neon_vdupd_laneq_f64:
12066 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
DoubleTy, 2));
12069 case NEON::BI__builtin_neon_vaddh_f16:
12071 return Builder.CreateFAdd(Ops[0], Ops[1],
"vaddh");
12072 case NEON::BI__builtin_neon_vsubh_f16:
12074 return Builder.CreateFSub(Ops[0], Ops[1],
"vsubh");
12075 case NEON::BI__builtin_neon_vmulh_f16:
12077 return Builder.CreateFMul(Ops[0], Ops[1],
"vmulh");
12078 case NEON::BI__builtin_neon_vdivh_f16:
12080 return Builder.CreateFDiv(Ops[0], Ops[1],
"vdivh");
12081 case NEON::BI__builtin_neon_vfmah_f16:
12084 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma,
HalfTy,
12086 case NEON::BI__builtin_neon_vfmsh_f16: {
12091 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma,
HalfTy,
12094 case NEON::BI__builtin_neon_vaddd_s64:
12095 case NEON::BI__builtin_neon_vaddd_u64:
12097 case NEON::BI__builtin_neon_vsubd_s64:
12098 case NEON::BI__builtin_neon_vsubd_u64:
12100 case NEON::BI__builtin_neon_vqdmlalh_s16:
12101 case NEON::BI__builtin_neon_vqdmlslh_s16: {
12105 auto *VTy = llvm::FixedVectorType::get(
Int32Ty, 4);
12107 ProductOps,
"vqdmlXl");
12108 Constant *CI = ConstantInt::get(
SizeTy, 0);
12109 Ops[1] =
Builder.CreateExtractElement(Ops[1], CI,
"lane0");
12111 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlalh_s16
12112 ? Intrinsic::aarch64_neon_sqadd
12113 : Intrinsic::aarch64_neon_sqsub;
12116 case NEON::BI__builtin_neon_vqshlud_n_s64: {
12122 case NEON::BI__builtin_neon_vqshld_n_u64:
12123 case NEON::BI__builtin_neon_vqshld_n_s64: {
12124 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vqshld_n_u64
12125 ? Intrinsic::aarch64_neon_uqshl
12126 : Intrinsic::aarch64_neon_sqshl;
12131 case NEON::BI__builtin_neon_vrshrd_n_u64:
12132 case NEON::BI__builtin_neon_vrshrd_n_s64: {
12133 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrshrd_n_u64
12134 ? Intrinsic::aarch64_neon_urshl
12135 : Intrinsic::aarch64_neon_srshl;
12137 int SV = cast<ConstantInt>(Ops[1])->getSExtValue();
12138 Ops[1] = ConstantInt::get(
Int64Ty, -SV);
12141 case NEON::BI__builtin_neon_vrsrad_n_u64:
12142 case NEON::BI__builtin_neon_vrsrad_n_s64: {
12143 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrsrad_n_u64
12144 ? Intrinsic::aarch64_neon_urshl
12145 : Intrinsic::aarch64_neon_srshl;
12149 {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)});
12152 case NEON::BI__builtin_neon_vshld_n_s64:
12153 case NEON::BI__builtin_neon_vshld_n_u64: {
12156 Ops[0], ConstantInt::get(
Int64Ty, Amt->getZExtValue()),
"shld_n");
12158 case NEON::BI__builtin_neon_vshrd_n_s64: {
12161 Ops[0], ConstantInt::get(
Int64Ty, std::min(
static_cast<uint64_t>(63),
12162 Amt->getZExtValue())),
12165 case NEON::BI__builtin_neon_vshrd_n_u64: {
12167 uint64_t ShiftAmt = Amt->getZExtValue();
12169 if (ShiftAmt == 64)
12170 return ConstantInt::get(
Int64Ty, 0);
12171 return Builder.CreateLShr(Ops[0], ConstantInt::get(
Int64Ty, ShiftAmt),
12174 case NEON::BI__builtin_neon_vsrad_n_s64: {
12177 Ops[1], ConstantInt::get(
Int64Ty, std::min(
static_cast<uint64_t>(63),
12178 Amt->getZExtValue())),
12180 return Builder.CreateAdd(Ops[0], Ops[1]);
12182 case NEON::BI__builtin_neon_vsrad_n_u64: {
12184 uint64_t ShiftAmt = Amt->getZExtValue();
12187 if (ShiftAmt == 64)
12189 Ops[1] =
Builder.CreateLShr(Ops[1], ConstantInt::get(
Int64Ty, ShiftAmt),
12191 return Builder.CreateAdd(Ops[0], Ops[1]);
12193 case NEON::BI__builtin_neon_vqdmlalh_lane_s16:
12194 case NEON::BI__builtin_neon_vqdmlalh_laneq_s16:
12195 case NEON::BI__builtin_neon_vqdmlslh_lane_s16:
12196 case NEON::BI__builtin_neon_vqdmlslh_laneq_s16: {
12202 auto *VTy = llvm::FixedVectorType::get(
Int32Ty, 4);
12204 ProductOps,
"vqdmlXl");
12205 Constant *CI = ConstantInt::get(
SizeTy, 0);
12206 Ops[1] =
Builder.CreateExtractElement(Ops[1], CI,
"lane0");
12209 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlalh_lane_s16 ||
12210 BuiltinID == NEON::BI__builtin_neon_vqdmlalh_laneq_s16)
12211 ? Intrinsic::aarch64_neon_sqadd
12212 : Intrinsic::aarch64_neon_sqsub;
12215 case NEON::BI__builtin_neon_vqdmlals_s32:
12216 case NEON::BI__builtin_neon_vqdmlsls_s32: {
12218 ProductOps.push_back(Ops[1]);
12222 ProductOps,
"vqdmlXl");
12224 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlals_s32
12225 ? Intrinsic::aarch64_neon_sqadd
12226 : Intrinsic::aarch64_neon_sqsub;
12229 case NEON::BI__builtin_neon_vqdmlals_lane_s32:
12230 case NEON::BI__builtin_neon_vqdmlals_laneq_s32:
12231 case NEON::BI__builtin_neon_vqdmlsls_lane_s32:
12232 case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: {
12236 ProductOps.push_back(Ops[1]);
12237 ProductOps.push_back(Ops[2]);
12240 ProductOps,
"vqdmlXl");
12243 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlals_lane_s32 ||
12244 BuiltinID == NEON::BI__builtin_neon_vqdmlals_laneq_s32)
12245 ? Intrinsic::aarch64_neon_sqadd
12246 : Intrinsic::aarch64_neon_sqsub;
12249 case NEON::BI__builtin_neon_vget_lane_bf16:
12250 case NEON::BI__builtin_neon_vduph_lane_bf16:
12251 case NEON::BI__builtin_neon_vduph_lane_f16: {
12255 case NEON::BI__builtin_neon_vgetq_lane_bf16:
12256 case NEON::BI__builtin_neon_vduph_laneq_bf16:
12257 case NEON::BI__builtin_neon_vduph_laneq_f16: {
12262 case clang::AArch64::BI_InterlockedAdd:
12263 case clang::AArch64::BI_InterlockedAdd64: {
12266 AtomicRMWInst *RMWI =
12268 llvm::AtomicOrdering::SequentiallyConsistent);
12269 return Builder.CreateAdd(RMWI, Val);
12274 llvm::Type *Ty = VTy;
12285 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
12286 Builtin->NameHint, Builtin->TypeModifier, E, Ops,
12293 switch (BuiltinID) {
12294 default:
return nullptr;
12295 case NEON::BI__builtin_neon_vbsl_v:
12296 case NEON::BI__builtin_neon_vbslq_v: {
12297 llvm::Type *BitTy = llvm::VectorType::getInteger(VTy);
12298 Ops[0] =
Builder.CreateBitCast(Ops[0], BitTy,
"vbsl");
12299 Ops[1] =
Builder.CreateBitCast(Ops[1], BitTy,
"vbsl");
12300 Ops[2] =
Builder.CreateBitCast(Ops[2], BitTy,
"vbsl");
12302 Ops[1] =
Builder.CreateAnd(Ops[0], Ops[1],
"vbsl");
12303 Ops[2] =
Builder.CreateAnd(
Builder.CreateNot(Ops[0]), Ops[2],
"vbsl");
12304 Ops[0] =
Builder.CreateOr(Ops[1], Ops[2],
"vbsl");
12305 return Builder.CreateBitCast(Ops[0], Ty);
12307 case NEON::BI__builtin_neon_vfma_lane_v:
12308 case NEON::BI__builtin_neon_vfmaq_lane_v: {
12311 Value *Addend = Ops[0];
12312 Value *Multiplicand = Ops[1];
12313 Value *LaneSource = Ops[2];
12314 Ops[0] = Multiplicand;
12315 Ops[1] = LaneSource;
12319 auto *SourceTy = BuiltinID == NEON::BI__builtin_neon_vfmaq_lane_v
12320 ? llvm::FixedVectorType::get(VTy->getElementType(),
12321 VTy->getNumElements() / 2)
12323 llvm::Constant *cst = cast<Constant>(Ops[3]);
12324 Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(), cst);
12325 Ops[1] =
Builder.CreateBitCast(Ops[1], SourceTy);
12326 Ops[1] =
Builder.CreateShuffleVector(Ops[1], Ops[1], SV,
"lane");
12329 Int =
Builder.getIsFPConstrained() ? Intrinsic::experimental_constrained_fma
12333 case NEON::BI__builtin_neon_vfma_laneq_v: {
12334 auto *VTy = cast<llvm::FixedVectorType>(Ty);
12336 if (VTy && VTy->getElementType() ==
DoubleTy) {
12339 llvm::FixedVectorType *VTy =
12341 Ops[2] =
Builder.CreateBitCast(Ops[2], VTy);
12342 Ops[2] =
Builder.CreateExtractElement(Ops[2], Ops[3],
"extract");
12345 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma,
12346 DoubleTy, {Ops[1], Ops[2], Ops[0]});
12349 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
12350 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
12352 auto *STy = llvm::FixedVectorType::get(VTy->getElementType(),
12353 VTy->getNumElements() * 2);
12354 Ops[2] =
Builder.CreateBitCast(Ops[2], STy);
12355 Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(),
12356 cast<ConstantInt>(Ops[3]));
12357 Ops[2] =
Builder.CreateShuffleVector(Ops[2], Ops[2], SV,
"lane");
12360 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
12361 {Ops[2], Ops[1], Ops[0]});
12363 case NEON::BI__builtin_neon_vfmaq_laneq_v: {
12364 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
12365 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
12367 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
12370 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
12371 {Ops[2], Ops[1], Ops[0]});
12373 case NEON::BI__builtin_neon_vfmah_lane_f16:
12374 case NEON::BI__builtin_neon_vfmas_lane_f32:
12375 case NEON::BI__builtin_neon_vfmah_laneq_f16:
12376 case NEON::BI__builtin_neon_vfmas_laneq_f32:
12377 case NEON::BI__builtin_neon_vfmad_lane_f64:
12378 case NEON::BI__builtin_neon_vfmad_laneq_f64: {
12381 Ops[2] =
Builder.CreateExtractElement(Ops[2], Ops[3],
"extract");
12383 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
12384 {Ops[1], Ops[2], Ops[0]});
12386 case NEON::BI__builtin_neon_vmull_v:
12388 Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull;
12389 if (
Type.isPoly())
Int = Intrinsic::aarch64_neon_pmull;
12391 case NEON::BI__builtin_neon_vmax_v:
12392 case NEON::BI__builtin_neon_vmaxq_v:
12394 Int = usgn ? Intrinsic::aarch64_neon_umax : Intrinsic::aarch64_neon_smax;
12395 if (Ty->isFPOrFPVectorTy())
Int = Intrinsic::aarch64_neon_fmax;
12397 case NEON::BI__builtin_neon_vmaxh_f16: {
12399 Int = Intrinsic::aarch64_neon_fmax;
12402 case NEON::BI__builtin_neon_vmin_v:
12403 case NEON::BI__builtin_neon_vminq_v:
12405 Int = usgn ? Intrinsic::aarch64_neon_umin : Intrinsic::aarch64_neon_smin;
12406 if (Ty->isFPOrFPVectorTy())
Int = Intrinsic::aarch64_neon_fmin;
12408 case NEON::BI__builtin_neon_vminh_f16: {
12410 Int = Intrinsic::aarch64_neon_fmin;
12413 case NEON::BI__builtin_neon_vabd_v:
12414 case NEON::BI__builtin_neon_vabdq_v:
12416 Int = usgn ? Intrinsic::aarch64_neon_uabd : Intrinsic::aarch64_neon_sabd;
12417 if (Ty->isFPOrFPVectorTy())
Int = Intrinsic::aarch64_neon_fabd;
12419 case NEON::BI__builtin_neon_vpadal_v:
12420 case NEON::BI__builtin_neon_vpadalq_v: {
12421 unsigned ArgElts = VTy->getNumElements();
12422 llvm::IntegerType *EltTy = cast<IntegerType>(VTy->getElementType());
12423 unsigned BitWidth = EltTy->getBitWidth();
12424 auto *ArgTy = llvm::FixedVectorType::get(
12425 llvm::IntegerType::get(
getLLVMContext(), BitWidth / 2), 2 * ArgElts);
12426 llvm::Type* Tys[2] = { VTy, ArgTy };
12427 Int = usgn ? Intrinsic::aarch64_neon_uaddlp : Intrinsic::aarch64_neon_saddlp;
12429 TmpOps.push_back(Ops[1]);
12432 llvm::Value *addend =
Builder.CreateBitCast(Ops[0], tmp->getType());
12433 return Builder.CreateAdd(tmp, addend);
12435 case NEON::BI__builtin_neon_vpmin_v:
12436 case NEON::BI__builtin_neon_vpminq_v:
12438 Int = usgn ? Intrinsic::aarch64_neon_uminp : Intrinsic::aarch64_neon_sminp;
12439 if (Ty->isFPOrFPVectorTy())
Int = Intrinsic::aarch64_neon_fminp;
12441 case NEON::BI__builtin_neon_vpmax_v:
12442 case NEON::BI__builtin_neon_vpmaxq_v:
12444 Int = usgn ? Intrinsic::aarch64_neon_umaxp : Intrinsic::aarch64_neon_smaxp;
12445 if (Ty->isFPOrFPVectorTy())
Int = Intrinsic::aarch64_neon_fmaxp;
12447 case NEON::BI__builtin_neon_vminnm_v:
12448 case NEON::BI__builtin_neon_vminnmq_v:
12449 Int = Intrinsic::aarch64_neon_fminnm;
12451 case NEON::BI__builtin_neon_vminnmh_f16:
12453 Int = Intrinsic::aarch64_neon_fminnm;
12455 case NEON::BI__builtin_neon_vmaxnm_v:
12456 case NEON::BI__builtin_neon_vmaxnmq_v:
12457 Int = Intrinsic::aarch64_neon_fmaxnm;
12459 case NEON::BI__builtin_neon_vmaxnmh_f16:
12461 Int = Intrinsic::aarch64_neon_fmaxnm;
12463 case NEON::BI__builtin_neon_vrecpss_f32: {
12468 case NEON::BI__builtin_neon_vrecpsd_f64:
12472 case NEON::BI__builtin_neon_vrecpsh_f16:
12476 case NEON::BI__builtin_neon_vqshrun_n_v:
12477 Int = Intrinsic::aarch64_neon_sqshrun;
12479 case NEON::BI__builtin_neon_vqrshrun_n_v:
12480 Int = Intrinsic::aarch64_neon_sqrshrun;
12482 case NEON::BI__builtin_neon_vqshrn_n_v:
12483 Int = usgn ? Intrinsic::aarch64_neon_uqshrn : Intrinsic::aarch64_neon_sqshrn;
12485 case NEON::BI__builtin_neon_vrshrn_n_v:
12486 Int = Intrinsic::aarch64_neon_rshrn;
12488 case NEON::BI__builtin_neon_vqrshrn_n_v:
12489 Int = usgn ? Intrinsic::aarch64_neon_uqrshrn : Intrinsic::aarch64_neon_sqrshrn;
12491 case NEON::BI__builtin_neon_vrndah_f16: {
12494 ? Intrinsic::experimental_constrained_round
12495 : Intrinsic::round;
12498 case NEON::BI__builtin_neon_vrnda_v:
12499 case NEON::BI__builtin_neon_vrndaq_v: {
12501 ? Intrinsic::experimental_constrained_round
12502 : Intrinsic::round;
12505 case NEON::BI__builtin_neon_vrndih_f16: {
12508 ? Intrinsic::experimental_constrained_nearbyint
12509 : Intrinsic::nearbyint;
12512 case NEON::BI__builtin_neon_vrndmh_f16: {
12515 ? Intrinsic::experimental_constrained_floor
12516 : Intrinsic::floor;
12519 case NEON::BI__builtin_neon_vrndm_v:
12520 case NEON::BI__builtin_neon_vrndmq_v: {
12522 ? Intrinsic::experimental_constrained_floor
12523 : Intrinsic::floor;
12526 case NEON::BI__builtin_neon_vrndnh_f16: {
12529 ? Intrinsic::experimental_constrained_roundeven
12530 : Intrinsic::roundeven;
12533 case NEON::BI__builtin_neon_vrndn_v:
12534 case NEON::BI__builtin_neon_vrndnq_v: {
12536 ? Intrinsic::experimental_constrained_roundeven
12537 : Intrinsic::roundeven;
12540 case NEON::BI__builtin_neon_vrndns_f32: {
12543 ? Intrinsic::experimental_constrained_roundeven
12544 : Intrinsic::roundeven;
12547 case NEON::BI__builtin_neon_vrndph_f16: {
12550 ? Intrinsic::experimental_constrained_ceil
12554 case NEON::BI__builtin_neon_vrndp_v:
12555 case NEON::BI__builtin_neon_vrndpq_v: {
12557 ? Intrinsic::experimental_constrained_ceil
12561 case NEON::BI__builtin_neon_vrndxh_f16: {
12564 ? Intrinsic::experimental_constrained_rint
12568 case NEON::BI__builtin_neon_vrndx_v:
12569 case NEON::BI__builtin_neon_vrndxq_v: {
12571 ? Intrinsic::experimental_constrained_rint
12575 case NEON::BI__builtin_neon_vrndh_f16: {
12578 ? Intrinsic::experimental_constrained_trunc
12579 : Intrinsic::trunc;
12582 case NEON::BI__builtin_neon_vrnd32x_f32:
12583 case NEON::BI__builtin_neon_vrnd32xq_f32:
12584 case NEON::BI__builtin_neon_vrnd32x_f64:
12585 case NEON::BI__builtin_neon_vrnd32xq_f64: {
12587 Int = Intrinsic::aarch64_neon_frint32x;
12590 case NEON::BI__builtin_neon_vrnd32z_f32:
12591 case NEON::BI__builtin_neon_vrnd32zq_f32:
12592 case NEON::BI__builtin_neon_vrnd32z_f64:
12593 case NEON::BI__builtin_neon_vrnd32zq_f64: {
12595 Int = Intrinsic::aarch64_neon_frint32z;
12598 case NEON::BI__builtin_neon_vrnd64x_f32:
12599 case NEON::BI__builtin_neon_vrnd64xq_f32:
12600 case NEON::BI__builtin_neon_vrnd64x_f64:
12601 case NEON::BI__builtin_neon_vrnd64xq_f64: {
12603 Int = Intrinsic::aarch64_neon_frint64x;
12606 case NEON::BI__builtin_neon_vrnd64z_f32:
12607 case NEON::BI__builtin_neon_vrnd64zq_f32:
12608 case NEON::BI__builtin_neon_vrnd64z_f64:
12609 case NEON::BI__builtin_neon_vrnd64zq_f64: {
12611 Int = Intrinsic::aarch64_neon_frint64z;
12614 case NEON::BI__builtin_neon_vrnd_v:
12615 case NEON::BI__builtin_neon_vrndq_v: {
12617 ? Intrinsic::experimental_constrained_trunc
12618 : Intrinsic::trunc;
12621 case NEON::BI__builtin_neon_vcvt_f64_v:
12622 case NEON::BI__builtin_neon_vcvtq_f64_v:
12623 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
12625 return usgn ?
Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
12626 :
Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
12627 case NEON::BI__builtin_neon_vcvt_f64_f32: {
12629 "unexpected vcvt_f64_f32 builtin");
12633 return Builder.CreateFPExt(Ops[0], Ty,
"vcvt");
12635 case NEON::BI__builtin_neon_vcvt_f32_f64: {
12637 "unexpected vcvt_f32_f64 builtin");
12641 return Builder.CreateFPTrunc(Ops[0], Ty,
"vcvt");
12643 case NEON::BI__builtin_neon_vcvt_s32_v:
12644 case NEON::BI__builtin_neon_vcvt_u32_v:
12645 case NEON::BI__builtin_neon_vcvt_s64_v:
12646 case NEON::BI__builtin_neon_vcvt_u64_v:
12647 case NEON::BI__builtin_neon_vcvt_s16_f16:
12648 case NEON::BI__builtin_neon_vcvt_u16_f16:
12649 case NEON::BI__builtin_neon_vcvtq_s32_v:
12650 case NEON::BI__builtin_neon_vcvtq_u32_v:
12651 case NEON::BI__builtin_neon_vcvtq_s64_v:
12652 case NEON::BI__builtin_neon_vcvtq_u64_v:
12653 case NEON::BI__builtin_neon_vcvtq_s16_f16:
12654 case NEON::BI__builtin_neon_vcvtq_u16_f16: {
12656 usgn ? Intrinsic::aarch64_neon_fcvtzu : Intrinsic::aarch64_neon_fcvtzs;
12660 case NEON::BI__builtin_neon_vcvta_s16_f16:
12661 case NEON::BI__builtin_neon_vcvta_u16_f16:
12662 case NEON::BI__builtin_neon_vcvta_s32_v:
12663 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
12664 case NEON::BI__builtin_neon_vcvtaq_s32_v:
12665 case NEON::BI__builtin_neon_vcvta_u32_v:
12666 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
12667 case NEON::BI__builtin_neon_vcvtaq_u32_v:
12668 case NEON::BI__builtin_neon_vcvta_s64_v:
12669 case NEON::BI__builtin_neon_vcvtaq_s64_v:
12670 case NEON::BI__builtin_neon_vcvta_u64_v:
12671 case NEON::BI__builtin_neon_vcvtaq_u64_v: {
12672 Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas;
12676 case NEON::BI__builtin_neon_vcvtm_s16_f16:
12677 case NEON::BI__builtin_neon_vcvtm_s32_v:
12678 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
12679 case NEON::BI__builtin_neon_vcvtmq_s32_v:
12680 case NEON::BI__builtin_neon_vcvtm_u16_f16:
12681 case NEON::BI__builtin_neon_vcvtm_u32_v:
12682 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
12683 case NEON::BI__builtin_neon_vcvtmq_u32_v:
12684 case NEON::BI__builtin_neon_vcvtm_s64_v:
12685 case NEON::BI__builtin_neon_vcvtmq_s64_v:
12686 case NEON::BI__builtin_neon_vcvtm_u64_v:
12687 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
12688 Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms;
12692 case NEON::BI__builtin_neon_vcvtn_s16_f16:
12693 case NEON::BI__builtin_neon_vcvtn_s32_v:
12694 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
12695 case NEON::BI__builtin_neon_vcvtnq_s32_v:
12696 case NEON::BI__builtin_neon_vcvtn_u16_f16:
12697 case NEON::BI__builtin_neon_vcvtn_u32_v:
12698 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
12699 case NEON::BI__builtin_neon_vcvtnq_u32_v:
12700 case NEON::BI__builtin_neon_vcvtn_s64_v:
12701 case NEON::BI__builtin_neon_vcvtnq_s64_v:
12702 case NEON::BI__builtin_neon_vcvtn_u64_v:
12703 case NEON::BI__builtin_neon_vcvtnq_u64_v: {
12704 Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns;
12708 case NEON::BI__builtin_neon_vcvtp_s16_f16:
12709 case NEON::BI__builtin_neon_vcvtp_s32_v:
12710 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
12711 case NEON::BI__builtin_neon_vcvtpq_s32_v:
12712 case NEON::BI__builtin_neon_vcvtp_u16_f16:
12713 case NEON::BI__builtin_neon_vcvtp_u32_v:
12714 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
12715 case NEON::BI__builtin_neon_vcvtpq_u32_v:
12716 case NEON::BI__builtin_neon_vcvtp_s64_v:
12717 case NEON::BI__builtin_neon_vcvtpq_s64_v:
12718 case NEON::BI__builtin_neon_vcvtp_u64_v:
12719 case NEON::BI__builtin_neon_vcvtpq_u64_v: {
12720 Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps;
12724 case NEON::BI__builtin_neon_vmulx_v:
12725 case NEON::BI__builtin_neon_vmulxq_v: {
12726 Int = Intrinsic::aarch64_neon_fmulx;
12729 case NEON::BI__builtin_neon_vmulxh_lane_f16:
12730 case NEON::BI__builtin_neon_vmulxh_laneq_f16: {
12734 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2],
"extract");
12736 Int = Intrinsic::aarch64_neon_fmulx;
12739 case NEON::BI__builtin_neon_vmul_lane_v:
12740 case NEON::BI__builtin_neon_vmul_laneq_v: {
12743 if (BuiltinID == NEON::BI__builtin_neon_vmul_laneq_v)
12746 llvm::FixedVectorType *VTy =
12748 Ops[1] =
Builder.CreateBitCast(Ops[1], VTy);
12749 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2],
"extract");
12753 case NEON::BI__builtin_neon_vnegd_s64:
12755 case NEON::BI__builtin_neon_vnegh_f16:
12757 case NEON::BI__builtin_neon_vpmaxnm_v:
12758 case NEON::BI__builtin_neon_vpmaxnmq_v: {
12759 Int = Intrinsic::aarch64_neon_fmaxnmp;
12762 case NEON::BI__builtin_neon_vpminnm_v:
12763 case NEON::BI__builtin_neon_vpminnmq_v: {
12764 Int = Intrinsic::aarch64_neon_fminnmp;
12767 case NEON::BI__builtin_neon_vsqrth_f16: {
12770 ? Intrinsic::experimental_constrained_sqrt
12774 case NEON::BI__builtin_neon_vsqrt_v:
12775 case NEON::BI__builtin_neon_vsqrtq_v: {
12777 ? Intrinsic::experimental_constrained_sqrt
12779 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
12782 case NEON::BI__builtin_neon_vrbit_v:
12783 case NEON::BI__builtin_neon_vrbitq_v: {
12784 Int = Intrinsic::bitreverse;
12787 case NEON::BI__builtin_neon_vaddv_u8:
12791 case NEON::BI__builtin_neon_vaddv_s8: {
12792 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
12794 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
12795 llvm::Type *Tys[2] = { Ty, VTy };
12800 case NEON::BI__builtin_neon_vaddv_u16:
12803 case NEON::BI__builtin_neon_vaddv_s16: {
12804 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
12806 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
12807 llvm::Type *Tys[2] = { Ty, VTy };
12812 case NEON::BI__builtin_neon_vaddvq_u8:
12815 case NEON::BI__builtin_neon_vaddvq_s8: {
12816 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
12818 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
12819 llvm::Type *Tys[2] = { Ty, VTy };
12824 case NEON::BI__builtin_neon_vaddvq_u16:
12827 case NEON::BI__builtin_neon_vaddvq_s16: {
12828 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
12830 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
12831 llvm::Type *Tys[2] = { Ty, VTy };
12836 case NEON::BI__builtin_neon_vmaxv_u8: {
12837 Int = Intrinsic::aarch64_neon_umaxv;
12839 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
12840 llvm::Type *Tys[2] = { Ty, VTy };
12845 case NEON::BI__builtin_neon_vmaxv_u16: {
12846 Int = Intrinsic::aarch64_neon_umaxv;
12848 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
12849 llvm::Type *Tys[2] = { Ty, VTy };
12854 case NEON::BI__builtin_neon_vmaxvq_u8: {
12855 Int = Intrinsic::aarch64_neon_umaxv;
12857 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
12858 llvm::Type *Tys[2] = { Ty, VTy };
12863 case NEON::BI__builtin_neon_vmaxvq_u16: {
12864 Int = Intrinsic::aarch64_neon_umaxv;
12866 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
12867 llvm::Type *Tys[2] = { Ty, VTy };
12872 case NEON::BI__builtin_neon_vmaxv_s8: {
12873 Int = Intrinsic::aarch64_neon_smaxv;
12875 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
12876 llvm::Type *Tys[2] = { Ty, VTy };
12881 case NEON::BI__builtin_neon_vmaxv_s16: {
12882 Int = Intrinsic::aarch64_neon_smaxv;
12884 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
12885 llvm::Type *Tys[2] = { Ty, VTy };
12890 case NEON::BI__builtin_neon_vmaxvq_s8: {
12891 Int = Intrinsic::aarch64_neon_smaxv;
12893 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
12894 llvm::Type *Tys[2] = { Ty, VTy };
12899 case NEON::BI__builtin_neon_vmaxvq_s16: {
12900 Int = Intrinsic::aarch64_neon_smaxv;
12902 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
12903 llvm::Type *Tys[2] = { Ty, VTy };
12908 case NEON::BI__builtin_neon_vmaxv_f16: {
12909 Int = Intrinsic::aarch64_neon_fmaxv;
12911 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
12912 llvm::Type *Tys[2] = { Ty, VTy };
12917 case NEON::BI__builtin_neon_vmaxvq_f16: {
12918 Int = Intrinsic::aarch64_neon_fmaxv;
12920 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
12921 llvm::Type *Tys[2] = { Ty, VTy };
12926 case NEON::BI__builtin_neon_vminv_u8: {
12927 Int = Intrinsic::aarch64_neon_uminv;
12929 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
12930 llvm::Type *Tys[2] = { Ty, VTy };
12935 case NEON::BI__builtin_neon_vminv_u16: {
12936 Int = Intrinsic::aarch64_neon_uminv;
12938 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
12939 llvm::Type *Tys[2] = { Ty, VTy };
12944 case NEON::BI__builtin_neon_vminvq_u8: {
12945 Int = Intrinsic::aarch64_neon_uminv;
12947 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
12948 llvm::Type *Tys[2] = { Ty, VTy };
12953 case NEON::BI__builtin_neon_vminvq_u16: {
12954 Int = Intrinsic::aarch64_neon_uminv;
12956 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
12957 llvm::Type *Tys[2] = { Ty, VTy };
12962 case NEON::BI__builtin_neon_vminv_s8: {
12963 Int = Intrinsic::aarch64_neon_sminv;
12965 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
12966 llvm::Type *Tys[2] = { Ty, VTy };
12971 case NEON::BI__builtin_neon_vminv_s16: {
12972 Int = Intrinsic::aarch64_neon_sminv;
12974 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
12975 llvm::Type *Tys[2] = { Ty, VTy };
12980 case NEON::BI__builtin_neon_vminvq_s8: {
12981 Int = Intrinsic::aarch64_neon_sminv;
12983 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
12984 llvm::Type *Tys[2] = { Ty, VTy };
12989 case NEON::BI__builtin_neon_vminvq_s16: {
12990 Int = Intrinsic::aarch64_neon_sminv;
12992 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
12993 llvm::Type *Tys[2] = { Ty, VTy };
12998 case NEON::BI__builtin_neon_vminv_f16: {
12999 Int = Intrinsic::aarch64_neon_fminv;
13001 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
13002 llvm::Type *Tys[2] = { Ty, VTy };
13007 case NEON::BI__builtin_neon_vminvq_f16: {
13008 Int = Intrinsic::aarch64_neon_fminv;
13010 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
13011 llvm::Type *Tys[2] = { Ty, VTy };
13016 case NEON::BI__builtin_neon_vmaxnmv_f16: {
13017 Int = Intrinsic::aarch64_neon_fmaxnmv;
13019 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
13020 llvm::Type *Tys[2] = { Ty, VTy };
13025 case NEON::BI__builtin_neon_vmaxnmvq_f16: {
13026 Int = Intrinsic::aarch64_neon_fmaxnmv;
13028 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
13029 llvm::Type *Tys[2] = { Ty, VTy };
13034 case NEON::BI__builtin_neon_vminnmv_f16: {
13035 Int = Intrinsic::aarch64_neon_fminnmv;
13037 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
13038 llvm::Type *Tys[2] = { Ty, VTy };
13043 case NEON::BI__builtin_neon_vminnmvq_f16: {
13044 Int = Intrinsic::aarch64_neon_fminnmv;
13046 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
13047 llvm::Type *Tys[2] = { Ty, VTy };
13052 case NEON::BI__builtin_neon_vmul_n_f64: {
13055 return Builder.CreateFMul(Ops[0], RHS);
13057 case NEON::BI__builtin_neon_vaddlv_u8: {
13058 Int = Intrinsic::aarch64_neon_uaddlv;
13060 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
13061 llvm::Type *Tys[2] = { Ty, VTy };
13066 case NEON::BI__builtin_neon_vaddlv_u16: {
13067 Int = Intrinsic::aarch64_neon_uaddlv;
13069 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
13070 llvm::Type *Tys[2] = { Ty, VTy };
13074 case NEON::BI__builtin_neon_vaddlvq_u8: {
13075 Int = Intrinsic::aarch64_neon_uaddlv;
13077 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
13078 llvm::Type *Tys[2] = { Ty, VTy };
13083 case NEON::BI__builtin_neon_vaddlvq_u16: {
13084 Int = Intrinsic::aarch64_neon_uaddlv;
13086 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
13087 llvm::Type *Tys[2] = { Ty, VTy };
13091 case NEON::BI__builtin_neon_vaddlv_s8: {
13092 Int = Intrinsic::aarch64_neon_saddlv;
13094 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
13095 llvm::Type *Tys[2] = { Ty, VTy };
13100 case NEON::BI__builtin_neon_vaddlv_s16: {
13101 Int = Intrinsic::aarch64_neon_saddlv;
13103 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
13104 llvm::Type *Tys[2] = { Ty, VTy };
13108 case NEON::BI__builtin_neon_vaddlvq_s8: {
13109 Int = Intrinsic::aarch64_neon_saddlv;
13111 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
13112 llvm::Type *Tys[2] = { Ty, VTy };
13117 case NEON::BI__builtin_neon_vaddlvq_s16: {
13118 Int = Intrinsic::aarch64_neon_saddlv;
13120 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
13121 llvm::Type *Tys[2] = { Ty, VTy };
13125 case NEON::BI__builtin_neon_vsri_n_v:
13126 case NEON::BI__builtin_neon_vsriq_n_v: {
13127 Int = Intrinsic::aarch64_neon_vsri;
13131 case NEON::BI__builtin_neon_vsli_n_v:
13132 case NEON::BI__builtin_neon_vsliq_n_v: {
13133 Int = Intrinsic::aarch64_neon_vsli;
13137 case NEON::BI__builtin_neon_vsra_n_v:
13138 case NEON::BI__builtin_neon_vsraq_n_v:
13139 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
13141 return Builder.CreateAdd(Ops[0], Ops[1]);
13142 case NEON::BI__builtin_neon_vrsra_n_v:
13143 case NEON::BI__builtin_neon_vrsraq_n_v: {
13144 Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl;
13146 TmpOps.push_back(Ops[1]);
13147 TmpOps.push_back(Ops[2]);
13149 llvm::Value *tmp =
EmitNeonCall(F, TmpOps,
"vrshr_n", 1,
true);
13150 Ops[0] =
Builder.CreateBitCast(Ops[0], VTy);
13151 return Builder.CreateAdd(Ops[0], tmp);
13153 case NEON::BI__builtin_neon_vld1_v:
13154 case NEON::BI__builtin_neon_vld1q_v: {
13157 case NEON::BI__builtin_neon_vst1_v:
13158 case NEON::BI__builtin_neon_vst1q_v:
13159 Ops[1] =
Builder.CreateBitCast(Ops[1], VTy);
13161 case NEON::BI__builtin_neon_vld1_lane_v:
13162 case NEON::BI__builtin_neon_vld1q_lane_v: {
13163 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13166 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vld1_lane");
13168 case NEON::BI__builtin_neon_vldap1_lane_s64:
13169 case NEON::BI__builtin_neon_vldap1q_lane_s64: {
13170 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13172 VTy->getElementType(), Ops[0], PtrOp0.
getAlignment());
13173 LI->setAtomic(llvm::AtomicOrdering::Acquire);
13175 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vldap1_lane");
13177 case NEON::BI__builtin_neon_vld1_dup_v:
13178 case NEON::BI__builtin_neon_vld1q_dup_v: {
13179 Value *
V = PoisonValue::get(Ty);
13182 llvm::Constant *CI = ConstantInt::get(
Int32Ty, 0);
13183 Ops[0] =
Builder.CreateInsertElement(
V, Ops[0], CI);
13186 case NEON::BI__builtin_neon_vst1_lane_v:
13187 case NEON::BI__builtin_neon_vst1q_lane_v:
13188 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13189 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2]);
13191 case NEON::BI__builtin_neon_vstl1_lane_s64:
13192 case NEON::BI__builtin_neon_vstl1q_lane_s64: {
13193 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13194 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2]);
13195 llvm::StoreInst *SI =
13197 SI->setAtomic(llvm::AtomicOrdering::Release);
13200 case NEON::BI__builtin_neon_vld2_v:
13201 case NEON::BI__builtin_neon_vld2q_v: {
13204 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld2");
13207 case NEON::BI__builtin_neon_vld3_v:
13208 case NEON::BI__builtin_neon_vld3q_v: {
13211 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld3");
13214 case NEON::BI__builtin_neon_vld4_v:
13215 case NEON::BI__builtin_neon_vld4q_v: {
13218 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld4");
13221 case NEON::BI__builtin_neon_vld2_dup_v:
13222 case NEON::BI__builtin_neon_vld2q_dup_v: {
13225 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld2");
13228 case NEON::BI__builtin_neon_vld3_dup_v:
13229 case NEON::BI__builtin_neon_vld3q_dup_v: {
13232 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld3");
13235 case NEON::BI__builtin_neon_vld4_dup_v:
13236 case NEON::BI__builtin_neon_vld4q_dup_v: {
13239 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld4");
13242 case NEON::BI__builtin_neon_vld2_lane_v:
13243 case NEON::BI__builtin_neon_vld2q_lane_v: {
13244 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
13246 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
13247 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13248 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13253 case NEON::BI__builtin_neon_vld3_lane_v:
13254 case NEON::BI__builtin_neon_vld3q_lane_v: {
13255 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
13257 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
13258 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13259 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13260 Ops[3] =
Builder.CreateBitCast(Ops[3], Ty);
13265 case NEON::BI__builtin_neon_vld4_lane_v:
13266 case NEON::BI__builtin_neon_vld4q_lane_v: {
13267 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
13269 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
13270 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13271 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13272 Ops[3] =
Builder.CreateBitCast(Ops[3], Ty);
13273 Ops[4] =
Builder.CreateBitCast(Ops[4], Ty);
13278 case NEON::BI__builtin_neon_vst2_v:
13279 case NEON::BI__builtin_neon_vst2q_v: {
13280 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13281 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() };
13285 case NEON::BI__builtin_neon_vst2_lane_v:
13286 case NEON::BI__builtin_neon_vst2q_lane_v: {
13287 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13289 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
13293 case NEON::BI__builtin_neon_vst3_v:
13294 case NEON::BI__builtin_neon_vst3q_v: {
13295 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13296 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
13300 case NEON::BI__builtin_neon_vst3_lane_v:
13301 case NEON::BI__builtin_neon_vst3q_lane_v: {
13302 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13304 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
13308 case NEON::BI__builtin_neon_vst4_v:
13309 case NEON::BI__builtin_neon_vst4q_v: {
13310 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13311 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
13315 case NEON::BI__builtin_neon_vst4_lane_v:
13316 case NEON::BI__builtin_neon_vst4q_lane_v: {
13317 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13319 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() };
13323 case NEON::BI__builtin_neon_vtrn_v:
13324 case NEON::BI__builtin_neon_vtrnq_v: {
13325 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13326 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13327 Value *SV =
nullptr;
13329 for (
unsigned vi = 0; vi != 2; ++vi) {
13331 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
13332 Indices.push_back(i+vi);
13333 Indices.push_back(i+e+vi);
13335 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
13336 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vtrn");
13341 case NEON::BI__builtin_neon_vuzp_v:
13342 case NEON::BI__builtin_neon_vuzpq_v: {
13343 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13344 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13345 Value *SV =
nullptr;
13347 for (
unsigned vi = 0; vi != 2; ++vi) {
13349 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
13350 Indices.push_back(2*i+vi);
13352 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
13353 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vuzp");
13358 case NEON::BI__builtin_neon_vzip_v:
13359 case NEON::BI__builtin_neon_vzipq_v: {
13360 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13361 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13362 Value *SV =
nullptr;
13364 for (
unsigned vi = 0; vi != 2; ++vi) {
13366 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
13367 Indices.push_back((i + vi*e) >> 1);
13368 Indices.push_back(((i + vi*e) >> 1)+e);
13370 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
13371 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vzip");
13376 case NEON::BI__builtin_neon_vqtbl1q_v: {
13380 case NEON::BI__builtin_neon_vqtbl2q_v: {
13384 case NEON::BI__builtin_neon_vqtbl3q_v: {
13388 case NEON::BI__builtin_neon_vqtbl4q_v: {
13392 case NEON::BI__builtin_neon_vqtbx1q_v: {
13396 case NEON::BI__builtin_neon_vqtbx2q_v: {
13400 case NEON::BI__builtin_neon_vqtbx3q_v: {
13404 case NEON::BI__builtin_neon_vqtbx4q_v: {
13408 case NEON::BI__builtin_neon_vsqadd_v:
13409 case NEON::BI__builtin_neon_vsqaddq_v: {
13410 Int = Intrinsic::aarch64_neon_usqadd;
13413 case NEON::BI__builtin_neon_vuqadd_v:
13414 case NEON::BI__builtin_neon_vuqaddq_v: {
13415 Int = Intrinsic::aarch64_neon_suqadd;
13423 assert((BuiltinID == BPF::BI__builtin_preserve_field_info ||
13424 BuiltinID == BPF::BI__builtin_btf_type_id ||
13425 BuiltinID == BPF::BI__builtin_preserve_type_info ||
13426 BuiltinID == BPF::BI__builtin_preserve_enum_value) &&
13427 "unexpected BPF builtin");
13432 static uint32_t BuiltinSeqNum;
13434 switch (BuiltinID) {
13436 llvm_unreachable(
"Unexpected BPF builtin");
13437 case BPF::BI__builtin_preserve_field_info: {
13443 "using __builtin_preserve_field_info() without -g");
13456 Value *InfoKind = ConstantInt::get(
Int64Ty,
C->getSExtValue());
13459 llvm::Function *FnGetFieldInfo = llvm::Intrinsic::getDeclaration(
13460 &
CGM.
getModule(), llvm::Intrinsic::bpf_preserve_field_info,
13461 {FieldAddr->getType()});
13462 return Builder.CreateCall(FnGetFieldInfo, {FieldAddr, InfoKind});
13464 case BPF::BI__builtin_btf_type_id:
13465 case BPF::BI__builtin_preserve_type_info: {
13476 Value *FlagValue = ConstantInt::get(
Int64Ty, Flag->getSExtValue());
13477 Value *SeqNumVal = ConstantInt::get(
Int32Ty, BuiltinSeqNum++);
13479 llvm::Function *FnDecl;
13480 if (BuiltinID == BPF::BI__builtin_btf_type_id)
13481 FnDecl = llvm::Intrinsic::getDeclaration(
13482 &
CGM.
getModule(), llvm::Intrinsic::bpf_btf_type_id, {});
13484 FnDecl = llvm::Intrinsic::getDeclaration(
13485 &
CGM.
getModule(), llvm::Intrinsic::bpf_preserve_type_info, {});
13486 CallInst *Fn =
Builder.CreateCall(FnDecl, {SeqNumVal, FlagValue});
13487 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
13490 case BPF::BI__builtin_preserve_enum_value: {
13501 const auto *UO = cast<UnaryOperator>(Arg0->
IgnoreParens());
13502 const auto *CE = cast<CStyleCastExpr>(UO->getSubExpr());
13503 const auto *DR = cast<DeclRefExpr>(CE->getSubExpr());
13504 const auto *Enumerator = cast<EnumConstantDecl>(DR->getDecl());
13506 auto InitVal = Enumerator->getInitVal();
13507 std::string InitValStr;
13508 if (InitVal.isNegative() || InitVal >
uint64_t(INT64_MAX))
13509 InitValStr = std::to_string(InitVal.getSExtValue());
13511 InitValStr = std::to_string(InitVal.getZExtValue());
13512 std::string EnumStr = Enumerator->getNameAsString() +
":" + InitValStr;
13513 Value *EnumStrVal =
Builder.CreateGlobalStringPtr(EnumStr);
13516 Value *FlagValue = ConstantInt::get(
Int64Ty, Flag->getSExtValue());
13517 Value *SeqNumVal = ConstantInt::get(
Int32Ty, BuiltinSeqNum++);
13519 llvm::Function *IntrinsicFn = llvm::Intrinsic::getDeclaration(
13520 &
CGM.
getModule(), llvm::Intrinsic::bpf_preserve_enum_value, {});
13522 Builder.CreateCall(IntrinsicFn, {SeqNumVal, EnumStrVal, FlagValue});
13523 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
13531 assert((Ops.size() & (Ops.size() - 1)) == 0 &&
13532 "Not a power-of-two sized vector!");
13533 bool AllConstants =
true;
13534 for (
unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i)
13535 AllConstants &= isa<Constant>(Ops[i]);
13538 if (AllConstants) {
13540 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
13541 CstOps.push_back(cast<Constant>(Ops[i]));
13542 return llvm::ConstantVector::get(CstOps);
13547 llvm::FixedVectorType::get(Ops[0]->getType(), Ops.size()));
13549 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
13557 unsigned NumElts) {
13559 auto *MaskTy = llvm::FixedVectorType::get(
13561 cast<IntegerType>(Mask->
getType())->getBitWidth());
13562 Value *MaskVec = CGF.
Builder.CreateBitCast(Mask, MaskTy);
13568 for (
unsigned i = 0; i != NumElts; ++i)
13570 MaskVec = CGF.
Builder.CreateShuffleVector(
13571 MaskVec, MaskVec,
ArrayRef(Indices, NumElts),
"extract");
13578 Value *Ptr = Ops[0];
13582 cast<llvm::FixedVectorType>(Ops[1]->getType())->getNumElements());
13584 return CGF.
Builder.CreateMaskedStore(Ops[1], Ptr, Alignment, MaskVec);
13589 llvm::Type *Ty = Ops[1]->getType();
13590 Value *Ptr = Ops[0];
13593 CGF, Ops[2], cast<llvm::FixedVectorType>(Ty)->getNumElements());
13595 return CGF.
Builder.CreateMaskedLoad(Ty, Ptr, Alignment, MaskVec, Ops[1]);
13600 auto *ResultTy = cast<llvm::VectorType>(Ops[1]->getType());
13601 Value *Ptr = Ops[0];
13604 CGF, Ops[2], cast<FixedVectorType>(ResultTy)->getNumElements());
13606 llvm::Function *F = CGF.
CGM.
getIntrinsic(Intrinsic::masked_expandload,
13608 return CGF.
Builder.CreateCall(F, { Ptr, MaskVec, Ops[1] });
13614 auto *ResultTy = cast<llvm::FixedVectorType>(Ops[1]->getType());
13618 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
13619 : Intrinsic::x86_avx512_mask_expand;
13621 return CGF.
Builder.CreateCall(F, { Ops[0], Ops[1], MaskVec });
13626 auto *ResultTy = cast<llvm::FixedVectorType>(Ops[1]->getType());
13627 Value *Ptr = Ops[0];
13631 llvm::Function *F = CGF.
CGM.
getIntrinsic(Intrinsic::masked_compressstore,
13633 return CGF.
Builder.CreateCall(F, { Ops[1], Ptr, MaskVec });
13638 bool InvertLHS =
false) {
13639 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
13644 LHS = CGF.
Builder.CreateNot(LHS);
13646 return CGF.
Builder.CreateBitCast(CGF.
Builder.CreateBinOp(Opc, LHS, RHS),
13647 Ops[0]->getType());
13651 Value *Amt,
bool IsRight) {
13652 llvm::Type *Ty = Op0->
getType();
13658 unsigned NumElts = cast<llvm::FixedVectorType>(Ty)->getNumElements();
13659 Amt = CGF.
Builder.CreateIntCast(Amt, Ty->getScalarType(),
false);
13660 Amt = CGF.
Builder.CreateVectorSplat(NumElts, Amt);
13663 unsigned IID = IsRight ? Intrinsic::fshr : Intrinsic::fshl;
13665 return CGF.
Builder.CreateCall(F, {Op0, Op1, Amt});
13670 Value *Op0 = Ops[0];
13671 Value *Op1 = Ops[1];
13672 llvm::Type *Ty = Op0->
getType();
13673 uint64_t Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
13675 CmpInst::Predicate Pred;
13678 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
13681 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
13684 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
13687 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
13690 Pred = ICmpInst::ICMP_EQ;
13693 Pred = ICmpInst::ICMP_NE;
13696 return llvm::Constant::getNullValue(Ty);
13698 return llvm::Constant::getAllOnesValue(Ty);
13700 llvm_unreachable(
"Unexpected XOP vpcom/vpcomu predicate");
13712 if (
const auto *
C = dyn_cast<Constant>(Mask))
13713 if (
C->isAllOnesValue())
13717 CGF, Mask, cast<llvm::FixedVectorType>(Op0->
getType())->getNumElements());
13719 return CGF.
Builder.CreateSelect(Mask, Op0, Op1);
13725 if (
const auto *
C = dyn_cast<Constant>(Mask))
13726 if (
C->isAllOnesValue())
13729 auto *MaskTy = llvm::FixedVectorType::get(
13730 CGF.
Builder.getInt1Ty(), Mask->
getType()->getIntegerBitWidth());
13731 Mask = CGF.
Builder.CreateBitCast(Mask, MaskTy);
13732 Mask = CGF.
Builder.CreateExtractElement(Mask, (uint64_t)0);
13733 return CGF.
Builder.CreateSelect(Mask, Op0, Op1);
13737 unsigned NumElts,
Value *MaskIn) {
13739 const auto *
C = dyn_cast<Constant>(MaskIn);
13740 if (!
C || !
C->isAllOnesValue())
13746 for (
unsigned i = 0; i != NumElts; ++i)
13748 for (
unsigned i = NumElts; i != 8; ++i)
13749 Indices[i] = i % NumElts + NumElts;
13750 Cmp = CGF.
Builder.CreateShuffleVector(
13751 Cmp, llvm::Constant::getNullValue(Cmp->getType()), Indices);
13754 return CGF.
Builder.CreateBitCast(Cmp,
13756 std::max(NumElts, 8U)));
13761 assert((Ops.size() == 2 || Ops.size() == 4) &&
13762 "Unexpected number of arguments");
13764 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
13768 Cmp = Constant::getNullValue(
13769 llvm::FixedVectorType::get(CGF.
Builder.getInt1Ty(), NumElts));
13770 }
else if (CC == 7) {
13771 Cmp = Constant::getAllOnesValue(
13772 llvm::FixedVectorType::get(CGF.
Builder.getInt1Ty(), NumElts));
13774 ICmpInst::Predicate Pred;
13776 default: llvm_unreachable(
"Unknown condition code");
13777 case 0: Pred = ICmpInst::ICMP_EQ;
break;
13778 case 1: Pred =
Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
break;
13779 case 2: Pred =
Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
break;
13780 case 4: Pred = ICmpInst::ICMP_NE;
break;
13781 case 5: Pred =
Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
break;
13782 case 6: Pred =
Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
break;
13784 Cmp = CGF.
Builder.CreateICmp(Pred, Ops[0], Ops[1]);
13787 Value *MaskIn =
nullptr;
13788 if (Ops.size() == 4)
13795 Value *Zero = Constant::getNullValue(In->getType());
13801 unsigned Rnd = cast<llvm::ConstantInt>(Ops[3])->getZExtValue();
13802 llvm::Type *Ty = Ops[1]->getType();
13806 Intrinsic::ID IID = IsSigned ? Intrinsic::x86_avx512_sitofp_round
13807 : Intrinsic::x86_avx512_uitofp_round;
13809 Res = CGF.
Builder.CreateCall(F, { Ops[0], Ops[3] });
13811 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
13812 Res = IsSigned ? CGF.
Builder.CreateSIToFP(Ops[0], Ty)
13813 : CGF.
Builder.CreateUIToFP(Ops[0], Ty);
13824 bool Subtract =
false;
13825 Intrinsic::ID IID = Intrinsic::not_intrinsic;
13826 switch (BuiltinID) {
13828 case clang::X86::BI__builtin_ia32_vfmsubph512_mask3:
13831 case clang::X86::BI__builtin_ia32_vfmaddph512_mask:
13832 case clang::X86::BI__builtin_ia32_vfmaddph512_maskz:
13833 case clang::X86::BI__builtin_ia32_vfmaddph512_mask3:
13834 IID = llvm::Intrinsic::x86_avx512fp16_vfmadd_ph_512;
13836 case clang::X86::BI__builtin_ia32_vfmsubaddph512_mask3:
13839 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask:
13840 case clang::X86::BI__builtin_ia32_vfmaddsubph512_maskz:
13841 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask3:
13842 IID = llvm::Intrinsic::x86_avx512fp16_vfmaddsub_ph_512;
13844 case clang::X86::BI__builtin_ia32_vfmsubps512_mask3:
13847 case clang::X86::BI__builtin_ia32_vfmaddps512_mask:
13848 case clang::X86::BI__builtin_ia32_vfmaddps512_maskz:
13849 case clang::X86::BI__builtin_ia32_vfmaddps512_mask3:
13850 IID = llvm::Intrinsic::x86_avx512_vfmadd_ps_512;
break;
13851 case clang::X86::BI__builtin_ia32_vfmsubpd512_mask3:
13854 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask:
13855 case clang::X86::BI__builtin_ia32_vfmaddpd512_maskz:
13856 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask3:
13857 IID = llvm::Intrinsic::x86_avx512_vfmadd_pd_512;
break;
13858 case clang::X86::BI__builtin_ia32_vfmsubaddps512_mask3:
13861 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask:
13862 case clang::X86::BI__builtin_ia32_vfmaddsubps512_maskz:
13863 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask3:
13864 IID = llvm::Intrinsic::x86_avx512_vfmaddsub_ps_512;
13866 case clang::X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
13869 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask:
13870 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
13871 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
13872 IID = llvm::Intrinsic::x86_avx512_vfmaddsub_pd_512;
13886 if (IID != Intrinsic::not_intrinsic &&
13887 (cast<llvm::ConstantInt>(Ops.back())->getZExtValue() != (uint64_t)4 ||
13890 Res = CGF.
Builder.CreateCall(Intr, {A, B,
C, Ops.back() });
13892 llvm::Type *Ty = A->
getType();
13894 if (CGF.
Builder.getIsFPConstrained()) {
13895 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
13896 FMA = CGF.
CGM.
getIntrinsic(Intrinsic::experimental_constrained_fma, Ty);
13897 Res = CGF.
Builder.CreateConstrainedFPCall(FMA, {A, B,
C});
13900 Res = CGF.
Builder.CreateCall(FMA, {A, B,
C});
13905 Value *MaskFalseVal =
nullptr;
13906 switch (BuiltinID) {
13907 case clang::X86::BI__builtin_ia32_vfmaddph512_mask:
13908 case clang::X86::BI__builtin_ia32_vfmaddps512_mask:
13909 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask:
13910 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask:
13911 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask:
13912 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask:
13913 MaskFalseVal = Ops[0];
13915 case clang::X86::BI__builtin_ia32_vfmaddph512_maskz:
13916 case clang::X86::BI__builtin_ia32_vfmaddps512_maskz:
13917 case clang::X86::BI__builtin_ia32_vfmaddpd512_maskz:
13918 case clang::X86::BI__builtin_ia32_vfmaddsubph512_maskz:
13919 case clang::X86::BI__builtin_ia32_vfmaddsubps512_maskz:
13920 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
13921 MaskFalseVal = Constant::getNullValue(Ops[0]->getType());
13923 case clang::X86::BI__builtin_ia32_vfmsubph512_mask3:
13924 case clang::X86::BI__builtin_ia32_vfmaddph512_mask3:
13925 case clang::X86::BI__builtin_ia32_vfmsubps512_mask3:
13926 case clang::X86::BI__builtin_ia32_vfmaddps512_mask3:
13927 case clang::X86::BI__builtin_ia32_vfmsubpd512_mask3:
13928 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask3:
13929 case clang::X86::BI__builtin_ia32_vfmsubaddph512_mask3:
13930 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask3:
13931 case clang::X86::BI__builtin_ia32_vfmsubaddps512_mask3:
13932 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask3:
13933 case clang::X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
13934 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
13935 MaskFalseVal = Ops[2];
13947 bool ZeroMask =
false,
unsigned PTIdx = 0,
13948 bool NegAcc =
false) {
13950 if (Ops.size() > 4)
13951 Rnd = cast<llvm::ConstantInt>(Ops[4])->getZExtValue();
13954 Ops[2] = CGF.
Builder.CreateFNeg(Ops[2]);
13956 Ops[0] = CGF.
Builder.CreateExtractElement(Ops[0], (uint64_t)0);
13957 Ops[1] = CGF.
Builder.CreateExtractElement(Ops[1], (uint64_t)0);
13958 Ops[2] = CGF.
Builder.CreateExtractElement(Ops[2], (uint64_t)0);
13963 switch (Ops[0]->getType()->getPrimitiveSizeInBits()) {
13965 IID = Intrinsic::x86_avx512fp16_vfmadd_f16;
13968 IID = Intrinsic::x86_avx512_vfmadd_f32;
13971 IID = Intrinsic::x86_avx512_vfmadd_f64;
13974 llvm_unreachable(
"Unexpected size");
13977 {Ops[0], Ops[1], Ops[2], Ops[4]});
13978 }
else if (CGF.
Builder.getIsFPConstrained()) {
13979 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
13981 Intrinsic::experimental_constrained_fma, Ops[0]->getType());
13982 Res = CGF.
Builder.CreateConstrainedFPCall(FMA, Ops.slice(0, 3));
13985 Res = CGF.
Builder.CreateCall(FMA, Ops.slice(0, 3));
13988 if (Ops.size() > 3) {
13989 Value *PassThru = ZeroMask ? Constant::getNullValue(Res->
getType())
13995 if (NegAcc && PTIdx == 2)
13996 PassThru = CGF.
Builder.CreateExtractElement(Upper, (uint64_t)0);
14000 return CGF.
Builder.CreateInsertElement(Upper, Res, (uint64_t)0);
14005 llvm::Type *Ty = Ops[0]->getType();
14007 Ty = llvm::FixedVectorType::get(CGF.
Int64Ty,
14008 Ty->getPrimitiveSizeInBits() / 64);
14014 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
14015 LHS = CGF.
Builder.CreateShl(LHS, ShiftAmt);
14016 LHS = CGF.
Builder.CreateAShr(LHS, ShiftAmt);
14017 RHS = CGF.
Builder.CreateShl(RHS, ShiftAmt);
14018 RHS = CGF.
Builder.CreateAShr(RHS, ShiftAmt);
14021 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
14022 LHS = CGF.
Builder.CreateAnd(LHS, Mask);
14023 RHS = CGF.
Builder.CreateAnd(RHS, Mask);
14026 return CGF.
Builder.CreateMul(LHS, RHS);
14034 llvm::Type *Ty = Ops[0]->getType();
14036 unsigned VecWidth = Ty->getPrimitiveSizeInBits();
14037 unsigned EltWidth = Ty->getScalarSizeInBits();
14039 if (VecWidth == 128 && EltWidth == 32)
14040 IID = Intrinsic::x86_avx512_pternlog_d_128;
14041 else if (VecWidth == 256 && EltWidth == 32)
14042 IID = Intrinsic::x86_avx512_pternlog_d_256;
14043 else if (VecWidth == 512 && EltWidth == 32)
14044 IID = Intrinsic::x86_avx512_pternlog_d_512;
14045 else if (VecWidth == 128 && EltWidth == 64)
14046 IID = Intrinsic::x86_avx512_pternlog_q_128;
14047 else if (VecWidth == 256 && EltWidth == 64)
14048 IID = Intrinsic::x86_avx512_pternlog_q_256;
14049 else if (VecWidth == 512 && EltWidth == 64)
14050 IID = Intrinsic::x86_avx512_pternlog_q_512;
14052 llvm_unreachable(
"Unexpected intrinsic");
14056 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty) : Ops[0];
14061 llvm::Type *DstTy) {
14062 unsigned NumberOfElements =
14063 cast<llvm::FixedVectorType>(DstTy)->getNumElements();
14065 return CGF.
Builder.CreateSExt(Mask, DstTy,
"vpmovm2");
14070 StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
14071 return EmitX86CpuIs(CPUStr);
14077 llvm::Type *DstTy) {
14078 assert((Ops.size() == 1 || Ops.size() == 3 || Ops.size() == 4) &&
14079 "Unknown cvtph2ps intrinsic");
14082 if (Ops.size() == 4 && cast<llvm::ConstantInt>(Ops[3])->getZExtValue() != 4) {
14085 return CGF.
Builder.CreateCall(F, {Ops[0], Ops[1], Ops[2], Ops[3]});
14088 unsigned NumDstElts = cast<llvm::FixedVectorType>(DstTy)->getNumElements();
14089 Value *Src = Ops[0];
14093 cast<llvm::FixedVectorType>(Src->
getType())->getNumElements()) {
14094 assert(NumDstElts == 4 &&
"Unexpected vector size");
14099 auto *HalfTy = llvm::FixedVectorType::get(
14101 Src = CGF.
Builder.CreateBitCast(Src, HalfTy);
14104 Value *Res = CGF.
Builder.CreateFPExt(Src, DstTy,
"cvtph2ps");
14106 if (Ops.size() >= 3)
14111Value *CodeGenFunction::EmitX86CpuIs(StringRef CPUStr) {
14122 llvm::ArrayType::get(
Int32Ty, 1));
14126 cast<llvm::GlobalValue>(CpuModel)->setDSOLocal(
true);
14132 std::tie(Index,
Value) = StringSwitch<std::pair<unsigned, unsigned>>(CPUStr)
14134 .Case(STRING, {0u,
static_cast<unsigned>(llvm::X86::ENUM)})
14136 .Case(
ALIAS, {1u,
static_cast<unsigned>(llvm::X86::ENUM)})
14138 .Case(STR, {1u,
static_cast<unsigned>(llvm::X86::ENUM)})
14140 .Case(
ALIAS, {2u,
static_cast<unsigned>(llvm::X86::ENUM)})
14142 .Case(STR, {2u,
static_cast<unsigned>(llvm::X86::ENUM)})
14143#include
"llvm/TargetParser/X86TargetParser.def"
14145 assert(
Value != 0 &&
"Invalid CPUStr passed to CpuIs");
14148 llvm::Value *Idxs[] = {ConstantInt::get(
Int32Ty, 0),
14149 ConstantInt::get(
Int32Ty, Index)};
14155 return Builder.CreateICmpEQ(CpuValue,
14159Value *CodeGenFunction::EmitX86CpuSupports(
const CallExpr *E) {
14161 StringRef FeatureStr = cast<StringLiteral>(FeatureExpr)->getString();
14162 if (!
getContext().getTargetInfo().validateCpuSupports(FeatureStr))
14164 return EmitX86CpuSupports(FeatureStr);
14168 return EmitX86CpuSupports(llvm::X86::getCpuSupportsMask(FeatureStrs));
14172CodeGenFunction::EmitX86CpuSupports(std::array<uint32_t, 4> FeatureMask) {
14174 if (FeatureMask[0] != 0) {
14182 llvm::ArrayType::get(
Int32Ty, 1));
14186 cast<llvm::GlobalValue>(CpuModel)->setDSOLocal(
true);
14203 llvm::Type *ATy = llvm::ArrayType::get(
Int32Ty, 3);
14204 llvm::Constant *CpuFeatures2 =
14206 cast<llvm::GlobalValue>(CpuFeatures2)->setDSOLocal(
true);
14207 for (
int i = 1; i != 4; ++i) {
14208 const uint32_t M = FeatureMask[i];
14225Value *CodeGenFunction::EmitAArch64CpuInit() {
14226 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy,
false);
14227 llvm::FunctionCallee
Func =
14229 cast<llvm::GlobalValue>(
Func.getCallee())->setDSOLocal(
true);
14230 cast<llvm::GlobalValue>(
Func.getCallee())
14231 ->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
14235Value *CodeGenFunction::EmitX86CpuInit() {
14236 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy,
14238 llvm::FunctionCallee
Func =
14240 cast<llvm::GlobalValue>(
Func.getCallee())->setDSOLocal(
true);
14241 cast<llvm::GlobalValue>(
Func.getCallee())
14242 ->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
14246Value *CodeGenFunction::EmitAArch64CpuSupports(
const CallExpr *E) {
14248 StringRef ArgStr = cast<StringLiteral>(ArgExpr)->getString();
14250 ArgStr.split(Features,
"+");
14251 for (
auto &Feature : Features) {
14252 Feature = Feature.trim();
14253 if (!llvm::AArch64::parseArchExtension(Feature))
14255 if (Feature !=
"default")
14256 Features.push_back(Feature);
14258 return EmitAArch64CpuSupports(Features);
14263 uint64_t FeaturesMask = llvm::AArch64::getCpuSupportsMask(FeaturesStrs);
14265 if (FeaturesMask != 0) {
14270 llvm::Type *STy = llvm::StructType::get(
Int64Ty);
14271 llvm::Constant *AArch64CPUFeatures =
14273 cast<llvm::GlobalValue>(AArch64CPUFeatures)->setDSOLocal(
true);
14275 STy, AArch64CPUFeatures,
14289 if (BuiltinID == Builtin::BI__builtin_cpu_is)
14290 return EmitX86CpuIs(E);
14291 if (BuiltinID == Builtin::BI__builtin_cpu_supports)
14292 return EmitX86CpuSupports(E);
14293 if (BuiltinID == Builtin::BI__builtin_cpu_init)
14294 return EmitX86CpuInit();
14302 bool IsMaskFCmp =
false;
14303 bool IsConjFMA =
false;
14306 unsigned ICEArguments = 0;
14311 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; i++) {
14321 auto getCmpIntrinsicCall = [
this, &Ops](Intrinsic::ID
ID,
unsigned Imm) {
14322 Ops.push_back(llvm::ConstantInt::get(
Int8Ty, Imm));
14324 return Builder.CreateCall(F, Ops);
14332 auto getVectorFCmpIR = [
this, &Ops, E](CmpInst::Predicate Pred,
14333 bool IsSignaling) {
14334 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
14337 Cmp =
Builder.CreateFCmpS(Pred, Ops[0], Ops[1]);
14339 Cmp =
Builder.CreateFCmp(Pred, Ops[0], Ops[1]);
14340 llvm::VectorType *FPVecTy = cast<llvm::VectorType>(Ops[0]->getType());
14341 llvm::VectorType *IntVecTy = llvm::VectorType::getInteger(FPVecTy);
14343 return Builder.CreateBitCast(Sext, FPVecTy);
14346 switch (BuiltinID) {
14347 default:
return nullptr;
14348 case X86::BI_mm_prefetch: {
14350 ConstantInt *
C = cast<ConstantInt>(Ops[1]);
14351 Value *RW = ConstantInt::get(
Int32Ty, (
C->getZExtValue() >> 2) & 0x1);
14352 Value *Locality = ConstantInt::get(
Int32Ty,
C->getZExtValue() & 0x3);
14357 case X86::BI_mm_clflush: {
14361 case X86::BI_mm_lfence: {
14364 case X86::BI_mm_mfence: {
14367 case X86::BI_mm_sfence: {
14370 case X86::BI_mm_pause: {
14373 case X86::BI__rdtsc: {
14376 case X86::BI__builtin_ia32_rdtscp: {
14382 case X86::BI__builtin_ia32_lzcnt_u16:
14383 case X86::BI__builtin_ia32_lzcnt_u32:
14384 case X86::BI__builtin_ia32_lzcnt_u64: {
14388 case X86::BI__builtin_ia32_tzcnt_u16:
14389 case X86::BI__builtin_ia32_tzcnt_u32:
14390 case X86::BI__builtin_ia32_tzcnt_u64: {
14394 case X86::BI__builtin_ia32_undef128:
14395 case X86::BI__builtin_ia32_undef256:
14396 case X86::BI__builtin_ia32_undef512:
14403 case X86::BI__builtin_ia32_vec_init_v8qi:
14404 case X86::BI__builtin_ia32_vec_init_v4hi:
14405 case X86::BI__builtin_ia32_vec_init_v2si:
14408 case X86::BI__builtin_ia32_vec_ext_v2si:
14409 case X86::BI__builtin_ia32_vec_ext_v16qi:
14410 case X86::BI__builtin_ia32_vec_ext_v8hi:
14411 case X86::BI__builtin_ia32_vec_ext_v4si:
14412 case X86::BI__builtin_ia32_vec_ext_v4sf:
14413 case X86::BI__builtin_ia32_vec_ext_v2di:
14414 case X86::BI__builtin_ia32_vec_ext_v32qi:
14415 case X86::BI__builtin_ia32_vec_ext_v16hi:
14416 case X86::BI__builtin_ia32_vec_ext_v8si:
14417 case X86::BI__builtin_ia32_vec_ext_v4di: {
14419 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
14420 uint64_t Index = cast<ConstantInt>(Ops[1])->getZExtValue();
14421 Index &= NumElts - 1;
14424 return Builder.CreateExtractElement(Ops[0], Index);
14426 case X86::BI__builtin_ia32_vec_set_v16qi:
14427 case X86::BI__builtin_ia32_vec_set_v8hi:
14428 case X86::BI__builtin_ia32_vec_set_v4si:
14429 case X86::BI__builtin_ia32_vec_set_v2di:
14430 case X86::BI__builtin_ia32_vec_set_v32qi:
14431 case X86::BI__builtin_ia32_vec_set_v16hi:
14432 case X86::BI__builtin_ia32_vec_set_v8si:
14433 case X86::BI__builtin_ia32_vec_set_v4di: {
14435 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
14436 unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue();
14437 Index &= NumElts - 1;
14440 return Builder.CreateInsertElement(Ops[0], Ops[1], Index);
14442 case X86::BI_mm_setcsr:
14443 case X86::BI__builtin_ia32_ldmxcsr: {
14449 case X86::BI_mm_getcsr:
14450 case X86::BI__builtin_ia32_stmxcsr: {
14456 case X86::BI__builtin_ia32_xsave:
14457 case X86::BI__builtin_ia32_xsave64:
14458 case X86::BI__builtin_ia32_xrstor:
14459 case X86::BI__builtin_ia32_xrstor64:
14460 case X86::BI__builtin_ia32_xsaveopt:
14461 case X86::BI__builtin_ia32_xsaveopt64:
14462 case X86::BI__builtin_ia32_xrstors:
14463 case X86::BI__builtin_ia32_xrstors64:
14464 case X86::BI__builtin_ia32_xsavec:
14465 case X86::BI__builtin_ia32_xsavec64:
14466 case X86::BI__builtin_ia32_xsaves:
14467 case X86::BI__builtin_ia32_xsaves64:
14468 case X86::BI__builtin_ia32_xsetbv:
14469 case X86::BI_xsetbv: {
14471#define INTRINSIC_X86_XSAVE_ID(NAME) \
14472 case X86::BI__builtin_ia32_##NAME: \
14473 ID = Intrinsic::x86_##NAME; \
14475 switch (BuiltinID) {
14476 default: llvm_unreachable(
"Unsupported intrinsic!");
14490 case X86::BI_xsetbv:
14491 ID = Intrinsic::x86_xsetbv;
14494#undef INTRINSIC_X86_XSAVE_ID
14499 Ops.push_back(Mlo);
14502 case X86::BI__builtin_ia32_xgetbv:
14503 case X86::BI_xgetbv:
14505 case X86::BI__builtin_ia32_storedqudi128_mask:
14506 case X86::BI__builtin_ia32_storedqusi128_mask:
14507 case X86::BI__builtin_ia32_storedquhi128_mask:
14508 case X86::BI__builtin_ia32_storedquqi128_mask:
14509 case X86::BI__builtin_ia32_storeupd128_mask:
14510 case X86::BI__builtin_ia32_storeups128_mask:
14511 case X86::BI__builtin_ia32_storedqudi256_mask:
14512 case X86::BI__builtin_ia32_storedqusi256_mask:
14513 case X86::BI__builtin_ia32_storedquhi256_mask:
14514 case X86::BI__builtin_ia32_storedquqi256_mask:
14515 case X86::BI__builtin_ia32_storeupd256_mask:
14516 case X86::BI__builtin_ia32_storeups256_mask:
14517 case X86::BI__builtin_ia32_storedqudi512_mask:
14518 case X86::BI__builtin_ia32_storedqusi512_mask:
14519 case X86::BI__builtin_ia32_storedquhi512_mask:
14520 case X86::BI__builtin_ia32_storedquqi512_mask:
14521 case X86::BI__builtin_ia32_storeupd512_mask:
14522 case X86::BI__builtin_ia32_storeups512_mask:
14525 case X86::BI__builtin_ia32_storesh128_mask:
14526 case X86::BI__builtin_ia32_storess128_mask:
14527 case X86::BI__builtin_ia32_storesd128_mask:
14530 case X86::BI__builtin_ia32_vpopcntb_128:
14531 case X86::BI__builtin_ia32_vpopcntd_128:
14532 case X86::BI__builtin_ia32_vpopcntq_128:
14533 case X86::BI__builtin_ia32_vpopcntw_128:
14534 case X86::BI__builtin_ia32_vpopcntb_256:
14535 case X86::BI__builtin_ia32_vpopcntd_256:
14536 case X86::BI__builtin_ia32_vpopcntq_256:
14537 case X86::BI__builtin_ia32_vpopcntw_256:
14538 case X86::BI__builtin_ia32_vpopcntb_512:
14539 case X86::BI__builtin_ia32_vpopcntd_512:
14540 case X86::BI__builtin_ia32_vpopcntq_512:
14541 case X86::BI__builtin_ia32_vpopcntw_512: {
14544 return Builder.CreateCall(F, Ops);
14546 case X86::BI__builtin_ia32_cvtmask2b128:
14547 case X86::BI__builtin_ia32_cvtmask2b256:
14548 case X86::BI__builtin_ia32_cvtmask2b512:
14549 case X86::BI__builtin_ia32_cvtmask2w128:
14550 case X86::BI__builtin_ia32_cvtmask2w256:
14551 case X86::BI__builtin_ia32_cvtmask2w512:
14552 case X86::BI__builtin_ia32_cvtmask2d128:
14553 case X86::BI__builtin_ia32_cvtmask2d256:
14554 case X86::BI__builtin_ia32_cvtmask2d512:
14555 case X86::BI__builtin_ia32_cvtmask2q128:
14556 case X86::BI__builtin_ia32_cvtmask2q256:
14557 case X86::BI__builtin_ia32_cvtmask2q512:
14560 case X86::BI__builtin_ia32_cvtb2mask128:
14561 case X86::BI__builtin_ia32_cvtb2mask256:
14562 case X86::BI__builtin_ia32_cvtb2mask512:
14563 case X86::BI__builtin_ia32_cvtw2mask128:
14564 case X86::BI__builtin_ia32_cvtw2mask256:
14565 case X86::BI__builtin_ia32_cvtw2mask512:
14566 case X86::BI__builtin_ia32_cvtd2mask128:
14567 case X86::BI__builtin_ia32_cvtd2mask256:
14568 case X86::BI__builtin_ia32_cvtd2mask512:
14569 case X86::BI__builtin_ia32_cvtq2mask128:
14570 case X86::BI__builtin_ia32_cvtq2mask256:
14571 case X86::BI__builtin_ia32_cvtq2mask512:
14574 case X86::BI__builtin_ia32_cvtdq2ps512_mask:
14575 case X86::BI__builtin_ia32_cvtqq2ps512_mask:
14576 case X86::BI__builtin_ia32_cvtqq2pd512_mask:
14577 case X86::BI__builtin_ia32_vcvtw2ph512_mask:
14578 case X86::BI__builtin_ia32_vcvtdq2ph512_mask:
14579 case X86::BI__builtin_ia32_vcvtqq2ph512_mask:
14581 case X86::BI__builtin_ia32_cvtudq2ps512_mask:
14582 case X86::BI__builtin_ia32_cvtuqq2ps512_mask:
14583 case X86::BI__builtin_ia32_cvtuqq2pd512_mask:
14584 case X86::BI__builtin_ia32_vcvtuw2ph512_mask:
14585 case X86::BI__builtin_ia32_vcvtudq2ph512_mask:
14586 case X86::BI__builtin_ia32_vcvtuqq2ph512_mask:
14589 case X86::BI__builtin_ia32_vfmaddss3:
14590 case X86::BI__builtin_ia32_vfmaddsd3:
14591 case X86::BI__builtin_ia32_vfmaddsh3_mask:
14592 case X86::BI__builtin_ia32_vfmaddss3_mask:
14593 case X86::BI__builtin_ia32_vfmaddsd3_mask:
14595 case X86::BI__builtin_ia32_vfmaddss:
14596 case X86::BI__builtin_ia32_vfmaddsd:
14598 Constant::getNullValue(Ops[0]->getType()));
14599 case X86::BI__builtin_ia32_vfmaddsh3_maskz:
14600 case X86::BI__builtin_ia32_vfmaddss3_maskz:
14601 case X86::BI__builtin_ia32_vfmaddsd3_maskz:
14603 case X86::BI__builtin_ia32_vfmaddsh3_mask3:
14604 case X86::BI__builtin_ia32_vfmaddss3_mask3:
14605 case X86::BI__builtin_ia32_vfmaddsd3_mask3:
14607 case X86::BI__builtin_ia32_vfmsubsh3_mask3:
14608 case X86::BI__builtin_ia32_vfmsubss3_mask3:
14609 case X86::BI__builtin_ia32_vfmsubsd3_mask3:
14612 case X86::BI__builtin_ia32_vfmaddph:
14613 case X86::BI__builtin_ia32_vfmaddps:
14614 case X86::BI__builtin_ia32_vfmaddpd:
14615 case X86::BI__builtin_ia32_vfmaddph256:
14616 case X86::BI__builtin_ia32_vfmaddps256:
14617 case X86::BI__builtin_ia32_vfmaddpd256:
14618 case X86::BI__builtin_ia32_vfmaddph512_mask:
14619 case X86::BI__builtin_ia32_vfmaddph512_maskz:
14620 case X86::BI__builtin_ia32_vfmaddph512_mask3:
14621 case X86::BI__builtin_ia32_vfmaddps512_mask:
14622 case X86::BI__builtin_ia32_vfmaddps512_maskz:
14623 case X86::BI__builtin_ia32_vfmaddps512_mask3:
14624 case X86::BI__builtin_ia32_vfmsubps512_mask3:
14625 case X86::BI__builtin_ia32_vfmaddpd512_mask:
14626 case X86::BI__builtin_ia32_vfmaddpd512_maskz:
14627 case X86::BI__builtin_ia32_vfmaddpd512_mask3:
14628 case X86::BI__builtin_ia32_vfmsubpd512_mask3:
14629 case X86::BI__builtin_ia32_vfmsubph512_mask3:
14631 case X86::BI__builtin_ia32_vfmaddsubph512_mask:
14632 case X86::BI__builtin_ia32_vfmaddsubph512_maskz:
14633 case X86::BI__builtin_ia32_vfmaddsubph512_mask3:
14634 case X86::BI__builtin_ia32_vfmsubaddph512_mask3:
14635 case X86::BI__builtin_ia32_vfmaddsubps512_mask:
14636 case X86::BI__builtin_ia32_vfmaddsubps512_maskz:
14637 case X86::BI__builtin_ia32_vfmaddsubps512_mask3:
14638 case X86::BI__builtin_ia32_vfmsubaddps512_mask3:
14639 case X86::BI__builtin_ia32_vfmaddsubpd512_mask:
14640 case X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
14641 case X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
14642 case X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
14645 case X86::BI__builtin_ia32_movdqa32store128_mask:
14646 case X86::BI__builtin_ia32_movdqa64store128_mask:
14647 case X86::BI__builtin_ia32_storeaps128_mask:
14648 case X86::BI__builtin_ia32_storeapd128_mask:
14649 case X86::BI__builtin_ia32_movdqa32store256_mask:
14650 case X86::BI__builtin_ia32_movdqa64store256_mask:
14651 case X86::BI__builtin_ia32_storeaps256_mask:
14652 case X86::BI__builtin_ia32_storeapd256_mask:
14653 case X86::BI__builtin_ia32_movdqa32store512_mask:
14654 case X86::BI__builtin_ia32_movdqa64store512_mask:
14655 case X86::BI__builtin_ia32_storeaps512_mask:
14656 case X86::BI__builtin_ia32_storeapd512_mask:
14661 case X86::BI__builtin_ia32_loadups128_mask:
14662 case X86::BI__builtin_ia32_loadups256_mask:
14663 case X86::BI__builtin_ia32_loadups512_mask:
14664 case X86::BI__builtin_ia32_loadupd128_mask:
14665 case X86::BI__builtin_ia32_loadupd256_mask:
14666 case X86::BI__builtin_ia32_loadupd512_mask:
14667 case X86::BI__builtin_ia32_loaddquqi128_mask:
14668 case X86::BI__builtin_ia32_loaddquqi256_mask:
14669 case X86::BI__builtin_ia32_loaddquqi512_mask:
14670 case X86::BI__builtin_ia32_loaddquhi128_mask:
14671 case X86::BI__builtin_ia32_loaddquhi256_mask:
14672 case X86::BI__builtin_ia32_loaddquhi512_mask:
14673 case X86::BI__builtin_ia32_loaddqusi128_mask:
14674 case X86::BI__builtin_ia32_loaddqusi256_mask:
14675 case X86::BI__builtin_ia32_loaddqusi512_mask:
14676 case X86::BI__builtin_ia32_loaddqudi128_mask:
14677 case X86::BI__builtin_ia32_loaddqudi256_mask:
14678 case X86::BI__builtin_ia32_loaddqudi512_mask:
14681 case X86::BI__builtin_ia32_loadsh128_mask:
14682 case X86::BI__builtin_ia32_loadss128_mask:
14683 case X86::BI__builtin_ia32_loadsd128_mask:
14686 case X86::BI__builtin_ia32_loadaps128_mask:
14687 case X86::BI__builtin_ia32_loadaps256_mask:
14688 case X86::BI__builtin_ia32_loadaps512_mask:
14689 case X86::BI__builtin_ia32_loadapd128_mask:
14690 case X86::BI__builtin_ia32_loadapd256_mask:
14691 case X86::BI__builtin_ia32_loadapd512_mask:
14692 case X86::BI__builtin_ia32_movdqa32load128_mask:
14693 case X86::BI__builtin_ia32_movdqa32load256_mask:
14694 case X86::BI__builtin_ia32_movdqa32load512_mask:
14695 case X86::BI__builtin_ia32_movdqa64load128_mask:
14696 case X86::BI__builtin_ia32_movdqa64load256_mask:
14697 case X86::BI__builtin_ia32_movdqa64load512_mask:
14702 case X86::BI__builtin_ia32_expandloaddf128_mask:
14703 case X86::BI__builtin_ia32_expandloaddf256_mask:
14704 case X86::BI__builtin_ia32_expandloaddf512_mask:
14705 case X86::BI__builtin_ia32_expandloadsf128_mask:
14706 case X86::BI__builtin_ia32_expandloadsf256_mask:
14707 case X86::BI__builtin_ia32_expandloadsf512_mask:
14708 case X86::BI__builtin_ia32_expandloaddi128_mask:
14709 case X86::BI__builtin_ia32_expandloaddi256_mask:
14710 case X86::BI__builtin_ia32_expandloaddi512_mask:
14711 case X86::BI__builtin_ia32_expandloadsi128_mask:
14712 case X86::BI__builtin_ia32_expandloadsi256_mask:
14713 case X86::BI__builtin_ia32_expandloadsi512_mask:
14714 case X86::BI__builtin_ia32_expandloadhi128_mask:
14715 case X86::BI__builtin_ia32_expandloadhi256_mask:
14716 case X86::BI__builtin_ia32_expandloadhi512_mask:
14717 case X86::BI__builtin_ia32_expandloadqi128_mask:
14718 case X86::BI__builtin_ia32_expandloadqi256_mask:
14719 case X86::BI__builtin_ia32_expandloadqi512_mask:
14722 case X86::BI__builtin_ia32_compressstoredf128_mask:
14723 case X86::BI__builtin_ia32_compressstoredf256_mask:
14724 case X86::BI__builtin_ia32_compressstoredf512_mask:
14725 case X86::BI__builtin_ia32_compressstoresf128_mask:
14726 case X86::BI__builtin_ia32_compressstoresf256_mask:
14727 case X86::BI__builtin_ia32_compressstoresf512_mask:
14728 case X86::BI__builtin_ia32_compressstoredi128_mask:
14729 case X86::BI__builtin_ia32_compressstoredi256_mask:
14730 case X86::BI__builtin_ia32_compressstoredi512_mask:
14731 case X86::BI__builtin_ia32_compressstoresi128_mask:
14732 case X86::BI__builtin_ia32_compressstoresi256_mask:
14733 case X86::BI__builtin_ia32_compressstoresi512_mask:
14734 case X86::BI__builtin_ia32_compressstorehi128_mask:
14735 case X86::BI__builtin_ia32_compressstorehi256_mask:
14736 case X86::BI__builtin_ia32_compressstorehi512_mask:
14737 case X86::BI__builtin_ia32_compressstoreqi128_mask:
14738 case X86::BI__builtin_ia32_compressstoreqi256_mask:
14739 case X86::BI__builtin_ia32_compressstoreqi512_mask:
14742 case X86::BI__builtin_ia32_expanddf128_mask:
14743 case X86::BI__builtin_ia32_expanddf256_mask:
14744 case X86::BI__builtin_ia32_expanddf512_mask:
14745 case X86::BI__builtin_ia32_expandsf128_mask:
14746 case X86::BI__builtin_ia32_expandsf256_mask:
14747 case X86::BI__builtin_ia32_expandsf512_mask:
14748 case X86::BI__builtin_ia32_expanddi128_mask:
14749 case X86::BI__builtin_ia32_expanddi256_mask:
14750 case X86::BI__builtin_ia32_expanddi512_mask:
14751 case X86::BI__builtin_ia32_expandsi128_mask:
14752 case X86::BI__builtin_ia32_expandsi256_mask:
14753 case X86::BI__builtin_ia32_expandsi512_mask:
14754 case X86::BI__builtin_ia32_expandhi128_mask:
14755 case X86::BI__builtin_ia32_expandhi256_mask:
14756 case X86::BI__builtin_ia32_expandhi512_mask:
14757 case X86::BI__builtin_ia32_expandqi128_mask:
14758 case X86::BI__builtin_ia32_expandqi256_mask:
14759 case X86::BI__builtin_ia32_expandqi512_mask:
14762 case X86::BI__builtin_ia32_compressdf128_mask:
14763 case X86::BI__builtin_ia32_compressdf256_mask:
14764 case X86::BI__builtin_ia32_compressdf512_mask:
14765 case X86::BI__builtin_ia32_compresssf128_mask:
14766 case X86::BI__builtin_ia32_compresssf256_mask:
14767 case X86::BI__builtin_ia32_compresssf512_mask:
14768 case X86::BI__builtin_ia32_compressdi128_mask:
14769 case X86::BI__builtin_ia32_compressdi256_mask:
14770 case X86::BI__builtin_ia32_compressdi512_mask:
14771 case X86::BI__builtin_ia32_compresssi128_mask:
14772 case X86::BI__builtin_ia32_compresssi256_mask:
14773 case X86::BI__builtin_ia32_compresssi512_mask:
14774 case X86::BI__builtin_ia32_compresshi128_mask:
14775 case X86::BI__builtin_ia32_compresshi256_mask:
14776 case X86::BI__builtin_ia32_compresshi512_mask:
14777 case X86::BI__builtin_ia32_compressqi128_mask:
14778 case X86::BI__builtin_ia32_compressqi256_mask:
14779 case X86::BI__builtin_ia32_compressqi512_mask:
14782 case X86::BI__builtin_ia32_gather3div2df:
14783 case X86::BI__builtin_ia32_gather3div2di:
14784 case X86::BI__builtin_ia32_gather3div4df:
14785 case X86::BI__builtin_ia32_gather3div4di:
14786 case X86::BI__builtin_ia32_gather3div4sf:
14787 case X86::BI__builtin_ia32_gather3div4si:
14788 case X86::BI__builtin_ia32_gather3div8sf:
14789 case X86::BI__builtin_ia32_gather3div8si:
14790 case X86::BI__builtin_ia32_gather3siv2df:
14791 case X86::BI__builtin_ia32_gather3siv2di:
14792 case X86::BI__builtin_ia32_gather3siv4df:
14793 case X86::BI__builtin_ia32_gather3siv4di:
14794 case X86::BI__builtin_ia32_gather3siv4sf:
14795 case X86::BI__builtin_ia32_gather3siv4si:
14796 case X86::BI__builtin_ia32_gather3siv8sf:
14797 case X86::BI__builtin_ia32_gather3siv8si:
14798 case X86::BI__builtin_ia32_gathersiv8df:
14799 case X86::BI__builtin_ia32_gathersiv16sf:
14800 case X86::BI__builtin_ia32_gatherdiv8df:
14801 case X86::BI__builtin_ia32_gatherdiv16sf:
14802 case X86::BI__builtin_ia32_gathersiv8di:
14803 case X86::BI__builtin_ia32_gathersiv16si:
14804 case X86::BI__builtin_ia32_gatherdiv8di:
14805 case X86::BI__builtin_ia32_gatherdiv16si: {
14807 switch (BuiltinID) {
14808 default: llvm_unreachable(
"Unexpected builtin");
14809 case X86::BI__builtin_ia32_gather3div2df:
14810 IID = Intrinsic::x86_avx512_mask_gather3div2_df;
14812 case X86::BI__builtin_ia32_gather3div2di:
14813 IID = Intrinsic::x86_avx512_mask_gather3div2_di;
14815 case X86::BI__builtin_ia32_gather3div4df:
14816 IID = Intrinsic::x86_avx512_mask_gather3div4_df;
14818 case X86::BI__builtin_ia32_gather3div4di:
14819 IID = Intrinsic::x86_avx512_mask_gather3div4_di;
14821 case X86::BI__builtin_ia32_gather3div4sf:
14822 IID = Intrinsic::x86_avx512_mask_gather3div4_sf;
14824 case X86::BI__builtin_ia32_gather3div4si:
14825 IID = Intrinsic::x86_avx512_mask_gather3div4_si;
14827 case X86::BI__builtin_ia32_gather3div8sf:
14828 IID = Intrinsic::x86_avx512_mask_gather3div8_sf;
14830 case X86::BI__builtin_ia32_gather3div8si:
14831 IID = Intrinsic::x86_avx512_mask_gather3div8_si;
14833 case X86::BI__builtin_ia32_gather3siv2df:
14834 IID = Intrinsic::x86_avx512_mask_gather3siv2_df;
14836 case X86::BI__builtin_ia32_gather3siv2di:
14837 IID = Intrinsic::x86_avx512_mask_gather3siv2_di;
14839 case X86::BI__builtin_ia32_gather3siv4df:
14840 IID = Intrinsic::x86_avx512_mask_gather3siv4_df;
14842 case X86::BI__builtin_ia32_gather3siv4di:
14843 IID = Intrinsic::x86_avx512_mask_gather3siv4_di;
14845 case X86::BI__builtin_ia32_gather3siv4sf:
14846 IID = Intrinsic::x86_avx512_mask_gather3siv4_sf;
14848 case X86::BI__builtin_ia32_gather3siv4si:
14849 IID = Intrinsic::x86_avx512_mask_gather3siv4_si;
14851 case X86::BI__builtin_ia32_gather3siv8sf:
14852 IID = Intrinsic::x86_avx512_mask_gather3siv8_sf;
14854 case X86::BI__builtin_ia32_gather3siv8si:
14855 IID = Intrinsic::x86_avx512_mask_gather3siv8_si;
14857 case X86::BI__builtin_ia32_gathersiv8df:
14858 IID = Intrinsic::x86_avx512_mask_gather_dpd_512;
14860 case X86::BI__builtin_ia32_gathersiv16sf:
14861 IID = Intrinsic::x86_avx512_mask_gather_dps_512;
14863 case X86::BI__builtin_ia32_gatherdiv8df:
14864 IID = Intrinsic::x86_avx512_mask_gather_qpd_512;
14866 case X86::BI__builtin_ia32_gatherdiv16sf:
14867 IID = Intrinsic::x86_avx512_mask_gather_qps_512;
14869 case X86::BI__builtin_ia32_gathersiv8di:
14870 IID = Intrinsic::x86_avx512_mask_gather_dpq_512;
14872 case X86::BI__builtin_ia32_gathersiv16si:
14873 IID = Intrinsic::x86_avx512_mask_gather_dpi_512;
14875 case X86::BI__builtin_ia32_gatherdiv8di:
14876 IID = Intrinsic::x86_avx512_mask_gather_qpq_512;
14878 case X86::BI__builtin_ia32_gatherdiv16si:
14879 IID = Intrinsic::x86_avx512_mask_gather_qpi_512;
14883 unsigned MinElts = std::min(
14884 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(),
14885 cast<llvm::FixedVectorType>(Ops[2]->getType())->getNumElements());
14888 return Builder.CreateCall(Intr, Ops);
14891 case X86::BI__builtin_ia32_scattersiv8df:
14892 case X86::BI__builtin_ia32_scattersiv16sf:
14893 case X86::BI__builtin_ia32_scatterdiv8df:
14894 case X86::BI__builtin_ia32_scatterdiv16sf:
14895 case X86::BI__builtin_ia32_scattersiv8di:
14896 case X86::BI__builtin_ia32_scattersiv16si:
14897 case X86::BI__builtin_ia32_scatterdiv8di:
14898 case X86::BI__builtin_ia32_scatterdiv16si:
14899 case X86::BI__builtin_ia32_scatterdiv2df:
14900 case X86::BI__builtin_ia32_scatterdiv2di:
14901 case X86::BI__builtin_ia32_scatterdiv4df:
14902 case X86::BI__builtin_ia32_scatterdiv4di:
14903 case X86::BI__builtin_ia32_scatterdiv4sf:
14904 case X86::BI__builtin_ia32_scatterdiv4si:
14905 case X86::BI__builtin_ia32_scatterdiv8sf:
14906 case X86::BI__builtin_ia32_scatterdiv8si:
14907 case X86::BI__builtin_ia32_scattersiv2df:
14908 case X86::BI__builtin_ia32_scattersiv2di:
14909 case X86::BI__builtin_ia32_scattersiv4df:
14910 case X86::BI__builtin_ia32_scattersiv4di:
14911 case X86::BI__builtin_ia32_scattersiv4sf:
14912 case X86::BI__builtin_ia32_scattersiv4si:
14913 case X86::BI__builtin_ia32_scattersiv8sf:
14914 case X86::BI__builtin_ia32_scattersiv8si: {
14916 switch (BuiltinID) {
14917 default: llvm_unreachable(
"Unexpected builtin");
14918 case X86::BI__builtin_ia32_scattersiv8df:
14919 IID = Intrinsic::x86_avx512_mask_scatter_dpd_512;
14921 case X86::BI__builtin_ia32_scattersiv16sf:
14922 IID = Intrinsic::x86_avx512_mask_scatter_dps_512;
14924 case X86::BI__builtin_ia32_scatterdiv8df:
14925 IID = Intrinsic::x86_avx512_mask_scatter_qpd_512;
14927 case X86::BI__builtin_ia32_scatterdiv16sf:
14928 IID = Intrinsic::x86_avx512_mask_scatter_qps_512;
14930 case X86::BI__builtin_ia32_scattersiv8di:
14931 IID = Intrinsic::x86_avx512_mask_scatter_dpq_512;
14933 case X86::BI__builtin_ia32_scattersiv16si:
14934 IID = Intrinsic::x86_avx512_mask_scatter_dpi_512;
14936 case X86::BI__builtin_ia32_scatterdiv8di:
14937 IID = Intrinsic::x86_avx512_mask_scatter_qpq_512;
14939 case X86::BI__builtin_ia32_scatterdiv16si:
14940 IID = Intrinsic::x86_avx512_mask_scatter_qpi_512;
14942 case X86::BI__builtin_ia32_scatterdiv2df:
14943 IID = Intrinsic::x86_avx512_mask_scatterdiv2_df;
14945 case X86::BI__builtin_ia32_scatterdiv2di:
14946 IID = Intrinsic::x86_avx512_mask_scatterdiv2_di;
14948 case X86::BI__builtin_ia32_scatterdiv4df:
14949 IID = Intrinsic::x86_avx512_mask_scatterdiv4_df;
14951 case X86::BI__builtin_ia32_scatterdiv4di:
14952 IID = Intrinsic::x86_avx512_mask_scatterdiv4_di;
14954 case X86::BI__builtin_ia32_scatterdiv4sf:
14955 IID = Intrinsic::x86_avx512_mask_scatterdiv4_sf;
14957 case X86::BI__builtin_ia32_scatterdiv4si:
14958 IID = Intrinsic::x86_avx512_mask_scatterdiv4_si;
14960 case X86::BI__builtin_ia32_scatterdiv8sf:
14961 IID = Intrinsic::x86_avx512_mask_scatterdiv8_sf;
14963 case X86::BI__builtin_ia32_scatterdiv8si:
14964 IID = Intrinsic::x86_avx512_mask_scatterdiv8_si;
14966 case X86::BI__builtin_ia32_scattersiv2df:
14967 IID = Intrinsic::x86_avx512_mask_scattersiv2_df;
14969 case X86::BI__builtin_ia32_scattersiv2di:
14970 IID = Intrinsic::x86_avx512_mask_scattersiv2_di;
14972 case X86::BI__builtin_ia32_scattersiv4df:
14973 IID = Intrinsic::x86_avx512_mask_scattersiv4_df;
14975 case X86::BI__builtin_ia32_scattersiv4di:
14976 IID = Intrinsic::x86_avx512_mask_scattersiv4_di;
14978 case X86::BI__builtin_ia32_scattersiv4sf:
14979 IID = Intrinsic::x86_avx512_mask_scattersiv4_sf;
14981 case X86::BI__builtin_ia32_scattersiv4si:
14982 IID = Intrinsic::x86_avx512_mask_scattersiv4_si;
14984 case X86::BI__builtin_ia32_scattersiv8sf:
14985 IID = Intrinsic::x86_avx512_mask_scattersiv8_sf;
14987 case X86::BI__builtin_ia32_scattersiv8si:
14988 IID = Intrinsic::x86_avx512_mask_scattersiv8_si;
14992 unsigned MinElts = std::min(
14993 cast<llvm::FixedVectorType>(Ops[2]->getType())->getNumElements(),
14994 cast<llvm::FixedVectorType>(Ops[3]->getType())->getNumElements());
14997 return Builder.CreateCall(Intr, Ops);
15000 case X86::BI__builtin_ia32_vextractf128_pd256:
15001 case X86::BI__builtin_ia32_vextractf128_ps256:
15002 case X86::BI__builtin_ia32_vextractf128_si256:
15003 case X86::BI__builtin_ia32_extract128i256:
15004 case X86::BI__builtin_ia32_extractf64x4_mask:
15005 case X86::BI__builtin_ia32_extractf32x4_mask:
15006 case X86::BI__builtin_ia32_extracti64x4_mask:
15007 case X86::BI__builtin_ia32_extracti32x4_mask:
15008 case X86::BI__builtin_ia32_extractf32x8_mask:
15009 case X86::BI__builtin_ia32_extracti32x8_mask:
15010 case X86::BI__builtin_ia32_extractf32x4_256_mask:
15011 case X86::BI__builtin_ia32_extracti32x4_256_mask:
15012 case X86::BI__builtin_ia32_extractf64x2_256_mask:
15013 case X86::BI__builtin_ia32_extracti64x2_256_mask:
15014 case X86::BI__builtin_ia32_extractf64x2_512_mask:
15015 case X86::BI__builtin_ia32_extracti64x2_512_mask: {
15017 unsigned NumElts = DstTy->getNumElements();
15018 unsigned SrcNumElts =
15019 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15020 unsigned SubVectors = SrcNumElts / NumElts;
15021 unsigned Index = cast<ConstantInt>(Ops[1])->getZExtValue();
15022 assert(llvm::isPowerOf2_32(SubVectors) &&
"Expected power of 2 subvectors");
15023 Index &= SubVectors - 1;
15027 for (
unsigned i = 0; i != NumElts; ++i)
15028 Indices[i] = i + Index;
15033 if (Ops.size() == 4)
15038 case X86::BI__builtin_ia32_vinsertf128_pd256:
15039 case X86::BI__builtin_ia32_vinsertf128_ps256:
15040 case X86::BI__builtin_ia32_vinsertf128_si256:
15041 case X86::BI__builtin_ia32_insert128i256:
15042 case X86::BI__builtin_ia32_insertf64x4:
15043 case X86::BI__builtin_ia32_insertf32x4:
15044 case X86::BI__builtin_ia32_inserti64x4:
15045 case X86::BI__builtin_ia32_inserti32x4:
15046 case X86::BI__builtin_ia32_insertf32x8:
15047 case X86::BI__builtin_ia32_inserti32x8:
15048 case X86::BI__builtin_ia32_insertf32x4_256:
15049 case X86::BI__builtin_ia32_inserti32x4_256:
15050 case X86::BI__builtin_ia32_insertf64x2_256:
15051 case X86::BI__builtin_ia32_inserti64x2_256:
15052 case X86::BI__builtin_ia32_insertf64x2_512:
15053 case X86::BI__builtin_ia32_inserti64x2_512: {
15054 unsigned DstNumElts =
15055 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15056 unsigned SrcNumElts =
15057 cast<llvm::FixedVectorType>(Ops[1]->getType())->getNumElements();
15058 unsigned SubVectors = DstNumElts / SrcNumElts;
15059 unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue();
15060 assert(llvm::isPowerOf2_32(SubVectors) &&
"Expected power of 2 subvectors");
15061 Index &= SubVectors - 1;
15062 Index *= SrcNumElts;
15065 for (
unsigned i = 0; i != DstNumElts; ++i)
15066 Indices[i] = (i >= SrcNumElts) ? SrcNumElts + (i % SrcNumElts) : i;
15069 Ops[1],
ArrayRef(Indices, DstNumElts),
"widen");
15071 for (
unsigned i = 0; i != DstNumElts; ++i) {
15072 if (i >= Index && i < (Index + SrcNumElts))
15073 Indices[i] = (i - Index) + DstNumElts;
15078 return Builder.CreateShuffleVector(Ops[0], Op1,
15079 ArrayRef(Indices, DstNumElts),
"insert");
15081 case X86::BI__builtin_ia32_pmovqd512_mask:
15082 case X86::BI__builtin_ia32_pmovwb512_mask: {
15083 Value *Res =
Builder.CreateTrunc(Ops[0], Ops[1]->getType());
15086 case X86::BI__builtin_ia32_pmovdb512_mask:
15087 case X86::BI__builtin_ia32_pmovdw512_mask:
15088 case X86::BI__builtin_ia32_pmovqw512_mask: {
15089 if (
const auto *
C = dyn_cast<Constant>(Ops[2]))
15090 if (
C->isAllOnesValue())
15091 return Builder.CreateTrunc(Ops[0], Ops[1]->getType());
15094 switch (BuiltinID) {
15095 default: llvm_unreachable(
"Unsupported intrinsic!");
15096 case X86::BI__builtin_ia32_pmovdb512_mask:
15097 IID = Intrinsic::x86_avx512_mask_pmov_db_512;
15099 case X86::BI__builtin_ia32_pmovdw512_mask:
15100 IID = Intrinsic::x86_avx512_mask_pmov_dw_512;
15102 case X86::BI__builtin_ia32_pmovqw512_mask:
15103 IID = Intrinsic::x86_avx512_mask_pmov_qw_512;
15108 return Builder.CreateCall(Intr, Ops);
15110 case X86::BI__builtin_ia32_pblendw128:
15111 case X86::BI__builtin_ia32_blendpd:
15112 case X86::BI__builtin_ia32_blendps:
15113 case X86::BI__builtin_ia32_blendpd256:
15114 case X86::BI__builtin_ia32_blendps256:
15115 case X86::BI__builtin_ia32_pblendw256:
15116 case X86::BI__builtin_ia32_pblendd128:
15117 case X86::BI__builtin_ia32_pblendd256: {
15119 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15120 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
15125 for (
unsigned i = 0; i != NumElts; ++i)
15126 Indices[i] = ((Imm >> (i % 8)) & 0x1) ? NumElts + i : i;
15128 return Builder.CreateShuffleVector(Ops[0], Ops[1],
15129 ArrayRef(Indices, NumElts),
"blend");
15131 case X86::BI__builtin_ia32_pshuflw:
15132 case X86::BI__builtin_ia32_pshuflw256:
15133 case X86::BI__builtin_ia32_pshuflw512: {
15134 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
15135 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15136 unsigned NumElts = Ty->getNumElements();
15139 Imm = (Imm & 0xff) * 0x01010101;
15142 for (
unsigned l = 0; l != NumElts; l += 8) {
15143 for (
unsigned i = 0; i != 4; ++i) {
15144 Indices[l + i] = l + (Imm & 3);
15147 for (
unsigned i = 4; i != 8; ++i)
15148 Indices[l + i] = l + i;
15151 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
15154 case X86::BI__builtin_ia32_pshufhw:
15155 case X86::BI__builtin_ia32_pshufhw256:
15156 case X86::BI__builtin_ia32_pshufhw512: {
15157 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
15158 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15159 unsigned NumElts = Ty->getNumElements();
15162 Imm = (Imm & 0xff) * 0x01010101;
15165 for (
unsigned l = 0; l != NumElts; l += 8) {
15166 for (
unsigned i = 0; i != 4; ++i)
15167 Indices[l + i] = l + i;
15168 for (
unsigned i = 4; i != 8; ++i) {
15169 Indices[l + i] = l + 4 + (Imm & 3);
15174 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
15177 case X86::BI__builtin_ia32_pshufd:
15178 case X86::BI__builtin_ia32_pshufd256:
15179 case X86::BI__builtin_ia32_pshufd512:
15180 case X86::BI__builtin_ia32_vpermilpd:
15181 case X86::BI__builtin_ia32_vpermilps:
15182 case X86::BI__builtin_ia32_vpermilpd256:
15183 case X86::BI__builtin_ia32_vpermilps256:
15184 case X86::BI__builtin_ia32_vpermilpd512:
15185 case X86::BI__builtin_ia32_vpermilps512: {
15186 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
15187 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15188 unsigned NumElts = Ty->getNumElements();
15189 unsigned NumLanes = Ty->getPrimitiveSizeInBits() / 128;
15190 unsigned NumLaneElts = NumElts / NumLanes;
15193 Imm = (Imm & 0xff) * 0x01010101;
15196 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
15197 for (
unsigned i = 0; i != NumLaneElts; ++i) {
15198 Indices[i + l] = (Imm % NumLaneElts) + l;
15199 Imm /= NumLaneElts;
15203 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
15206 case X86::BI__builtin_ia32_shufpd:
15207 case X86::BI__builtin_ia32_shufpd256:
15208 case X86::BI__builtin_ia32_shufpd512:
15209 case X86::BI__builtin_ia32_shufps:
15210 case X86::BI__builtin_ia32_shufps256:
15211 case X86::BI__builtin_ia32_shufps512: {
15212 uint32_t Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
15213 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15214 unsigned NumElts = Ty->getNumElements();
15215 unsigned NumLanes = Ty->getPrimitiveSizeInBits() / 128;
15216 unsigned NumLaneElts = NumElts / NumLanes;
15219 Imm = (Imm & 0xff) * 0x01010101;
15222 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
15223 for (
unsigned i = 0; i != NumLaneElts; ++i) {
15224 unsigned Index = Imm % NumLaneElts;
15225 Imm /= NumLaneElts;
15226 if (i >= (NumLaneElts / 2))
15228 Indices[l + i] = l + Index;
15232 return Builder.CreateShuffleVector(Ops[0], Ops[1],
15233 ArrayRef(Indices, NumElts),
"shufp");
15235 case X86::BI__builtin_ia32_permdi256:
15236 case X86::BI__builtin_ia32_permdf256:
15237 case X86::BI__builtin_ia32_permdi512:
15238 case X86::BI__builtin_ia32_permdf512: {
15239 unsigned Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
15240 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15241 unsigned NumElts = Ty->getNumElements();
15245 for (
unsigned l = 0; l != NumElts; l += 4)
15246 for (
unsigned i = 0; i != 4; ++i)
15247 Indices[l + i] = l + ((Imm >> (2 * i)) & 0x3);
15249 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
15252 case X86::BI__builtin_ia32_palignr128:
15253 case X86::BI__builtin_ia32_palignr256:
15254 case X86::BI__builtin_ia32_palignr512: {
15255 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff;
15258 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15259 assert(NumElts % 16 == 0);
15263 if (ShiftVal >= 32)
15268 if (ShiftVal > 16) {
15271 Ops[0] = llvm::Constant::getNullValue(Ops[0]->getType());
15276 for (
unsigned l = 0; l != NumElts; l += 16) {
15277 for (
unsigned i = 0; i != 16; ++i) {
15278 unsigned Idx = ShiftVal + i;
15280 Idx += NumElts - 16;
15281 Indices[l + i] = Idx + l;
15285 return Builder.CreateShuffleVector(Ops[1], Ops[0],
15286 ArrayRef(Indices, NumElts),
"palignr");
15288 case X86::BI__builtin_ia32_alignd128:
15289 case X86::BI__builtin_ia32_alignd256:
15290 case X86::BI__builtin_ia32_alignd512:
15291 case X86::BI__builtin_ia32_alignq128:
15292 case X86::BI__builtin_ia32_alignq256:
15293 case X86::BI__builtin_ia32_alignq512: {
15295 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15296 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff;
15299 ShiftVal &= NumElts - 1;
15302 for (
unsigned i = 0; i != NumElts; ++i)
15303 Indices[i] = i + ShiftVal;
15305 return Builder.CreateShuffleVector(Ops[1], Ops[0],
15306 ArrayRef(Indices, NumElts),
"valign");
15308 case X86::BI__builtin_ia32_shuf_f32x4_256:
15309 case X86::BI__builtin_ia32_shuf_f64x2_256:
15310 case X86::BI__builtin_ia32_shuf_i32x4_256:
15311 case X86::BI__builtin_ia32_shuf_i64x2_256:
15312 case X86::BI__builtin_ia32_shuf_f32x4:
15313 case X86::BI__builtin_ia32_shuf_f64x2:
15314 case X86::BI__builtin_ia32_shuf_i32x4:
15315 case X86::BI__builtin_ia32_shuf_i64x2: {
15316 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
15317 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15318 unsigned NumElts = Ty->getNumElements();
15319 unsigned NumLanes = Ty->getPrimitiveSizeInBits() == 512 ? 4 : 2;
15320 unsigned NumLaneElts = NumElts / NumLanes;
15323 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
15324 unsigned Index = (Imm % NumLanes) * NumLaneElts;
15326 if (l >= (NumElts / 2))
15328 for (
unsigned i = 0; i != NumLaneElts; ++i) {
15329 Indices[l + i] = Index + i;
15333 return Builder.CreateShuffleVector(Ops[0], Ops[1],
15334 ArrayRef(Indices, NumElts),
"shuf");
15337 case X86::BI__builtin_ia32_vperm2f128_pd256:
15338 case X86::BI__builtin_ia32_vperm2f128_ps256:
15339 case X86::BI__builtin_ia32_vperm2f128_si256:
15340 case X86::BI__builtin_ia32_permti256: {
15341 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
15343 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15352 for (
unsigned l = 0; l != 2; ++l) {
15354 if (Imm & (1 << ((l * 4) + 3)))
15355 OutOps[l] = llvm::ConstantAggregateZero::get(Ops[0]->getType());
15356 else if (Imm & (1 << ((l * 4) + 1)))
15357 OutOps[l] = Ops[1];
15359 OutOps[l] = Ops[0];
15361 for (
unsigned i = 0; i != NumElts/2; ++i) {
15363 unsigned Idx = (l * NumElts) + i;
15366 if (Imm & (1 << (l * 4)))
15368 Indices[(l * (NumElts/2)) + i] = Idx;
15372 return Builder.CreateShuffleVector(OutOps[0], OutOps[1],
15373 ArrayRef(Indices, NumElts),
"vperm");
15376 case X86::BI__builtin_ia32_pslldqi128_byteshift:
15377 case X86::BI__builtin_ia32_pslldqi256_byteshift:
15378 case X86::BI__builtin_ia32_pslldqi512_byteshift: {
15379 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
15380 auto *ResultType = cast<llvm::FixedVectorType>(Ops[0]->getType());
15382 unsigned NumElts = ResultType->getNumElements() * 8;
15385 if (ShiftVal >= 16)
15386 return llvm::Constant::getNullValue(ResultType);
15390 for (
unsigned l = 0; l != NumElts; l += 16) {
15391 for (
unsigned i = 0; i != 16; ++i) {
15392 unsigned Idx = NumElts + i - ShiftVal;
15393 if (Idx < NumElts) Idx -= NumElts - 16;
15394 Indices[l + i] = Idx + l;
15398 auto *VecTy = llvm::FixedVectorType::get(
Int8Ty, NumElts);
15400 Value *
Zero = llvm::Constant::getNullValue(VecTy);
15402 Zero, Cast,
ArrayRef(Indices, NumElts),
"pslldq");
15403 return Builder.CreateBitCast(SV, Ops[0]->getType(),
"cast");
15405 case X86::BI__builtin_ia32_psrldqi128_byteshift:
15406 case X86::BI__builtin_ia32_psrldqi256_byteshift:
15407 case X86::BI__builtin_ia32_psrldqi512_byteshift: {
15408 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
15409 auto *ResultType = cast<llvm::FixedVectorType>(Ops[0]->getType());
15411 unsigned NumElts = ResultType->getNumElements() * 8;
15414 if (ShiftVal >= 16)
15415 return llvm::Constant::getNullValue(ResultType);
15419 for (
unsigned l = 0; l != NumElts; l += 16) {
15420 for (
unsigned i = 0; i != 16; ++i) {
15421 unsigned Idx = i + ShiftVal;
15422 if (Idx >= 16) Idx += NumElts - 16;
15423 Indices[l + i] = Idx + l;
15427 auto *VecTy = llvm::FixedVectorType::get(
Int8Ty, NumElts);
15429 Value *
Zero = llvm::Constant::getNullValue(VecTy);
15431 Cast, Zero,
ArrayRef(Indices, NumElts),
"psrldq");
15432 return Builder.CreateBitCast(SV, ResultType,
"cast");
15434 case X86::BI__builtin_ia32_kshiftliqi:
15435 case X86::BI__builtin_ia32_kshiftlihi:
15436 case X86::BI__builtin_ia32_kshiftlisi:
15437 case X86::BI__builtin_ia32_kshiftlidi: {
15438 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
15439 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15441 if (ShiftVal >= NumElts)
15442 return llvm::Constant::getNullValue(Ops[0]->getType());
15447 for (
unsigned i = 0; i != NumElts; ++i)
15448 Indices[i] = NumElts + i - ShiftVal;
15450 Value *
Zero = llvm::Constant::getNullValue(
In->getType());
15452 Zero, In,
ArrayRef(Indices, NumElts),
"kshiftl");
15453 return Builder.CreateBitCast(SV, Ops[0]->getType());
15455 case X86::BI__builtin_ia32_kshiftriqi:
15456 case X86::BI__builtin_ia32_kshiftrihi:
15457 case X86::BI__builtin_ia32_kshiftrisi:
15458 case X86::BI__builtin_ia32_kshiftridi: {
15459 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
15460 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15462 if (ShiftVal >= NumElts)
15463 return llvm::Constant::getNullValue(Ops[0]->getType());
15468 for (
unsigned i = 0; i != NumElts; ++i)
15469 Indices[i] = i + ShiftVal;
15471 Value *
Zero = llvm::Constant::getNullValue(
In->getType());
15473 In, Zero,
ArrayRef(Indices, NumElts),
"kshiftr");
15474 return Builder.CreateBitCast(SV, Ops[0]->getType());
15476 case X86::BI__builtin_ia32_movnti:
15477 case X86::BI__builtin_ia32_movnti64:
15478 case X86::BI__builtin_ia32_movntsd:
15479 case X86::BI__builtin_ia32_movntss: {
15480 llvm::MDNode *
Node = llvm::MDNode::get(
15483 Value *Ptr = Ops[0];
15484 Value *Src = Ops[1];
15487 if (BuiltinID == X86::BI__builtin_ia32_movntsd ||
15488 BuiltinID == X86::BI__builtin_ia32_movntss)
15489 Src =
Builder.CreateExtractElement(Src, (uint64_t)0,
"extract");
15493 SI->setMetadata(llvm::LLVMContext::MD_nontemporal,
Node);
15494 SI->setAlignment(llvm::Align(1));
15498 case X86::BI__builtin_ia32_vprotb:
15499 case X86::BI__builtin_ia32_vprotw:
15500 case X86::BI__builtin_ia32_vprotd:
15501 case X86::BI__builtin_ia32_vprotq:
15502 case X86::BI__builtin_ia32_vprotbi:
15503 case X86::BI__builtin_ia32_vprotwi:
15504 case X86::BI__builtin_ia32_vprotdi:
15505 case X86::BI__builtin_ia32_vprotqi:
15506 case X86::BI__builtin_ia32_prold128:
15507 case X86::BI__builtin_ia32_prold256:
15508 case X86::BI__builtin_ia32_prold512:
15509 case X86::BI__builtin_ia32_prolq128:
15510 case X86::BI__builtin_ia32_prolq256:
15511 case X86::BI__builtin_ia32_prolq512:
15512 case X86::BI__builtin_ia32_prolvd128:
15513 case X86::BI__builtin_ia32_prolvd256:
15514 case X86::BI__builtin_ia32_prolvd512:
15515 case X86::BI__builtin_ia32_prolvq128:
15516 case X86::BI__builtin_ia32_prolvq256:
15517 case X86::BI__builtin_ia32_prolvq512:
15519 case X86::BI__builtin_ia32_prord128:
15520 case X86::BI__builtin_ia32_prord256:
15521 case X86::BI__builtin_ia32_prord512:
15522 case X86::BI__builtin_ia32_prorq128:
15523 case X86::BI__builtin_ia32_prorq256:
15524 case X86::BI__builtin_ia32_prorq512:
15525 case X86::BI__builtin_ia32_prorvd128:
15526 case X86::BI__builtin_ia32_prorvd256:
15527 case X86::BI__builtin_ia32_prorvd512:
15528 case X86::BI__builtin_ia32_prorvq128:
15529 case X86::BI__builtin_ia32_prorvq256:
15530 case X86::BI__builtin_ia32_prorvq512:
15532 case X86::BI__builtin_ia32_selectb_128:
15533 case X86::BI__builtin_ia32_selectb_256:
15534 case X86::BI__builtin_ia32_selectb_512:
15535 case X86::BI__builtin_ia32_selectw_128:
15536 case X86::BI__builtin_ia32_selectw_256:
15537 case X86::BI__builtin_ia32_selectw_512:
15538 case X86::BI__builtin_ia32_selectd_128:
15539 case X86::BI__builtin_ia32_selectd_256:
15540 case X86::BI__builtin_ia32_selectd_512:
15541 case X86::BI__builtin_ia32_selectq_128:
15542 case X86::BI__builtin_ia32_selectq_256:
15543 case X86::BI__builtin_ia32_selectq_512:
15544 case X86::BI__builtin_ia32_selectph_128:
15545 case X86::BI__builtin_ia32_selectph_256:
15546 case X86::BI__builtin_ia32_selectph_512:
15547 case X86::BI__builtin_ia32_selectpbf_128:
15548 case X86::BI__builtin_ia32_selectpbf_256:
15549 case X86::BI__builtin_ia32_selectpbf_512:
15550 case X86::BI__builtin_ia32_selectps_128:
15551 case X86::BI__builtin_ia32_selectps_256:
15552 case X86::BI__builtin_ia32_selectps_512:
15553 case X86::BI__builtin_ia32_selectpd_128:
15554 case X86::BI__builtin_ia32_selectpd_256:
15555 case X86::BI__builtin_ia32_selectpd_512:
15557 case X86::BI__builtin_ia32_selectsh_128:
15558 case X86::BI__builtin_ia32_selectsbf_128:
15559 case X86::BI__builtin_ia32_selectss_128:
15560 case X86::BI__builtin_ia32_selectsd_128: {
15561 Value *A =
Builder.CreateExtractElement(Ops[1], (uint64_t)0);
15562 Value *B =
Builder.CreateExtractElement(Ops[2], (uint64_t)0);
15564 return Builder.CreateInsertElement(Ops[1], A, (uint64_t)0);
15566 case X86::BI__builtin_ia32_cmpb128_mask:
15567 case X86::BI__builtin_ia32_cmpb256_mask:
15568 case X86::BI__builtin_ia32_cmpb512_mask:
15569 case X86::BI__builtin_ia32_cmpw128_mask:
15570 case X86::BI__builtin_ia32_cmpw256_mask:
15571 case X86::BI__builtin_ia32_cmpw512_mask:
15572 case X86::BI__builtin_ia32_cmpd128_mask:
15573 case X86::BI__builtin_ia32_cmpd256_mask:
15574 case X86::BI__builtin_ia32_cmpd512_mask:
15575 case X86::BI__builtin_ia32_cmpq128_mask:
15576 case X86::BI__builtin_ia32_cmpq256_mask:
15577 case X86::BI__builtin_ia32_cmpq512_mask: {
15578 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
15581 case X86::BI__builtin_ia32_ucmpb128_mask:
15582 case X86::BI__builtin_ia32_ucmpb256_mask:
15583 case X86::BI__builtin_ia32_ucmpb512_mask:
15584 case X86::BI__builtin_ia32_ucmpw128_mask:
15585 case X86::BI__builtin_ia32_ucmpw256_mask:
15586 case X86::BI__builtin_ia32_ucmpw512_mask:
15587 case X86::BI__builtin_ia32_ucmpd128_mask:
15588 case X86::BI__builtin_ia32_ucmpd256_mask:
15589 case X86::BI__builtin_ia32_ucmpd512_mask:
15590 case X86::BI__builtin_ia32_ucmpq128_mask:
15591 case X86::BI__builtin_ia32_ucmpq256_mask:
15592 case X86::BI__builtin_ia32_ucmpq512_mask: {
15593 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
15596 case X86::BI__builtin_ia32_vpcomb:
15597 case X86::BI__builtin_ia32_vpcomw:
15598 case X86::BI__builtin_ia32_vpcomd:
15599 case X86::BI__builtin_ia32_vpcomq:
15601 case X86::BI__builtin_ia32_vpcomub:
15602 case X86::BI__builtin_ia32_vpcomuw:
15603 case X86::BI__builtin_ia32_vpcomud:
15604 case X86::BI__builtin_ia32_vpcomuq:
15607 case X86::BI__builtin_ia32_kortestcqi:
15608 case X86::BI__builtin_ia32_kortestchi:
15609 case X86::BI__builtin_ia32_kortestcsi:
15610 case X86::BI__builtin_ia32_kortestcdi: {
15612 Value *
C = llvm::Constant::getAllOnesValue(Ops[0]->getType());
15616 case X86::BI__builtin_ia32_kortestzqi:
15617 case X86::BI__builtin_ia32_kortestzhi:
15618 case X86::BI__builtin_ia32_kortestzsi:
15619 case X86::BI__builtin_ia32_kortestzdi: {
15621 Value *
C = llvm::Constant::getNullValue(Ops[0]->getType());
15626 case X86::BI__builtin_ia32_ktestcqi:
15627 case X86::BI__builtin_ia32_ktestzqi:
15628 case X86::BI__builtin_ia32_ktestchi:
15629 case X86::BI__builtin_ia32_ktestzhi:
15630 case X86::BI__builtin_ia32_ktestcsi:
15631 case X86::BI__builtin_ia32_ktestzsi:
15632 case X86::BI__builtin_ia32_ktestcdi:
15633 case X86::BI__builtin_ia32_ktestzdi: {
15635 switch (BuiltinID) {
15636 default: llvm_unreachable(
"Unsupported intrinsic!");
15637 case X86::BI__builtin_ia32_ktestcqi:
15638 IID = Intrinsic::x86_avx512_ktestc_b;
15640 case X86::BI__builtin_ia32_ktestzqi:
15641 IID = Intrinsic::x86_avx512_ktestz_b;
15643 case X86::BI__builtin_ia32_ktestchi:
15644 IID = Intrinsic::x86_avx512_ktestc_w;
15646 case X86::BI__builtin_ia32_ktestzhi:
15647 IID = Intrinsic::x86_avx512_ktestz_w;
15649 case X86::BI__builtin_ia32_ktestcsi:
15650 IID = Intrinsic::x86_avx512_ktestc_d;
15652 case X86::BI__builtin_ia32_ktestzsi:
15653 IID = Intrinsic::x86_avx512_ktestz_d;
15655 case X86::BI__builtin_ia32_ktestcdi:
15656 IID = Intrinsic::x86_avx512_ktestc_q;
15658 case X86::BI__builtin_ia32_ktestzdi:
15659 IID = Intrinsic::x86_avx512_ktestz_q;
15663 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15667 return Builder.CreateCall(Intr, {LHS, RHS});
15670 case X86::BI__builtin_ia32_kaddqi:
15671 case X86::BI__builtin_ia32_kaddhi:
15672 case X86::BI__builtin_ia32_kaddsi:
15673 case X86::BI__builtin_ia32_kadddi: {
15675 switch (BuiltinID) {
15676 default: llvm_unreachable(
"Unsupported intrinsic!");
15677 case X86::BI__builtin_ia32_kaddqi:
15678 IID = Intrinsic::x86_avx512_kadd_b;
15680 case X86::BI__builtin_ia32_kaddhi:
15681 IID = Intrinsic::x86_avx512_kadd_w;
15683 case X86::BI__builtin_ia32_kaddsi:
15684 IID = Intrinsic::x86_avx512_kadd_d;
15686 case X86::BI__builtin_ia32_kadddi:
15687 IID = Intrinsic::x86_avx512_kadd_q;
15691 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15696 return Builder.CreateBitCast(Res, Ops[0]->getType());
15698 case X86::BI__builtin_ia32_kandqi:
15699 case X86::BI__builtin_ia32_kandhi:
15700 case X86::BI__builtin_ia32_kandsi:
15701 case X86::BI__builtin_ia32_kanddi:
15703 case X86::BI__builtin_ia32_kandnqi:
15704 case X86::BI__builtin_ia32_kandnhi:
15705 case X86::BI__builtin_ia32_kandnsi:
15706 case X86::BI__builtin_ia32_kandndi:
15708 case X86::BI__builtin_ia32_korqi:
15709 case X86::BI__builtin_ia32_korhi:
15710 case X86::BI__builtin_ia32_korsi:
15711 case X86::BI__builtin_ia32_kordi:
15713 case X86::BI__builtin_ia32_kxnorqi:
15714 case X86::BI__builtin_ia32_kxnorhi:
15715 case X86::BI__builtin_ia32_kxnorsi:
15716 case X86::BI__builtin_ia32_kxnordi:
15718 case X86::BI__builtin_ia32_kxorqi:
15719 case X86::BI__builtin_ia32_kxorhi:
15720 case X86::BI__builtin_ia32_kxorsi:
15721 case X86::BI__builtin_ia32_kxordi:
15723 case X86::BI__builtin_ia32_knotqi:
15724 case X86::BI__builtin_ia32_knothi:
15725 case X86::BI__builtin_ia32_knotsi:
15726 case X86::BI__builtin_ia32_knotdi: {
15727 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15730 Ops[0]->getType());
15732 case X86::BI__builtin_ia32_kmovb:
15733 case X86::BI__builtin_ia32_kmovw:
15734 case X86::BI__builtin_ia32_kmovd:
15735 case X86::BI__builtin_ia32_kmovq: {
15739 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15741 return Builder.CreateBitCast(Res, Ops[0]->getType());
15744 case X86::BI__builtin_ia32_kunpckdi:
15745 case X86::BI__builtin_ia32_kunpcksi:
15746 case X86::BI__builtin_ia32_kunpckhi: {
15747 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15751 for (
unsigned i = 0; i != NumElts; ++i)
15756 LHS =
Builder.CreateShuffleVector(LHS, LHS,
ArrayRef(Indices, NumElts / 2));
15757 RHS =
Builder.CreateShuffleVector(RHS, RHS,
ArrayRef(Indices, NumElts / 2));
15762 return Builder.CreateBitCast(Res, Ops[0]->getType());
15765 case X86::BI__builtin_ia32_vplzcntd_128:
15766 case X86::BI__builtin_ia32_vplzcntd_256:
15767 case X86::BI__builtin_ia32_vplzcntd_512:
15768 case X86::BI__builtin_ia32_vplzcntq_128:
15769 case X86::BI__builtin_ia32_vplzcntq_256:
15770 case X86::BI__builtin_ia32_vplzcntq_512: {
15774 case X86::BI__builtin_ia32_sqrtss:
15775 case X86::BI__builtin_ia32_sqrtsd: {
15776 Value *A =
Builder.CreateExtractElement(Ops[0], (uint64_t)0);
15778 if (
Builder.getIsFPConstrained()) {
15779 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
15782 A =
Builder.CreateConstrainedFPCall(F, {A});
15785 A =
Builder.CreateCall(F, {A});
15787 return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0);
15789 case X86::BI__builtin_ia32_sqrtsh_round_mask:
15790 case X86::BI__builtin_ia32_sqrtsd_round_mask:
15791 case X86::BI__builtin_ia32_sqrtss_round_mask: {
15792 unsigned CC = cast<llvm::ConstantInt>(Ops[4])->getZExtValue();
15798 switch (BuiltinID) {
15800 llvm_unreachable(
"Unsupported intrinsic!");
15801 case X86::BI__builtin_ia32_sqrtsh_round_mask:
15802 IID = Intrinsic::x86_avx512fp16_mask_sqrt_sh;
15804 case X86::BI__builtin_ia32_sqrtsd_round_mask:
15805 IID = Intrinsic::x86_avx512_mask_sqrt_sd;
15807 case X86::BI__builtin_ia32_sqrtss_round_mask:
15808 IID = Intrinsic::x86_avx512_mask_sqrt_ss;
15813 Value *A =
Builder.CreateExtractElement(Ops[1], (uint64_t)0);
15815 if (
Builder.getIsFPConstrained()) {
15816 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
15819 A =
Builder.CreateConstrainedFPCall(F, A);
15822 A =
Builder.CreateCall(F, A);
15824 Value *Src =
Builder.CreateExtractElement(Ops[2], (uint64_t)0);
15826 return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0);
15828 case X86::BI__builtin_ia32_sqrtpd256:
15829 case X86::BI__builtin_ia32_sqrtpd:
15830 case X86::BI__builtin_ia32_sqrtps256:
15831 case X86::BI__builtin_ia32_sqrtps:
15832 case X86::BI__builtin_ia32_sqrtph256:
15833 case X86::BI__builtin_ia32_sqrtph:
15834 case X86::BI__builtin_ia32_sqrtph512:
15835 case X86::BI__builtin_ia32_sqrtps512:
15836 case X86::BI__builtin_ia32_sqrtpd512: {
15837 if (Ops.size() == 2) {
15838 unsigned CC = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
15844 switch (BuiltinID) {
15846 llvm_unreachable(
"Unsupported intrinsic!");
15847 case X86::BI__builtin_ia32_sqrtph512:
15848 IID = Intrinsic::x86_avx512fp16_sqrt_ph_512;
15850 case X86::BI__builtin_ia32_sqrtps512:
15851 IID = Intrinsic::x86_avx512_sqrt_ps_512;
15853 case X86::BI__builtin_ia32_sqrtpd512:
15854 IID = Intrinsic::x86_avx512_sqrt_pd_512;
15860 if (
Builder.getIsFPConstrained()) {
15861 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
15863 Ops[0]->getType());
15864 return Builder.CreateConstrainedFPCall(F, Ops[0]);
15867 return Builder.CreateCall(F, Ops[0]);
15871 case X86::BI__builtin_ia32_pmuludq128:
15872 case X86::BI__builtin_ia32_pmuludq256:
15873 case X86::BI__builtin_ia32_pmuludq512:
15876 case X86::BI__builtin_ia32_pmuldq128:
15877 case X86::BI__builtin_ia32_pmuldq256:
15878 case X86::BI__builtin_ia32_pmuldq512:
15881 case X86::BI__builtin_ia32_pternlogd512_mask:
15882 case X86::BI__builtin_ia32_pternlogq512_mask:
15883 case X86::BI__builtin_ia32_pternlogd128_mask:
15884 case X86::BI__builtin_ia32_pternlogd256_mask:
15885 case X86::BI__builtin_ia32_pternlogq128_mask:
15886 case X86::BI__builtin_ia32_pternlogq256_mask:
15889 case X86::BI__builtin_ia32_pternlogd512_maskz:
15890 case X86::BI__builtin_ia32_pternlogq512_maskz:
15891 case X86::BI__builtin_ia32_pternlogd128_maskz:
15892 case X86::BI__builtin_ia32_pternlogd256_maskz:
15893 case X86::BI__builtin_ia32_pternlogq128_maskz:
15894 case X86::BI__builtin_ia32_pternlogq256_maskz:
15897 case X86::BI__builtin_ia32_vpshldd128:
15898 case X86::BI__builtin_ia32_vpshldd256:
15899 case X86::BI__builtin_ia32_vpshldd512:
15900 case X86::BI__builtin_ia32_vpshldq128:
15901 case X86::BI__builtin_ia32_vpshldq256:
15902 case X86::BI__builtin_ia32_vpshldq512:
15903 case X86::BI__builtin_ia32_vpshldw128:
15904 case X86::BI__builtin_ia32_vpshldw256:
15905 case X86::BI__builtin_ia32_vpshldw512:
15908 case X86::BI__builtin_ia32_vpshrdd128:
15909 case X86::BI__builtin_ia32_vpshrdd256:
15910 case X86::BI__builtin_ia32_vpshrdd512:
15911 case X86::BI__builtin_ia32_vpshrdq128:
15912 case X86::BI__builtin_ia32_vpshrdq256:
15913 case X86::BI__builtin_ia32_vpshrdq512:
15914 case X86::BI__builtin_ia32_vpshrdw128:
15915 case X86::BI__builtin_ia32_vpshrdw256:
15916 case X86::BI__builtin_ia32_vpshrdw512:
15920 case X86::BI__builtin_ia32_vpshldvd128:
15921 case X86::BI__builtin_ia32_vpshldvd256:
15922 case X86::BI__builtin_ia32_vpshldvd512:
15923 case X86::BI__builtin_ia32_vpshldvq128:
15924 case X86::BI__builtin_ia32_vpshldvq256:
15925 case X86::BI__builtin_ia32_vpshldvq512:
15926 case X86::BI__builtin_ia32_vpshldvw128:
15927 case X86::BI__builtin_ia32_vpshldvw256:
15928 case X86::BI__builtin_ia32_vpshldvw512:
15931 case X86::BI__builtin_ia32_vpshrdvd128:
15932 case X86::BI__builtin_ia32_vpshrdvd256:
15933 case X86::BI__builtin_ia32_vpshrdvd512:
15934 case X86::BI__builtin_ia32_vpshrdvq128:
15935 case X86::BI__builtin_ia32_vpshrdvq256:
15936 case X86::BI__builtin_ia32_vpshrdvq512:
15937 case X86::BI__builtin_ia32_vpshrdvw128:
15938 case X86::BI__builtin_ia32_vpshrdvw256:
15939 case X86::BI__builtin_ia32_vpshrdvw512:
15944 case X86::BI__builtin_ia32_reduce_fadd_pd512:
15945 case X86::BI__builtin_ia32_reduce_fadd_ps512:
15946 case X86::BI__builtin_ia32_reduce_fadd_ph512:
15947 case X86::BI__builtin_ia32_reduce_fadd_ph256:
15948 case X86::BI__builtin_ia32_reduce_fadd_ph128: {
15951 IRBuilder<>::FastMathFlagGuard FMFGuard(
Builder);
15952 Builder.getFastMathFlags().setAllowReassoc();
15953 return Builder.CreateCall(F, {Ops[0], Ops[1]});
15955 case X86::BI__builtin_ia32_reduce_fmul_pd512:
15956 case X86::BI__builtin_ia32_reduce_fmul_ps512:
15957 case X86::BI__builtin_ia32_reduce_fmul_ph512:
15958 case X86::BI__builtin_ia32_reduce_fmul_ph256:
15959 case X86::BI__builtin_ia32_reduce_fmul_ph128: {
15962 IRBuilder<>::FastMathFlagGuard FMFGuard(
Builder);
15963 Builder.getFastMathFlags().setAllowReassoc();
15964 return Builder.CreateCall(F, {Ops[0], Ops[1]});
15966 case X86::BI__builtin_ia32_reduce_fmax_pd512:
15967 case X86::BI__builtin_ia32_reduce_fmax_ps512:
15968 case X86::BI__builtin_ia32_reduce_fmax_ph512:
15969 case X86::BI__builtin_ia32_reduce_fmax_ph256:
15970 case X86::BI__builtin_ia32_reduce_fmax_ph128: {
15973 IRBuilder<>::FastMathFlagGuard FMFGuard(
Builder);
15974 Builder.getFastMathFlags().setNoNaNs();
15975 return Builder.CreateCall(F, {Ops[0]});
15977 case X86::BI__builtin_ia32_reduce_fmin_pd512:
15978 case X86::BI__builtin_ia32_reduce_fmin_ps512:
15979 case X86::BI__builtin_ia32_reduce_fmin_ph512:
15980 case X86::BI__builtin_ia32_reduce_fmin_ph256:
15981 case X86::BI__builtin_ia32_reduce_fmin_ph128: {
15984 IRBuilder<>::FastMathFlagGuard FMFGuard(
Builder);
15985 Builder.getFastMathFlags().setNoNaNs();
15986 return Builder.CreateCall(F, {Ops[0]});
15990 case X86::BI__builtin_ia32_pswapdsf:
15991 case X86::BI__builtin_ia32_pswapdsi: {
15993 Ops[0] =
Builder.CreateBitCast(Ops[0], MMXTy,
"cast");
15995 return Builder.CreateCall(F, Ops,
"pswapd");
15997 case X86::BI__builtin_ia32_rdrand16_step:
15998 case X86::BI__builtin_ia32_rdrand32_step:
15999 case X86::BI__builtin_ia32_rdrand64_step:
16000 case X86::BI__builtin_ia32_rdseed16_step:
16001 case X86::BI__builtin_ia32_rdseed32_step:
16002 case X86::BI__builtin_ia32_rdseed64_step: {
16004 switch (BuiltinID) {
16005 default: llvm_unreachable(
"Unsupported intrinsic!");
16006 case X86::BI__builtin_ia32_rdrand16_step:
16007 ID = Intrinsic::x86_rdrand_16;
16009 case X86::BI__builtin_ia32_rdrand32_step:
16010 ID = Intrinsic::x86_rdrand_32;
16012 case X86::BI__builtin_ia32_rdrand64_step:
16013 ID = Intrinsic::x86_rdrand_64;
16015 case X86::BI__builtin_ia32_rdseed16_step:
16016 ID = Intrinsic::x86_rdseed_16;
16018 case X86::BI__builtin_ia32_rdseed32_step:
16019 ID = Intrinsic::x86_rdseed_32;
16021 case X86::BI__builtin_ia32_rdseed64_step:
16022 ID = Intrinsic::x86_rdseed_64;
16031 case X86::BI__builtin_ia32_addcarryx_u32:
16032 case X86::BI__builtin_ia32_addcarryx_u64:
16033 case X86::BI__builtin_ia32_subborrow_u32:
16034 case X86::BI__builtin_ia32_subborrow_u64: {
16036 switch (BuiltinID) {
16037 default: llvm_unreachable(
"Unsupported intrinsic!");
16038 case X86::BI__builtin_ia32_addcarryx_u32:
16039 IID = Intrinsic::x86_addcarry_32;
16041 case X86::BI__builtin_ia32_addcarryx_u64:
16042 IID = Intrinsic::x86_addcarry_64;
16044 case X86::BI__builtin_ia32_subborrow_u32:
16045 IID = Intrinsic::x86_subborrow_32;
16047 case X86::BI__builtin_ia32_subborrow_u64:
16048 IID = Intrinsic::x86_subborrow_64;
16053 { Ops[0], Ops[1], Ops[2] });
16059 case X86::BI__builtin_ia32_fpclassps128_mask:
16060 case X86::BI__builtin_ia32_fpclassps256_mask:
16061 case X86::BI__builtin_ia32_fpclassps512_mask:
16062 case X86::BI__builtin_ia32_fpclassph128_mask:
16063 case X86::BI__builtin_ia32_fpclassph256_mask:
16064 case X86::BI__builtin_ia32_fpclassph512_mask:
16065 case X86::BI__builtin_ia32_fpclasspd128_mask:
16066 case X86::BI__builtin_ia32_fpclasspd256_mask:
16067 case X86::BI__builtin_ia32_fpclasspd512_mask: {
16069 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16070 Value *MaskIn = Ops[2];
16071 Ops.erase(&Ops[2]);
16074 switch (BuiltinID) {
16075 default: llvm_unreachable(
"Unsupported intrinsic!");
16076 case X86::BI__builtin_ia32_fpclassph128_mask:
16077 ID = Intrinsic::x86_avx512fp16_fpclass_ph_128;
16079 case X86::BI__builtin_ia32_fpclassph256_mask:
16080 ID = Intrinsic::x86_avx512fp16_fpclass_ph_256;
16082 case X86::BI__builtin_ia32_fpclassph512_mask:
16083 ID = Intrinsic::x86_avx512fp16_fpclass_ph_512;
16085 case X86::BI__builtin_ia32_fpclassps128_mask:
16086 ID = Intrinsic::x86_avx512_fpclass_ps_128;
16088 case X86::BI__builtin_ia32_fpclassps256_mask:
16089 ID = Intrinsic::x86_avx512_fpclass_ps_256;
16091 case X86::BI__builtin_ia32_fpclassps512_mask:
16092 ID = Intrinsic::x86_avx512_fpclass_ps_512;
16094 case X86::BI__builtin_ia32_fpclasspd128_mask:
16095 ID = Intrinsic::x86_avx512_fpclass_pd_128;
16097 case X86::BI__builtin_ia32_fpclasspd256_mask:
16098 ID = Intrinsic::x86_avx512_fpclass_pd_256;
16100 case X86::BI__builtin_ia32_fpclasspd512_mask:
16101 ID = Intrinsic::x86_avx512_fpclass_pd_512;
16109 case X86::BI__builtin_ia32_vp2intersect_q_512:
16110 case X86::BI__builtin_ia32_vp2intersect_q_256:
16111 case X86::BI__builtin_ia32_vp2intersect_q_128:
16112 case X86::BI__builtin_ia32_vp2intersect_d_512:
16113 case X86::BI__builtin_ia32_vp2intersect_d_256:
16114 case X86::BI__builtin_ia32_vp2intersect_d_128: {
16116 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16119 switch (BuiltinID) {
16120 default: llvm_unreachable(
"Unsupported intrinsic!");
16121 case X86::BI__builtin_ia32_vp2intersect_q_512:
16122 ID = Intrinsic::x86_avx512_vp2intersect_q_512;
16124 case X86::BI__builtin_ia32_vp2intersect_q_256:
16125 ID = Intrinsic::x86_avx512_vp2intersect_q_256;
16127 case X86::BI__builtin_ia32_vp2intersect_q_128:
16128 ID = Intrinsic::x86_avx512_vp2intersect_q_128;
16130 case X86::BI__builtin_ia32_vp2intersect_d_512:
16131 ID = Intrinsic::x86_avx512_vp2intersect_d_512;
16133 case X86::BI__builtin_ia32_vp2intersect_d_256:
16134 ID = Intrinsic::x86_avx512_vp2intersect_d_256;
16136 case X86::BI__builtin_ia32_vp2intersect_d_128:
16137 ID = Intrinsic::x86_avx512_vp2intersect_d_128;
16151 case X86::BI__builtin_ia32_vpmultishiftqb128:
16152 case X86::BI__builtin_ia32_vpmultishiftqb256:
16153 case X86::BI__builtin_ia32_vpmultishiftqb512: {
16155 switch (BuiltinID) {
16156 default: llvm_unreachable(
"Unsupported intrinsic!");
16157 case X86::BI__builtin_ia32_vpmultishiftqb128:
16158 ID = Intrinsic::x86_avx512_pmultishift_qb_128;
16160 case X86::BI__builtin_ia32_vpmultishiftqb256:
16161 ID = Intrinsic::x86_avx512_pmultishift_qb_256;
16163 case X86::BI__builtin_ia32_vpmultishiftqb512:
16164 ID = Intrinsic::x86_avx512_pmultishift_qb_512;
16171 case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
16172 case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
16173 case X86::BI__builtin_ia32_vpshufbitqmb512_mask: {
16175 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16176 Value *MaskIn = Ops[2];
16177 Ops.erase(&Ops[2]);
16180 switch (BuiltinID) {
16181 default: llvm_unreachable(
"Unsupported intrinsic!");
16182 case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
16183 ID = Intrinsic::x86_avx512_vpshufbitqmb_128;
16185 case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
16186 ID = Intrinsic::x86_avx512_vpshufbitqmb_256;
16188 case X86::BI__builtin_ia32_vpshufbitqmb512_mask:
16189 ID = Intrinsic::x86_avx512_vpshufbitqmb_512;
16198 case X86::BI__builtin_ia32_cmpeqps:
16199 case X86::BI__builtin_ia32_cmpeqpd:
16200 return getVectorFCmpIR(CmpInst::FCMP_OEQ,
false);
16201 case X86::BI__builtin_ia32_cmpltps:
16202 case X86::BI__builtin_ia32_cmpltpd:
16203 return getVectorFCmpIR(CmpInst::FCMP_OLT,
true);
16204 case X86::BI__builtin_ia32_cmpleps:
16205 case X86::BI__builtin_ia32_cmplepd:
16206 return getVectorFCmpIR(CmpInst::FCMP_OLE,
true);
16207 case X86::BI__builtin_ia32_cmpunordps:
16208 case X86::BI__builtin_ia32_cmpunordpd:
16209 return getVectorFCmpIR(CmpInst::FCMP_UNO,
false);
16210 case X86::BI__builtin_ia32_cmpneqps:
16211 case X86::BI__builtin_ia32_cmpneqpd:
16212 return getVectorFCmpIR(CmpInst::FCMP_UNE,
false);
16213 case X86::BI__builtin_ia32_cmpnltps:
16214 case X86::BI__builtin_ia32_cmpnltpd:
16215 return getVectorFCmpIR(CmpInst::FCMP_UGE,
true);
16216 case X86::BI__builtin_ia32_cmpnleps:
16217 case X86::BI__builtin_ia32_cmpnlepd:
16218 return getVectorFCmpIR(CmpInst::FCMP_UGT,
true);
16219 case X86::BI__builtin_ia32_cmpordps:
16220 case X86::BI__builtin_ia32_cmpordpd:
16221 return getVectorFCmpIR(CmpInst::FCMP_ORD,
false);
16222 case X86::BI__builtin_ia32_cmpph128_mask:
16223 case X86::BI__builtin_ia32_cmpph256_mask:
16224 case X86::BI__builtin_ia32_cmpph512_mask:
16225 case X86::BI__builtin_ia32_cmpps128_mask:
16226 case X86::BI__builtin_ia32_cmpps256_mask:
16227 case X86::BI__builtin_ia32_cmpps512_mask:
16228 case X86::BI__builtin_ia32_cmppd128_mask:
16229 case X86::BI__builtin_ia32_cmppd256_mask:
16230 case X86::BI__builtin_ia32_cmppd512_mask:
16233 case X86::BI__builtin_ia32_cmpps:
16234 case X86::BI__builtin_ia32_cmpps256:
16235 case X86::BI__builtin_ia32_cmppd:
16236 case X86::BI__builtin_ia32_cmppd256: {
16244 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x1f;
16249 FCmpInst::Predicate Pred;
16253 switch (CC & 0xf) {
16254 case 0x00: Pred = FCmpInst::FCMP_OEQ; IsSignaling =
false;
break;
16255 case 0x01: Pred = FCmpInst::FCMP_OLT; IsSignaling =
true;
break;
16256 case 0x02: Pred = FCmpInst::FCMP_OLE; IsSignaling =
true;
break;
16257 case 0x03: Pred = FCmpInst::FCMP_UNO; IsSignaling =
false;
break;
16258 case 0x04: Pred = FCmpInst::FCMP_UNE; IsSignaling =
false;
break;
16259 case 0x05: Pred = FCmpInst::FCMP_UGE; IsSignaling =
true;
break;
16260 case 0x06: Pred = FCmpInst::FCMP_UGT; IsSignaling =
true;
break;
16261 case 0x07: Pred = FCmpInst::FCMP_ORD; IsSignaling =
false;
break;
16262 case 0x08: Pred = FCmpInst::FCMP_UEQ; IsSignaling =
false;
break;
16263 case 0x09: Pred = FCmpInst::FCMP_ULT; IsSignaling =
true;
break;
16264 case 0x0a: Pred = FCmpInst::FCMP_ULE; IsSignaling =
true;
break;
16265 case 0x0b: Pred = FCmpInst::FCMP_FALSE; IsSignaling =
false;
break;
16266 case 0x0c: Pred = FCmpInst::FCMP_ONE; IsSignaling =
false;
break;
16267 case 0x0d: Pred = FCmpInst::FCMP_OGE; IsSignaling =
true;
break;
16268 case 0x0e: Pred = FCmpInst::FCMP_OGT; IsSignaling =
true;
break;
16269 case 0x0f: Pred = FCmpInst::FCMP_TRUE; IsSignaling =
false;
break;
16270 default: llvm_unreachable(
"Unhandled CC");
16275 IsSignaling = !IsSignaling;
16282 if (
Builder.getIsFPConstrained() &&
16283 (Pred == FCmpInst::FCMP_TRUE || Pred == FCmpInst::FCMP_FALSE ||
16287 switch (BuiltinID) {
16288 default: llvm_unreachable(
"Unexpected builtin");
16289 case X86::BI__builtin_ia32_cmpps:
16290 IID = Intrinsic::x86_sse_cmp_ps;
16292 case X86::BI__builtin_ia32_cmpps256:
16293 IID = Intrinsic::x86_avx_cmp_ps_256;
16295 case X86::BI__builtin_ia32_cmppd:
16296 IID = Intrinsic::x86_sse2_cmp_pd;
16298 case X86::BI__builtin_ia32_cmppd256:
16299 IID = Intrinsic::x86_avx_cmp_pd_256;
16301 case X86::BI__builtin_ia32_cmpph128_mask:
16302 IID = Intrinsic::x86_avx512fp16_mask_cmp_ph_128;
16304 case X86::BI__builtin_ia32_cmpph256_mask:
16305 IID = Intrinsic::x86_avx512fp16_mask_cmp_ph_256;
16307 case X86::BI__builtin_ia32_cmpph512_mask:
16308 IID = Intrinsic::x86_avx512fp16_mask_cmp_ph_512;
16310 case X86::BI__builtin_ia32_cmpps512_mask:
16311 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
16313 case X86::BI__builtin_ia32_cmppd512_mask:
16314 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
16316 case X86::BI__builtin_ia32_cmpps128_mask:
16317 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
16319 case X86::BI__builtin_ia32_cmpps256_mask:
16320 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
16322 case X86::BI__builtin_ia32_cmppd128_mask:
16323 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
16325 case X86::BI__builtin_ia32_cmppd256_mask:
16326 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
16333 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16339 return Builder.CreateCall(Intr, Ops);
16350 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16353 Cmp =
Builder.CreateFCmpS(Pred, Ops[0], Ops[1]);
16355 Cmp =
Builder.CreateFCmp(Pred, Ops[0], Ops[1]);
16359 return getVectorFCmpIR(Pred, IsSignaling);
16363 case X86::BI__builtin_ia32_cmpeqss:
16364 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 0);
16365 case X86::BI__builtin_ia32_cmpltss:
16366 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 1);
16367 case X86::BI__builtin_ia32_cmpless:
16368 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 2);
16369 case X86::BI__builtin_ia32_cmpunordss:
16370 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 3);
16371 case X86::BI__builtin_ia32_cmpneqss:
16372 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 4);
16373 case X86::BI__builtin_ia32_cmpnltss:
16374 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 5);
16375 case X86::BI__builtin_ia32_cmpnless:
16376 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 6);
16377 case X86::BI__builtin_ia32_cmpordss:
16378 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 7);
16379 case X86::BI__builtin_ia32_cmpeqsd:
16380 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 0);
16381 case X86::BI__builtin_ia32_cmpltsd:
16382 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 1);
16383 case X86::BI__builtin_ia32_cmplesd:
16384 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 2);
16385 case X86::BI__builtin_ia32_cmpunordsd:
16386 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 3);
16387 case X86::BI__builtin_ia32_cmpneqsd:
16388 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 4);
16389 case X86::BI__builtin_ia32_cmpnltsd:
16390 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 5);
16391 case X86::BI__builtin_ia32_cmpnlesd:
16392 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 6);
16393 case X86::BI__builtin_ia32_cmpordsd:
16394 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 7);
16397 case X86::BI__builtin_ia32_vcvtph2ps:
16398 case X86::BI__builtin_ia32_vcvtph2ps256:
16399 case X86::BI__builtin_ia32_vcvtph2ps_mask:
16400 case X86::BI__builtin_ia32_vcvtph2ps256_mask:
16401 case X86::BI__builtin_ia32_vcvtph2ps512_mask: {
16402 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
16407 case X86::BI__builtin_ia32_cvtneps2bf16_128_mask: {
16410 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements());
16411 Intrinsic::ID IID = Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128;
16414 case X86::BI__builtin_ia32_cvtsbf162ss_32:
16417 case X86::BI__builtin_ia32_cvtneps2bf16_256_mask:
16418 case X86::BI__builtin_ia32_cvtneps2bf16_512_mask: {
16420 switch (BuiltinID) {
16421 default: llvm_unreachable(
"Unsupported intrinsic!");
16422 case X86::BI__builtin_ia32_cvtneps2bf16_256_mask:
16423 IID = Intrinsic::x86_avx512bf16_cvtneps2bf16_256;
16425 case X86::BI__builtin_ia32_cvtneps2bf16_512_mask:
16426 IID = Intrinsic::x86_avx512bf16_cvtneps2bf16_512;
16433 case X86::BI__cpuid:
16434 case X86::BI__cpuidex: {
16436 Value *SubFuncId = BuiltinID == X86::BI__cpuidex
16440 llvm::StructType *CpuidRetTy =
16442 llvm::FunctionType *FTy =
16445 StringRef
Asm, Constraints;
16446 if (
getTarget().getTriple().getArch() == llvm::Triple::x86) {
16448 Constraints =
"={ax},={bx},={cx},={dx},{ax},{cx}";
16451 Asm =
"xchgq %rbx, ${1:q}\n"
16453 "xchgq %rbx, ${1:q}";
16454 Constraints =
"={ax},=r,={cx},={dx},0,2";
16457 llvm::InlineAsm *IA = llvm::InlineAsm::get(FTy,
Asm, Constraints,
16459 Value *IACall =
Builder.CreateCall(IA, {FuncId, SubFuncId});
16462 for (
unsigned i = 0; i < 4; i++) {
16463 Value *Extracted =
Builder.CreateExtractValue(IACall, i);
16473 case X86::BI__emul:
16474 case X86::BI__emulu: {
16476 bool isSigned = (BuiltinID == X86::BI__emul);
16479 return Builder.CreateMul(LHS, RHS,
"", !isSigned, isSigned);
16481 case X86::BI__mulh:
16482 case X86::BI__umulh:
16483 case X86::BI_mul128:
16484 case X86::BI_umul128: {
16486 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
16488 bool IsSigned = (BuiltinID == X86::BI__mulh || BuiltinID == X86::BI_mul128);
16489 Value *LHS =
Builder.CreateIntCast(Ops[0], Int128Ty, IsSigned);
16490 Value *RHS =
Builder.CreateIntCast(Ops[1], Int128Ty, IsSigned);
16492 Value *MulResult, *HigherBits;
16494 MulResult =
Builder.CreateNSWMul(LHS, RHS);
16495 HigherBits =
Builder.CreateAShr(MulResult, 64);
16497 MulResult =
Builder.CreateNUWMul(LHS, RHS);
16498 HigherBits =
Builder.CreateLShr(MulResult, 64);
16500 HigherBits =
Builder.CreateIntCast(HigherBits, ResType, IsSigned);
16502 if (BuiltinID == X86::BI__mulh || BuiltinID == X86::BI__umulh)
16507 return Builder.CreateIntCast(MulResult, ResType, IsSigned);
16510 case X86::BI__faststorefence: {
16511 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
16512 llvm::SyncScope::System);
16514 case X86::BI__shiftleft128:
16515 case X86::BI__shiftright128: {
16517 BuiltinID == X86::BI__shiftleft128 ? Intrinsic::fshl : Intrinsic::fshr,
16522 std::swap(Ops[0], Ops[1]);
16524 return Builder.CreateCall(F, Ops);
16526 case X86::BI_ReadWriteBarrier:
16527 case X86::BI_ReadBarrier:
16528 case X86::BI_WriteBarrier: {
16529 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
16530 llvm::SyncScope::SingleThread);
16533 case X86::BI_AddressOfReturnAddress: {
16536 return Builder.CreateCall(F);
16538 case X86::BI__stosb: {
16546 case X86::BI__int2c: {
16548 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy,
false);
16549 llvm::InlineAsm *IA =
16550 llvm::InlineAsm::get(FTy,
"int $$0x2c",
"",
true);
16551 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
16553 llvm::Attribute::NoReturn);
16554 llvm::CallInst *CI =
Builder.CreateCall(IA);
16555 CI->setAttributes(NoReturnAttr);
16558 case X86::BI__readfsbyte:
16559 case X86::BI__readfsword:
16560 case X86::BI__readfsdword:
16561 case X86::BI__readfsqword: {
16567 Load->setVolatile(
true);
16570 case X86::BI__readgsbyte:
16571 case X86::BI__readgsword:
16572 case X86::BI__readgsdword:
16573 case X86::BI__readgsqword: {
16579 Load->setVolatile(
true);
16582 case X86::BI__builtin_ia32_encodekey128_u32: {
16583 Intrinsic::ID IID = Intrinsic::x86_encodekey128;
16587 for (
int i = 0; i < 3; ++i) {
16595 case X86::BI__builtin_ia32_encodekey256_u32: {
16596 Intrinsic::ID IID = Intrinsic::x86_encodekey256;
16601 for (
int i = 0; i < 4; ++i) {
16609 case X86::BI__builtin_ia32_aesenc128kl_u8:
16610 case X86::BI__builtin_ia32_aesdec128kl_u8:
16611 case X86::BI__builtin_ia32_aesenc256kl_u8:
16612 case X86::BI__builtin_ia32_aesdec256kl_u8: {
16614 StringRef BlockName;
16615 switch (BuiltinID) {
16617 llvm_unreachable(
"Unexpected builtin");
16618 case X86::BI__builtin_ia32_aesenc128kl_u8:
16619 IID = Intrinsic::x86_aesenc128kl;
16620 BlockName =
"aesenc128kl";
16622 case X86::BI__builtin_ia32_aesdec128kl_u8:
16623 IID = Intrinsic::x86_aesdec128kl;
16624 BlockName =
"aesdec128kl";
16626 case X86::BI__builtin_ia32_aesenc256kl_u8:
16627 IID = Intrinsic::x86_aesenc256kl;
16628 BlockName =
"aesenc256kl";
16630 case X86::BI__builtin_ia32_aesdec256kl_u8:
16631 IID = Intrinsic::x86_aesdec256kl;
16632 BlockName =
"aesdec256kl";
16638 BasicBlock *NoError =
16646 Builder.CreateCondBr(Succ, NoError, Error);
16648 Builder.SetInsertPoint(NoError);
16652 Builder.SetInsertPoint(Error);
16653 Constant *
Zero = llvm::Constant::getNullValue(Out->getType());
16660 case X86::BI__builtin_ia32_aesencwide128kl_u8:
16661 case X86::BI__builtin_ia32_aesdecwide128kl_u8:
16662 case X86::BI__builtin_ia32_aesencwide256kl_u8:
16663 case X86::BI__builtin_ia32_aesdecwide256kl_u8: {
16665 StringRef BlockName;
16666 switch (BuiltinID) {
16667 case X86::BI__builtin_ia32_aesencwide128kl_u8:
16668 IID = Intrinsic::x86_aesencwide128kl;
16669 BlockName =
"aesencwide128kl";
16671 case X86::BI__builtin_ia32_aesdecwide128kl_u8:
16672 IID = Intrinsic::x86_aesdecwide128kl;
16673 BlockName =
"aesdecwide128kl";
16675 case X86::BI__builtin_ia32_aesencwide256kl_u8:
16676 IID = Intrinsic::x86_aesencwide256kl;
16677 BlockName =
"aesencwide256kl";
16679 case X86::BI__builtin_ia32_aesdecwide256kl_u8:
16680 IID = Intrinsic::x86_aesdecwide256kl;
16681 BlockName =
"aesdecwide256kl";
16685 llvm::Type *Ty = FixedVectorType::get(
Builder.getInt64Ty(), 2);
16688 for (
int i = 0; i != 8; ++i) {
16689 Value *Ptr =
Builder.CreateConstGEP1_32(Ty, Ops[1], i);
16695 BasicBlock *NoError =
16702 Builder.CreateCondBr(Succ, NoError, Error);
16704 Builder.SetInsertPoint(NoError);
16705 for (
int i = 0; i != 8; ++i) {
16712 Builder.SetInsertPoint(Error);
16713 for (
int i = 0; i != 8; ++i) {
16715 Constant *
Zero = llvm::Constant::getNullValue(Out->getType());
16716 Value *Ptr =
Builder.CreateConstGEP1_32(Out->getType(), Ops[0], i);
16724 case X86::BI__builtin_ia32_vfcmaddcph512_mask:
16727 case X86::BI__builtin_ia32_vfmaddcph512_mask: {
16728 Intrinsic::ID IID = IsConjFMA
16729 ? Intrinsic::x86_avx512fp16_mask_vfcmadd_cph_512
16730 : Intrinsic::x86_avx512fp16_mask_vfmadd_cph_512;
16734 case X86::BI__builtin_ia32_vfcmaddcsh_round_mask:
16737 case X86::BI__builtin_ia32_vfmaddcsh_round_mask: {
16738 Intrinsic::ID IID = IsConjFMA ? Intrinsic::x86_avx512fp16_mask_vfcmadd_csh
16739 : Intrinsic::x86_avx512fp16_mask_vfmadd_csh;
16744 case X86::BI__builtin_ia32_vfcmaddcsh_round_mask3:
16747 case X86::BI__builtin_ia32_vfmaddcsh_round_mask3: {
16748 Intrinsic::ID IID = IsConjFMA ? Intrinsic::x86_avx512fp16_mask_vfcmadd_csh
16749 : Intrinsic::x86_avx512fp16_mask_vfmadd_csh;
16751 static constexpr int Mask[] = {0, 5, 6, 7};
16752 return Builder.CreateShuffleVector(
Call, Ops[2], Mask);
16754 case X86::BI__builtin_ia32_prefetchi:
16757 {Ops[0], llvm::ConstantInt::get(Int32Ty, 0), Ops[1],
16758 llvm::ConstantInt::get(Int32Ty, 0)});
16776 Intrinsic::ID
ID = Intrinsic::not_intrinsic;
16778#include "llvm/TargetParser/PPCTargetParser.def"
16779 auto GenAIXPPCBuiltinCpuExpr = [&](
unsigned SupportMethod,
unsigned FieldIdx,
16781 unsigned OpValue) ->
Value * {
16782 if (SupportMethod == AIX_BUILTIN_PPC_FALSE)
16785 if (SupportMethod == AIX_BUILTIN_PPC_TRUE)
16788 assert(SupportMethod <= USE_SYS_CONF &&
"Invalid value for SupportMethod.");
16789 assert((CompOp == COMP_EQ) &&
"Only equal comparisons are supported.");
16791 llvm::Type *STy = llvm::StructType::get(PPC_SYSTEMCONFIG_TYPE);
16792 llvm::Constant *SysConf =
16796 llvm::Value *Idxs[] = {ConstantInt::get(
Int32Ty, 0),
16797 ConstantInt::get(
Int32Ty, FieldIdx)};
16802 assert(FieldValue->getType()->isIntegerTy(32) &&
16803 "Only 32-bit integers are supported in GenAIXPPCBuiltinCpuExpr().");
16804 return Builder.CreateICmp(ICmpInst::ICMP_EQ, FieldValue,
16805 ConstantInt::get(
Int32Ty, OpValue));
16808 switch (BuiltinID) {
16809 default:
return nullptr;
16811 case Builtin::BI__builtin_cpu_is: {
16813 StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
16816 if (Triple.isOSAIX()) {
16817 unsigned IsCpuSupport, FieldIdx, CompareOp, CpuIdValue;
16818 typedef std::tuple<unsigned, unsigned, unsigned, unsigned> CPUType;
16819 std::tie(IsCpuSupport, FieldIdx, CompareOp, CpuIdValue) =
16820 static_cast<CPUType
>(StringSwitch<CPUType>(CPUStr)
16821#define PPC_AIX_CPU(NAME, SUPPORT_MAGIC, INDEX, COMPARE_OP, VALUE) \
16822 .Case(NAME, {SUPPORT_MAGIC, INDEX, COMPARE_OP, VALUE})
16823#include "llvm/TargetParser/PPCTargetParser.def"
16825 return GenAIXPPCBuiltinCpuExpr(IsCpuSupport, FieldIdx, CompareOp,
16829 assert(Triple.isOSLinux() &&
16830 "__builtin_cpu_is() is only supported for AIX and Linux.");
16831 unsigned NumCPUID = StringSwitch<unsigned>(CPUStr)
16832#define PPC_LNX_CPU(Name, NumericID) .Case(Name, NumericID)
16833#include "llvm/TargetParser/PPCTargetParser.def"
16835 assert(NumCPUID < -1U &&
"Invalid CPU name. Missed by SemaChecking?");
16836 Value *Op0 = llvm::ConstantInt::get(
Int32Ty, PPC_FAWORD_CPUID);
16838 Value *TheCall =
Builder.CreateCall(F, {Op0},
"cpu_is");
16839 return Builder.CreateICmpEQ(TheCall,
16840 llvm::ConstantInt::get(
Int32Ty, NumCPUID));
16842 case Builtin::BI__builtin_cpu_supports: {
16843 unsigned FeatureWord;
16846 StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
16847 std::tie(FeatureWord, BitMask) =
16848 StringSwitch<std::pair<unsigned, unsigned>>(CPUStr)
16849#define
PPC_LNX_FEATURE(Name, Description, EnumName, Bitmask, FA_WORD) \
16850 .Case(Name, {FA_WORD, Bitmask})
16851#include
"llvm/TargetParser/PPCTargetParser.def"
16855 Value *Op0 = llvm::ConstantInt::get(
Int32Ty, FeatureWord);
16857 Value *TheCall =
Builder.CreateCall(F, {Op0},
"cpu_supports");
16859 Builder.CreateAnd(TheCall, llvm::ConstantInt::get(
Int32Ty, BitMask));
16860 return Builder.CreateICmpNE(Mask, llvm::Constant::getNullValue(
Int32Ty));
16861#undef PPC_FAWORD_HWCAP
16862#undef PPC_FAWORD_HWCAP2
16863#undef PPC_FAWORD_CPUID
16868 case PPC::BI__builtin_ppc_get_timebase:
16872 case PPC::BI__builtin_altivec_lvx:
16873 case PPC::BI__builtin_altivec_lvxl:
16874 case PPC::BI__builtin_altivec_lvebx:
16875 case PPC::BI__builtin_altivec_lvehx:
16876 case PPC::BI__builtin_altivec_lvewx:
16877 case PPC::BI__builtin_altivec_lvsl:
16878 case PPC::BI__builtin_altivec_lvsr:
16879 case PPC::BI__builtin_vsx_lxvd2x:
16880 case PPC::BI__builtin_vsx_lxvw4x:
16881 case PPC::BI__builtin_vsx_lxvd2x_be:
16882 case PPC::BI__builtin_vsx_lxvw4x_be:
16883 case PPC::BI__builtin_vsx_lxvl:
16884 case PPC::BI__builtin_vsx_lxvll:
16889 if (!(BuiltinID == PPC::BI__builtin_vsx_lxvl ||
16890 BuiltinID == PPC::BI__builtin_vsx_lxvll)) {
16895 switch (BuiltinID) {
16896 default: llvm_unreachable(
"Unsupported ld/lvsl/lvsr intrinsic!");
16897 case PPC::BI__builtin_altivec_lvx:
16898 ID = Intrinsic::ppc_altivec_lvx;
16900 case PPC::BI__builtin_altivec_lvxl:
16901 ID = Intrinsic::ppc_altivec_lvxl;
16903 case PPC::BI__builtin_altivec_lvebx:
16904 ID = Intrinsic::ppc_altivec_lvebx;
16906 case PPC::BI__builtin_altivec_lvehx:
16907 ID = Intrinsic::ppc_altivec_lvehx;
16909 case PPC::BI__builtin_altivec_lvewx:
16910 ID = Intrinsic::ppc_altivec_lvewx;
16912 case PPC::BI__builtin_altivec_lvsl:
16913 ID = Intrinsic::ppc_altivec_lvsl;
16915 case PPC::BI__builtin_altivec_lvsr:
16916 ID = Intrinsic::ppc_altivec_lvsr;
16918 case PPC::BI__builtin_vsx_lxvd2x:
16919 ID = Intrinsic::ppc_vsx_lxvd2x;
16921 case PPC::BI__builtin_vsx_lxvw4x:
16922 ID = Intrinsic::ppc_vsx_lxvw4x;
16924 case PPC::BI__builtin_vsx_lxvd2x_be:
16925 ID = Intrinsic::ppc_vsx_lxvd2x_be;
16927 case PPC::BI__builtin_vsx_lxvw4x_be:
16928 ID = Intrinsic::ppc_vsx_lxvw4x_be;
16930 case PPC::BI__builtin_vsx_lxvl:
16931 ID = Intrinsic::ppc_vsx_lxvl;
16933 case PPC::BI__builtin_vsx_lxvll:
16934 ID = Intrinsic::ppc_vsx_lxvll;
16938 return Builder.CreateCall(F, Ops,
"");
16942 case PPC::BI__builtin_altivec_stvx:
16943 case PPC::BI__builtin_altivec_stvxl:
16944 case PPC::BI__builtin_altivec_stvebx:
16945 case PPC::BI__builtin_altivec_stvehx:
16946 case PPC::BI__builtin_altivec_stvewx:
16947 case PPC::BI__builtin_vsx_stxvd2x:
16948 case PPC::BI__builtin_vsx_stxvw4x:
16949 case PPC::BI__builtin_vsx_stxvd2x_be:
16950 case PPC::BI__builtin_vsx_stxvw4x_be:
16951 case PPC::BI__builtin_vsx_stxvl:
16952 case PPC::BI__builtin_vsx_stxvll:
16958 if (!(BuiltinID == PPC::BI__builtin_vsx_stxvl ||
16959 BuiltinID == PPC::BI__builtin_vsx_stxvll)) {
16964 switch (BuiltinID) {
16965 default: llvm_unreachable(
"Unsupported st intrinsic!");
16966 case PPC::BI__builtin_altivec_stvx:
16967 ID = Intrinsic::ppc_altivec_stvx;
16969 case PPC::BI__builtin_altivec_stvxl:
16970 ID = Intrinsic::ppc_altivec_stvxl;
16972 case PPC::BI__builtin_altivec_stvebx:
16973 ID = Intrinsic::ppc_altivec_stvebx;
16975 case PPC::BI__builtin_altivec_stvehx:
16976 ID = Intrinsic::ppc_altivec_stvehx;
16978 case PPC::BI__builtin_altivec_stvewx:
16979 ID = Intrinsic::ppc_altivec_stvewx;
16981 case PPC::BI__builtin_vsx_stxvd2x:
16982 ID = Intrinsic::ppc_vsx_stxvd2x;
16984 case PPC::BI__builtin_vsx_stxvw4x:
16985 ID = Intrinsic::ppc_vsx_stxvw4x;
16987 case PPC::BI__builtin_vsx_stxvd2x_be:
16988 ID = Intrinsic::ppc_vsx_stxvd2x_be;
16990 case PPC::BI__builtin_vsx_stxvw4x_be:
16991 ID = Intrinsic::ppc_vsx_stxvw4x_be;
16993 case PPC::BI__builtin_vsx_stxvl:
16994 ID = Intrinsic::ppc_vsx_stxvl;
16996 case PPC::BI__builtin_vsx_stxvll:
16997 ID = Intrinsic::ppc_vsx_stxvll;
17001 return Builder.CreateCall(F, Ops,
"");
17003 case PPC::BI__builtin_vsx_ldrmb: {
17009 int64_t NumBytes = cast<ConstantInt>(Op1)->getZExtValue();
17014 if (NumBytes == 16) {
17022 for (
int Idx = 0; Idx < 16; Idx++)
17023 RevMask.push_back(15 - Idx);
17024 return Builder.CreateShuffleVector(LD, LD, RevMask);
17028 llvm::Function *Lvs =
CGM.
getIntrinsic(IsLE ? Intrinsic::ppc_altivec_lvsr
17029 : Intrinsic::ppc_altivec_lvsl);
17030 llvm::Function *Vperm =
CGM.
getIntrinsic(Intrinsic::ppc_altivec_vperm);
17032 Int8Ty, Op0, ConstantInt::get(Op1->
getType(), NumBytes - 1));
17034 Value *HiLd =
Builder.CreateCall(Lvx, HiMem,
"ld.hi");
17037 Op0 = IsLE ? HiLd : LoLd;
17038 Op1 = IsLE ? LoLd : HiLd;
17039 Value *AllElts =
Builder.CreateCall(Vperm, {Op0, Op1, Mask1},
"shuffle1");
17040 Constant *
Zero = llvm::Constant::getNullValue(IsLE ? ResTy : AllElts->
getType());
17044 for (
int Idx = 0; Idx < 16; Idx++) {
17045 int Val = (NumBytes - Idx - 1 >= 0) ? (NumBytes - Idx - 1)
17046 : 16 - (NumBytes - Idx);
17047 Consts.push_back(Val);
17049 return Builder.CreateShuffleVector(
Builder.CreateBitCast(AllElts, ResTy),
17053 for (
int Idx = 0; Idx < 16; Idx++)
17054 Consts.push_back(
Builder.getInt8(NumBytes + Idx));
17055 Value *Mask2 = ConstantVector::get(Consts);
17056 return Builder.CreateBitCast(
17057 Builder.CreateCall(Vperm, {Zero, AllElts, Mask2},
"shuffle2"), ResTy);
17059 case PPC::BI__builtin_vsx_strmb: {
17063 int64_t NumBytes = cast<ConstantInt>(Op1)->getZExtValue();
17065 auto StoreSubVec = [&](
unsigned Width,
unsigned Offset,
unsigned EltNo) {
17069 Value *StVec = Op2;
17072 for (
int Idx = 0; Idx < 16; Idx++)
17073 RevMask.push_back(15 - Idx);
17074 StVec =
Builder.CreateShuffleVector(Op2, Op2, RevMask);
17080 unsigned NumElts = 0;
17083 llvm_unreachable(
"width for stores must be a power of 2");
17102 Op2, llvm::FixedVectorType::get(ConvTy, NumElts));
17105 Value *Elt =
Builder.CreateExtractElement(Vec, EltNo);
17106 if (IsLE && Width > 1) {
17108 Elt =
Builder.CreateCall(F, Elt);
17113 unsigned Stored = 0;
17114 unsigned RemainingBytes = NumBytes;
17116 if (NumBytes == 16)
17117 return StoreSubVec(16, 0, 0);
17118 if (NumBytes >= 8) {
17119 Result = StoreSubVec(8, NumBytes - 8, IsLE ? 0 : 1);
17120 RemainingBytes -= 8;
17123 if (RemainingBytes >= 4) {
17124 Result = StoreSubVec(4, NumBytes - Stored - 4,
17125 IsLE ? (Stored >> 2) : 3 - (Stored >> 2));
17126 RemainingBytes -= 4;
17129 if (RemainingBytes >= 2) {
17130 Result = StoreSubVec(2, NumBytes - Stored - 2,
17131 IsLE ? (Stored >> 1) : 7 - (Stored >> 1));
17132 RemainingBytes -= 2;
17135 if (RemainingBytes)
17137 StoreSubVec(1, NumBytes - Stored - 1, IsLE ? Stored : 15 - Stored);
17141 case PPC::BI__builtin_vsx_xvsqrtsp:
17142 case PPC::BI__builtin_vsx_xvsqrtdp: {
17145 if (
Builder.getIsFPConstrained()) {
17147 Intrinsic::experimental_constrained_sqrt, ResultType);
17148 return Builder.CreateConstrainedFPCall(F,
X);
17155 case PPC::BI__builtin_altivec_vclzb:
17156 case PPC::BI__builtin_altivec_vclzh:
17157 case PPC::BI__builtin_altivec_vclzw:
17158 case PPC::BI__builtin_altivec_vclzd: {
17161 Value *Undef = ConstantInt::get(
Builder.getInt1Ty(),
false);
17163 return Builder.CreateCall(F, {
X, Undef});
17165 case PPC::BI__builtin_altivec_vctzb:
17166 case PPC::BI__builtin_altivec_vctzh:
17167 case PPC::BI__builtin_altivec_vctzw:
17168 case PPC::BI__builtin_altivec_vctzd: {
17171 Value *Undef = ConstantInt::get(
Builder.getInt1Ty(),
false);
17173 return Builder.CreateCall(F, {
X, Undef});
17175 case PPC::BI__builtin_altivec_vinsd:
17176 case PPC::BI__builtin_altivec_vinsw:
17177 case PPC::BI__builtin_altivec_vinsd_elt:
17178 case PPC::BI__builtin_altivec_vinsw_elt: {
17184 bool IsUnaligned = (BuiltinID == PPC::BI__builtin_altivec_vinsw ||
17185 BuiltinID == PPC::BI__builtin_altivec_vinsd);
17187 bool Is32bit = (BuiltinID == PPC::BI__builtin_altivec_vinsw ||
17188 BuiltinID == PPC::BI__builtin_altivec_vinsw_elt);
17191 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
17193 "Third Arg to vinsw/vinsd intrinsic must be a constant integer!");
17197 int ValidMaxValue = 0;
17199 ValidMaxValue = (Is32bit) ? 12 : 8;
17201 ValidMaxValue = (Is32bit) ? 3 : 1;
17204 int64_t ConstArg = ArgCI->getSExtValue();
17207 std::string RangeErrMsg = IsUnaligned ?
"byte" :
"element";
17208 RangeErrMsg +=
" number " + llvm::to_string(ConstArg);
17209 RangeErrMsg +=
" is outside of the valid range [0, ";
17210 RangeErrMsg += llvm::to_string(ValidMaxValue) +
"]";
17213 if (ConstArg < 0 || ConstArg > ValidMaxValue)
17217 if (!IsUnaligned) {
17218 ConstArg *= Is32bit ? 4 : 8;
17221 ConstArg = (Is32bit ? 12 : 8) - ConstArg;
17224 ID = Is32bit ? Intrinsic::ppc_altivec_vinsw : Intrinsic::ppc_altivec_vinsd;
17225 Op2 = ConstantInt::getSigned(
Int32Ty, ConstArg);
17229 ?
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int32Ty, 4))
17231 llvm::FixedVectorType::get(
Int64Ty, 2));
17232 return Builder.CreateBitCast(
17235 case PPC::BI__builtin_altivec_vpopcntb:
17236 case PPC::BI__builtin_altivec_vpopcnth:
17237 case PPC::BI__builtin_altivec_vpopcntw:
17238 case PPC::BI__builtin_altivec_vpopcntd: {
17244 case PPC::BI__builtin_altivec_vadduqm:
17245 case PPC::BI__builtin_altivec_vsubuqm: {
17248 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
17249 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(Int128Ty, 1));
17250 Op1 =
Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(Int128Ty, 1));
17251 if (BuiltinID == PPC::BI__builtin_altivec_vadduqm)
17252 return Builder.CreateAdd(Op0, Op1,
"vadduqm");
17254 return Builder.CreateSub(Op0, Op1,
"vsubuqm");
17256 case PPC::BI__builtin_altivec_vaddcuq_c:
17257 case PPC::BI__builtin_altivec_vsubcuq_c: {
17261 llvm::Type *V1I128Ty = llvm::FixedVectorType::get(
17263 Ops.push_back(
Builder.CreateBitCast(Op0, V1I128Ty));
17264 Ops.push_back(
Builder.CreateBitCast(Op1, V1I128Ty));
17265 ID = (BuiltinID == PPC::BI__builtin_altivec_vaddcuq_c)
17266 ? Intrinsic::ppc_altivec_vaddcuq
17267 : Intrinsic::ppc_altivec_vsubcuq;
17270 case PPC::BI__builtin_altivec_vaddeuqm_c:
17271 case PPC::BI__builtin_altivec_vaddecuq_c:
17272 case PPC::BI__builtin_altivec_vsubeuqm_c:
17273 case PPC::BI__builtin_altivec_vsubecuq_c: {
17278 llvm::Type *V1I128Ty = llvm::FixedVectorType::get(
17280 Ops.push_back(
Builder.CreateBitCast(Op0, V1I128Ty));
17281 Ops.push_back(
Builder.CreateBitCast(Op1, V1I128Ty));
17282 Ops.push_back(
Builder.CreateBitCast(Op2, V1I128Ty));
17283 switch (BuiltinID) {
17285 llvm_unreachable(
"Unsupported intrinsic!");
17286 case PPC::BI__builtin_altivec_vaddeuqm_c:
17287 ID = Intrinsic::ppc_altivec_vaddeuqm;
17289 case PPC::BI__builtin_altivec_vaddecuq_c:
17290 ID = Intrinsic::ppc_altivec_vaddecuq;
17292 case PPC::BI__builtin_altivec_vsubeuqm_c:
17293 ID = Intrinsic::ppc_altivec_vsubeuqm;
17295 case PPC::BI__builtin_altivec_vsubecuq_c:
17296 ID = Intrinsic::ppc_altivec_vsubecuq;
17301 case PPC::BI__builtin_ppc_rldimi:
17302 case PPC::BI__builtin_ppc_rlwimi: {
17309 if (BuiltinID == PPC::BI__builtin_ppc_rldimi &&
17319 ? Intrinsic::ppc_rldimi
17320 : Intrinsic::ppc_rlwimi),
17321 {Op0, Op1, Op2, Op3});
17323 case PPC::BI__builtin_ppc_rlwnm: {
17330 case PPC::BI__builtin_ppc_poppar4:
17331 case PPC::BI__builtin_ppc_poppar8: {
17333 llvm::Type *ArgType = Op0->
getType();
17339 if (
Result->getType() != ResultType)
17344 case PPC::BI__builtin_ppc_cmpb: {
17347 if (
getTarget().getTriple().isPPC64()) {
17350 return Builder.CreateCall(F, {Op0, Op1},
"cmpb");
17370 Constant *ShiftAmt = ConstantInt::get(
Int64Ty, 32);
17379 Value *ResHi =
Builder.CreateShl(ResHiShift, ShiftAmt);
17380 return Builder.CreateOr(ResLo, ResHi);
17383 case PPC::BI__builtin_vsx_xvcpsgnsp:
17384 case PPC::BI__builtin_vsx_xvcpsgndp: {
17388 ID = Intrinsic::copysign;
17390 return Builder.CreateCall(F, {
X, Y});
17393 case PPC::BI__builtin_vsx_xvrspip:
17394 case PPC::BI__builtin_vsx_xvrdpip:
17395 case PPC::BI__builtin_vsx_xvrdpim:
17396 case PPC::BI__builtin_vsx_xvrspim:
17397 case PPC::BI__builtin_vsx_xvrdpi:
17398 case PPC::BI__builtin_vsx_xvrspi:
17399 case PPC::BI__builtin_vsx_xvrdpic:
17400 case PPC::BI__builtin_vsx_xvrspic:
17401 case PPC::BI__builtin_vsx_xvrdpiz:
17402 case PPC::BI__builtin_vsx_xvrspiz: {
17405 if (BuiltinID == PPC::BI__builtin_vsx_xvrdpim ||
17406 BuiltinID == PPC::BI__builtin_vsx_xvrspim)
17408 ? Intrinsic::experimental_constrained_floor
17409 : Intrinsic::floor;
17410 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpi ||
17411 BuiltinID == PPC::BI__builtin_vsx_xvrspi)
17413 ? Intrinsic::experimental_constrained_round
17414 : Intrinsic::round;
17415 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpic ||
17416 BuiltinID == PPC::BI__builtin_vsx_xvrspic)
17418 ? Intrinsic::experimental_constrained_rint
17420 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpip ||
17421 BuiltinID == PPC::BI__builtin_vsx_xvrspip)
17423 ? Intrinsic::experimental_constrained_ceil
17425 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpiz ||
17426 BuiltinID == PPC::BI__builtin_vsx_xvrspiz)
17428 ? Intrinsic::experimental_constrained_trunc
17429 : Intrinsic::trunc;
17431 return Builder.getIsFPConstrained() ?
Builder.CreateConstrainedFPCall(F,
X)
17436 case PPC::BI__builtin_vsx_xvabsdp:
17437 case PPC::BI__builtin_vsx_xvabssp: {
17445 case PPC::BI__builtin_ppc_recipdivf:
17446 case PPC::BI__builtin_ppc_recipdivd:
17447 case PPC::BI__builtin_ppc_rsqrtf:
17448 case PPC::BI__builtin_ppc_rsqrtd: {
17449 FastMathFlags FMF =
Builder.getFastMathFlags();
17450 Builder.getFastMathFlags().setFast();
17454 if (BuiltinID == PPC::BI__builtin_ppc_recipdivf ||
17455 BuiltinID == PPC::BI__builtin_ppc_recipdivd) {
17458 Builder.getFastMathFlags() &= (FMF);
17461 auto *One = ConstantFP::get(ResultType, 1.0);
17464 Builder.getFastMathFlags() &= (FMF);
17467 case PPC::BI__builtin_ppc_alignx: {
17470 ConstantInt *AlignmentCI = cast<ConstantInt>(Op0);
17471 if (AlignmentCI->getValue().ugt(llvm::Value::MaximumAlignment))
17472 AlignmentCI = ConstantInt::get(AlignmentCI->getIntegerType(),
17473 llvm::Value::MaximumAlignment);
17477 AlignmentCI,
nullptr);
17480 case PPC::BI__builtin_ppc_rdlam: {
17484 llvm::Type *Ty = Op0->
getType();
17485 Value *ShiftAmt =
Builder.CreateIntCast(Op1, Ty,
false);
17487 Value *Rotate =
Builder.CreateCall(F, {Op0, Op0, ShiftAmt});
17488 return Builder.CreateAnd(Rotate, Op2);
17490 case PPC::BI__builtin_ppc_load2r: {
17497 case PPC::BI__builtin_ppc_fnmsub:
17498 case PPC::BI__builtin_ppc_fnmsubs:
17499 case PPC::BI__builtin_vsx_xvmaddadp:
17500 case PPC::BI__builtin_vsx_xvmaddasp:
17501 case PPC::BI__builtin_vsx_xvnmaddadp:
17502 case PPC::BI__builtin_vsx_xvnmaddasp:
17503 case PPC::BI__builtin_vsx_xvmsubadp:
17504 case PPC::BI__builtin_vsx_xvmsubasp:
17505 case PPC::BI__builtin_vsx_xvnmsubadp:
17506 case PPC::BI__builtin_vsx_xvnmsubasp: {
17512 if (
Builder.getIsFPConstrained())
17513 F =
CGM.
getIntrinsic(Intrinsic::experimental_constrained_fma, ResultType);
17516 switch (BuiltinID) {
17517 case PPC::BI__builtin_vsx_xvmaddadp:
17518 case PPC::BI__builtin_vsx_xvmaddasp:
17519 if (
Builder.getIsFPConstrained())
17520 return Builder.CreateConstrainedFPCall(F, {
X, Y, Z});
17522 return Builder.CreateCall(F, {
X, Y, Z});
17523 case PPC::BI__builtin_vsx_xvnmaddadp:
17524 case PPC::BI__builtin_vsx_xvnmaddasp:
17525 if (
Builder.getIsFPConstrained())
17527 Builder.CreateConstrainedFPCall(F, {X, Y, Z}),
"neg");
17529 return Builder.CreateFNeg(
Builder.CreateCall(F, {X, Y, Z}),
"neg");
17530 case PPC::BI__builtin_vsx_xvmsubadp:
17531 case PPC::BI__builtin_vsx_xvmsubasp:
17532 if (
Builder.getIsFPConstrained())
17533 return Builder.CreateConstrainedFPCall(
17534 F, {
X, Y,
Builder.CreateFNeg(Z,
"neg")});
17537 case PPC::BI__builtin_ppc_fnmsub:
17538 case PPC::BI__builtin_ppc_fnmsubs:
17539 case PPC::BI__builtin_vsx_xvnmsubadp:
17540 case PPC::BI__builtin_vsx_xvnmsubasp:
17541 if (
Builder.getIsFPConstrained())
17543 Builder.CreateConstrainedFPCall(
17544 F, {X, Y, Builder.CreateFNeg(Z,
"neg")}),
17550 llvm_unreachable(
"Unknown FMA operation");
17554 case PPC::BI__builtin_vsx_insertword: {
17562 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
17564 "Third arg to xxinsertw intrinsic must be constant integer");
17566 int64_t Index = std::clamp(ArgCI->getSExtValue(), (int64_t)0, MaxIndex);
17573 std::swap(Op0, Op1);
17577 Op1 =
Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(
Int64Ty, 2));
17581 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int64Ty, 2));
17585 Index = MaxIndex - Index;
17589 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int32Ty, 4));
17590 Op2 = ConstantInt::getSigned(
Int32Ty, Index);
17591 return Builder.CreateCall(F, {Op0, Op1, Op2});
17594 case PPC::BI__builtin_vsx_extractuword: {
17597 llvm::Function *F =
CGM.
getIntrinsic(Intrinsic::ppc_vsx_xxextractuw);
17600 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int64Ty, 2));
17604 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op1);
17606 "Second Arg to xxextractuw intrinsic must be a constant integer!");
17608 int64_t Index = std::clamp(ArgCI->getSExtValue(), (int64_t)0, MaxIndex);
17612 Index = MaxIndex - Index;
17613 Op1 = ConstantInt::getSigned(
Int32Ty, Index);
17618 Value *ShuffleCall =
17620 return ShuffleCall;
17622 Op1 = ConstantInt::getSigned(
Int32Ty, Index);
17623 return Builder.CreateCall(F, {Op0, Op1});
17627 case PPC::BI__builtin_vsx_xxpermdi: {
17631 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
17632 assert(ArgCI &&
"Third arg must be constant integer!");
17634 unsigned Index = ArgCI->getZExtValue();
17635 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int64Ty, 2));
17636 Op1 =
Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(
Int64Ty, 2));
17641 int ElemIdx0 = (Index & 2) >> 1;
17642 int ElemIdx1 = 2 + (Index & 1);
17644 int ShuffleElts[2] = {ElemIdx0, ElemIdx1};
17645 Value *ShuffleCall =
Builder.CreateShuffleVector(Op0, Op1, ShuffleElts);
17648 return Builder.CreateBitCast(ShuffleCall, RetTy);
17651 case PPC::BI__builtin_vsx_xxsldwi: {
17655 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
17656 assert(ArgCI &&
"Third argument must be a compile time constant");
17657 unsigned Index = ArgCI->getZExtValue() & 0x3;
17658 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int32Ty, 4));
17659 Op1 =
Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(
Int32Ty, 4));
17670 ElemIdx0 = (8 - Index) % 8;
17671 ElemIdx1 = (9 - Index) % 8;
17672 ElemIdx2 = (10 - Index) % 8;
17673 ElemIdx3 = (11 - Index) % 8;
17677 ElemIdx1 = Index + 1;
17678 ElemIdx2 = Index + 2;
17679 ElemIdx3 = Index + 3;
17682 int ShuffleElts[4] = {ElemIdx0, ElemIdx1, ElemIdx2, ElemIdx3};
17683 Value *ShuffleCall =
Builder.CreateShuffleVector(Op0, Op1, ShuffleElts);
17686 return Builder.CreateBitCast(ShuffleCall, RetTy);
17689 case PPC::BI__builtin_pack_vector_int128: {
17693 Value *PoisonValue =
17694 llvm::PoisonValue::get(llvm::FixedVectorType::get(Op0->
getType(), 2));
17696 PoisonValue, Op0, (uint64_t)(isLittleEndian ? 1 : 0));
17697 Res =
Builder.CreateInsertElement(Res, Op1,
17698 (uint64_t)(isLittleEndian ? 0 : 1));
17702 case PPC::BI__builtin_unpack_vector_int128: {
17705 ConstantInt *Index = cast<ConstantInt>(Op1);
17711 ConstantInt::get(Index->getIntegerType(), 1 - Index->getZExtValue());
17713 return Builder.CreateExtractElement(Unpacked, Index);
17716 case PPC::BI__builtin_ppc_sthcx: {
17720 return Builder.CreateCall(F, {Op0, Op1});
17729#define CUSTOM_BUILTIN(Name, Intr, Types, Accumulate, Feature) \
17730 case PPC::BI__builtin_##Name:
17731#include "clang/Basic/BuiltinsPPC.def"
17734 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; i++)
17744 if (BuiltinID == PPC::BI__builtin_mma_disassemble_acc ||
17745 BuiltinID == PPC::BI__builtin_vsx_disassemble_pair ||
17746 BuiltinID == PPC::BI__builtin_mma_disassemble_pair) {
17747 unsigned NumVecs = 2;
17748 auto Intrinsic = Intrinsic::ppc_vsx_disassemble_pair;
17749 if (BuiltinID == PPC::BI__builtin_mma_disassemble_acc) {
17751 Intrinsic = Intrinsic::ppc_mma_disassemble_acc;
17757 llvm::Type *VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
17758 Value *Ptr = Ops[0];
17759 for (
unsigned i=0; i<NumVecs; i++) {
17761 llvm::ConstantInt* Index = llvm::ConstantInt::get(
IntTy, i);
17767 if (BuiltinID == PPC::BI__builtin_vsx_build_pair ||
17768 BuiltinID == PPC::BI__builtin_mma_build_acc) {
17776 std::reverse(Ops.begin() + 1, Ops.end());
17779 switch (BuiltinID) {
17780 #define CUSTOM_BUILTIN(Name, Intr, Types, Acc, Feature) \
17781 case PPC::BI__builtin_##Name: \
17782 ID = Intrinsic::ppc_##Intr; \
17783 Accumulate = Acc; \
17785 #include "clang/Basic/BuiltinsPPC.def"
17787 if (BuiltinID == PPC::BI__builtin_vsx_lxvp ||
17788 BuiltinID == PPC::BI__builtin_vsx_stxvp ||
17789 BuiltinID == PPC::BI__builtin_mma_lxvp ||
17790 BuiltinID == PPC::BI__builtin_mma_stxvp) {
17791 if (BuiltinID == PPC::BI__builtin_vsx_lxvp ||
17792 BuiltinID == PPC::BI__builtin_mma_lxvp) {
17799 return Builder.CreateCall(F, Ops,
"");
17805 CallOps.push_back(Acc);
17807 for (
unsigned i=1; i<Ops.size(); i++)
17808 CallOps.push_back(Ops[i]);
17814 case PPC::BI__builtin_ppc_compare_and_swap:
17815 case PPC::BI__builtin_ppc_compare_and_swaplp: {
17824 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Monotonic,
true);
17832 Value *LoadedVal = Pair.first.getScalarVal();
17836 case PPC::BI__builtin_ppc_fetch_and_add:
17837 case PPC::BI__builtin_ppc_fetch_and_addlp: {
17839 llvm::AtomicOrdering::Monotonic);
17841 case PPC::BI__builtin_ppc_fetch_and_and:
17842 case PPC::BI__builtin_ppc_fetch_and_andlp: {
17844 llvm::AtomicOrdering::Monotonic);
17847 case PPC::BI__builtin_ppc_fetch_and_or:
17848 case PPC::BI__builtin_ppc_fetch_and_orlp: {
17850 llvm::AtomicOrdering::Monotonic);
17852 case PPC::BI__builtin_ppc_fetch_and_swap:
17853 case PPC::BI__builtin_ppc_fetch_and_swaplp: {
17855 llvm::AtomicOrdering::Monotonic);
17857 case PPC::BI__builtin_ppc_ldarx:
17858 case PPC::BI__builtin_ppc_lwarx:
17859 case PPC::BI__builtin_ppc_lharx:
17860 case PPC::BI__builtin_ppc_lbarx:
17862 case PPC::BI__builtin_ppc_mfspr: {
17868 return Builder.CreateCall(F, {Op0});
17870 case PPC::BI__builtin_ppc_mtspr: {
17877 return Builder.CreateCall(F, {Op0, Op1});
17879 case PPC::BI__builtin_ppc_popcntb: {
17881 llvm::Type *ArgType = ArgValue->
getType();
17883 return Builder.CreateCall(F, {ArgValue},
"popcntb");
17885 case PPC::BI__builtin_ppc_mtfsf: {
17895 case PPC::BI__builtin_ppc_swdiv_nochk:
17896 case PPC::BI__builtin_ppc_swdivs_nochk: {
17899 FastMathFlags FMF =
Builder.getFastMathFlags();
17900 Builder.getFastMathFlags().setFast();
17901 Value *FDiv =
Builder.CreateFDiv(Op0, Op1,
"swdiv_nochk");
17902 Builder.getFastMathFlags() &= (FMF);
17905 case PPC::BI__builtin_ppc_fric:
17907 *
this, E, Intrinsic::rint,
17908 Intrinsic::experimental_constrained_rint))
17910 case PPC::BI__builtin_ppc_frim:
17911 case PPC::BI__builtin_ppc_frims:
17913 *
this, E, Intrinsic::floor,
17914 Intrinsic::experimental_constrained_floor))
17916 case PPC::BI__builtin_ppc_frin:
17917 case PPC::BI__builtin_ppc_frins:
17919 *
this, E, Intrinsic::round,
17920 Intrinsic::experimental_constrained_round))
17922 case PPC::BI__builtin_ppc_frip:
17923 case PPC::BI__builtin_ppc_frips:
17925 *
this, E, Intrinsic::ceil,
17926 Intrinsic::experimental_constrained_ceil))
17928 case PPC::BI__builtin_ppc_friz:
17929 case PPC::BI__builtin_ppc_frizs:
17931 *
this, E, Intrinsic::trunc,
17932 Intrinsic::experimental_constrained_trunc))
17934 case PPC::BI__builtin_ppc_fsqrt:
17935 case PPC::BI__builtin_ppc_fsqrts:
17937 *
this, E, Intrinsic::sqrt,
17938 Intrinsic::experimental_constrained_sqrt))
17940 case PPC::BI__builtin_ppc_test_data_class: {
17945 {Op0, Op1},
"test_data_class");
17947 case PPC::BI__builtin_ppc_maxfe: {
17953 {Op0, Op1, Op2, Op3});
17955 case PPC::BI__builtin_ppc_maxfl: {
17961 {Op0, Op1, Op2, Op3});
17963 case PPC::BI__builtin_ppc_maxfs: {
17969 {Op0, Op1, Op2, Op3});
17971 case PPC::BI__builtin_ppc_minfe: {
17977 {Op0, Op1, Op2, Op3});
17979 case PPC::BI__builtin_ppc_minfl: {
17985 {Op0, Op1, Op2, Op3});
17987 case PPC::BI__builtin_ppc_minfs: {
17993 {Op0, Op1, Op2, Op3});
17995 case PPC::BI__builtin_ppc_swdiv:
17996 case PPC::BI__builtin_ppc_swdivs: {
17999 return Builder.CreateFDiv(Op0, Op1,
"swdiv");
18001 case PPC::BI__builtin_ppc_set_fpscr_rn:
18003 {EmitScalarExpr(E->getArg(0))});
18004 case PPC::BI__builtin_ppc_mffs:
18017 Attribute::getWithDereferenceableBytes(
Call->getContext(), 64));
18018 Call->addRetAttr(Attribute::getWithAlignment(
Call->getContext(), Align(4)));
18022 auto *RetTy = cast<llvm::PointerType>(CGF.
ConvertType(BuiltinRetType));
18023 if (RetTy ==
Call->getType())
18032 Attribute::getWithDereferenceableBytes(
Call->getContext(), 256));
18033 Call->addRetAttr(Attribute::getWithAlignment(
Call->getContext(), Align(8)));
18048 llvm::LoadInst *LD;
18052 if (Cov == CodeObjectVersionKind::COV_None) {
18053 StringRef Name =
"__oclc_ABI_version";
18054 auto *ABIVersionC = CGF.
CGM.
getModule().getNamedGlobal(Name);
18056 ABIVersionC =
new llvm::GlobalVariable(
18058 llvm::GlobalValue::ExternalLinkage,
nullptr, Name,
nullptr,
18059 llvm::GlobalVariable::NotThreadLocal,
18070 llvm::ConstantInt::get(CGF.
Int32Ty, CodeObjectVersionKind::COV_5));
18074 CGF.
Int8Ty, EmitAMDGPUImplicitArgPtr(CGF), 12 + Index * 2);
18078 CGF.
Int8Ty, EmitAMDGPUDispatchPtr(CGF), 4 + Index * 2);
18080 auto Result = CGF.
Builder.CreateSelect(IsCOV5, ImplicitGEP, DispatchGEP);
18084 Value *GEP =
nullptr;
18085 if (Cov >= CodeObjectVersionKind::COV_5) {
18087 GEP = CGF.
Builder.CreateConstGEP1_32(
18088 CGF.
Int8Ty, EmitAMDGPUImplicitArgPtr(CGF), 12 + Index * 2);
18091 GEP = CGF.
Builder.CreateConstGEP1_32(
18092 CGF.
Int8Ty, EmitAMDGPUDispatchPtr(CGF), 4 + Index * 2);
18099 llvm::MDNode *RNode = MDHelper.createRange(
APInt(16, 1),
18101 LD->setMetadata(llvm::LLVMContext::MD_range, RNode);
18102 LD->setMetadata(llvm::LLVMContext::MD_noundef,
18104 LD->setMetadata(llvm::LLVMContext::MD_invariant_load,
18111 const unsigned XOffset = 12;
18112 auto *DP = EmitAMDGPUDispatchPtr(CGF);
18114 auto *Offset = llvm::ConstantInt::get(CGF.
Int32Ty, XOffset + Index * 4);
18118 LD->setMetadata(llvm::LLVMContext::MD_invariant_load,
18131 llvm::AtomicOrdering &AO,
18132 llvm::SyncScope::ID &SSID) {
18133 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
18136 assert(llvm::isValidAtomicOrderingCABI(ord));
18137 switch (
static_cast<llvm::AtomicOrderingCABI
>(ord)) {
18138 case llvm::AtomicOrderingCABI::acquire:
18139 case llvm::AtomicOrderingCABI::consume:
18140 AO = llvm::AtomicOrdering::Acquire;
18142 case llvm::AtomicOrderingCABI::release:
18143 AO = llvm::AtomicOrdering::Release;
18145 case llvm::AtomicOrderingCABI::acq_rel:
18146 AO = llvm::AtomicOrdering::AcquireRelease;
18148 case llvm::AtomicOrderingCABI::seq_cst:
18149 AO = llvm::AtomicOrdering::SequentiallyConsistent;
18151 case llvm::AtomicOrderingCABI::relaxed:
18152 AO = llvm::AtomicOrdering::Monotonic;
18157 llvm::getConstantStringInfo(
Scope, scp);
18164 llvm::Value *Arg =
nullptr;
18165 if ((ICEArguments & (1 << Idx)) == 0) {
18170 std::optional<llvm::APSInt>
Result =
18172 assert(
Result &&
"Expected argument to be a constant");
18180 switch (elementCount) {
18182 return Intrinsic::dx_dot2;
18184 return Intrinsic::dx_dot3;
18186 return Intrinsic::dx_dot4;
18190 return Intrinsic::dx_sdot;
18193 return Intrinsic::dx_udot;
18201 switch (BuiltinID) {
18202 case Builtin::BI__builtin_hlsl_elementwise_all: {
18204 return Builder.CreateIntrinsic(
18209 case Builtin::BI__builtin_hlsl_elementwise_any: {
18211 return Builder.CreateIntrinsic(
18216 case Builtin::BI__builtin_hlsl_elementwise_clamp: {
18222 bool IsUnsigned =
false;
18224 Ty = VecTy->getElementType();
18226 return Builder.CreateIntrinsic(
18228 IsUnsigned ? Intrinsic::dx_uclamp : Intrinsic::dx_clamp,
18231 case Builtin::BI__builtin_hlsl_dot: {
18234 llvm::Type *T0 = Op0->
getType();
18235 llvm::Type *T1 = Op1->
getType();
18236 if (!T0->isVectorTy() && !T1->isVectorTy()) {
18237 if (T0->isFloatingPointTy())
18238 return Builder.CreateFMul(Op0, Op1,
"dx.dot");
18240 if (T0->isIntegerTy())
18241 return Builder.CreateMul(Op0, Op1,
"dx.dot");
18245 "Scalar dot product is only supported on ints and floats.");
18248 assert(T0->isVectorTy() && T1->isVectorTy() &&
18249 "Dot product of vector and scalar is not supported.");
18252 assert(T0->getScalarType() == T1->getScalarType() &&
18253 "Dot product of vectors need the same element types.");
18256 [[maybe_unused]]
auto *VecTy1 =
18260 "Dot product requires vectors to be of the same size.");
18262 return Builder.CreateIntrinsic(
18263 T0->getScalarType(),
18265 VecTy0->getNumElements()),
18268 case Builtin::BI__builtin_hlsl_lerp: {
18273 llvm_unreachable(
"lerp operand must have a float representation");
18274 return Builder.CreateIntrinsic(
18278 case Builtin::BI__builtin_hlsl_elementwise_frac: {
18281 llvm_unreachable(
"frac operand must have a float representation");
18282 return Builder.CreateIntrinsic(
18283 Op0->
getType(), Intrinsic::dx_frac,
18286 case Builtin::BI__builtin_hlsl_elementwise_isinf: {
18288 llvm::Type *Xty = Op0->
getType();
18289 llvm::Type *retType = llvm::Type::getInt1Ty(this->
getLLVMContext());
18290 if (Xty->isVectorTy()) {
18292 retType = llvm::VectorType::get(
18293 retType, ElementCount::getFixed(XVecTy->getNumElements()));
18296 llvm_unreachable(
"isinf operand must have a float representation");
18297 return Builder.CreateIntrinsic(retType, Intrinsic::dx_isinf,
18300 case Builtin::BI__builtin_hlsl_mad: {
18305 return Builder.CreateIntrinsic(
18306 M->
getType(), Intrinsic::fmuladd,
18311 return Builder.CreateIntrinsic(
18312 M->
getType(), Intrinsic::dx_imad,
18316 return Builder.CreateNSWAdd(Mul, B);
18320 return Builder.CreateIntrinsic(
18321 M->
getType(), Intrinsic::dx_umad,
18325 return Builder.CreateNUWAdd(Mul, B);
18327 case Builtin::BI__builtin_hlsl_elementwise_rcp: {
18330 llvm_unreachable(
"rcp operand must have a float representation");
18331 llvm::Type *Ty = Op0->
getType();
18332 llvm::Type *EltTy = Ty->getScalarType();
18335 ? ConstantVector::getSplat(
18336 ElementCount::getFixed(
18337 dyn_cast<FixedVectorType>(Ty)->getNumElements()),
18338 ConstantFP::get(EltTy, 1.0))
18339 : ConstantFP::get(EltTy, 1.0);
18340 return Builder.CreateFDiv(One, Op0,
"hlsl.rcp");
18342 case Builtin::BI__builtin_hlsl_elementwise_rsqrt: {
18345 llvm_unreachable(
"rsqrt operand must have a float representation");
18346 return Builder.CreateIntrinsic(
18347 Op0->
getType(), Intrinsic::dx_rsqrt,
18350 case Builtin::BI__builtin_hlsl_wave_get_lane_index: {
18352 llvm::FunctionType::get(
IntTy, {},
false),
"__hlsl_wave_get_lane_index",
18354 if (
getTarget().getTriple().isSPIRVLogical())
18364 llvm::AtomicOrdering AO = llvm::AtomicOrdering::SequentiallyConsistent;
18365 llvm::SyncScope::ID SSID;
18366 switch (BuiltinID) {
18367 case AMDGPU::BI__builtin_amdgcn_div_scale:
18368 case AMDGPU::BI__builtin_amdgcn_div_scalef: {
18381 llvm::Value *Tmp =
Builder.CreateCall(Callee, {
X, Y, Z});
18384 llvm::Value *Flag =
Builder.CreateExtractValue(Tmp, 1);
18388 llvm::Value *FlagExt =
Builder.CreateZExt(Flag, RealFlagType);
18392 case AMDGPU::BI__builtin_amdgcn_div_fmas:
18393 case AMDGPU::BI__builtin_amdgcn_div_fmasf: {
18401 llvm::Value *Src3ToBool =
Builder.CreateIsNotNull(Src3);
18402 return Builder.CreateCall(F, {Src0, Src1, Src2, Src3ToBool});
18405 case AMDGPU::BI__builtin_amdgcn_ds_swizzle:
18407 case AMDGPU::BI__builtin_amdgcn_mov_dpp8:
18409 case AMDGPU::BI__builtin_amdgcn_mov_dpp:
18410 case AMDGPU::BI__builtin_amdgcn_update_dpp: {
18414 unsigned ICEArguments = 0;
18418 for (
unsigned I = 0; I != E->
getNumArgs(); ++I) {
18421 assert(Args.size() == 5 || Args.size() == 6);
18422 if (Args.size() == 5)
18423 Args.insert(Args.begin(), llvm::PoisonValue::get(Args[0]->getType()));
18426 return Builder.CreateCall(F, Args);
18428 case AMDGPU::BI__builtin_amdgcn_div_fixup:
18429 case AMDGPU::BI__builtin_amdgcn_div_fixupf:
18430 case AMDGPU::BI__builtin_amdgcn_div_fixuph:
18432 case AMDGPU::BI__builtin_amdgcn_trig_preop:
18433 case AMDGPU::BI__builtin_amdgcn_trig_preopf:
18435 case AMDGPU::BI__builtin_amdgcn_rcp:
18436 case AMDGPU::BI__builtin_amdgcn_rcpf:
18437 case AMDGPU::BI__builtin_amdgcn_rcph:
18439 case AMDGPU::BI__builtin_amdgcn_sqrt:
18440 case AMDGPU::BI__builtin_amdgcn_sqrtf:
18441 case AMDGPU::BI__builtin_amdgcn_sqrth:
18443 case AMDGPU::BI__builtin_amdgcn_rsq:
18444 case AMDGPU::BI__builtin_amdgcn_rsqf:
18445 case AMDGPU::BI__builtin_amdgcn_rsqh:
18447 case AMDGPU::BI__builtin_amdgcn_rsq_clamp:
18448 case AMDGPU::BI__builtin_amdgcn_rsq_clampf:
18450 case AMDGPU::BI__builtin_amdgcn_sinf:
18451 case AMDGPU::BI__builtin_amdgcn_sinh:
18453 case AMDGPU::BI__builtin_amdgcn_cosf:
18454 case AMDGPU::BI__builtin_amdgcn_cosh:
18456 case AMDGPU::BI__builtin_amdgcn_dispatch_ptr:
18457 return EmitAMDGPUDispatchPtr(*
this, E);
18458 case AMDGPU::BI__builtin_amdgcn_logf:
18460 case AMDGPU::BI__builtin_amdgcn_exp2f:
18462 case AMDGPU::BI__builtin_amdgcn_log_clampf:
18464 case AMDGPU::BI__builtin_amdgcn_ldexp:
18465 case AMDGPU::BI__builtin_amdgcn_ldexpf: {
18468 llvm::Function *F =
18469 CGM.
getIntrinsic(Intrinsic::ldexp, {Src0->getType(), Src1->getType()});
18470 return Builder.CreateCall(F, {Src0, Src1});
18472 case AMDGPU::BI__builtin_amdgcn_ldexph: {
18477 llvm::Function *F =
18481 case AMDGPU::BI__builtin_amdgcn_frexp_mant:
18482 case AMDGPU::BI__builtin_amdgcn_frexp_mantf:
18483 case AMDGPU::BI__builtin_amdgcn_frexp_manth:
18485 case AMDGPU::BI__builtin_amdgcn_frexp_exp:
18486 case AMDGPU::BI__builtin_amdgcn_frexp_expf: {
18490 return Builder.CreateCall(F, Src0);
18492 case AMDGPU::BI__builtin_amdgcn_frexp_exph: {
18496 return Builder.CreateCall(F, Src0);
18498 case AMDGPU::BI__builtin_amdgcn_fract:
18499 case AMDGPU::BI__builtin_amdgcn_fractf:
18500 case AMDGPU::BI__builtin_amdgcn_fracth:
18502 case AMDGPU::BI__builtin_amdgcn_lerp:
18504 case AMDGPU::BI__builtin_amdgcn_ubfe:
18506 case AMDGPU::BI__builtin_amdgcn_sbfe:
18508 case AMDGPU::BI__builtin_amdgcn_ballot_w32:
18509 case AMDGPU::BI__builtin_amdgcn_ballot_w64: {
18513 return Builder.CreateCall(F, { Src });
18515 case AMDGPU::BI__builtin_amdgcn_uicmp:
18516 case AMDGPU::BI__builtin_amdgcn_uicmpl:
18517 case AMDGPU::BI__builtin_amdgcn_sicmp:
18518 case AMDGPU::BI__builtin_amdgcn_sicmpl: {
18525 {
Builder.getInt64Ty(), Src0->getType() });
18526 return Builder.CreateCall(F, { Src0, Src1, Src2 });
18528 case AMDGPU::BI__builtin_amdgcn_fcmp:
18529 case AMDGPU::BI__builtin_amdgcn_fcmpf: {
18536 {
Builder.getInt64Ty(), Src0->getType() });
18537 return Builder.CreateCall(F, { Src0, Src1, Src2 });
18539 case AMDGPU::BI__builtin_amdgcn_class:
18540 case AMDGPU::BI__builtin_amdgcn_classf:
18541 case AMDGPU::BI__builtin_amdgcn_classh:
18543 case AMDGPU::BI__builtin_amdgcn_fmed3f:
18544 case AMDGPU::BI__builtin_amdgcn_fmed3h:
18546 case AMDGPU::BI__builtin_amdgcn_ds_append:
18547 case AMDGPU::BI__builtin_amdgcn_ds_consume: {
18548 Intrinsic::ID Intrin = BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_append ?
18549 Intrinsic::amdgcn_ds_append : Intrinsic::amdgcn_ds_consume;
18554 case AMDGPU::BI__builtin_amdgcn_ds_faddf:
18555 case AMDGPU::BI__builtin_amdgcn_ds_fminf:
18556 case AMDGPU::BI__builtin_amdgcn_ds_fmaxf: {
18557 Intrinsic::ID Intrin;
18558 switch (BuiltinID) {
18559 case AMDGPU::BI__builtin_amdgcn_ds_faddf:
18560 Intrin = Intrinsic::amdgcn_ds_fadd;
18562 case AMDGPU::BI__builtin_amdgcn_ds_fminf:
18563 Intrin = Intrinsic::amdgcn_ds_fmin;
18565 case AMDGPU::BI__builtin_amdgcn_ds_fmaxf:
18566 Intrin = Intrinsic::amdgcn_ds_fmax;
18575 llvm::FunctionType *FTy = F->getFunctionType();
18576 llvm::Type *PTy = FTy->getParamType(0);
18578 return Builder.CreateCall(F, { Src0, Src1, Src2, Src3, Src4 });
18580 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f64:
18581 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f32:
18582 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2f16:
18583 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmin_f64:
18584 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmax_f64:
18585 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f64:
18586 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmin_f64:
18587 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmax_f64:
18588 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f32:
18589 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2f16: {
18592 switch (BuiltinID) {
18593 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f32:
18595 IID = Intrinsic::amdgcn_global_atomic_fadd;
18597 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2f16:
18598 ArgTy = llvm::FixedVectorType::get(
18600 IID = Intrinsic::amdgcn_global_atomic_fadd;
18602 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f64:
18603 IID = Intrinsic::amdgcn_global_atomic_fadd;
18605 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmin_f64:
18606 IID = Intrinsic::amdgcn_global_atomic_fmin;
18608 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmax_f64:
18609 IID = Intrinsic::amdgcn_global_atomic_fmax;
18611 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f64:
18612 IID = Intrinsic::amdgcn_flat_atomic_fadd;
18614 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmin_f64:
18615 IID = Intrinsic::amdgcn_flat_atomic_fmin;
18617 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmax_f64:
18618 IID = Intrinsic::amdgcn_flat_atomic_fmax;
18620 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f32:
18622 IID = Intrinsic::amdgcn_flat_atomic_fadd;
18624 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2f16:
18625 ArgTy = llvm::FixedVectorType::get(
18627 IID = Intrinsic::amdgcn_flat_atomic_fadd;
18632 llvm::Function *F =
18634 return Builder.CreateCall(F, {Addr, Val});
18636 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2bf16:
18637 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2bf16: {
18639 switch (BuiltinID) {
18640 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2bf16:
18641 IID = Intrinsic::amdgcn_global_atomic_fadd_v2bf16;
18643 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2bf16:
18644 IID = Intrinsic::amdgcn_flat_atomic_fadd_v2bf16;
18650 return Builder.CreateCall(F, {Addr, Val});
18652 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f64:
18653 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f32:
18654 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2f16: {
18657 switch (BuiltinID) {
18658 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f32:
18660 IID = Intrinsic::amdgcn_ds_fadd;
18662 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f64:
18664 IID = Intrinsic::amdgcn_ds_fadd;
18666 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2f16:
18667 ArgTy = llvm::FixedVectorType::get(
18669 IID = Intrinsic::amdgcn_ds_fadd;
18674 llvm::Constant *ZeroI32 = llvm::ConstantInt::getIntegerValue(
18676 llvm::Constant *ZeroI1 = llvm::ConstantInt::getIntegerValue(
18679 return Builder.CreateCall(F, {Addr, Val, ZeroI32, ZeroI32, ZeroI1});
18681 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_i32:
18682 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_v2i32:
18683 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4i16:
18684 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16: {
18687 switch (BuiltinID) {
18688 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_i32:
18689 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_v2i32:
18690 IID = Intrinsic::amdgcn_global_load_tr_b64;
18692 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4i16:
18693 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16:
18694 IID = Intrinsic::amdgcn_global_load_tr_b128;
18700 return Builder.CreateCall(F, {Addr});
18702 case AMDGPU::BI__builtin_amdgcn_get_fpenv: {
18705 return Builder.CreateCall(F);
18707 case AMDGPU::BI__builtin_amdgcn_set_fpenv: {
18713 case AMDGPU::BI__builtin_amdgcn_read_exec:
18715 case AMDGPU::BI__builtin_amdgcn_read_exec_lo:
18717 case AMDGPU::BI__builtin_amdgcn_read_exec_hi:
18719 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray:
18720 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_h:
18721 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_l:
18722 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_lh: {
18732 RayOrigin =
Builder.CreateShuffleVector(RayOrigin, RayOrigin,
18736 RayInverseDir =
Builder.CreateShuffleVector(RayInverseDir, RayInverseDir,
18740 {NodePtr->getType(), RayDir->getType()});
18741 return Builder.CreateCall(F, {NodePtr, RayExtent, RayOrigin, RayDir,
18742 RayInverseDir, TextureDescr});
18745 case AMDGPU::BI__builtin_amdgcn_ds_bvh_stack_rtn: {
18747 for (
int i = 0, e = E->
getNumArgs(); i != e; ++i)
18755 Value *I0 =
Builder.CreateInsertElement(PoisonValue::get(RetTy), Rtn,
18757 return Builder.CreateInsertElement(I0, A, 1);
18760 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32:
18761 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w32:
18762 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64:
18763 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w64:
18764 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32:
18765 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w32:
18766 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64:
18767 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w64:
18768 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32:
18769 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64:
18770 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32:
18771 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64:
18772 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32:
18773 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64:
18774 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32:
18775 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64:
18776 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32_gfx12:
18777 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64_gfx12:
18778 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32_gfx12:
18779 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64_gfx12:
18780 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32_gfx12:
18781 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64_gfx12:
18782 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12:
18783 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64_gfx12:
18784 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32_gfx12:
18785 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64_gfx12:
18786 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12:
18787 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64_gfx12:
18788 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w32_gfx12:
18789 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w64_gfx12:
18790 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w32_gfx12:
18791 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w64_gfx12:
18792 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w32_gfx12:
18793 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w64_gfx12:
18794 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w32_gfx12:
18795 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w64_gfx12:
18796 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w32_gfx12:
18797 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w64_gfx12:
18798 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w32:
18799 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w64:
18800 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w32:
18801 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w64:
18802 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w32:
18803 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w64:
18804 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w32:
18805 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w64:
18806 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w32:
18807 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w64:
18808 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w32:
18809 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w64:
18810 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w32:
18811 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w64:
18812 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w32:
18813 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w64:
18814 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w32:
18815 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w64:
18816 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w32:
18817 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w64:
18818 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w32:
18819 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w64: {
18832 bool AppendFalseForOpselArg =
false;
18833 unsigned BuiltinWMMAOp;
18835 switch (BuiltinID) {
18836 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32:
18837 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64:
18838 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12:
18839 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64_gfx12:
18840 ArgsForMatchingMatrixTypes = {2, 0};
18841 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_f16;
18843 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32:
18844 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64:
18845 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32_gfx12:
18846 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64_gfx12:
18847 ArgsForMatchingMatrixTypes = {2, 0};
18848 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_bf16;
18850 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32_gfx12:
18851 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64_gfx12:
18852 AppendFalseForOpselArg =
true;
18854 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32:
18855 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64:
18856 ArgsForMatchingMatrixTypes = {2, 0};
18857 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f16_16x16x16_f16;
18859 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32_gfx12:
18860 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64_gfx12:
18861 AppendFalseForOpselArg =
true;
18863 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32:
18864 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64:
18865 ArgsForMatchingMatrixTypes = {2, 0};
18866 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_bf16_16x16x16_bf16;
18868 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w32:
18869 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w64:
18870 ArgsForMatchingMatrixTypes = {2, 0};
18871 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f16_16x16x16_f16_tied;
18873 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w32:
18874 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w64:
18875 ArgsForMatchingMatrixTypes = {2, 0};
18876 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_bf16_16x16x16_bf16_tied;
18878 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32:
18879 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64:
18880 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12:
18881 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64_gfx12:
18882 ArgsForMatchingMatrixTypes = {4, 1};
18883 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x16_iu8;
18885 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32:
18886 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64:
18887 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32_gfx12:
18888 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64_gfx12:
18889 ArgsForMatchingMatrixTypes = {4, 1};
18890 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x16_iu4;
18892 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w32_gfx12:
18893 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w64_gfx12:
18894 ArgsForMatchingMatrixTypes = {2, 0};
18895 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_fp8_fp8;
18897 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w32_gfx12:
18898 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w64_gfx12:
18899 ArgsForMatchingMatrixTypes = {2, 0};
18900 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_fp8_bf8;
18902 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w32_gfx12:
18903 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w64_gfx12:
18904 ArgsForMatchingMatrixTypes = {2, 0};
18905 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_bf8_fp8;
18907 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w32_gfx12:
18908 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w64_gfx12:
18909 ArgsForMatchingMatrixTypes = {2, 0};
18910 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_bf8_bf8;
18912 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w32_gfx12:
18913 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w64_gfx12:
18914 ArgsForMatchingMatrixTypes = {4, 1};
18915 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x32_iu4;
18917 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w32:
18918 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w64:
18919 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
18920 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_f16;
18922 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w32:
18923 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w64:
18924 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
18925 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_bf16;
18927 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w32:
18928 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w64:
18929 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
18930 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f16_16x16x32_f16;
18932 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w32:
18933 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w64:
18934 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
18935 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_bf16_16x16x32_bf16;
18937 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w32:
18938 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w64:
18939 ArgsForMatchingMatrixTypes = {4, 1, 3, 5};
18940 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_i32_16x16x32_iu8;
18942 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w32:
18943 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w64:
18944 ArgsForMatchingMatrixTypes = {4, 1, 3, 5};
18945 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_i32_16x16x32_iu4;
18947 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w32:
18948 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w64:
18949 ArgsForMatchingMatrixTypes = {4, 1, 3, 5};
18950 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_i32_16x16x64_iu4;
18952 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w32:
18953 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w64:
18954 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
18955 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_fp8;
18957 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w32:
18958 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w64:
18959 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
18960 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_bf8;
18962 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w32:
18963 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w64:
18964 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
18965 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_fp8;
18967 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w32:
18968 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w64:
18969 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
18970 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_bf8;
18975 for (
int i = 0, e = E->
getNumArgs(); i != e; ++i)
18977 if (AppendFalseForOpselArg)
18978 Args.push_back(
Builder.getFalse());
18981 for (
auto ArgIdx : ArgsForMatchingMatrixTypes)
18982 ArgTypes.push_back(Args[ArgIdx]->getType());
18985 return Builder.CreateCall(F, Args);
18989 case AMDGPU::BI__builtin_amdgcn_workitem_id_x:
18991 case AMDGPU::BI__builtin_amdgcn_workitem_id_y:
18993 case AMDGPU::BI__builtin_amdgcn_workitem_id_z:
18997 case AMDGPU::BI__builtin_amdgcn_workgroup_size_x:
18998 return EmitAMDGPUWorkGroupSize(*
this, 0);
18999 case AMDGPU::BI__builtin_amdgcn_workgroup_size_y:
19000 return EmitAMDGPUWorkGroupSize(*
this, 1);
19001 case AMDGPU::BI__builtin_amdgcn_workgroup_size_z:
19002 return EmitAMDGPUWorkGroupSize(*
this, 2);
19005 case AMDGPU::BI__builtin_amdgcn_grid_size_x:
19006 return EmitAMDGPUGridSize(*
this, 0);
19007 case AMDGPU::BI__builtin_amdgcn_grid_size_y:
19008 return EmitAMDGPUGridSize(*
this, 1);
19009 case AMDGPU::BI__builtin_amdgcn_grid_size_z:
19010 return EmitAMDGPUGridSize(*
this, 2);
19013 case AMDGPU::BI__builtin_r600_recipsqrt_ieee:
19014 case AMDGPU::BI__builtin_r600_recipsqrt_ieeef:
19016 case AMDGPU::BI__builtin_r600_read_tidig_x:
19018 case AMDGPU::BI__builtin_r600_read_tidig_y:
19020 case AMDGPU::BI__builtin_r600_read_tidig_z:
19022 case AMDGPU::BI__builtin_amdgcn_alignbit: {
19027 return Builder.CreateCall(F, { Src0, Src1, Src2 });
19029 case AMDGPU::BI__builtin_amdgcn_fence: {
19032 return Builder.CreateFence(AO, SSID);
19034 case AMDGPU::BI__builtin_amdgcn_atomic_inc32:
19035 case AMDGPU::BI__builtin_amdgcn_atomic_inc64:
19036 case AMDGPU::BI__builtin_amdgcn_atomic_dec32:
19037 case AMDGPU::BI__builtin_amdgcn_atomic_dec64: {
19038 llvm::AtomicRMWInst::BinOp BinOp;
19039 switch (BuiltinID) {
19040 case AMDGPU::BI__builtin_amdgcn_atomic_inc32:
19041 case AMDGPU::BI__builtin_amdgcn_atomic_inc64:
19042 BinOp = llvm::AtomicRMWInst::UIncWrap;
19044 case AMDGPU::BI__builtin_amdgcn_atomic_dec32:
19045 case AMDGPU::BI__builtin_amdgcn_atomic_dec64:
19046 BinOp = llvm::AtomicRMWInst::UDecWrap;
19060 llvm::AtomicRMWInst *RMW =
19063 RMW->setVolatile(
true);
19066 case AMDGPU::BI__builtin_amdgcn_s_sendmsg_rtn:
19067 case AMDGPU::BI__builtin_amdgcn_s_sendmsg_rtnl: {
19073 return Builder.CreateCall(F, {Arg});
19084 unsigned IntrinsicID,
19088 for (
unsigned I = 0; I < NumArgs; ++I)
19100 switch (BuiltinID) {
19101 case SystemZ::BI__builtin_tbegin: {
19103 Value *Control = llvm::ConstantInt::get(
Int32Ty, 0xff0c);
19105 return Builder.CreateCall(F, {TDB, Control});
19107 case SystemZ::BI__builtin_tbegin_nofloat: {
19109 Value *Control = llvm::ConstantInt::get(
Int32Ty, 0xff0c);
19111 return Builder.CreateCall(F, {TDB, Control});
19113 case SystemZ::BI__builtin_tbeginc: {
19115 Value *Control = llvm::ConstantInt::get(
Int32Ty, 0xff08);
19117 return Builder.CreateCall(F, {TDB, Control});
19119 case SystemZ::BI__builtin_tabort: {
19124 case SystemZ::BI__builtin_non_tx_store: {
19136 case SystemZ::BI__builtin_s390_vpopctb:
19137 case SystemZ::BI__builtin_s390_vpopcth:
19138 case SystemZ::BI__builtin_s390_vpopctf:
19139 case SystemZ::BI__builtin_s390_vpopctg: {
19146 case SystemZ::BI__builtin_s390_vclzb:
19147 case SystemZ::BI__builtin_s390_vclzh:
19148 case SystemZ::BI__builtin_s390_vclzf:
19149 case SystemZ::BI__builtin_s390_vclzg: {
19152 Value *Undef = ConstantInt::get(
Builder.getInt1Ty(),
false);
19154 return Builder.CreateCall(F, {
X, Undef});
19157 case SystemZ::BI__builtin_s390_vctzb:
19158 case SystemZ::BI__builtin_s390_vctzh:
19159 case SystemZ::BI__builtin_s390_vctzf:
19160 case SystemZ::BI__builtin_s390_vctzg: {
19163 Value *Undef = ConstantInt::get(
Builder.getInt1Ty(),
false);
19165 return Builder.CreateCall(F, {
X, Undef});
19168 case SystemZ::BI__builtin_s390_verllb:
19169 case SystemZ::BI__builtin_s390_verllh:
19170 case SystemZ::BI__builtin_s390_verllf:
19171 case SystemZ::BI__builtin_s390_verllg: {
19176 unsigned NumElts = cast<llvm::FixedVectorType>(ResultType)->getNumElements();
19177 Amt =
Builder.CreateIntCast(Amt, ResultType->getScalarType(),
false);
19178 Amt =
Builder.CreateVectorSplat(NumElts, Amt);
19180 return Builder.CreateCall(F, { Src, Src, Amt });
19183 case SystemZ::BI__builtin_s390_verllvb:
19184 case SystemZ::BI__builtin_s390_verllvh:
19185 case SystemZ::BI__builtin_s390_verllvf:
19186 case SystemZ::BI__builtin_s390_verllvg: {
19191 return Builder.CreateCall(F, { Src, Src, Amt });
19194 case SystemZ::BI__builtin_s390_vfsqsb:
19195 case SystemZ::BI__builtin_s390_vfsqdb: {
19198 if (
Builder.getIsFPConstrained()) {
19200 return Builder.CreateConstrainedFPCall(F, {
X });
19206 case SystemZ::BI__builtin_s390_vfmasb:
19207 case SystemZ::BI__builtin_s390_vfmadb: {
19212 if (
Builder.getIsFPConstrained()) {
19214 return Builder.CreateConstrainedFPCall(F, {
X, Y, Z});
19217 return Builder.CreateCall(F, {
X, Y, Z});
19220 case SystemZ::BI__builtin_s390_vfmssb:
19221 case SystemZ::BI__builtin_s390_vfmsdb: {
19226 if (
Builder.getIsFPConstrained()) {
19228 return Builder.CreateConstrainedFPCall(F, {
X, Y,
Builder.CreateFNeg(Z,
"neg")});
19234 case SystemZ::BI__builtin_s390_vfnmasb:
19235 case SystemZ::BI__builtin_s390_vfnmadb: {
19240 if (
Builder.getIsFPConstrained()) {
19242 return Builder.CreateFNeg(
Builder.CreateConstrainedFPCall(F, {X, Y, Z}),
"neg");
19245 return Builder.CreateFNeg(
Builder.CreateCall(F, {X, Y, Z}),
"neg");
19248 case SystemZ::BI__builtin_s390_vfnmssb:
19249 case SystemZ::BI__builtin_s390_vfnmsdb: {
19254 if (
Builder.getIsFPConstrained()) {
19257 return Builder.CreateFNeg(
Builder.CreateConstrainedFPCall(F, {X, Y, NegZ}));
19264 case SystemZ::BI__builtin_s390_vflpsb:
19265 case SystemZ::BI__builtin_s390_vflpdb: {
19271 case SystemZ::BI__builtin_s390_vflnsb:
19272 case SystemZ::BI__builtin_s390_vflndb: {
19278 case SystemZ::BI__builtin_s390_vfisb:
19279 case SystemZ::BI__builtin_s390_vfidb: {
19287 Intrinsic::ID
ID = Intrinsic::not_intrinsic;
19289 switch (M4.getZExtValue()) {
19292 switch (M5.getZExtValue()) {
19294 case 0:
ID = Intrinsic::rint;
19295 CI = Intrinsic::experimental_constrained_rint;
break;
19299 switch (M5.getZExtValue()) {
19301 case 0:
ID = Intrinsic::nearbyint;
19302 CI = Intrinsic::experimental_constrained_nearbyint;
break;
19303 case 1:
ID = Intrinsic::round;
19304 CI = Intrinsic::experimental_constrained_round;
break;
19305 case 5:
ID = Intrinsic::trunc;
19306 CI = Intrinsic::experimental_constrained_trunc;
break;
19307 case 6:
ID = Intrinsic::ceil;
19308 CI = Intrinsic::experimental_constrained_ceil;
break;
19309 case 7:
ID = Intrinsic::floor;
19310 CI = Intrinsic::experimental_constrained_floor;
break;
19314 if (ID != Intrinsic::not_intrinsic) {
19315 if (
Builder.getIsFPConstrained()) {
19317 return Builder.CreateConstrainedFPCall(F,
X);
19323 switch (BuiltinID) {
19324 case SystemZ::BI__builtin_s390_vfisb:
ID = Intrinsic::s390_vfisb;
break;
19325 case SystemZ::BI__builtin_s390_vfidb:
ID = Intrinsic::s390_vfidb;
break;
19326 default: llvm_unreachable(
"Unknown BuiltinID");
19331 return Builder.CreateCall(F, {
X, M4Value, M5Value});
19333 case SystemZ::BI__builtin_s390_vfmaxsb:
19334 case SystemZ::BI__builtin_s390_vfmaxdb: {
19342 Intrinsic::ID
ID = Intrinsic::not_intrinsic;
19344 switch (M4.getZExtValue()) {
19346 case 4:
ID = Intrinsic::maxnum;
19347 CI = Intrinsic::experimental_constrained_maxnum;
break;
19349 if (ID != Intrinsic::not_intrinsic) {
19350 if (
Builder.getIsFPConstrained()) {
19352 return Builder.CreateConstrainedFPCall(F, {
X, Y});
19355 return Builder.CreateCall(F, {
X, Y});
19358 switch (BuiltinID) {
19359 case SystemZ::BI__builtin_s390_vfmaxsb:
ID = Intrinsic::s390_vfmaxsb;
break;
19360 case SystemZ::BI__builtin_s390_vfmaxdb:
ID = Intrinsic::s390_vfmaxdb;
break;
19361 default: llvm_unreachable(
"Unknown BuiltinID");
19365 return Builder.CreateCall(F, {
X, Y, M4Value});
19367 case SystemZ::BI__builtin_s390_vfminsb:
19368 case SystemZ::BI__builtin_s390_vfmindb: {
19376 Intrinsic::ID
ID = Intrinsic::not_intrinsic;
19378 switch (M4.getZExtValue()) {
19380 case 4:
ID = Intrinsic::minnum;
19381 CI = Intrinsic::experimental_constrained_minnum;
break;
19383 if (ID != Intrinsic::not_intrinsic) {
19384 if (
Builder.getIsFPConstrained()) {
19386 return Builder.CreateConstrainedFPCall(F, {
X, Y});
19389 return Builder.CreateCall(F, {
X, Y});
19392 switch (BuiltinID) {
19393 case SystemZ::BI__builtin_s390_vfminsb:
ID = Intrinsic::s390_vfminsb;
break;
19394 case SystemZ::BI__builtin_s390_vfmindb:
ID = Intrinsic::s390_vfmindb;
break;
19395 default: llvm_unreachable(
"Unknown BuiltinID");
19399 return Builder.CreateCall(F, {
X, Y, M4Value});
19402 case SystemZ::BI__builtin_s390_vlbrh:
19403 case SystemZ::BI__builtin_s390_vlbrf:
19404 case SystemZ::BI__builtin_s390_vlbrg: {
19413#define INTRINSIC_WITH_CC(NAME) \
19414 case SystemZ::BI__builtin_##NAME: \
19415 return EmitSystemZIntrinsicWithCC(*this, Intrinsic::NAME, E)
19494#undef INTRINSIC_WITH_CC
19503struct NVPTXMmaLdstInfo {
19504 unsigned NumResults;
19510#define MMA_INTR(geom_op_type, layout) \
19511 Intrinsic::nvvm_wmma_##geom_op_type##_##layout##_stride
19512#define MMA_LDST(n, geom_op_type) \
19513 { n, MMA_INTR(geom_op_type, col), MMA_INTR(geom_op_type, row) }
19515static NVPTXMmaLdstInfo getNVPTXMmaLdstInfo(
unsigned BuiltinID) {
19516 switch (BuiltinID) {
19518 case NVPTX::BI__hmma_m16n16k16_ld_a:
19519 return MMA_LDST(8, m16n16k16_load_a_f16);
19520 case NVPTX::BI__hmma_m16n16k16_ld_b:
19521 return MMA_LDST(8, m16n16k16_load_b_f16);
19522 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
19523 return MMA_LDST(4, m16n16k16_load_c_f16);
19524 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
19525 return MMA_LDST(8, m16n16k16_load_c_f32);
19526 case NVPTX::BI__hmma_m32n8k16_ld_a:
19527 return MMA_LDST(8, m32n8k16_load_a_f16);
19528 case NVPTX::BI__hmma_m32n8k16_ld_b:
19529 return MMA_LDST(8, m32n8k16_load_b_f16);
19530 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
19531 return MMA_LDST(4, m32n8k16_load_c_f16);
19532 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
19533 return MMA_LDST(8, m32n8k16_load_c_f32);
19534 case NVPTX::BI__hmma_m8n32k16_ld_a:
19535 return MMA_LDST(8, m8n32k16_load_a_f16);
19536 case NVPTX::BI__hmma_m8n32k16_ld_b:
19537 return MMA_LDST(8, m8n32k16_load_b_f16);
19538 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
19539 return MMA_LDST(4, m8n32k16_load_c_f16);
19540 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
19541 return MMA_LDST(8, m8n32k16_load_c_f32);
19544 case NVPTX::BI__imma_m16n16k16_ld_a_s8:
19545 return MMA_LDST(2, m16n16k16_load_a_s8);
19546 case NVPTX::BI__imma_m16n16k16_ld_a_u8:
19547 return MMA_LDST(2, m16n16k16_load_a_u8);
19548 case NVPTX::BI__imma_m16n16k16_ld_b_s8:
19549 return MMA_LDST(2, m16n16k16_load_b_s8);
19550 case NVPTX::BI__imma_m16n16k16_ld_b_u8:
19551 return MMA_LDST(2, m16n16k16_load_b_u8);
19552 case NVPTX::BI__imma_m16n16k16_ld_c:
19553 return MMA_LDST(8, m16n16k16_load_c_s32);
19554 case NVPTX::BI__imma_m32n8k16_ld_a_s8:
19555 return MMA_LDST(4, m32n8k16_load_a_s8);
19556 case NVPTX::BI__imma_m32n8k16_ld_a_u8:
19557 return MMA_LDST(4, m32n8k16_load_a_u8);
19558 case NVPTX::BI__imma_m32n8k16_ld_b_s8:
19559 return MMA_LDST(1, m32n8k16_load_b_s8);
19560 case NVPTX::BI__imma_m32n8k16_ld_b_u8:
19561 return MMA_LDST(1, m32n8k16_load_b_u8);
19562 case NVPTX::BI__imma_m32n8k16_ld_c:
19563 return MMA_LDST(8, m32n8k16_load_c_s32);
19564 case NVPTX::BI__imma_m8n32k16_ld_a_s8:
19565 return MMA_LDST(1, m8n32k16_load_a_s8);
19566 case NVPTX::BI__imma_m8n32k16_ld_a_u8:
19567 return MMA_LDST(1, m8n32k16_load_a_u8);
19568 case NVPTX::BI__imma_m8n32k16_ld_b_s8:
19569 return MMA_LDST(4, m8n32k16_load_b_s8);
19570 case NVPTX::BI__imma_m8n32k16_ld_b_u8:
19571 return MMA_LDST(4, m8n32k16_load_b_u8);
19572 case NVPTX::BI__imma_m8n32k16_ld_c:
19573 return MMA_LDST(8, m8n32k16_load_c_s32);
19577 case NVPTX::BI__imma_m8n8k32_ld_a_s4:
19578 return {1, 0,
MMA_INTR(m8n8k32_load_a_s4, row)};
19579 case NVPTX::BI__imma_m8n8k32_ld_a_u4:
19580 return {1, 0,
MMA_INTR(m8n8k32_load_a_u4, row)};
19581 case NVPTX::BI__imma_m8n8k32_ld_b_s4:
19582 return {1,
MMA_INTR(m8n8k32_load_b_s4, col), 0};
19583 case NVPTX::BI__imma_m8n8k32_ld_b_u4:
19584 return {1,
MMA_INTR(m8n8k32_load_b_u4, col), 0};
19585 case NVPTX::BI__imma_m8n8k32_ld_c:
19586 return MMA_LDST(2, m8n8k32_load_c_s32);
19587 case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
19588 return {1, 0,
MMA_INTR(m8n8k128_load_a_b1, row)};
19589 case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
19590 return {1,
MMA_INTR(m8n8k128_load_b_b1, col), 0};
19591 case NVPTX::BI__bmma_m8n8k128_ld_c:
19592 return MMA_LDST(2, m8n8k128_load_c_s32);
19595 case NVPTX::BI__dmma_m8n8k4_ld_a:
19596 return MMA_LDST(1, m8n8k4_load_a_f64);
19597 case NVPTX::BI__dmma_m8n8k4_ld_b:
19598 return MMA_LDST(1, m8n8k4_load_b_f64);
19599 case NVPTX::BI__dmma_m8n8k4_ld_c:
19600 return MMA_LDST(2, m8n8k4_load_c_f64);
19603 case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
19604 return MMA_LDST(4, m16n16k16_load_a_bf16);
19605 case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
19606 return MMA_LDST(4, m16n16k16_load_b_bf16);
19607 case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
19608 return MMA_LDST(2, m8n32k16_load_a_bf16);
19609 case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
19610 return MMA_LDST(8, m8n32k16_load_b_bf16);
19611 case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
19612 return MMA_LDST(8, m32n8k16_load_a_bf16);
19613 case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
19614 return MMA_LDST(2, m32n8k16_load_b_bf16);
19615 case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
19616 return MMA_LDST(4, m16n16k8_load_a_tf32);
19617 case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
19618 return MMA_LDST(4, m16n16k8_load_b_tf32);
19619 case NVPTX::BI__mma_tf32_m16n16k8_ld_c:
19620 return MMA_LDST(8, m16n16k8_load_c_f32);
19626 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
19627 return MMA_LDST(4, m16n16k16_store_d_f16);
19628 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
19629 return MMA_LDST(8, m16n16k16_store_d_f32);
19630 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
19631 return MMA_LDST(4, m32n8k16_store_d_f16);
19632 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
19633 return MMA_LDST(8, m32n8k16_store_d_f32);
19634 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
19635 return MMA_LDST(4, m8n32k16_store_d_f16);
19636 case NVPTX::BI__hmma_m8n32k16_st_c_f32:
19637 return MMA_LDST(8, m8n32k16_store_d_f32);
19642 case NVPTX::BI__imma_m16n16k16_st_c_i32:
19643 return MMA_LDST(8, m16n16k16_store_d_s32);
19644 case NVPTX::BI__imma_m32n8k16_st_c_i32:
19645 return MMA_LDST(8, m32n8k16_store_d_s32);
19646 case NVPTX::BI__imma_m8n32k16_st_c_i32:
19647 return MMA_LDST(8, m8n32k16_store_d_s32);
19648 case NVPTX::BI__imma_m8n8k32_st_c_i32:
19649 return MMA_LDST(2, m8n8k32_store_d_s32);
19650 case NVPTX::BI__bmma_m8n8k128_st_c_i32:
19651 return MMA_LDST(2, m8n8k128_store_d_s32);
19654 case NVPTX::BI__dmma_m8n8k4_st_c_f64:
19655 return MMA_LDST(2, m8n8k4_store_d_f64);
19658 case NVPTX::BI__mma_m16n16k8_st_c_f32:
19659 return MMA_LDST(8, m16n16k8_store_d_f32);
19662 llvm_unreachable(
"Unknown MMA builtin");
19669struct NVPTXMmaInfo {
19678 std::array<unsigned, 8> Variants;
19680 unsigned getMMAIntrinsic(
int Layout,
bool Satf) {
19681 unsigned Index = Layout + 4 * Satf;
19682 if (Index >= Variants.size())
19684 return Variants[Index];
19690static NVPTXMmaInfo getNVPTXMmaInfo(
unsigned BuiltinID) {
19692#define MMA_VARIANTS(geom, type) \
19693 Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type, \
19694 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type, \
19695 Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type, \
19696 Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type
19697#define MMA_SATF_VARIANTS(geom, type) \
19698 MMA_VARIANTS(geom, type), \
19699 Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type##_satfinite, \
19700 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type##_satfinite, \
19701 Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type##_satfinite, \
19702 Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type##_satfinite
19704#define MMA_VARIANTS_I4(geom, type) \
19706 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type, \
19710 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type##_satfinite, \
19714#define MMA_VARIANTS_B1_XOR(geom, type) \
19716 Intrinsic::nvvm_wmma_##geom##_mma_xor_popc_row_col_##type, \
19723#define MMA_VARIANTS_B1_AND(geom, type) \
19725 Intrinsic::nvvm_wmma_##geom##_mma_and_popc_row_col_##type, \
19733 switch (BuiltinID) {
19737 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
19739 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
19741 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
19743 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
19745 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
19747 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
19749 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
19751 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
19753 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
19755 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
19757 case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
19759 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
19763 case NVPTX::BI__imma_m16n16k16_mma_s8:
19765 case NVPTX::BI__imma_m16n16k16_mma_u8:
19767 case NVPTX::BI__imma_m32n8k16_mma_s8:
19769 case NVPTX::BI__imma_m32n8k16_mma_u8:
19771 case NVPTX::BI__imma_m8n32k16_mma_s8:
19773 case NVPTX::BI__imma_m8n32k16_mma_u8:
19777 case NVPTX::BI__imma_m8n8k32_mma_s4:
19779 case NVPTX::BI__imma_m8n8k32_mma_u4:
19781 case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1:
19783 case NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1:
19787 case NVPTX::BI__dmma_m8n8k4_mma_f64:
19791 case NVPTX::BI__mma_bf16_m16n16k16_mma_f32:
19792 return {4, 4, 8, 8, {{
MMA_VARIANTS(m16n16k16, bf16)}}};
19793 case NVPTX::BI__mma_bf16_m8n32k16_mma_f32:
19794 return {2, 8, 8, 8, {{
MMA_VARIANTS(m8n32k16, bf16)}}};
19795 case NVPTX::BI__mma_bf16_m32n8k16_mma_f32:
19796 return {8, 2, 8, 8, {{
MMA_VARIANTS(m32n8k16, bf16)}}};
19797 case NVPTX::BI__mma_tf32_m16n16k8_mma_f32:
19798 return {4, 4, 8, 8, {{
MMA_VARIANTS(m16n16k8, tf32)}}};
19800 llvm_unreachable(
"Unexpected builtin ID.");
19803#undef MMA_SATF_VARIANTS
19804#undef MMA_VARIANTS_I4
19805#undef MMA_VARIANTS_B1_AND
19806#undef MMA_VARIANTS_B1_XOR
19815 return CGF.
Builder.CreateCall(
19817 {Ptr, ConstantInt::get(CGF.Builder.getInt32Ty(), Align.getQuantity())});
19823 llvm::Type *ElemTy =
19825 return CGF.
Builder.CreateCall(
19827 {Ptr, CGF.EmitScalarExpr(E->getArg(1))});
19830static Value *MakeCpAsync(
unsigned IntrinsicID,
unsigned IntrinsicIDS,
19835 {CGF.EmitScalarExpr(E->getArg(0)),
19836 CGF.EmitScalarExpr(E->getArg(1)),
19837 CGF.EmitScalarExpr(E->getArg(2))})
19839 {CGF.EmitScalarExpr(E->getArg(0)),
19840 CGF.EmitScalarExpr(E->getArg(1))});
19843static Value *MakeHalfType(
unsigned IntrinsicID,
unsigned BuiltinID,
19846 if (!(
C.getLangOpts().NativeHalfType ||
19847 !
C.getTargetInfo().useFP16ConversionIntrinsics())) {
19849 " requires native half type support.");
19853 if (IntrinsicID == Intrinsic::nvvm_ldg_global_f ||
19854 IntrinsicID == Intrinsic::nvvm_ldu_global_f)
19855 return MakeLdgLdu(IntrinsicID, CGF, E);
19859 auto *FTy = F->getFunctionType();
19860 unsigned ICEArguments = 0;
19862 C.GetBuiltinType(BuiltinID, Error, &ICEArguments);
19864 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; ++i) {
19865 assert((ICEArguments & (1 << i)) == 0);
19867 auto *PTy = FTy->getParamType(i);
19868 if (PTy != ArgValue->
getType())
19869 ArgValue = CGF.
Builder.CreateBitCast(ArgValue, PTy);
19870 Args.push_back(ArgValue);
19873 return CGF.
Builder.CreateCall(F, Args);
19879 switch (BuiltinID) {
19880 case NVPTX::BI__nvvm_atom_add_gen_i:
19881 case NVPTX::BI__nvvm_atom_add_gen_l:
19882 case NVPTX::BI__nvvm_atom_add_gen_ll:
19885 case NVPTX::BI__nvvm_atom_sub_gen_i:
19886 case NVPTX::BI__nvvm_atom_sub_gen_l:
19887 case NVPTX::BI__nvvm_atom_sub_gen_ll:
19890 case NVPTX::BI__nvvm_atom_and_gen_i:
19891 case NVPTX::BI__nvvm_atom_and_gen_l:
19892 case NVPTX::BI__nvvm_atom_and_gen_ll:
19895 case NVPTX::BI__nvvm_atom_or_gen_i:
19896 case NVPTX::BI__nvvm_atom_or_gen_l:
19897 case NVPTX::BI__nvvm_atom_or_gen_ll:
19900 case NVPTX::BI__nvvm_atom_xor_gen_i:
19901 case NVPTX::BI__nvvm_atom_xor_gen_l:
19902 case NVPTX::BI__nvvm_atom_xor_gen_ll:
19905 case NVPTX::BI__nvvm_atom_xchg_gen_i:
19906 case NVPTX::BI__nvvm_atom_xchg_gen_l:
19907 case NVPTX::BI__nvvm_atom_xchg_gen_ll:
19910 case NVPTX::BI__nvvm_atom_max_gen_i:
19911 case NVPTX::BI__nvvm_atom_max_gen_l:
19912 case NVPTX::BI__nvvm_atom_max_gen_ll:
19915 case NVPTX::BI__nvvm_atom_max_gen_ui:
19916 case NVPTX::BI__nvvm_atom_max_gen_ul:
19917 case NVPTX::BI__nvvm_atom_max_gen_ull:
19920 case NVPTX::BI__nvvm_atom_min_gen_i:
19921 case NVPTX::BI__nvvm_atom_min_gen_l:
19922 case NVPTX::BI__nvvm_atom_min_gen_ll:
19925 case NVPTX::BI__nvvm_atom_min_gen_ui:
19926 case NVPTX::BI__nvvm_atom_min_gen_ul:
19927 case NVPTX::BI__nvvm_atom_min_gen_ull:
19930 case NVPTX::BI__nvvm_atom_cas_gen_i:
19931 case NVPTX::BI__nvvm_atom_cas_gen_l:
19932 case NVPTX::BI__nvvm_atom_cas_gen_ll:
19937 case NVPTX::BI__nvvm_atom_add_gen_f:
19938 case NVPTX::BI__nvvm_atom_add_gen_d: {
19943 AtomicOrdering::SequentiallyConsistent);
19946 case NVPTX::BI__nvvm_atom_inc_gen_ui: {
19951 return Builder.CreateCall(FnALI32, {Ptr, Val});
19954 case NVPTX::BI__nvvm_atom_dec_gen_ui: {
19959 return Builder.CreateCall(FnALD32, {Ptr, Val});
19962 case NVPTX::BI__nvvm_ldg_c:
19963 case NVPTX::BI__nvvm_ldg_sc:
19964 case NVPTX::BI__nvvm_ldg_c2:
19965 case NVPTX::BI__nvvm_ldg_sc2:
19966 case NVPTX::BI__nvvm_ldg_c4:
19967 case NVPTX::BI__nvvm_ldg_sc4:
19968 case NVPTX::BI__nvvm_ldg_s:
19969 case NVPTX::BI__nvvm_ldg_s2:
19970 case NVPTX::BI__nvvm_ldg_s4:
19971 case NVPTX::BI__nvvm_ldg_i:
19972 case NVPTX::BI__nvvm_ldg_i2:
19973 case NVPTX::BI__nvvm_ldg_i4:
19974 case NVPTX::BI__nvvm_ldg_l:
19975 case NVPTX::BI__nvvm_ldg_l2:
19976 case NVPTX::BI__nvvm_ldg_ll:
19977 case NVPTX::BI__nvvm_ldg_ll2:
19978 case NVPTX::BI__nvvm_ldg_uc:
19979 case NVPTX::BI__nvvm_ldg_uc2:
19980 case NVPTX::BI__nvvm_ldg_uc4:
19981 case NVPTX::BI__nvvm_ldg_us:
19982 case NVPTX::BI__nvvm_ldg_us2:
19983 case NVPTX::BI__nvvm_ldg_us4:
19984 case NVPTX::BI__nvvm_ldg_ui:
19985 case NVPTX::BI__nvvm_ldg_ui2:
19986 case NVPTX::BI__nvvm_ldg_ui4:
19987 case NVPTX::BI__nvvm_ldg_ul:
19988 case NVPTX::BI__nvvm_ldg_ul2:
19989 case NVPTX::BI__nvvm_ldg_ull:
19990 case NVPTX::BI__nvvm_ldg_ull2:
19994 return MakeLdgLdu(Intrinsic::nvvm_ldg_global_i, *
this, E);
19995 case NVPTX::BI__nvvm_ldg_f:
19996 case NVPTX::BI__nvvm_ldg_f2:
19997 case NVPTX::BI__nvvm_ldg_f4:
19998 case NVPTX::BI__nvvm_ldg_d:
19999 case NVPTX::BI__nvvm_ldg_d2:
20000 return MakeLdgLdu(Intrinsic::nvvm_ldg_global_f, *
this, E);
20002 case NVPTX::BI__nvvm_ldu_c:
20003 case NVPTX::BI__nvvm_ldu_sc:
20004 case NVPTX::BI__nvvm_ldu_c2:
20005 case NVPTX::BI__nvvm_ldu_sc2:
20006 case NVPTX::BI__nvvm_ldu_c4:
20007 case NVPTX::BI__nvvm_ldu_sc4:
20008 case NVPTX::BI__nvvm_ldu_s:
20009 case NVPTX::BI__nvvm_ldu_s2:
20010 case NVPTX::BI__nvvm_ldu_s4:
20011 case NVPTX::BI__nvvm_ldu_i:
20012 case NVPTX::BI__nvvm_ldu_i2:
20013 case NVPTX::BI__nvvm_ldu_i4:
20014 case NVPTX::BI__nvvm_ldu_l:
20015 case NVPTX::BI__nvvm_ldu_l2:
20016 case NVPTX::BI__nvvm_ldu_ll:
20017 case NVPTX::BI__nvvm_ldu_ll2:
20018 case NVPTX::BI__nvvm_ldu_uc:
20019 case NVPTX::BI__nvvm_ldu_uc2:
20020 case NVPTX::BI__nvvm_ldu_uc4:
20021 case NVPTX::BI__nvvm_ldu_us:
20022 case NVPTX::BI__nvvm_ldu_us2:
20023 case NVPTX::BI__nvvm_ldu_us4:
20024 case NVPTX::BI__nvvm_ldu_ui:
20025 case NVPTX::BI__nvvm_ldu_ui2:
20026 case NVPTX::BI__nvvm_ldu_ui4:
20027 case NVPTX::BI__nvvm_ldu_ul:
20028 case NVPTX::BI__nvvm_ldu_ul2:
20029 case NVPTX::BI__nvvm_ldu_ull:
20030 case NVPTX::BI__nvvm_ldu_ull2:
20031 return MakeLdgLdu(Intrinsic::nvvm_ldu_global_i, *
this, E);
20032 case NVPTX::BI__nvvm_ldu_f:
20033 case NVPTX::BI__nvvm_ldu_f2:
20034 case NVPTX::BI__nvvm_ldu_f4:
20035 case NVPTX::BI__nvvm_ldu_d:
20036 case NVPTX::BI__nvvm_ldu_d2:
20037 return MakeLdgLdu(Intrinsic::nvvm_ldu_global_f, *
this, E);
20039 case NVPTX::BI__nvvm_atom_cta_add_gen_i:
20040 case NVPTX::BI__nvvm_atom_cta_add_gen_l:
20041 case NVPTX::BI__nvvm_atom_cta_add_gen_ll:
20042 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_cta, *
this, E);
20043 case NVPTX::BI__nvvm_atom_sys_add_gen_i:
20044 case NVPTX::BI__nvvm_atom_sys_add_gen_l:
20045 case NVPTX::BI__nvvm_atom_sys_add_gen_ll:
20046 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_sys, *
this, E);
20047 case NVPTX::BI__nvvm_atom_cta_add_gen_f:
20048 case NVPTX::BI__nvvm_atom_cta_add_gen_d:
20049 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_cta, *
this, E);
20050 case NVPTX::BI__nvvm_atom_sys_add_gen_f:
20051 case NVPTX::BI__nvvm_atom_sys_add_gen_d:
20052 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_sys, *
this, E);
20053 case NVPTX::BI__nvvm_atom_cta_xchg_gen_i:
20054 case NVPTX::BI__nvvm_atom_cta_xchg_gen_l:
20055 case NVPTX::BI__nvvm_atom_cta_xchg_gen_ll:
20056 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_cta, *
this, E);
20057 case NVPTX::BI__nvvm_atom_sys_xchg_gen_i:
20058 case NVPTX::BI__nvvm_atom_sys_xchg_gen_l:
20059 case NVPTX::BI__nvvm_atom_sys_xchg_gen_ll:
20060 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_sys, *
this, E);
20061 case NVPTX::BI__nvvm_atom_cta_max_gen_i:
20062 case NVPTX::BI__nvvm_atom_cta_max_gen_ui:
20063 case NVPTX::BI__nvvm_atom_cta_max_gen_l:
20064 case NVPTX::BI__nvvm_atom_cta_max_gen_ul:
20065 case NVPTX::BI__nvvm_atom_cta_max_gen_ll:
20066 case NVPTX::BI__nvvm_atom_cta_max_gen_ull:
20067 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_cta, *
this, E);
20068 case NVPTX::BI__nvvm_atom_sys_max_gen_i:
20069 case NVPTX::BI__nvvm_atom_sys_max_gen_ui:
20070 case NVPTX::BI__nvvm_atom_sys_max_gen_l:
20071 case NVPTX::BI__nvvm_atom_sys_max_gen_ul:
20072 case NVPTX::BI__nvvm_atom_sys_max_gen_ll:
20073 case NVPTX::BI__nvvm_atom_sys_max_gen_ull:
20074 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_sys, *
this, E);
20075 case NVPTX::BI__nvvm_atom_cta_min_gen_i:
20076 case NVPTX::BI__nvvm_atom_cta_min_gen_ui:
20077 case NVPTX::BI__nvvm_atom_cta_min_gen_l:
20078 case NVPTX::BI__nvvm_atom_cta_min_gen_ul:
20079 case NVPTX::BI__nvvm_atom_cta_min_gen_ll:
20080 case NVPTX::BI__nvvm_atom_cta_min_gen_ull:
20081 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_cta, *
this, E);
20082 case NVPTX::BI__nvvm_atom_sys_min_gen_i:
20083 case NVPTX::BI__nvvm_atom_sys_min_gen_ui:
20084 case NVPTX::BI__nvvm_atom_sys_min_gen_l:
20085 case NVPTX::BI__nvvm_atom_sys_min_gen_ul:
20086 case NVPTX::BI__nvvm_atom_sys_min_gen_ll:
20087 case NVPTX::BI__nvvm_atom_sys_min_gen_ull:
20088 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_sys, *
this, E);
20089 case NVPTX::BI__nvvm_atom_cta_inc_gen_ui:
20090 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_cta, *
this, E);
20091 case NVPTX::BI__nvvm_atom_cta_dec_gen_ui:
20092 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_cta, *
this, E);
20093 case NVPTX::BI__nvvm_atom_sys_inc_gen_ui:
20094 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_sys, *
this, E);
20095 case NVPTX::BI__nvvm_atom_sys_dec_gen_ui:
20096 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_sys, *
this, E);
20097 case NVPTX::BI__nvvm_atom_cta_and_gen_i:
20098 case NVPTX::BI__nvvm_atom_cta_and_gen_l:
20099 case NVPTX::BI__nvvm_atom_cta_and_gen_ll:
20100 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_cta, *
this, E);
20101 case NVPTX::BI__nvvm_atom_sys_and_gen_i:
20102 case NVPTX::BI__nvvm_atom_sys_and_gen_l:
20103 case NVPTX::BI__nvvm_atom_sys_and_gen_ll:
20104 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_sys, *
this, E);
20105 case NVPTX::BI__nvvm_atom_cta_or_gen_i:
20106 case NVPTX::BI__nvvm_atom_cta_or_gen_l:
20107 case NVPTX::BI__nvvm_atom_cta_or_gen_ll:
20108 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_cta, *
this, E);
20109 case NVPTX::BI__nvvm_atom_sys_or_gen_i:
20110 case NVPTX::BI__nvvm_atom_sys_or_gen_l:
20111 case NVPTX::BI__nvvm_atom_sys_or_gen_ll:
20112 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_sys, *
this, E);
20113 case NVPTX::BI__nvvm_atom_cta_xor_gen_i:
20114 case NVPTX::BI__nvvm_atom_cta_xor_gen_l:
20115 case NVPTX::BI__nvvm_atom_cta_xor_gen_ll:
20116 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_cta, *
this, E);
20117 case NVPTX::BI__nvvm_atom_sys_xor_gen_i:
20118 case NVPTX::BI__nvvm_atom_sys_xor_gen_l:
20119 case NVPTX::BI__nvvm_atom_sys_xor_gen_ll:
20120 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_sys, *
this, E);
20121 case NVPTX::BI__nvvm_atom_cta_cas_gen_i:
20122 case NVPTX::BI__nvvm_atom_cta_cas_gen_l:
20123 case NVPTX::BI__nvvm_atom_cta_cas_gen_ll: {
20125 llvm::Type *ElemTy =
20129 Intrinsic::nvvm_atomic_cas_gen_i_cta, {ElemTy, Ptr->getType()}),
20130 {Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))});
20132 case NVPTX::BI__nvvm_atom_sys_cas_gen_i:
20133 case NVPTX::BI__nvvm_atom_sys_cas_gen_l:
20134 case NVPTX::BI__nvvm_atom_sys_cas_gen_ll: {
20136 llvm::Type *ElemTy =
20140 Intrinsic::nvvm_atomic_cas_gen_i_sys, {ElemTy, Ptr->getType()}),
20141 {Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))});
20143 case NVPTX::BI__nvvm_match_all_sync_i32p:
20144 case NVPTX::BI__nvvm_match_all_sync_i64p: {
20150 ? Intrinsic::nvvm_match_all_sync_i32p
20151 : Intrinsic::nvvm_match_all_sync_i64p),
20156 return Builder.CreateExtractValue(ResultPair, 0);
20160 case NVPTX::BI__hmma_m16n16k16_ld_a:
20161 case NVPTX::BI__hmma_m16n16k16_ld_b:
20162 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
20163 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
20164 case NVPTX::BI__hmma_m32n8k16_ld_a:
20165 case NVPTX::BI__hmma_m32n8k16_ld_b:
20166 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
20167 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
20168 case NVPTX::BI__hmma_m8n32k16_ld_a:
20169 case NVPTX::BI__hmma_m8n32k16_ld_b:
20170 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
20171 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
20173 case NVPTX::BI__imma_m16n16k16_ld_a_s8:
20174 case NVPTX::BI__imma_m16n16k16_ld_a_u8:
20175 case NVPTX::BI__imma_m16n16k16_ld_b_s8:
20176 case NVPTX::BI__imma_m16n16k16_ld_b_u8:
20177 case NVPTX::BI__imma_m16n16k16_ld_c:
20178 case NVPTX::BI__imma_m32n8k16_ld_a_s8:
20179 case NVPTX::BI__imma_m32n8k16_ld_a_u8:
20180 case NVPTX::BI__imma_m32n8k16_ld_b_s8:
20181 case NVPTX::BI__imma_m32n8k16_ld_b_u8:
20182 case NVPTX::BI__imma_m32n8k16_ld_c:
20183 case NVPTX::BI__imma_m8n32k16_ld_a_s8:
20184 case NVPTX::BI__imma_m8n32k16_ld_a_u8:
20185 case NVPTX::BI__imma_m8n32k16_ld_b_s8:
20186 case NVPTX::BI__imma_m8n32k16_ld_b_u8:
20187 case NVPTX::BI__imma_m8n32k16_ld_c:
20189 case NVPTX::BI__imma_m8n8k32_ld_a_s4:
20190 case NVPTX::BI__imma_m8n8k32_ld_a_u4:
20191 case NVPTX::BI__imma_m8n8k32_ld_b_s4:
20192 case NVPTX::BI__imma_m8n8k32_ld_b_u4:
20193 case NVPTX::BI__imma_m8n8k32_ld_c:
20194 case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
20195 case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
20196 case NVPTX::BI__bmma_m8n8k128_ld_c:
20198 case NVPTX::BI__dmma_m8n8k4_ld_a:
20199 case NVPTX::BI__dmma_m8n8k4_ld_b:
20200 case NVPTX::BI__dmma_m8n8k4_ld_c:
20202 case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
20203 case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
20204 case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
20205 case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
20206 case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
20207 case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
20208 case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
20209 case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
20210 case NVPTX::BI__mma_tf32_m16n16k8_ld_c: {
20214 std::optional<llvm::APSInt> isColMajorArg =
20216 if (!isColMajorArg)
20218 bool isColMajor = isColMajorArg->getSExtValue();
20219 NVPTXMmaLdstInfo II = getNVPTXMmaLdstInfo(BuiltinID);
20220 unsigned IID = isColMajor ? II.IID_col : II.IID_row;
20228 assert(II.NumResults);
20229 if (II.NumResults == 1) {
20233 for (
unsigned i = 0; i < II.NumResults; ++i) {
20238 llvm::ConstantInt::get(
IntTy, i)),
20245 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
20246 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
20247 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
20248 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
20249 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
20250 case NVPTX::BI__hmma_m8n32k16_st_c_f32:
20251 case NVPTX::BI__imma_m16n16k16_st_c_i32:
20252 case NVPTX::BI__imma_m32n8k16_st_c_i32:
20253 case NVPTX::BI__imma_m8n32k16_st_c_i32:
20254 case NVPTX::BI__imma_m8n8k32_st_c_i32:
20255 case NVPTX::BI__bmma_m8n8k128_st_c_i32:
20256 case NVPTX::BI__dmma_m8n8k4_st_c_f64:
20257 case NVPTX::BI__mma_m16n16k8_st_c_f32: {
20261 std::optional<llvm::APSInt> isColMajorArg =
20263 if (!isColMajorArg)
20265 bool isColMajor = isColMajorArg->getSExtValue();
20266 NVPTXMmaLdstInfo II = getNVPTXMmaLdstInfo(BuiltinID);
20267 unsigned IID = isColMajor ? II.IID_col : II.IID_row;
20272 llvm::Type *ParamType = Intrinsic->getFunctionType()->getParamType(1);
20274 for (
unsigned i = 0; i < II.NumResults; ++i) {
20278 llvm::ConstantInt::get(
IntTy, i)),
20280 Values.push_back(
Builder.CreateBitCast(
V, ParamType));
20282 Values.push_back(Ldm);
20289 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
20290 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
20291 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
20292 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
20293 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
20294 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
20295 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
20296 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
20297 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
20298 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
20299 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
20300 case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
20301 case NVPTX::BI__imma_m16n16k16_mma_s8:
20302 case NVPTX::BI__imma_m16n16k16_mma_u8:
20303 case NVPTX::BI__imma_m32n8k16_mma_s8:
20304 case NVPTX::BI__imma_m32n8k16_mma_u8:
20305 case NVPTX::BI__imma_m8n32k16_mma_s8:
20306 case NVPTX::BI__imma_m8n32k16_mma_u8:
20307 case NVPTX::BI__imma_m8n8k32_mma_s4:
20308 case NVPTX::BI__imma_m8n8k32_mma_u4:
20309 case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1:
20310 case NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1:
20311 case NVPTX::BI__dmma_m8n8k4_mma_f64:
20312 case NVPTX::BI__mma_bf16_m16n16k16_mma_f32:
20313 case NVPTX::BI__mma_bf16_m8n32k16_mma_f32:
20314 case NVPTX::BI__mma_bf16_m32n8k16_mma_f32:
20315 case NVPTX::BI__mma_tf32_m16n16k8_mma_f32: {
20320 std::optional<llvm::APSInt> LayoutArg =
20324 int Layout = LayoutArg->getSExtValue();
20325 if (Layout < 0 || Layout > 3)
20327 llvm::APSInt SatfArg;
20328 if (BuiltinID == NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1 ||
20329 BuiltinID == NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1)
20331 else if (std::optional<llvm::APSInt> OptSatfArg =
20333 SatfArg = *OptSatfArg;
20336 bool Satf = SatfArg.getSExtValue();
20337 NVPTXMmaInfo MI = getNVPTXMmaInfo(BuiltinID);
20338 unsigned IID = MI.getMMAIntrinsic(Layout, Satf);
20344 llvm::Type *AType = Intrinsic->getFunctionType()->getParamType(0);
20346 for (
unsigned i = 0; i < MI.NumEltsA; ++i) {
20350 llvm::ConstantInt::get(
IntTy, i)),
20352 Values.push_back(
Builder.CreateBitCast(
V, AType));
20355 llvm::Type *BType = Intrinsic->getFunctionType()->getParamType(MI.NumEltsA);
20356 for (
unsigned i = 0; i < MI.NumEltsB; ++i) {
20360 llvm::ConstantInt::get(
IntTy, i)),
20362 Values.push_back(
Builder.CreateBitCast(
V, BType));
20365 llvm::Type *CType =
20366 Intrinsic->getFunctionType()->getParamType(MI.NumEltsA + MI.NumEltsB);
20367 for (
unsigned i = 0; i < MI.NumEltsC; ++i) {
20371 llvm::ConstantInt::get(
IntTy, i)),
20373 Values.push_back(
Builder.CreateBitCast(
V, CType));
20377 for (
unsigned i = 0; i < MI.NumEltsD; ++i)
20381 llvm::ConstantInt::get(
IntTy, i)),
20386 case NVPTX::BI__nvvm_ex2_approx_f16:
20387 return MakeHalfType(Intrinsic::nvvm_ex2_approx_f16, BuiltinID, E, *
this);
20388 case NVPTX::BI__nvvm_ex2_approx_f16x2:
20389 return MakeHalfType(Intrinsic::nvvm_ex2_approx_f16x2, BuiltinID, E, *
this);
20390 case NVPTX::BI__nvvm_ff2f16x2_rn:
20391 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rn, BuiltinID, E, *
this);
20392 case NVPTX::BI__nvvm_ff2f16x2_rn_relu:
20393 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rn_relu, BuiltinID, E, *
this);
20394 case NVPTX::BI__nvvm_ff2f16x2_rz:
20395 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rz, BuiltinID, E, *
this);
20396 case NVPTX::BI__nvvm_ff2f16x2_rz_relu:
20397 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rz_relu, BuiltinID, E, *
this);
20398 case NVPTX::BI__nvvm_fma_rn_f16:
20399 return MakeHalfType(Intrinsic::nvvm_fma_rn_f16, BuiltinID, E, *
this);
20400 case NVPTX::BI__nvvm_fma_rn_f16x2:
20401 return MakeHalfType(Intrinsic::nvvm_fma_rn_f16x2, BuiltinID, E, *
this);
20402 case NVPTX::BI__nvvm_fma_rn_ftz_f16:
20403 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_f16, BuiltinID, E, *
this);
20404 case NVPTX::BI__nvvm_fma_rn_ftz_f16x2:
20405 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_f16x2, BuiltinID, E, *
this);
20406 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16:
20407 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_relu_f16, BuiltinID, E,
20409 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16x2:
20410 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_relu_f16x2, BuiltinID, E,
20412 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16:
20413 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_sat_f16, BuiltinID, E,
20415 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16x2:
20416 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_sat_f16x2, BuiltinID, E,
20418 case NVPTX::BI__nvvm_fma_rn_relu_f16:
20419 return MakeHalfType(Intrinsic::nvvm_fma_rn_relu_f16, BuiltinID, E, *
this);
20420 case NVPTX::BI__nvvm_fma_rn_relu_f16x2:
20421 return MakeHalfType(Intrinsic::nvvm_fma_rn_relu_f16x2, BuiltinID, E, *
this);
20422 case NVPTX::BI__nvvm_fma_rn_sat_f16:
20423 return MakeHalfType(Intrinsic::nvvm_fma_rn_sat_f16, BuiltinID, E, *
this);
20424 case NVPTX::BI__nvvm_fma_rn_sat_f16x2:
20425 return MakeHalfType(Intrinsic::nvvm_fma_rn_sat_f16x2, BuiltinID, E, *
this);
20426 case NVPTX::BI__nvvm_fmax_f16:
20427 return MakeHalfType(Intrinsic::nvvm_fmax_f16, BuiltinID, E, *
this);
20428 case NVPTX::BI__nvvm_fmax_f16x2:
20429 return MakeHalfType(Intrinsic::nvvm_fmax_f16x2, BuiltinID, E, *
this);
20430 case NVPTX::BI__nvvm_fmax_ftz_f16:
20431 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_f16, BuiltinID, E, *
this);
20432 case NVPTX::BI__nvvm_fmax_ftz_f16x2:
20433 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_f16x2, BuiltinID, E, *
this);
20434 case NVPTX::BI__nvvm_fmax_ftz_nan_f16:
20435 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_f16, BuiltinID, E, *
this);
20436 case NVPTX::BI__nvvm_fmax_ftz_nan_f16x2:
20437 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_f16x2, BuiltinID, E,
20439 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16:
20440 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f16, BuiltinID,
20442 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16x2:
20443 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f16x2,
20444 BuiltinID, E, *
this);
20445 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16:
20446 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_xorsign_abs_f16, BuiltinID, E,
20448 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16x2:
20449 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_xorsign_abs_f16x2, BuiltinID,
20451 case NVPTX::BI__nvvm_fmax_nan_f16:
20452 return MakeHalfType(Intrinsic::nvvm_fmax_nan_f16, BuiltinID, E, *
this);
20453 case NVPTX::BI__nvvm_fmax_nan_f16x2:
20454 return MakeHalfType(Intrinsic::nvvm_fmax_nan_f16x2, BuiltinID, E, *
this);
20455 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16:
20456 return MakeHalfType(Intrinsic::nvvm_fmax_nan_xorsign_abs_f16, BuiltinID, E,
20458 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16x2:
20459 return MakeHalfType(Intrinsic::nvvm_fmax_nan_xorsign_abs_f16x2, BuiltinID,
20461 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16:
20462 return MakeHalfType(Intrinsic::nvvm_fmax_xorsign_abs_f16, BuiltinID, E,
20464 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16x2:
20465 return MakeHalfType(Intrinsic::nvvm_fmax_xorsign_abs_f16x2, BuiltinID, E,
20467 case NVPTX::BI__nvvm_fmin_f16:
20468 return MakeHalfType(Intrinsic::nvvm_fmin_f16, BuiltinID, E, *
this);
20469 case NVPTX::BI__nvvm_fmin_f16x2:
20470 return MakeHalfType(Intrinsic::nvvm_fmin_f16x2, BuiltinID, E, *
this);
20471 case NVPTX::BI__nvvm_fmin_ftz_f16:
20472 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_f16, BuiltinID, E, *
this);
20473 case NVPTX::BI__nvvm_fmin_ftz_f16x2:
20474 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_f16x2, BuiltinID, E, *
this);
20475 case NVPTX::BI__nvvm_fmin_ftz_nan_f16:
20476 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_f16, BuiltinID, E, *
this);
20477 case NVPTX::BI__nvvm_fmin_ftz_nan_f16x2:
20478 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_f16x2, BuiltinID, E,
20480 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16:
20481 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f16, BuiltinID,
20483 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16x2:
20484 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f16x2,
20485 BuiltinID, E, *
this);
20486 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16:
20487 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_xorsign_abs_f16, BuiltinID, E,
20489 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16x2:
20490 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_xorsign_abs_f16x2, BuiltinID,
20492 case NVPTX::BI__nvvm_fmin_nan_f16:
20493 return MakeHalfType(Intrinsic::nvvm_fmin_nan_f16, BuiltinID, E, *
this);
20494 case NVPTX::BI__nvvm_fmin_nan_f16x2:
20495 return MakeHalfType(Intrinsic::nvvm_fmin_nan_f16x2, BuiltinID, E, *
this);
20496 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16:
20497 return MakeHalfType(Intrinsic::nvvm_fmin_nan_xorsign_abs_f16, BuiltinID, E,
20499 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16x2:
20500 return MakeHalfType(Intrinsic::nvvm_fmin_nan_xorsign_abs_f16x2, BuiltinID,
20502 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16:
20503 return MakeHalfType(Intrinsic::nvvm_fmin_xorsign_abs_f16, BuiltinID, E,
20505 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16x2:
20506 return MakeHalfType(Intrinsic::nvvm_fmin_xorsign_abs_f16x2, BuiltinID, E,
20508 case NVPTX::BI__nvvm_ldg_h:
20509 return MakeHalfType(Intrinsic::nvvm_ldg_global_f, BuiltinID, E, *
this);
20510 case NVPTX::BI__nvvm_ldg_h2:
20511 return MakeHalfType(Intrinsic::nvvm_ldg_global_f, BuiltinID, E, *
this);
20512 case NVPTX::BI__nvvm_ldu_h:
20513 return MakeHalfType(Intrinsic::nvvm_ldu_global_f, BuiltinID, E, *
this);
20514 case NVPTX::BI__nvvm_ldu_h2: {
20515 return MakeHalfType(Intrinsic::nvvm_ldu_global_f, BuiltinID, E, *
this);
20517 case NVPTX::BI__nvvm_cp_async_ca_shared_global_4:
20518 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_4,
20519 Intrinsic::nvvm_cp_async_ca_shared_global_4_s, *
this, E,
20521 case NVPTX::BI__nvvm_cp_async_ca_shared_global_8:
20522 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_8,
20523 Intrinsic::nvvm_cp_async_ca_shared_global_8_s, *
this, E,
20525 case NVPTX::BI__nvvm_cp_async_ca_shared_global_16:
20526 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_16,
20527 Intrinsic::nvvm_cp_async_ca_shared_global_16_s, *
this, E,
20529 case NVPTX::BI__nvvm_cp_async_cg_shared_global_16:
20530 return MakeCpAsync(Intrinsic::nvvm_cp_async_cg_shared_global_16,
20531 Intrinsic::nvvm_cp_async_cg_shared_global_16_s, *
this, E,
20533 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_x:
20536 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_y:
20539 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_z:
20542 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_w:
20545 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_x:
20548 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_y:
20551 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_z:
20554 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_w:
20557 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_x:
20560 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_y:
20563 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_z:
20566 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_w:
20569 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_x:
20572 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_y:
20575 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_z:
20578 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_w:
20581 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctarank:
20584 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctarank:
20587 case NVPTX::BI__nvvm_is_explicit_cluster:
20590 case NVPTX::BI__nvvm_isspacep_shared_cluster:
20594 case NVPTX::BI__nvvm_mapa:
20597 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
20598 case NVPTX::BI__nvvm_mapa_shared_cluster:
20601 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
20602 case NVPTX::BI__nvvm_getctarank:
20606 case NVPTX::BI__nvvm_getctarank_shared_cluster:
20610 case NVPTX::BI__nvvm_barrier_cluster_arrive:
20613 case NVPTX::BI__nvvm_barrier_cluster_arrive_relaxed:
20616 case NVPTX::BI__nvvm_barrier_cluster_wait:
20619 case NVPTX::BI__nvvm_fence_sc_cluster:
20628struct BuiltinAlignArgs {
20629 llvm::Value *Src =
nullptr;
20630 llvm::Type *SrcType =
nullptr;
20631 llvm::Value *Alignment =
nullptr;
20632 llvm::Value *Mask =
nullptr;
20633 llvm::IntegerType *IntType =
nullptr;
20641 SrcType = Src->getType();
20642 if (SrcType->isPointerTy()) {
20643 IntType = IntegerType::get(
20647 assert(SrcType->isIntegerTy());
20648 IntType = cast<llvm::IntegerType>(SrcType);
20651 Alignment = CGF.
Builder.CreateZExtOrTrunc(Alignment, IntType,
"alignment");
20652 auto *One = llvm::ConstantInt::get(IntType, 1);
20653 Mask = CGF.
Builder.CreateSub(Alignment, One,
"mask");
20660 BuiltinAlignArgs Args(E, *
this);
20661 llvm::Value *SrcAddress = Args.Src;
20662 if (Args.SrcType->isPointerTy())
20664 Builder.CreateBitOrPointerCast(Args.Src, Args.IntType,
"src_addr");
20666 Builder.CreateAnd(SrcAddress, Args.Mask,
"set_bits"),
20667 llvm::Constant::getNullValue(Args.IntType),
"is_aligned"));
20674 BuiltinAlignArgs Args(E, *
this);
20675 llvm::Value *SrcForMask = Args.Src;
20681 if (Args.Src->getType()->isPointerTy()) {
20691 SrcForMask =
Builder.CreateAdd(SrcForMask, Args.Mask,
"over_boundary");
20695 llvm::Value *InvertedMask =
Builder.CreateNot(Args.Mask,
"inverted_mask");
20696 llvm::Value *
Result =
nullptr;
20697 if (Args.Src->getType()->isPointerTy()) {
20699 Intrinsic::ptrmask, {Args.SrcType, Args.IntType},
20700 {SrcForMask, InvertedMask},
nullptr,
"aligned_result");
20702 Result =
Builder.CreateAnd(SrcForMask, InvertedMask,
"aligned_result");
20704 assert(
Result->getType() == Args.SrcType);
20710 switch (BuiltinID) {
20711 case WebAssembly::BI__builtin_wasm_memory_size: {
20716 return Builder.CreateCall(Callee, I);
20718 case WebAssembly::BI__builtin_wasm_memory_grow: {
20724 return Builder.CreateCall(Callee, Args);
20726 case WebAssembly::BI__builtin_wasm_tls_size: {
20729 return Builder.CreateCall(Callee);
20731 case WebAssembly::BI__builtin_wasm_tls_align: {
20734 return Builder.CreateCall(Callee);
20736 case WebAssembly::BI__builtin_wasm_tls_base: {
20738 return Builder.CreateCall(Callee);
20740 case WebAssembly::BI__builtin_wasm_throw: {
20744 return Builder.CreateCall(Callee, {Tag, Obj});
20746 case WebAssembly::BI__builtin_wasm_rethrow: {
20748 return Builder.CreateCall(Callee);
20750 case WebAssembly::BI__builtin_wasm_memory_atomic_wait32: {
20757 case WebAssembly::BI__builtin_wasm_memory_atomic_wait64: {
20764 case WebAssembly::BI__builtin_wasm_memory_atomic_notify: {
20768 return Builder.CreateCall(Callee, {Addr, Count});
20770 case WebAssembly::BI__builtin_wasm_trunc_s_i32_f32:
20771 case WebAssembly::BI__builtin_wasm_trunc_s_i32_f64:
20772 case WebAssembly::BI__builtin_wasm_trunc_s_i64_f32:
20773 case WebAssembly::BI__builtin_wasm_trunc_s_i64_f64: {
20778 return Builder.CreateCall(Callee, {Src});
20780 case WebAssembly::BI__builtin_wasm_trunc_u_i32_f32:
20781 case WebAssembly::BI__builtin_wasm_trunc_u_i32_f64:
20782 case WebAssembly::BI__builtin_wasm_trunc_u_i64_f32:
20783 case WebAssembly::BI__builtin_wasm_trunc_u_i64_f64: {
20788 return Builder.CreateCall(Callee, {Src});
20790 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32_f32:
20791 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32_f64:
20792 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i64_f32:
20793 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i64_f64:
20794 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32x4_f32x4: {
20799 return Builder.CreateCall(Callee, {Src});
20801 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32_f32:
20802 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32_f64:
20803 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i64_f32:
20804 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i64_f64:
20805 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32x4_f32x4: {
20810 return Builder.CreateCall(Callee, {Src});
20812 case WebAssembly::BI__builtin_wasm_min_f32:
20813 case WebAssembly::BI__builtin_wasm_min_f64:
20814 case WebAssembly::BI__builtin_wasm_min_f32x4:
20815 case WebAssembly::BI__builtin_wasm_min_f64x2: {
20820 return Builder.CreateCall(Callee, {LHS, RHS});
20822 case WebAssembly::BI__builtin_wasm_max_f32:
20823 case WebAssembly::BI__builtin_wasm_max_f64:
20824 case WebAssembly::BI__builtin_wasm_max_f32x4:
20825 case WebAssembly::BI__builtin_wasm_max_f64x2: {
20830 return Builder.CreateCall(Callee, {LHS, RHS});
20832 case WebAssembly::BI__builtin_wasm_pmin_f32x4:
20833 case WebAssembly::BI__builtin_wasm_pmin_f64x2: {
20838 return Builder.CreateCall(Callee, {LHS, RHS});
20840 case WebAssembly::BI__builtin_wasm_pmax_f32x4:
20841 case WebAssembly::BI__builtin_wasm_pmax_f64x2: {
20846 return Builder.CreateCall(Callee, {LHS, RHS});
20848 case WebAssembly::BI__builtin_wasm_ceil_f32x4:
20849 case WebAssembly::BI__builtin_wasm_floor_f32x4:
20850 case WebAssembly::BI__builtin_wasm_trunc_f32x4:
20851 case WebAssembly::BI__builtin_wasm_nearest_f32x4:
20852 case WebAssembly::BI__builtin_wasm_ceil_f64x2:
20853 case WebAssembly::BI__builtin_wasm_floor_f64x2:
20854 case WebAssembly::BI__builtin_wasm_trunc_f64x2:
20855 case WebAssembly::BI__builtin_wasm_nearest_f64x2: {
20857 switch (BuiltinID) {
20858 case WebAssembly::BI__builtin_wasm_ceil_f32x4:
20859 case WebAssembly::BI__builtin_wasm_ceil_f64x2:
20860 IntNo = Intrinsic::ceil;
20862 case WebAssembly::BI__builtin_wasm_floor_f32x4:
20863 case WebAssembly::BI__builtin_wasm_floor_f64x2:
20864 IntNo = Intrinsic::floor;
20866 case WebAssembly::BI__builtin_wasm_trunc_f32x4:
20867 case WebAssembly::BI__builtin_wasm_trunc_f64x2:
20868 IntNo = Intrinsic::trunc;
20870 case WebAssembly::BI__builtin_wasm_nearest_f32x4:
20871 case WebAssembly::BI__builtin_wasm_nearest_f64x2:
20872 IntNo = Intrinsic::nearbyint;
20875 llvm_unreachable(
"unexpected builtin ID");
20881 case WebAssembly::BI__builtin_wasm_ref_null_extern: {
20883 return Builder.CreateCall(Callee);
20885 case WebAssembly::BI__builtin_wasm_ref_null_func: {
20887 return Builder.CreateCall(Callee);
20889 case WebAssembly::BI__builtin_wasm_swizzle_i8x16: {
20893 return Builder.CreateCall(Callee, {Src, Indices});
20895 case WebAssembly::BI__builtin_wasm_add_sat_s_i8x16:
20896 case WebAssembly::BI__builtin_wasm_add_sat_u_i8x16:
20897 case WebAssembly::BI__builtin_wasm_add_sat_s_i16x8:
20898 case WebAssembly::BI__builtin_wasm_add_sat_u_i16x8:
20899 case WebAssembly::BI__builtin_wasm_sub_sat_s_i8x16:
20900 case WebAssembly::BI__builtin_wasm_sub_sat_u_i8x16:
20901 case WebAssembly::BI__builtin_wasm_sub_sat_s_i16x8:
20902 case WebAssembly::BI__builtin_wasm_sub_sat_u_i16x8: {
20904 switch (BuiltinID) {
20905 case WebAssembly::BI__builtin_wasm_add_sat_s_i8x16:
20906 case WebAssembly::BI__builtin_wasm_add_sat_s_i16x8:
20907 IntNo = Intrinsic::sadd_sat;
20909 case WebAssembly::BI__builtin_wasm_add_sat_u_i8x16:
20910 case WebAssembly::BI__builtin_wasm_add_sat_u_i16x8:
20911 IntNo = Intrinsic::uadd_sat;
20913 case WebAssembly::BI__builtin_wasm_sub_sat_s_i8x16:
20914 case WebAssembly::BI__builtin_wasm_sub_sat_s_i16x8:
20915 IntNo = Intrinsic::wasm_sub_sat_signed;
20917 case WebAssembly::BI__builtin_wasm_sub_sat_u_i8x16:
20918 case WebAssembly::BI__builtin_wasm_sub_sat_u_i16x8:
20919 IntNo = Intrinsic::wasm_sub_sat_unsigned;
20922 llvm_unreachable(
"unexpected builtin ID");
20927 return Builder.CreateCall(Callee, {LHS, RHS});
20929 case WebAssembly::BI__builtin_wasm_abs_i8x16:
20930 case WebAssembly::BI__builtin_wasm_abs_i16x8:
20931 case WebAssembly::BI__builtin_wasm_abs_i32x4:
20932 case WebAssembly::BI__builtin_wasm_abs_i64x2: {
20935 Constant *
Zero = llvm::Constant::getNullValue(Vec->
getType());
20936 Value *ICmp =
Builder.CreateICmpSLT(Vec, Zero,
"abscond");
20937 return Builder.CreateSelect(ICmp, Neg, Vec,
"abs");
20939 case WebAssembly::BI__builtin_wasm_min_s_i8x16:
20940 case WebAssembly::BI__builtin_wasm_min_u_i8x16:
20941 case WebAssembly::BI__builtin_wasm_max_s_i8x16:
20942 case WebAssembly::BI__builtin_wasm_max_u_i8x16:
20943 case WebAssembly::BI__builtin_wasm_min_s_i16x8:
20944 case WebAssembly::BI__builtin_wasm_min_u_i16x8:
20945 case WebAssembly::BI__builtin_wasm_max_s_i16x8:
20946 case WebAssembly::BI__builtin_wasm_max_u_i16x8:
20947 case WebAssembly::BI__builtin_wasm_min_s_i32x4:
20948 case WebAssembly::BI__builtin_wasm_min_u_i32x4:
20949 case WebAssembly::BI__builtin_wasm_max_s_i32x4:
20950 case WebAssembly::BI__builtin_wasm_max_u_i32x4: {
20954 switch (BuiltinID) {
20955 case WebAssembly::BI__builtin_wasm_min_s_i8x16:
20956 case WebAssembly::BI__builtin_wasm_min_s_i16x8:
20957 case WebAssembly::BI__builtin_wasm_min_s_i32x4:
20958 ICmp =
Builder.CreateICmpSLT(LHS, RHS);
20960 case WebAssembly::BI__builtin_wasm_min_u_i8x16:
20961 case WebAssembly::BI__builtin_wasm_min_u_i16x8:
20962 case WebAssembly::BI__builtin_wasm_min_u_i32x4:
20963 ICmp =
Builder.CreateICmpULT(LHS, RHS);
20965 case WebAssembly::BI__builtin_wasm_max_s_i8x16:
20966 case WebAssembly::BI__builtin_wasm_max_s_i16x8:
20967 case WebAssembly::BI__builtin_wasm_max_s_i32x4:
20968 ICmp =
Builder.CreateICmpSGT(LHS, RHS);
20970 case WebAssembly::BI__builtin_wasm_max_u_i8x16:
20971 case WebAssembly::BI__builtin_wasm_max_u_i16x8:
20972 case WebAssembly::BI__builtin_wasm_max_u_i32x4:
20973 ICmp =
Builder.CreateICmpUGT(LHS, RHS);
20976 llvm_unreachable(
"unexpected builtin ID");
20978 return Builder.CreateSelect(ICmp, LHS, RHS);
20980 case WebAssembly::BI__builtin_wasm_avgr_u_i8x16:
20981 case WebAssembly::BI__builtin_wasm_avgr_u_i16x8: {
20986 return Builder.CreateCall(Callee, {LHS, RHS});
20988 case WebAssembly::BI__builtin_wasm_q15mulr_sat_s_i16x8: {
20992 return Builder.CreateCall(Callee, {LHS, RHS});
20994 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_s_i16x8:
20995 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_u_i16x8:
20996 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_s_i32x4:
20997 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_u_i32x4: {
21000 switch (BuiltinID) {
21001 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_s_i16x8:
21002 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_s_i32x4:
21003 IntNo = Intrinsic::wasm_extadd_pairwise_signed;
21005 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_u_i16x8:
21006 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_u_i32x4:
21007 IntNo = Intrinsic::wasm_extadd_pairwise_unsigned;
21010 llvm_unreachable(
"unexpected builtin ID");
21014 return Builder.CreateCall(Callee, Vec);
21016 case WebAssembly::BI__builtin_wasm_bitselect: {
21022 return Builder.CreateCall(Callee, {V1, V2,
C});
21024 case WebAssembly::BI__builtin_wasm_dot_s_i32x4_i16x8: {
21028 return Builder.CreateCall(Callee, {LHS, RHS});
21030 case WebAssembly::BI__builtin_wasm_popcnt_i8x16: {
21034 return Builder.CreateCall(Callee, {Vec});
21036 case WebAssembly::BI__builtin_wasm_any_true_v128:
21037 case WebAssembly::BI__builtin_wasm_all_true_i8x16:
21038 case WebAssembly::BI__builtin_wasm_all_true_i16x8:
21039 case WebAssembly::BI__builtin_wasm_all_true_i32x4:
21040 case WebAssembly::BI__builtin_wasm_all_true_i64x2: {
21042 switch (BuiltinID) {
21043 case WebAssembly::BI__builtin_wasm_any_true_v128:
21044 IntNo = Intrinsic::wasm_anytrue;
21046 case WebAssembly::BI__builtin_wasm_all_true_i8x16:
21047 case WebAssembly::BI__builtin_wasm_all_true_i16x8:
21048 case WebAssembly::BI__builtin_wasm_all_true_i32x4:
21049 case WebAssembly::BI__builtin_wasm_all_true_i64x2:
21050 IntNo = Intrinsic::wasm_alltrue;
21053 llvm_unreachable(
"unexpected builtin ID");
21057 return Builder.CreateCall(Callee, {Vec});
21059 case WebAssembly::BI__builtin_wasm_bitmask_i8x16:
21060 case WebAssembly::BI__builtin_wasm_bitmask_i16x8:
21061 case WebAssembly::BI__builtin_wasm_bitmask_i32x4:
21062 case WebAssembly::BI__builtin_wasm_bitmask_i64x2: {
21066 return Builder.CreateCall(Callee, {Vec});
21068 case WebAssembly::BI__builtin_wasm_abs_f32x4:
21069 case WebAssembly::BI__builtin_wasm_abs_f64x2: {
21072 return Builder.CreateCall(Callee, {Vec});
21074 case WebAssembly::BI__builtin_wasm_sqrt_f32x4:
21075 case WebAssembly::BI__builtin_wasm_sqrt_f64x2: {
21078 return Builder.CreateCall(Callee, {Vec});
21080 case WebAssembly::BI__builtin_wasm_narrow_s_i8x16_i16x8:
21081 case WebAssembly::BI__builtin_wasm_narrow_u_i8x16_i16x8:
21082 case WebAssembly::BI__builtin_wasm_narrow_s_i16x8_i32x4:
21083 case WebAssembly::BI__builtin_wasm_narrow_u_i16x8_i32x4: {
21087 switch (BuiltinID) {
21088 case WebAssembly::BI__builtin_wasm_narrow_s_i8x16_i16x8:
21089 case WebAssembly::BI__builtin_wasm_narrow_s_i16x8_i32x4:
21090 IntNo = Intrinsic::wasm_narrow_signed;
21092 case WebAssembly::BI__builtin_wasm_narrow_u_i8x16_i16x8:
21093 case WebAssembly::BI__builtin_wasm_narrow_u_i16x8_i32x4:
21094 IntNo = Intrinsic::wasm_narrow_unsigned;
21097 llvm_unreachable(
"unexpected builtin ID");
21101 return Builder.CreateCall(Callee, {Low, High});
21103 case WebAssembly::BI__builtin_wasm_trunc_sat_s_zero_f64x2_i32x4:
21104 case WebAssembly::BI__builtin_wasm_trunc_sat_u_zero_f64x2_i32x4: {
21107 switch (BuiltinID) {
21108 case WebAssembly::BI__builtin_wasm_trunc_sat_s_zero_f64x2_i32x4:
21109 IntNo = Intrinsic::fptosi_sat;
21111 case WebAssembly::BI__builtin_wasm_trunc_sat_u_zero_f64x2_i32x4:
21112 IntNo = Intrinsic::fptoui_sat;
21115 llvm_unreachable(
"unexpected builtin ID");
21117 llvm::Type *SrcT = Vec->
getType();
21118 llvm::Type *TruncT = SrcT->getWithNewType(
Builder.getInt32Ty());
21121 Value *Splat = Constant::getNullValue(TruncT);
21124 case WebAssembly::BI__builtin_wasm_shuffle_i8x16: {
21129 while (OpIdx < 18) {
21130 std::optional<llvm::APSInt> LaneConst =
21132 assert(LaneConst &&
"Constant arg isn't actually constant?");
21133 Ops[OpIdx++] = llvm::ConstantInt::get(
getLLVMContext(), *LaneConst);
21136 return Builder.CreateCall(Callee, Ops);
21138 case WebAssembly::BI__builtin_wasm_relaxed_madd_f32x4:
21139 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f32x4:
21140 case WebAssembly::BI__builtin_wasm_relaxed_madd_f64x2:
21141 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f64x2: {
21146 switch (BuiltinID) {
21147 case WebAssembly::BI__builtin_wasm_relaxed_madd_f32x4:
21148 case WebAssembly::BI__builtin_wasm_relaxed_madd_f64x2:
21149 IntNo = Intrinsic::wasm_relaxed_madd;
21151 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f32x4:
21152 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f64x2:
21153 IntNo = Intrinsic::wasm_relaxed_nmadd;
21156 llvm_unreachable(
"unexpected builtin ID");
21159 return Builder.CreateCall(Callee, {A, B,
C});
21161 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i8x16:
21162 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i16x8:
21163 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i32x4:
21164 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i64x2: {
21170 return Builder.CreateCall(Callee, {A, B,
C});
21172 case WebAssembly::BI__builtin_wasm_relaxed_swizzle_i8x16: {
21176 return Builder.CreateCall(Callee, {Src, Indices});
21178 case WebAssembly::BI__builtin_wasm_relaxed_min_f32x4:
21179 case WebAssembly::BI__builtin_wasm_relaxed_max_f32x4:
21180 case WebAssembly::BI__builtin_wasm_relaxed_min_f64x2:
21181 case WebAssembly::BI__builtin_wasm_relaxed_max_f64x2: {
21185 switch (BuiltinID) {
21186 case WebAssembly::BI__builtin_wasm_relaxed_min_f32x4:
21187 case WebAssembly::BI__builtin_wasm_relaxed_min_f64x2:
21188 IntNo = Intrinsic::wasm_relaxed_min;
21190 case WebAssembly::BI__builtin_wasm_relaxed_max_f32x4:
21191 case WebAssembly::BI__builtin_wasm_relaxed_max_f64x2:
21192 IntNo = Intrinsic::wasm_relaxed_max;
21195 llvm_unreachable(
"unexpected builtin ID");
21198 return Builder.CreateCall(Callee, {LHS, RHS});
21200 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_i32x4_f32x4:
21201 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_i32x4_f32x4:
21202 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_zero_i32x4_f64x2:
21203 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_zero_i32x4_f64x2: {
21206 switch (BuiltinID) {
21207 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_i32x4_f32x4:
21208 IntNo = Intrinsic::wasm_relaxed_trunc_signed;
21210 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_i32x4_f32x4:
21211 IntNo = Intrinsic::wasm_relaxed_trunc_unsigned;
21213 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_zero_i32x4_f64x2:
21214 IntNo = Intrinsic::wasm_relaxed_trunc_signed_zero;
21216 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_zero_i32x4_f64x2:
21217 IntNo = Intrinsic::wasm_relaxed_trunc_unsigned_zero;
21220 llvm_unreachable(
"unexpected builtin ID");
21223 return Builder.CreateCall(Callee, {Vec});
21225 case WebAssembly::BI__builtin_wasm_relaxed_q15mulr_s_i16x8: {
21229 return Builder.CreateCall(Callee, {LHS, RHS});
21231 case WebAssembly::BI__builtin_wasm_relaxed_dot_i8x16_i7x16_s_i16x8: {
21236 return Builder.CreateCall(Callee, {LHS, RHS});
21238 case WebAssembly::BI__builtin_wasm_relaxed_dot_i8x16_i7x16_add_s_i32x4: {
21243 CGM.
getIntrinsic(Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed);
21244 return Builder.CreateCall(Callee, {LHS, RHS, Acc});
21246 case WebAssembly::BI__builtin_wasm_relaxed_dot_bf16x8_add_f32_f32x4: {
21252 return Builder.CreateCall(Callee, {LHS, RHS, Acc});
21254 case WebAssembly::BI__builtin_wasm_table_get: {
21265 "Unexpected reference type for __builtin_wasm_table_get");
21266 return Builder.CreateCall(Callee, {Table, Index});
21268 case WebAssembly::BI__builtin_wasm_table_set: {
21280 "Unexpected reference type for __builtin_wasm_table_set");
21281 return Builder.CreateCall(Callee, {Table, Index, Val});
21283 case WebAssembly::BI__builtin_wasm_table_size: {
21289 case WebAssembly::BI__builtin_wasm_table_grow: {
21302 "Unexpected reference type for __builtin_wasm_table_grow");
21304 return Builder.CreateCall(Callee, {Table, Val, NElems});
21306 case WebAssembly::BI__builtin_wasm_table_fill: {
21320 "Unexpected reference type for __builtin_wasm_table_fill");
21322 return Builder.CreateCall(Callee, {Table, Index, Val, NElems});
21324 case WebAssembly::BI__builtin_wasm_table_copy: {
21334 return Builder.CreateCall(Callee, {TableX, TableY, SrcIdx, DstIdx, NElems});
21341static std::pair<Intrinsic::ID, unsigned>
21344 unsigned BuiltinID;
21345 Intrinsic::ID IntrinsicID;
21348 static Info Infos[] = {
21349#define CUSTOM_BUILTIN_MAPPING(x,s) \
21350 { Hexagon::BI__builtin_HEXAGON_##x, Intrinsic::hexagon_##x, s },
21382#include "clang/Basic/BuiltinsHexagonMapCustomDep.def"
21383#undef CUSTOM_BUILTIN_MAPPING
21386 auto CmpInfo = [] (Info A, Info B) {
return A.BuiltinID < B.BuiltinID; };
21387 static const bool SortOnce = (llvm::sort(Infos, CmpInfo),
true);
21390 const Info *F = llvm::lower_bound(Infos, Info{BuiltinID, 0, 0}, CmpInfo);
21391 if (F == std::end(Infos) || F->BuiltinID != BuiltinID)
21392 return {Intrinsic::not_intrinsic, 0};
21394 return {F->IntrinsicID, F->VecLen};
21403 auto MakeCircOp = [
this, E](
unsigned IntID,
bool IsLoad) {
21417 for (
unsigned i = 1, e = E->
getNumArgs(); i != e; ++i)
21423 llvm::Value *NewBase = IsLoad ?
Builder.CreateExtractValue(
Result, 1)
21427 llvm::Value *RetVal =
21437 auto MakeBrevLd = [
this, E](
unsigned IntID, llvm::Type *DestTy) {
21454 CGM.
getIntrinsic(IntID), {BaseAddress, EmitScalarExpr(E->getArg(2))});
21457 llvm::Value *DestVal =
Builder.CreateExtractValue(
Result, 0);
21462 DestVal =
Builder.CreateTrunc(DestVal, DestTy);
21469 auto V2Q = [
this, VecLen] (llvm::Value *Vec) {
21470 Intrinsic::ID
ID = VecLen == 128 ? Intrinsic::hexagon_V6_vandvrt_128B
21471 : Intrinsic::hexagon_V6_vandvrt;
21473 {Vec,
Builder.getInt32(-1)});
21475 auto Q2V = [
this, VecLen] (llvm::Value *Pred) {
21476 Intrinsic::ID
ID = VecLen == 128 ? Intrinsic::hexagon_V6_vandqrt_128B
21477 : Intrinsic::hexagon_V6_vandqrt;
21479 {Pred,
Builder.getInt32(-1)});
21482 switch (BuiltinID) {
21486 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarry:
21487 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarry_128B:
21488 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarry:
21489 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarry_128B: {
21496 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1)), PredIn});
21498 llvm::Value *PredOut =
Builder.CreateExtractValue(
Result, 1);
21506 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarryo:
21507 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarryo_128B:
21508 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarryo:
21509 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarryo_128B: {
21515 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
21517 llvm::Value *PredOut =
Builder.CreateExtractValue(
Result, 1);
21523 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstoreq:
21524 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorenq:
21525 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentq:
21526 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentnq:
21527 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstoreq_128B:
21528 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorenq_128B:
21529 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentq_128B:
21530 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentnq_128B: {
21534 if (
auto *Cast = dyn_cast<ImplicitCastExpr>(PredOp)) {
21535 if (
Cast->getCastKind() == CK_BitCast)
21536 PredOp =
Cast->getSubExpr();
21539 for (
int i = 1, e = E->
getNumArgs(); i != e; ++i)
21544 case Hexagon::BI__builtin_HEXAGON_L2_loadrub_pci:
21545 case Hexagon::BI__builtin_HEXAGON_L2_loadrb_pci:
21546 case Hexagon::BI__builtin_HEXAGON_L2_loadruh_pci:
21547 case Hexagon::BI__builtin_HEXAGON_L2_loadrh_pci:
21548 case Hexagon::BI__builtin_HEXAGON_L2_loadri_pci:
21549 case Hexagon::BI__builtin_HEXAGON_L2_loadrd_pci:
21550 case Hexagon::BI__builtin_HEXAGON_L2_loadrub_pcr:
21551 case Hexagon::BI__builtin_HEXAGON_L2_loadrb_pcr:
21552 case Hexagon::BI__builtin_HEXAGON_L2_loadruh_pcr:
21553 case Hexagon::BI__builtin_HEXAGON_L2_loadrh_pcr:
21554 case Hexagon::BI__builtin_HEXAGON_L2_loadri_pcr:
21555 case Hexagon::BI__builtin_HEXAGON_L2_loadrd_pcr:
21556 return MakeCircOp(ID,
true);
21557 case Hexagon::BI__builtin_HEXAGON_S2_storerb_pci:
21558 case Hexagon::BI__builtin_HEXAGON_S2_storerh_pci:
21559 case Hexagon::BI__builtin_HEXAGON_S2_storerf_pci:
21560 case Hexagon::BI__builtin_HEXAGON_S2_storeri_pci:
21561 case Hexagon::BI__builtin_HEXAGON_S2_storerd_pci:
21562 case Hexagon::BI__builtin_HEXAGON_S2_storerb_pcr:
21563 case Hexagon::BI__builtin_HEXAGON_S2_storerh_pcr:
21564 case Hexagon::BI__builtin_HEXAGON_S2_storerf_pcr:
21565 case Hexagon::BI__builtin_HEXAGON_S2_storeri_pcr:
21566 case Hexagon::BI__builtin_HEXAGON_S2_storerd_pcr:
21567 return MakeCircOp(ID,
false);
21568 case Hexagon::BI__builtin_brev_ldub:
21569 return MakeBrevLd(Intrinsic::hexagon_L2_loadrub_pbr,
Int8Ty);
21570 case Hexagon::BI__builtin_brev_ldb:
21571 return MakeBrevLd(Intrinsic::hexagon_L2_loadrb_pbr,
Int8Ty);
21572 case Hexagon::BI__builtin_brev_lduh:
21573 return MakeBrevLd(Intrinsic::hexagon_L2_loadruh_pbr,
Int16Ty);
21574 case Hexagon::BI__builtin_brev_ldh:
21575 return MakeBrevLd(Intrinsic::hexagon_L2_loadrh_pbr,
Int16Ty);
21576 case Hexagon::BI__builtin_brev_ldw:
21577 return MakeBrevLd(Intrinsic::hexagon_L2_loadri_pbr,
Int32Ty);
21578 case Hexagon::BI__builtin_brev_ldd:
21579 return MakeBrevLd(Intrinsic::hexagon_L2_loadrd_pbr,
Int64Ty);
21592 unsigned ICEArguments = 0;
21600 if (BuiltinID == RISCVVector::BI__builtin_rvv_vget_v ||
21601 BuiltinID == RISCVVector::BI__builtin_rvv_vset_v)
21602 ICEArguments = 1 << 1;
21607 if (BuiltinID == RISCV::BI__builtin_riscv_ntl_load)
21608 ICEArguments |= (1 << 1);
21609 if (BuiltinID == RISCV::BI__builtin_riscv_ntl_store)
21610 ICEArguments |= (1 << 2);
21612 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; i++) {
21617 Ops.push_back(AggValue);
21623 Intrinsic::ID
ID = Intrinsic::not_intrinsic;
21627 constexpr unsigned RVV_VTA = 0x1;
21628 constexpr unsigned RVV_VMA = 0x2;
21629 int PolicyAttrs = 0;
21630 bool IsMasked =
false;
21634 switch (BuiltinID) {
21635 default: llvm_unreachable(
"unexpected builtin ID");
21636 case RISCV::BI__builtin_riscv_orc_b_32:
21637 case RISCV::BI__builtin_riscv_orc_b_64:
21638 case RISCV::BI__builtin_riscv_clz_32:
21639 case RISCV::BI__builtin_riscv_clz_64:
21640 case RISCV::BI__builtin_riscv_ctz_32:
21641 case RISCV::BI__builtin_riscv_ctz_64:
21642 case RISCV::BI__builtin_riscv_clmul_32:
21643 case RISCV::BI__builtin_riscv_clmul_64:
21644 case RISCV::BI__builtin_riscv_clmulh_32:
21645 case RISCV::BI__builtin_riscv_clmulh_64:
21646 case RISCV::BI__builtin_riscv_clmulr_32:
21647 case RISCV::BI__builtin_riscv_clmulr_64:
21648 case RISCV::BI__builtin_riscv_xperm4_32:
21649 case RISCV::BI__builtin_riscv_xperm4_64:
21650 case RISCV::BI__builtin_riscv_xperm8_32:
21651 case RISCV::BI__builtin_riscv_xperm8_64:
21652 case RISCV::BI__builtin_riscv_brev8_32:
21653 case RISCV::BI__builtin_riscv_brev8_64:
21654 case RISCV::BI__builtin_riscv_zip_32:
21655 case RISCV::BI__builtin_riscv_unzip_32: {
21656 switch (BuiltinID) {
21657 default: llvm_unreachable(
"unexpected builtin ID");
21659 case RISCV::BI__builtin_riscv_orc_b_32:
21660 case RISCV::BI__builtin_riscv_orc_b_64:
21661 ID = Intrinsic::riscv_orc_b;
21663 case RISCV::BI__builtin_riscv_clz_32:
21664 case RISCV::BI__builtin_riscv_clz_64: {
21667 if (
Result->getType() != ResultType)
21672 case RISCV::BI__builtin_riscv_ctz_32:
21673 case RISCV::BI__builtin_riscv_ctz_64: {
21676 if (
Result->getType() != ResultType)
21683 case RISCV::BI__builtin_riscv_clmul_32:
21684 case RISCV::BI__builtin_riscv_clmul_64:
21685 ID = Intrinsic::riscv_clmul;
21687 case RISCV::BI__builtin_riscv_clmulh_32:
21688 case RISCV::BI__builtin_riscv_clmulh_64:
21689 ID = Intrinsic::riscv_clmulh;
21691 case RISCV::BI__builtin_riscv_clmulr_32:
21692 case RISCV::BI__builtin_riscv_clmulr_64:
21693 ID = Intrinsic::riscv_clmulr;
21697 case RISCV::BI__builtin_riscv_xperm8_32:
21698 case RISCV::BI__builtin_riscv_xperm8_64:
21699 ID = Intrinsic::riscv_xperm8;
21701 case RISCV::BI__builtin_riscv_xperm4_32:
21702 case RISCV::BI__builtin_riscv_xperm4_64:
21703 ID = Intrinsic::riscv_xperm4;
21707 case RISCV::BI__builtin_riscv_brev8_32:
21708 case RISCV::BI__builtin_riscv_brev8_64:
21709 ID = Intrinsic::riscv_brev8;
21711 case RISCV::BI__builtin_riscv_zip_32:
21712 ID = Intrinsic::riscv_zip;
21714 case RISCV::BI__builtin_riscv_unzip_32:
21715 ID = Intrinsic::riscv_unzip;
21719 IntrinsicTypes = {ResultType};
21726 case RISCV::BI__builtin_riscv_sha256sig0:
21727 ID = Intrinsic::riscv_sha256sig0;
21729 case RISCV::BI__builtin_riscv_sha256sig1:
21730 ID = Intrinsic::riscv_sha256sig1;
21732 case RISCV::BI__builtin_riscv_sha256sum0:
21733 ID = Intrinsic::riscv_sha256sum0;
21735 case RISCV::BI__builtin_riscv_sha256sum1:
21736 ID = Intrinsic::riscv_sha256sum1;
21740 case RISCV::BI__builtin_riscv_sm4ks:
21741 ID = Intrinsic::riscv_sm4ks;
21743 case RISCV::BI__builtin_riscv_sm4ed:
21744 ID = Intrinsic::riscv_sm4ed;
21748 case RISCV::BI__builtin_riscv_sm3p0:
21749 ID = Intrinsic::riscv_sm3p0;
21751 case RISCV::BI__builtin_riscv_sm3p1:
21752 ID = Intrinsic::riscv_sm3p1;
21756 case RISCV::BI__builtin_riscv_ntl_load: {
21758 unsigned DomainVal = 5;
21759 if (Ops.size() == 2)
21760 DomainVal = cast<ConstantInt>(Ops[1])->getZExtValue();
21762 llvm::MDNode *RISCVDomainNode = llvm::MDNode::get(
21764 llvm::ConstantAsMetadata::get(
Builder.getInt32(DomainVal)));
21765 llvm::MDNode *NontemporalNode = llvm::MDNode::get(
21769 if(ResTy->isScalableTy()) {
21770 const ScalableVectorType *SVTy = cast<ScalableVectorType>(ResTy);
21771 llvm::Type *ScalarTy = ResTy->getScalarType();
21772 Width = ScalarTy->getPrimitiveSizeInBits() *
21773 SVTy->getElementCount().getKnownMinValue();
21775 Width = ResTy->getPrimitiveSizeInBits();
21779 Load->setMetadata(llvm::LLVMContext::MD_nontemporal, NontemporalNode);
21780 Load->setMetadata(
CGM.
getModule().getMDKindID(
"riscv-nontemporal-domain"),
21785 case RISCV::BI__builtin_riscv_ntl_store: {
21786 unsigned DomainVal = 5;
21787 if (Ops.size() == 3)
21788 DomainVal = cast<ConstantInt>(Ops[2])->getZExtValue();
21790 llvm::MDNode *RISCVDomainNode = llvm::MDNode::get(
21792 llvm::ConstantAsMetadata::get(
Builder.getInt32(DomainVal)));
21793 llvm::MDNode *NontemporalNode = llvm::MDNode::get(
21797 Store->setMetadata(llvm::LLVMContext::MD_nontemporal, NontemporalNode);
21805#include "clang/Basic/riscv_vector_builtin_cg.inc"
21807#include "clang/Basic/riscv_sifive_vector_builtin_cg.inc"
21810 assert(ID != Intrinsic::not_intrinsic);
21813 return Builder.CreateCall(F, Ops,
"");
Defines the clang::ASTContext interface.
#define PPC_LNX_FEATURE(NAME, DESC, ENUMNAME, ENUMVAL, HWCAPN)
#define X86_CPU_SUBTYPE(ENUM, STR)
#define X86_CPU_SUBTYPE_ALIAS(ENUM, ALIAS)
#define X86_VENDOR(ENUM, STRING)
#define X86_CPU_TYPE_ALIAS(ENUM, ALIAS)
#define X86_CPU_TYPE(ENUM, STR)
static constexpr Builtin::Info BuiltinInfo[]
static void Accumulate(SMap &SM, CFGBlock *B)
static Value * EmitSpecialRegisterBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Type *RegisterType, llvm::Type *ValueType, SpecialRegisterAccessKind AccessKind, StringRef SysReg="")
static llvm::Value * ARMMVEVectorReinterpret(CGBuilderTy &Builder, CodeGenFunction *CGF, llvm::Value *V, llvm::Type *DestType)
static Value * MakeBinaryAtomicValue(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E, AtomicOrdering Ordering=AtomicOrdering::SequentiallyConsistent)
Utility to insert an atomic instruction based on Intrinsic::ID and the expression node.
static char bitActionToX86BTCode(BitTest::ActionKind A)
#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier)
static Value * EmitAtomicCmpXchg128ForMSIntrin(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering SuccessOrdering)
#define INTRINSIC_X86_XSAVE_ID(NAME)
static CanQualType getOSLogArgType(ASTContext &C, int Size)
Get the argument type for arguments to os_log_helper.
static Value * EmitOverflowCheckedAbs(CodeGenFunction &CGF, const CallExpr *E, bool SanitizeOverflow)
static llvm::VectorType * GetFloatNeonType(CodeGenFunction *CGF, NeonTypeFlags IntTypeFlags)
static Value * tryUseTestFPKind(CodeGenFunction &CGF, unsigned BuiltinID, Value *V)
static llvm::Value * MVEImmediateShr(CGBuilderTy &Builder, llvm::Value *V, uint32_t Shift, bool Unsigned)
static bool areBOSTypesCompatible(int From, int To)
Checks if using the result of __builtin_object_size(p, From) in place of __builtin_object_size(p,...
static llvm::Value * SignOrZeroExtend(CGBuilderTy &Builder, llvm::Value *V, llvm::Type *T, bool Unsigned)
static void InsertExplicitZeroOperand(CGBuilderTy &Builder, llvm::Type *Ty, SmallVectorImpl< Value * > &Ops)
static const ARMVectorIntrinsicInfo AArch64SMEIntrinsicMap[]
static Value * EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E, SmallVectorImpl< Value * > &Ops, llvm::Triple::ArchType Arch)
#define MMA_VARIANTS_B1_AND(geom, type)
static void swapCommutativeSMEOperands(unsigned BuiltinID, SmallVectorImpl< Value * > &Ops)
static bool AArch64SISDIntrinsicsProvenSorted
static Value * EmitX86CompressExpand(CodeGenFunction &CGF, ArrayRef< Value * > Ops, bool IsCompress)
static const ARMVectorIntrinsicInfo AArch64SVEIntrinsicMap[]
static bool HasExtraNeonArgument(unsigned BuiltinID)
Return true if BuiltinID is an overloaded Neon intrinsic with an extra argument that specifies the ve...
static bool TypeRequiresBuiltinLaunderImp(const ASTContext &Ctx, QualType Ty, llvm::SmallPtrSetImpl< const Decl * > &Seen)
static Value * EmitAtomicIncrementValue(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering Ordering=AtomicOrdering::SequentiallyConsistent)
static std::pair< Intrinsic::ID, unsigned > getIntrinsicForHexagonNonClangBuiltin(unsigned BuiltinID)
static Value * emitRangedBuiltin(CodeGenFunction &CGF, unsigned IntrinsicID, int low, int high)
#define MMA_INTR(geom_op_type, layout)
static Value * EmitX86MaskedCompare(CodeGenFunction &CGF, unsigned CC, bool Signed, ArrayRef< Value * > Ops)
static Value * emitUnaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier)
static bool AArch64SVEIntrinsicsProvenSorted
static RValue EmitMSVCRTSetJmp(CodeGenFunction &CGF, MSVCSetJmpKind SJKind, const CallExpr *E)
MSVC handles setjmp a bit differently on different platforms.
static const ARMVectorIntrinsicInfo * findARMVectorIntrinsicInMap(ArrayRef< ARMVectorIntrinsicInfo > IntrinsicMap, unsigned BuiltinID, bool &MapProvenSorted)
static Value * EmitScalarFMAExpr(CodeGenFunction &CGF, const CallExpr *E, MutableArrayRef< Value * > Ops, Value *Upper, bool ZeroMask=false, unsigned PTIdx=0, bool NegAcc=false)
#define MUTATE_LDBL(func)
static Value * EmitX86ExpandLoad(CodeGenFunction &CGF, ArrayRef< Value * > Ops)
static unsigned CountCountedByAttrs(const RecordDecl *RD)
static Value * emitMaybeConstrainedFPToIntRoundBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
static Value * EmitX86MaskedLoad(CodeGenFunction &CGF, ArrayRef< Value * > Ops, Align Alignment)
static bool TypeRequiresBuiltinLaunder(CodeGenModule &CGM, QualType Ty)
Determine if the specified type requires laundering by checking if it is a dynamic class type or cont...
static void InsertExplicitUndefOperand(CGBuilderTy &Builder, llvm::Type *Ty, SmallVectorImpl< Value * > &Ops)
static Value * EmitISOVolatileLoad(CodeGenFunction &CGF, const CallExpr *E)
static struct WidthAndSignedness EncompassingIntegerType(ArrayRef< struct WidthAndSignedness > Types)
static Integer GetIntegerConstantValue(const Expr *E, ASTContext &Context)
#define MMA_VARIANTS(geom, type)
static bool AArch64SMEIntrinsicsProvenSorted
static llvm::Value * VectorZip(CGBuilderTy &Builder, llvm::Value *V0, llvm::Value *V1)
static Value * EmitTargetArchBuiltinExpr(CodeGenFunction *CGF, unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
constexpr unsigned SVEBitsPerBlock
static std::optional< CodeGenFunction::MSVCIntrin > translateX86ToMsvcIntrin(unsigned BuiltinID)
static const std::pair< unsigned, unsigned > NEONEquivalentIntrinsicMap[]
#define NEONMAP0(NameBase)
static Value * EmitX86MaskedStore(CodeGenFunction &CGF, ArrayRef< Value * > Ops, Align Alignment)
static Value * emitBinaryBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
static Value * emitBinaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E, Instruction::BinaryOps Op, bool Invert=false)
Utility to insert an atomic instruction based Intrinsic::ID and the expression node,...
static Value * EmitX86Muldq(CodeGenFunction &CGF, bool IsSigned, ArrayRef< Value * > Ops)
static llvm::Value * ARMMVEVectorElementReverse(CGBuilderTy &Builder, llvm::Value *V, unsigned ReverseWidth)
#define MMA_SATF_VARIANTS(geom, type)
static std::optional< CodeGenFunction::MSVCIntrin > translateAarch64ToMsvcIntrin(unsigned BuiltinID)
static std::optional< CodeGenFunction::MSVCIntrin > translateArmToMsvcIntrin(unsigned BuiltinID)
static llvm::Value * EmitBitTestIntrinsic(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E)
Emit a _bittest* intrinsic.
static const ARMVectorIntrinsicInfo ARMSIMDIntrinsicMap[]
static Value * EmitSignBit(CodeGenFunction &CGF, Value *V)
Emit the computation of the sign bit for a floating point value.
static Value * EmitFAbs(CodeGenFunction &CGF, Value *V)
EmitFAbs - Emit a call to @llvm.fabs().
#define CUSTOM_BUILTIN_MAPPING(x, s)
static Value * EmitX86CvtF16ToFloatExpr(CodeGenFunction &CGF, ArrayRef< Value * > Ops, llvm::Type *DstTy)
static bool isSpecialUnsignedMultiplySignedResult(unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info, WidthAndSignedness ResultInfo)
static llvm::Value * getDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType)
static llvm::Value * emitPPCLoadReserveIntrinsic(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E)
static llvm::Value * VectorUnzip(CGBuilderTy &Builder, llvm::Value *V, bool Odd)
static Value * EmitX86SExtMask(CodeGenFunction &CGF, Value *Op, llvm::Type *DstTy)
static Value * emitTernaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
static WidthAndSignedness getIntegerWidthAndSignedness(const clang::ASTContext &context, const clang::QualType Type)
static Value * EmitX86FunnelShift(CodeGenFunction &CGF, Value *Op0, Value *Op1, Value *Amt, bool IsRight)
static RValue EmitCheckedMixedSignMultiply(CodeGenFunction &CGF, const clang::Expr *Op1, WidthAndSignedness Op1Info, const clang::Expr *Op2, WidthAndSignedness Op2Info, const clang::Expr *ResultArg, QualType ResultQTy, WidthAndSignedness ResultInfo)
Emit a checked mixed-sign multiply.
static llvm::ScalableVectorType * getSVEVectorForElementType(llvm::Type *EltTy)
static unsigned mutateLongDoubleBuiltin(unsigned BuiltinID)
#define INTRINSIC_WITH_CC(NAME)
static llvm::FixedVectorType * GetNeonType(CodeGenFunction *CGF, NeonTypeFlags TypeFlags, bool HasLegalHalfType=true, bool V1Ty=false, bool AllowBFloatArgsAndRet=true)
static RValue EmitBinaryAtomic(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E)
static llvm::Value * ARMMVEConstantSplat(CGBuilderTy &Builder, llvm::Type *VT)
static Value * EmitX86MaskLogic(CodeGenFunction &CGF, Instruction::BinaryOps Opc, ArrayRef< Value * > Ops, bool InvertLHS=false)
static Value * EmitFromInt(CodeGenFunction &CGF, llvm::Value *V, QualType T, llvm::Type *ResultType)
static Value * EmitAMDGCNBallotForExec(CodeGenFunction &CGF, const CallExpr *E, llvm::Type *RegisterType, llvm::Type *ValueType, bool isExecHi)
static void initializeAlloca(CodeGenFunction &CGF, AllocaInst *AI, Value *Size, Align AlignmentInBytes)
static Value * EmitX86Select(CodeGenFunction &CGF, Value *Mask, Value *Op0, Value *Op1)
SpecialRegisterAccessKind
static Value * EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering SuccessOrdering=AtomicOrdering::SequentiallyConsistent)
This function should be invoked to emit atomic cmpxchg for Microsoft's _InterlockedCompareExchange* i...
static Address CheckAtomicAlignment(CodeGenFunction &CGF, const CallExpr *E)
static Value * EmitX86FMAExpr(CodeGenFunction &CGF, const CallExpr *E, ArrayRef< Value * > Ops, unsigned BuiltinID, bool IsAddSub)
static Value * getMaskVecValue(CodeGenFunction &CGF, Value *Mask, unsigned NumElts)
static bool isSpecialMixedSignMultiply(unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info, WidthAndSignedness ResultInfo)
Determine if a binop is a checked mixed-sign multiply we can specialize.
static Value * MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E, bool ReturnBool)
Utility to insert an atomic cmpxchg instruction.
static Value * emitBinaryExpMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Intrinsic::ID IntrinsicID, llvm::Intrinsic::ID ConstrainedIntrinsicID)
static Value * EmitToInt(CodeGenFunction &CGF, llvm::Value *V, QualType T, llvm::IntegerType *IntType)
Emit the conversions required to turn the given value into an integer of the given size.
static llvm::Value * ARMMVEVectorSplat(CGBuilderTy &Builder, llvm::Value *V)
static Value * EmitX86MaskedCompareResult(CodeGenFunction &CGF, Value *Cmp, unsigned NumElts, Value *MaskIn)
static Value * EmitX86CompressStore(CodeGenFunction &CGF, ArrayRef< Value * > Ops)
static bool NEONSIMDIntrinsicsProvenSorted
static const ARMVectorIntrinsicInfo AArch64SIMDIntrinsicMap[]
static Value * EmitNontemporalStore(CodeGenFunction &CGF, const CallExpr *E)
static llvm::Value * EmitOverflowIntrinsic(CodeGenFunction &CGF, const llvm::Intrinsic::ID IntrinsicID, llvm::Value *X, llvm::Value *Y, llvm::Value *&Carry)
Emit a call to llvm.
static Value * EmitCommonNeonSISDBuiltinExpr(CodeGenFunction &CGF, const ARMVectorIntrinsicInfo &SISDInfo, SmallVectorImpl< Value * > &Ops, const CallExpr *E)
static Value * emitFPIntBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
static Value * emitTernaryBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
#define MMA_LDST(n, geom_op_type)
static Value * EmitX86vpcom(CodeGenFunction &CGF, ArrayRef< Value * > Ops, bool IsSigned)
static Value * emitFrexpBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Intrinsic::ID IntrinsicID)
@ _InterlockedExchangeAdd_rel
@ _InterlockedIncrement_acq
@ _InterlockedExchange_nf
@ _InterlockedIncrement_nf
@ _InterlockedExchange_acq
@ _InterlockedCompareExchange128_rel
@ _InterlockedCompareExchange128_acq
@ _InterlockedCompareExchange_acq
@ _InterlockedExchangeAdd_nf
@ _InterlockedCompareExchange_nf
@ _InterlockedDecrement_rel
@ _InterlockedExchangeSub
@ _InterlockedExchangeAdd_acq
@ _InterlockedIncrement_rel
@ _InterlockedCompareExchange128_nf
@ _InterlockedCompareExchange128
@ _InterlockedExchange_rel
@ _InterlockedDecrement_nf
@ _InterlockedExchangeAdd
@ _InterlockedDecrement_acq
@ _InterlockedCompareExchange_rel
static Value * EmitX86ConvertToMask(CodeGenFunction &CGF, Value *In)
static Value * EmitISOVolatileStore(CodeGenFunction &CGF, const CallExpr *E)
static Value * EmitSystemZIntrinsicWithCC(CodeGenFunction &CGF, unsigned IntrinsicID, const CallExpr *E)
Handle a SystemZ function in which the final argument is a pointer to an int that receives the post-i...
static RValue EmitHipStdParUnsupportedBuiltin(CodeGenFunction *CGF, const FunctionDecl *FD)
static llvm::Value * EmitX86BitTestIntrinsic(CodeGenFunction &CGF, BitTest BT, const CallExpr *E, Value *BitBase, Value *BitPos)
static RValue EmitCheckedUnsignedMultiplySignedResult(CodeGenFunction &CGF, const clang::Expr *Op1, WidthAndSignedness Op1Info, const clang::Expr *Op2, WidthAndSignedness Op2Info, const clang::Expr *ResultArg, QualType ResultQTy, WidthAndSignedness ResultInfo)
static Value * emitUnaryBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, llvm::StringRef Name="")
static Value * emitCallMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID, llvm::Type *Ty, ArrayRef< Value * > Args)
static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *FD, const CallExpr *E, llvm::Constant *calleeValue)
static Value * EmitNontemporalLoad(CodeGenFunction &CGF, const CallExpr *E)
static Value * EmitX86Ternlog(CodeGenFunction &CGF, bool ZeroMask, ArrayRef< Value * > Ops)
static llvm::AtomicOrdering getBitTestAtomicOrdering(BitTest::InterlockingKind I)
#define MMA_VARIANTS_B1_XOR(geom, type)
#define MMA_VARIANTS_I4(geom, type)
Intrinsic::ID getDotProductIntrinsic(QualType QT, int elementCount)
static Value * EmitX86ConvertIntToFp(CodeGenFunction &CGF, const CallExpr *E, ArrayRef< Value * > Ops, bool IsSigned)
static Value * packTBLDVectorList(CodeGenFunction &CGF, ArrayRef< Value * > Ops, Value *ExtOp, Value *IndexOp, llvm::Type *ResTy, unsigned IntID, const char *Name)
static Value * EmitAbs(CodeGenFunction &CGF, Value *ArgValue, bool HasNSW)
static Value * EmitX86ScalarSelect(CodeGenFunction &CGF, Value *Mask, Value *Op0, Value *Op1)
static Value * EmitAtomicDecrementValue(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering Ordering=AtomicOrdering::SequentiallyConsistent)
static bool AArch64SIMDIntrinsicsProvenSorted
static const ARMVectorIntrinsicInfo AArch64SISDIntrinsicMap[]
CodeGenFunction::ComplexPairTy ComplexPairTy
#define ALIAS(NAME, TOK, FLAGS)
llvm::MachO::Record Record
static std::string getName(const CallEvent &Call)
static std::string toString(const clang::SanitizerSet &Sanitizers)
Produce a string containing comma-separated names of sanitizers in Sanitizers set.
Enumerates target-specific builtins in their own namespaces within namespace clang.
Defines the clang::TargetOptions class.
__DEVICE__ float modf(float __x, float *__iptr)
__DEVICE__ double nan(const char *)
__device__ __2f16 float __ockl_bool s
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
CharUnits getTypeAlignInChars(QualType T) const
Return the ABI-specified alignment of a (complete) type T, in characters.
unsigned getIntWidth(QualType T) const
const ASTRecordLayout & getASTRecordLayout(const RecordDecl *D) const
Get or compute information about the layout of the specified record (struct/union/class) D,...
Builtin::Context & BuiltinInfo
QualType getConstantArrayType(QualType EltTy, const llvm::APInt &ArySize, const Expr *SizeExpr, ArraySizeModifier ASM, unsigned IndexTypeQuals) const
Return the unique reference to the type for a constant array of the specified element type.
QualType getBaseElementType(const ArrayType *VAT) const
Return the innermost element type of an array type.
TypeInfo getTypeInfo(const Type *T) const
Get the size and alignment of the specified complete type in bits.
QualType getObjCIdType() const
Represents the Objective-CC id type.
bool hasSameUnqualifiedType(QualType T1, QualType T2) const
Determine whether the given types are equivalent after cvr-qualifiers have been removed.
const ArrayType * getAsArrayType(QualType T) const
Type Query functions.
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
QualType GetBuiltinType(unsigned ID, GetBuiltinTypeError &Error, unsigned *IntegerConstantArgs=nullptr) const
Return the type for the specified builtin.
const TargetInfo & getTargetInfo() const
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
unsigned getTargetAddressSpace(LangAS AS) const
@ GE_Missing_type
Missing a type.
ASTRecordLayout - This class contains layout information for one RecordDecl, which is a struct/union/...
CharUnits getSize() const
getSize - Get the record size in characters.
uint64_t getFieldOffset(unsigned FieldNo) const
getFieldOffset - Get the offset of the given field index, in bits.
Represents an array type, per C99 6.7.5.2 - Array Declarators.
QualType getElementType() const
bool isLibFunction(unsigned ID) const
Return true if this is a builtin for a libc/libm function, with a "__builtin_" prefix (e....
llvm::StringRef getName(unsigned ID) const
Return the identifier name for the specified builtin, e.g.
bool isConstWithoutErrnoAndExceptions(unsigned ID) const
Return true if this function has no side effects and doesn't read memory, except for possibly errno o...
bool isConstWithoutExceptions(unsigned ID) const
bool isConst(unsigned ID) const
Return true if this function has no side effects and doesn't read memory.
CallExpr - Represents a function call (C99 6.5.2.2, C++ [expr.call]).
Expr * getArg(unsigned Arg)
getArg - Return the specified argument.
bool hasStoredFPFeatures() const
SourceLocation getBeginLoc() const LLVM_READONLY
FunctionDecl * getDirectCallee()
If the callee is a FunctionDecl, return it. Otherwise return null.
FPOptionsOverride getFPFeatures() const
unsigned getNumArgs() const
getNumArgs - Return the number of actual arguments to this call.
QualType getCallReturnType(const ASTContext &Ctx) const
getCallReturnType - Get the return type of the call expr.
CharUnits - This is an opaque type for sizes expressed in character units.
llvm::Align getAsAlign() const
getAsAlign - Returns Quantity as a valid llvm::Align, Beware llvm::Align assumes power of two 8-bit b...
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
static CharUnits One()
One - Construct a CharUnits quantity of one.
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
XRayInstrSet XRayInstrumentationBundle
Set of XRay instrumentation kinds to emit.
Like RawAddress, an abstract representation of an aligned address, but the pointer contained in this ...
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
Return the pointer contained in this class after authenticating it and adding offset to it if necessa...
CharUnits getAlignment() const
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
Address withAlignment(CharUnits NewAlignment) const
Return address with different alignment, but same pointer and element type.
llvm::PointerType * getType() const
Return the type of the pointer value.
Address getAddress() const
static ApplyDebugLocation CreateArtificial(CodeGenFunction &CGF)
Apply TemporaryLocation if it is valid.
static ApplyDebugLocation CreateEmpty(CodeGenFunction &CGF)
Set the IRBuilder to not attach debug locations.
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
llvm::StoreInst * CreateAlignedStore(llvm::Value *Val, llvm::Value *Addr, CharUnits Align, bool IsVolatile=false)
Address CreateGEP(CodeGenFunction &CGF, Address Addr, llvm::Value *Index, const llvm::Twine &Name="")
llvm::CallInst * CreateMemMove(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
llvm::CallInst * CreateMemCpyInline(Address Dest, Address Src, uint64_t Size)
llvm::AtomicRMWInst * CreateAtomicRMW(llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val, llvm::AtomicOrdering Ordering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
llvm::CallInst * CreateMemSetInline(Address Dest, llvm::Value *Value, uint64_t Size)
llvm::StoreInst * CreateDefaultAlignedStore(llvm::Value *Val, llvm::Value *Addr, bool IsVolatile=false)
llvm::CallInst * CreateMemSet(Address Dest, llvm::Value *Value, llvm::Value *Size, bool IsVolatile=false)
llvm::AtomicCmpXchgInst * CreateAtomicCmpXchg(Address Addr, llvm::Value *Cmp, llvm::Value *New, llvm::AtomicOrdering SuccessOrdering, llvm::AtomicOrdering FailureOrdering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
Address CreateConstByteGEP(Address Addr, CharUnits Offset, const llvm::Twine &Name="")
Address CreateLaunderInvariantGroup(Address Addr)
llvm::CallInst * CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
llvm::LoadInst * CreateAlignedLoad(llvm::Type *Ty, llvm::Value *Addr, CharUnits Align, const llvm::Twine &Name="")
Address CreateAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
Address CreateConstInBoundsGEP(Address Addr, uint64_t Index, const llvm::Twine &Name="")
Given addr = T* ... produce name = getelementptr inbounds addr, i64 index where i64 is actually the t...
Address CreateInBoundsGEP(Address Addr, ArrayRef< llvm::Value * > IdxList, llvm::Type *ElementType, CharUnits Align, const Twine &Name="")
virtual std::string getDeviceSideName(const NamedDecl *ND)=0
Returns function or variable name on device side even if the current compilation is for host.
virtual llvm::GlobalVariable * getThrowInfo(QualType T)
All available information about a concrete callee.
static CGCallee forDirect(llvm::Constant *functionPtr, const CGCalleeInfo &abstractInfo=CGCalleeInfo())
llvm::DIType * getOrCreateStandaloneType(QualType Ty, SourceLocation Loc)
Emit standalone debug info for a type.
CGFunctionInfo - Class to encapsulate the information about a function definition.
virtual void EmitGCMemmoveCollectable(CodeGen::CodeGenFunction &CGF, Address DestPtr, Address SrcPtr, llvm::Value *Size)=0
EnqueuedBlockInfo emitOpenCLEnqueuedBlock(CodeGenFunction &CGF, const Expr *E)
CallArgList - Type for representing both the value and type of arguments in a call.
void add(RValue rvalue, QualType type)
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
llvm::Value * EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E, llvm::Triple::ArchType Arch)
llvm::Value * EmitFromMemory(llvm::Value *Value, QualType Ty)
EmitFromMemory - Change a scalar value from its memory representation to its value representation.
llvm::Value * EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID, const CallExpr *E)
void FinishFunction(SourceLocation EndLoc=SourceLocation())
FinishFunction - Complete IR generation of the current function.
llvm::Value * EmitLifetimeStart(llvm::TypeSize Size, llvm::Value *Addr)
std::pair< RValue, llvm::Value * > EmitAtomicCompareExchange(LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, llvm::AtomicOrdering Success=llvm::AtomicOrdering::SequentiallyConsistent, llvm::AtomicOrdering Failure=llvm::AtomicOrdering::SequentiallyConsistent, bool IsWeak=false, AggValueSlot Slot=AggValueSlot::ignored())
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
llvm::Value * EmitSVEPredicateCast(llvm::Value *Pred, llvm::ScalableVectorType *VTy)
llvm::CallInst * EmitTrapCall(llvm::Intrinsic::ID IntrID)
Emit a call to trap or debugtrap and attach function attribute "trap-func-name" if specified.
llvm::CallBase * addControlledConvergenceToken(llvm::CallBase *Input)
SanitizerSet SanOpts
Sanitizers enabled for this function.
RValue EmitBuiltinIsAligned(const CallExpr *E)
Emit IR for __builtin_is_aligned.
LValue EmitAggExprToLValue(const Expr *E)
EmitAggExprToLValue - Emit the computation of the specified expression of aggregate type into a tempo...
void EmitNonNullArgCheck(RValue RV, QualType ArgType, SourceLocation ArgLoc, AbstractCallee AC, unsigned ParmNum)
Create a check for a function parameter that may potentially be declared as non-null.
llvm::Value * EmitHexagonBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
void GetAArch64SVEProcessedOperands(unsigned BuiltinID, const CallExpr *E, SmallVectorImpl< llvm::Value * > &Ops, SVETypeFlags TypeFlags)
llvm::Value * EmitAMDGPUBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
void EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr)
void pushLifetimeExtendedDestroy(CleanupKind kind, Address addr, QualType type, Destroyer *destroyer, bool useEHCleanupForArray)
llvm::Value * EmitCheckedArgForBuiltin(const Expr *E, BuiltinCheckKind Kind)
Emits an argument for a call to a builtin.
llvm::Value * EmitSVEGatherLoad(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
CleanupKind getARCCleanupKind()
Retrieves the default cleanup kind for an ARC cleanup.
llvm::Value * EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitHLSLBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitAArch64SVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
RValue emitBuiltinOSLogFormat(const CallExpr &E)
Emit IR for __builtin_os_log_format.
LValue EmitLValue(const Expr *E, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitLValue - Emit code to compute a designator that specifies the location of the expression.
llvm::Value * EmitVAStartEnd(llvm::Value *ArgValue, bool IsStart)
Emits a call to an LLVM variable-argument intrinsic, either llvm.va_start or llvm....
llvm::Value * EmitSVEMaskedStore(const CallExpr *, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
llvm::Value * EmitSVEReinterpret(llvm::Value *Val, llvm::Type *Ty)
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
llvm::Value * EmitSEHExceptionInfo()
RValue EmitBuiltinAlignTo(const CallExpr *E, bool AlignUp)
Emit IR for __builtin_align_up/__builtin_align_down.
const LangOptions & getLangOpts() const
void ProcessOrderScopeAMDGCN(llvm::Value *Order, llvm::Value *Scope, llvm::AtomicOrdering &AO, llvm::SyncScope::ID &SSID)
llvm::Constant * EmitCheckTypeDescriptor(QualType T)
Emit a description of a type in a format suitable for passing to a runtime sanitizer handler.
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
void EmitTrapCheck(llvm::Value *Checked, SanitizerHandler CheckHandlerID)
Create a basic block that will call the trap intrinsic, and emit a conditional branch to it,...
void EmitUnreachable(SourceLocation Loc)
Emit a reached-unreachable diagnostic if Loc is valid and runtime checking is enabled.
llvm::Value * EmitSVETupleCreate(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType, ArrayRef< llvm::Value * > Ops)
Address makeNaturalAddressForPointer(llvm::Value *Ptr, QualType T, CharUnits Alignment=CharUnits::Zero(), bool ForPointeeType=false, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
Construct an address with the natural alignment of T.
ComplexPairTy EmitComplexExpr(const Expr *E, bool IgnoreReal=false, bool IgnoreImag=false)
EmitComplexExpr - Emit the computation of the specified expression of complex type,...
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
TypeCheckKind
Situations in which we might emit a check for the suitability of a pointer or glvalue.
@ TCK_Store
Checking the destination of a store. Must be suitably sized and aligned.
@ TCK_Load
Checking the operand of a load. Must be suitably sized and aligned.
llvm::Value * EmitSMELdrStr(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee, ReturnValueSlot ReturnValue, const CallArgList &Args, llvm::CallBase **callOrInvoke, bool IsMustTail, SourceLocation Loc)
EmitCall - Generate a call of the given function, expecting the given result type,...
llvm::Value * FormSVEBuiltinResult(llvm::Value *Call)
FormSVEBuiltinResult - Returns the struct of scalable vectors as a wider vector.
llvm::Value * EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx, const llvm::ElementCount &Count)
llvm::Type * ConvertTypeForMem(QualType T)
llvm::Value * EmitSVEMaskedLoad(const CallExpr *, llvm::Type *ReturnTy, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID, bool IsZExtReturn)
bool AlwaysEmitXRayCustomEvents() const
AlwaysEmitXRayCustomEvents - Return true if we must unconditionally emit XRay custom event handling c...
llvm::Value * EmitSVEDupX(llvm::Value *Scalar)
RawAddress CreateMemTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
@ Default
! No language constraints on evaluation order.
const TargetInfo & getTarget() const
llvm::Value * vectorWrapScalar16(llvm::Value *Op)
llvm::Function * LookupNeonLLVMIntrinsic(unsigned IntrinsicID, unsigned Modifier, llvm::Type *ArgTy, const CallExpr *E)
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
llvm::Value * EmitSEHExceptionCode()
Address EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitPointerWithAlignment - Given an expression with a pointer type, emit the value and compute our be...
llvm::Value * EmitTargetBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
EmitTargetBuiltinExpr - Emit the given builtin call.
llvm::Value * EmitCountedByFieldExpr(const Expr *Base, const FieldDecl *FAMDecl, const FieldDecl *CountDecl)
Build an expression accessing the "counted_by" field.
RValue EmitAnyExprToTemp(const Expr *E)
EmitAnyExprToTemp - Similarly to EmitAnyExpr(), however, the result will always be accessible even if...
RValue EmitCoroutineIntrinsic(const CallExpr *E, unsigned int IID)
llvm::Value * EmitAArch64SMEBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
RValue EmitAMDGPUDevicePrintfCallExpr(const CallExpr *E)
Address EmitArrayToPointerDecay(const Expr *Array, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
void EmitCheck(ArrayRef< std::pair< llvm::Value *, SanitizerMask > > Checked, SanitizerHandler Check, ArrayRef< llvm::Constant * > StaticArgs, ArrayRef< llvm::Value * > DynamicArgs)
Create a basic block that will either trap or call a handler function in the UBSan runtime with the p...
RValue EmitBuiltinNewDeleteCall(const FunctionProtoType *Type, const CallExpr *TheCallExpr, bool IsDelete)
llvm::Value * EmitSVETupleSetOrGet(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType, ArrayRef< llvm::Value * > Ops)
RValue EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
llvm::Value * EmitSVEPrefetchLoad(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
llvm::Value * EmitAArch64CompareBuiltinExpr(llvm::Value *Op, llvm::Type *Ty, const llvm::CmpInst::Predicate Fp, const llvm::CmpInst::Predicate Ip, const llvm::Twine &Name="")
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
AggValueSlot CreateAggTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateAggTemp - Create a temporary memory object for the given aggregate type.
llvm::ScalableVectorType * getSVEType(const SVETypeFlags &TypeFlags)
RValue emitRotate(const CallExpr *E, bool IsRotateRight)
llvm::Constant * EmitCheckSourceLocation(SourceLocation Loc)
Emit a description of a source location in a format suitable for passing to a runtime sanitizer handl...
void ErrorUnsupported(const Stmt *S, const char *Type)
ErrorUnsupported - Print out an error that codegen doesn't support the specified stmt yet.
CGDebugInfo * getDebugInfo()
const FieldDecl * FindFlexibleArrayMemberFieldAndOffset(ASTContext &Ctx, const RecordDecl *RD, const FieldDecl *FAMDecl, uint64_t &Offset)
Address EmitVAListRef(const Expr *E)
llvm::Value * EmitNeonShiftVector(llvm::Value *V, llvm::Type *Ty, bool negateForRightShift)
llvm::Value * EmitSVEMovl(const SVETypeFlags &TypeFlags, llvm::ArrayRef< llvm::Value * > Ops, unsigned BuiltinID)
void emitAlignmentAssumption(llvm::Value *PtrValue, QualType Ty, SourceLocation Loc, SourceLocation AssumptionLoc, llvm::Value *Alignment, llvm::Value *OffsetValue=nullptr)
const TargetCodeGenInfo & getTargetHooks() const
llvm::Value * EmitARMBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
llvm::Type * getEltType(const SVETypeFlags &TypeFlags)
void EmitAggExpr(const Expr *E, AggValueSlot AS)
EmitAggExpr - Emit the computation of the specified expression of aggregate type.
bool ShouldXRayInstrumentFunction() const
ShouldXRayInstrument - Return true if the current function should be instrumented with XRay nop sleds...
llvm::Value * EmitSVEPMull(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
llvm::Value * EmitToMemory(llvm::Value *Value, QualType Ty)
EmitToMemory - Change a scalar value from its value representation to its in-memory representation.
RValue EmitOpenMPDevicePrintfCallExpr(const CallExpr *E)
bool IsInPreservedAIRegion
True if CodeGen currently emits code inside presereved access index region.
llvm::Value * EmitARCRetain(QualType type, llvm::Value *value)
bool AlwaysEmitXRayTypedEvents() const
AlwaysEmitXRayTypedEvents - Return true if clang must unconditionally emit XRay typed event handling ...
void SetSqrtFPAccuracy(llvm::Value *Val)
Set the minimum required accuracy of the given sqrt operation based on CodeGenOpts.
llvm::Value * EmitSVEScatterStore(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::CallInst * EmitNounwindRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
ASTContext & getContext() const
llvm::Function * generateBuiltinOSLogHelperFunction(const analyze_os_log::OSLogBufferLayout &Layout, CharUnits BufferAlignment)
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
CGCallee EmitCallee(const Expr *E)
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
llvm::Value * EmitScalarOrConstFoldImmArg(unsigned ICEArguments, unsigned Idx, const CallExpr *E)
void checkTargetFeatures(const CallExpr *E, const FunctionDecl *TargetDecl)
llvm::Value * BuildVector(ArrayRef< llvm::Value * > Ops)
llvm::Value * EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitARMCDEBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
llvm::CallInst * EmitRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
llvm::Type * ConvertType(QualType T)
llvm::CallBase * EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee, ArrayRef< llvm::Value * > args, const Twine &name="")
llvm::Value * EmitSystemZBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitSMEReadWrite(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, LValue LV, QualType Type, SanitizerSet SkippedChecks=SanitizerSet(), llvm::Value *ArraySize=nullptr)
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T)
llvm::Value * EmitSMELd1St1(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::Value * EmitPPCBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
static Destroyer destroyARCStrongPrecise
void EmitARCIntrinsicUse(ArrayRef< llvm::Value * > values)
RValue EmitNVPTXDevicePrintfCallExpr(const CallExpr *E)
llvm::Value * EmitSVEStructLoad(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Address EmitMSVAListRef(const Expr *E)
Emit a "reference" to a __builtin_ms_va_list; this is always the value of the expression,...
llvm::Value * EmitCheckedInBoundsGEP(llvm::Type *ElemTy, llvm::Value *Ptr, ArrayRef< llvm::Value * > IdxList, bool SignedIndices, bool IsSubtraction, SourceLocation Loc, const Twine &Name="")
Same as IRBuilder::CreateInBoundsGEP, but additionally emits a check to detect undefined behavior whe...
llvm::Value * EmitNeonRShiftImm(llvm::Value *Vec, llvm::Value *Amt, llvm::Type *Ty, bool usgn, const char *name)
SmallVector< llvm::Type *, 2 > getSVEOverloadTypes(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType, ArrayRef< llvm::Value * > Ops)
static bool hasAggregateEvaluationKind(QualType T)
const FieldDecl * FindCountedByField(const FieldDecl *FD)
Find the FieldDecl specified in a FAM's "counted_by" attribute.
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
llvm::Value * EmitARMMVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
llvm::Value * EmitSVEStructStore(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
llvm::Value * EmitSEHAbnormalTermination()
llvm::Value * EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E)
Address ReturnValue
ReturnValue - The temporary alloca to hold the return value.
llvm::Value * EmitSVEAllTruePred(const SVETypeFlags &TypeFlags)
RValue GetUndefRValue(QualType Ty)
GetUndefRValue - Get an appropriate 'undef' rvalue for the given type.
llvm::Type * SVEBuiltinMemEltTy(const SVETypeFlags &TypeFlags)
SVEBuiltinMemEltTy - Returns the memory element type for this memory access builtin.
llvm::LLVMContext & getLLVMContext()
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
llvm::Value * EmitSMEZero(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::Value * EmitRISCVBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
llvm::Value * EmitCommonNeonBuiltinExpr(unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic, const char *NameHint, unsigned Modifier, const CallExpr *E, SmallVectorImpl< llvm::Value * > &Ops, Address PtrOp0, Address PtrOp1, llvm::Triple::ArchType Arch)
llvm::Value * EmitNeonCall(llvm::Function *F, SmallVectorImpl< llvm::Value * > &O, const char *name, unsigned shift=0, bool rightshift=false)
llvm::Value * EmitAnnotationCall(llvm::Function *AnnotationFn, llvm::Value *AnnotatedVal, StringRef AnnotationStr, SourceLocation Location, const AnnotateAttr *Attr)
Emit an annotation call (intrinsic).
llvm::ScalableVectorType * getSVEPredType(const SVETypeFlags &TypeFlags)
llvm::Value * EmitSVEGatherPrefetch(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
llvm::Value * EmitBPFBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
This class organizes the cross-function state that is used while generating LLVM code.
CGHLSLRuntime & getHLSLRuntime()
Return a reference to the configured HLSL runtime.
llvm::Module & getModule() const
llvm::FunctionCallee CreateRuntimeFunction(llvm::FunctionType *Ty, StringRef Name, llvm::AttributeList ExtraAttrs=llvm::AttributeList(), bool Local=false, bool AssumeConvergent=false)
Create or return a runtime function declaration with the specified type and name.
llvm::Constant * getBuiltinLibFunction(const FunctionDecl *FD, unsigned BuiltinID)
Given a builtin id for a function like "__builtin_fabsf", return a Function* for "fabsf".
DiagnosticsEngine & getDiags() const
void ErrorUnsupported(const Stmt *S, const char *Type)
Print out an error that codegen doesn't support the specified stmt yet.
CGCUDARuntime & getCUDARuntime()
Return a reference to the configured CUDA runtime.
CodeGenTypes & getTypes()
CGOpenCLRuntime & getOpenCLRuntime()
Return a reference to the configured OpenCL runtime.
const TargetInfo & getTarget() const
const llvm::DataLayout & getDataLayout() const
void Error(SourceLocation loc, StringRef error)
Emit a general error that something can't be done.
CGCXXABI & getCXXABI() const
llvm::Constant * GetFunctionStart(const ValueDecl *Decl)
const llvm::Triple & getTriple() const
void DecorateInstructionWithTBAA(llvm::Instruction *Inst, TBAAAccessInfo TBAAInfo)
DecorateInstructionWithTBAA - Decorate the instruction with a TBAA tag.
llvm::Constant * CreateRuntimeVariable(llvm::Type *Ty, StringRef Name)
Create a new runtime global variable with the specified type and name.
TBAAAccessInfo getTBAAAccessInfo(QualType AccessType)
getTBAAAccessInfo - Get TBAA information that describes an access to an object of the given type.
ASTContext & getContext() const
const CodeGenOptions & getCodeGenOpts() const
StringRef getMangledName(GlobalDecl GD)
CharUnits getNaturalPointeeTypeAlignment(QualType T, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
llvm::LLVMContext & getLLVMContext()
CGObjCRuntime & getObjCRuntime()
Return a reference to the configured Objective-C runtime.
void SetLLVMFunctionAttributes(GlobalDecl GD, const CGFunctionInfo &Info, llvm::Function *F, bool IsThunk)
Set the LLVM function attributes (sext, zext, etc).
void SetLLVMFunctionAttributesForDefinition(const Decl *D, llvm::Function *F)
Set the LLVM function attributes which only apply to a function definition.
llvm::Function * getIntrinsic(unsigned IID, ArrayRef< llvm::Type * > Tys=std::nullopt)
ConstantAddress GetAddrOfConstantCString(const std::string &Str, const char *GlobalName=nullptr)
Returns a pointer to a character array containing the literal and a terminating '\0' character.
llvm::Type * ConvertType(QualType T)
ConvertType - Convert type T into a llvm::Type.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
const CGFunctionInfo & arrangeBuiltinFunctionCall(QualType resultType, const CallArgList &args)
llvm::Constant * emitAbstract(const Expr *E, QualType T)
Emit the result of the given expression as an abstract constant, asserting that it succeeded.
Information for lazily generating a cleanup.
FunctionArgList - Type for representing both the decl and type of parameters to a function.
LValue - This represents an lvalue references.
llvm::Value * getRawBitFieldPointer(CodeGenFunction &CGF) const
Address getAddress(CodeGenFunction &CGF) const
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
void setNontemporal(bool Value)
llvm::Value * getPointer(CodeGenFunction &CGF) const
bool hasInfo() const
Returns true if there is LoopInfo on the stack.
const LoopInfo & getInfo() const
Return the LoopInfo for the current loop.
Information used when generating a structured loop.
llvm::BasicBlock * getHeader() const
Get the header block of this loop.
const LoopInfo * getParent() const
Returns the first outer loop containing this loop if any, nullptr otherwise.
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
static RValue getIgnored()
static RValue get(llvm::Value *V)
static RValue getAggregate(Address addr, bool isVolatile=false)
Convert an Address to an RValue.
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
An abstract representation of an aligned address.
llvm::Value * getPointer() const
static RawAddress invalid()
ReturnValueSlot - Contains the address where the return value of a function can be stored,...
virtual llvm::Value * encodeReturnAddress(CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const
Performs the code-generation required to convert the address of an instruction into a return address ...
virtual llvm::Value * decodeReturnAddress(CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const
Performs the code-generation required to convert a return address as stored by the system into the ac...
const T & getABIInfo() const
virtual int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const
Determines the DWARF register number for the stack pointer, for exception-handling purposes.
virtual llvm::Value * testFPKind(llvm::Value *V, unsigned BuiltinID, CGBuilderTy &Builder, CodeGenModule &CGM) const
Performs a target specific test of a floating point value for things like IsNaN, Infinity,...
Complex values, per C99 6.2.5p11.
Represents a concrete matrix type with constant number of rows and columns.
RecordDecl * getOuterLexicalRecordContext()
Retrieve the outermost lexically enclosing record context.
bool isImplicit() const
isImplicit - Indicates whether the declaration was implicitly generated by the implementation.
FunctionDecl * getAsFunction() LLVM_READONLY
Returns the function itself, or the templated function if this is a function template.
DeclContext * getDeclContext()
static bool isFlexibleArrayMemberLike(ASTContext &Context, const Decl *D, QualType Ty, LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel, bool IgnoreTemplateOrMacroSubstitution)
Whether it resembles a flexible array member.
Concrete class used by the front-end to report problems and issues.
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
This represents one expression.
bool EvaluateAsInt(EvalResult &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer,...
Expr * IgnoreParenCasts() LLVM_READONLY
Skip past any parentheses and casts which might surround this expression until reaching a fixed point...
llvm::APSInt EvaluateKnownConstInt(const ASTContext &Ctx, SmallVectorImpl< PartialDiagnosticAt > *Diag=nullptr) const
EvaluateKnownConstInt - Call EvaluateAsRValue and return the folded integer.
Expr * IgnoreParenImpCasts() LLVM_READONLY
Skip past any parentheses and implicit casts which might surround this expression until reaching a fi...
bool EvaluateAsFloat(llvm::APFloat &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsFloat - Return true if this is a constant which we can fold and convert to a floating point...
Expr * IgnoreParens() LLVM_READONLY
Skip past any parentheses which might surround this expression until reaching a fixed point.
@ NPC_ValueDependentIsNotNull
Specifies that a value-dependent expression should be considered to never be a null pointer constant.
ExprObjectKind getObjectKind() const
getObjectKind - The object kind that this expression produces.
bool EvaluateAsRValue(EvalResult &Result, const ASTContext &Ctx, bool InConstantContext=false) const
EvaluateAsRValue - Return true if this is a constant which we can fold to an rvalue using any crazy t...
bool HasSideEffects(const ASTContext &Ctx, bool IncludePossibleEffects=true) const
HasSideEffects - This routine returns true for all those expressions which have any effect other than...
Expr * IgnoreImpCasts() LLVM_READONLY
Skip past any implicit casts which might surround this expression until reaching a fixed point.
NullPointerConstantKind isNullPointerConstant(ASTContext &Ctx, NullPointerConstantValueDependence NPC) const
isNullPointerConstant - C99 6.3.2.3p3 - Test if this reduces down to a Null pointer constant.
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
std::optional< llvm::APSInt > getIntegerConstantExpr(const ASTContext &Ctx, SourceLocation *Loc=nullptr) const
isIntegerConstantExpr - Return the value if this expression is a valid integer constant expression.
bool tryEvaluateObjectSize(uint64_t &Result, ASTContext &Ctx, unsigned Type) const
If the current Expr is a pointer, this will try to statically determine the number of bytes available...
const ValueDecl * getAsBuiltinConstantDeclRef(const ASTContext &Context) const
If this expression is an unambiguous reference to a single declaration, in the style of __builtin_fun...
Represents difference between two FPOptions values.
Represents a member of a struct/union/class.
Represents a function declaration or definition.
const ParmVarDecl * getParamDecl(unsigned i) const
Represents a prototype with parameter type info, e.g.
GlobalDecl - represents a global declaration.
const Decl * getDecl() const
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
StrictFlexArraysLevelKind
StringRef getName() const
Get the name of identifier for this declaration as a StringRef.
std::string getNameAsString() const
Get a human-readable name for the declaration, even if it is one of the special kinds of names (C++ c...
Flags to identify the types for overloaded Neon builtins.
EltType getEltType() const
PointerType - C99 6.7.5.1 - Pointer Declarators.
QualType getPointeeType() const
A (possibly-)qualified type.
bool isVolatileQualified() const
Determine whether this type is volatile-qualified.
bool isWebAssemblyFuncrefType() const
Returns true if it is a WebAssembly Funcref Type.
LangAS getAddressSpace() const
Return the address space of this type.
bool isWebAssemblyExternrefType() const
Returns true if it is a WebAssembly Externref Type.
The collection of all-type qualifiers we support.
Represents a struct/union/class.
field_range fields() const
Flags to identify the types for overloaded SVE builtins.
bool isZExtReturn() const
bool isReverseUSDOT() const
bool isOverloadNone() const
MemEltType getMemEltType() const
bool isGatherLoad() const
bool isOverloadCvt() const
EltType getEltType() const
bool isOverloadDefault() const
bool isOverloadWhileRW() const
bool isReverseMergeAnyAccOp() const
bool isReductionQV() const
bool isInsertOp1SVALL() const
bool isAppendSVALL() const
bool isReverseMergeAnyBinOp() const
bool isStructStore() const
bool isTupleCreate() const
bool isGatherPrefetch() const
bool hasSplatOperand() const
MergeType getMergeType() const
bool isByteIndexed() const
bool isStructLoad() const
bool isOverloadWhileOrMultiVecCvt() const
unsigned getSplatOperand() const
bool isScatterStore() const
bool isReverseCompare() const
Scope - A scope is a transient data structure that is used while parsing the program.
Encodes a location in the source.
SourceLocation getBeginLoc() const LLVM_READONLY
Exposes information about the current target.
TargetOptions & getTargetOpts() const
Retrieve the target options.
virtual bool hasLegalHalfType() const
Determine whether _Float16 is supported on this target.
const llvm::Triple & getTriple() const
Returns the target triple of the primary target.
bool isLittleEndian() const
unsigned getMaxOpenCLWorkGroupSize() const
virtual bool checkArithmeticFenceSupported() const
Controls if __arithmetic_fence is supported in the targeted backend.
unsigned getSuitableAlign() const
Return the alignment that is the largest alignment ever used for any scalar/SIMD data type on the tar...
virtual std::string_view getClobbers() const =0
Returns a string of target-specific clobbers, in LLVM format.
llvm::CodeObjectVersionKind CodeObjectVersion
Code object version for AMDGPU.
The base class of the type hierarchy.
CXXRecordDecl * getAsCXXRecordDecl() const
Retrieves the CXXRecordDecl that this type refers to, either because the type is a RecordType or beca...
bool isBlockPointerType() const
bool isBooleanType() const
bool isSignedIntegerType() const
Return true if this is an integer type that is signed, according to C99 6.2.5p4 [char,...
bool isComplexType() const
isComplexType() does not include complex integers (a GCC extension).
bool isCountAttributedType() const
bool isPointerType() const
bool isIntegerType() const
isIntegerType() does not include complex integers (a GCC extension).
const T * castAs() const
Member-template castAs<specific type>.
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
bool isIntegralOrEnumerationType() const
Determine whether this type is an integral or enumeration type.
bool hasUnsignedIntegerRepresentation() const
Determine whether this type has an unsigned integer representation of some sort, e....
bool isBitIntType() const
bool hasSignedIntegerRepresentation() const
Determine whether this type has an signed integer representation of some sort, e.g....
bool isObjCObjectPointerType() const
bool hasFloatingRepresentation() const
Determine whether this type has a floating-point representation of some sort, e.g....
bool isFloatingType() const
bool isUnsignedIntegerType() const
Return true if this is an integer type that is unsigned, according to C99 6.2.5p6 [which returns true...
const T * getAs() const
Member-template getAs<specific type>'.
bool isRecordType() const
RecordDecl * getAsRecordDecl() const
Retrieves the RecordDecl this type refers to.
Represent the declaration of a variable (in which case it is an lvalue) a function (in which case it ...
Represents a GCC generic vector type.
unsigned getNumElements() const
SmallVector< OSLogBufferItem, 4 > Items
unsigned char getNumArgsByte() const
unsigned char getSummaryByte() const
Defines the clang::TargetInfo interface.
@ Type
The l-value was considered opaque, so the alignment was determined from a type.
llvm::Constant * initializationPatternFor(CodeGenModule &, llvm::Type *)
TypeEvaluationKind
The kind of evaluation to perform on values of a particular type.
@ EHCleanup
Denotes a cleanup that should run when a scope is exited using exceptional control flow (a throw stat...
constexpr XRayInstrMask Typed
constexpr XRayInstrMask Custom
bool computeOSLogBufferLayout(clang::ASTContext &Ctx, const clang::CallExpr *E, OSLogBufferLayout &layout)
const void * Store
Store - This opaque type encapsulates an immutable mapping from locations to values.
bool Dup(InterpState &S, CodePtr OpPC)
bool Ret(InterpState &S, CodePtr &PC, APValue &Result)
bool Zero(InterpState &S, CodePtr OpPC)
bool Mul(InterpState &S, CodePtr OpPC)
bool Neg(InterpState &S, CodePtr OpPC)
bool Load(InterpState &S, CodePtr OpPC)
bool Cast(InterpState &S, CodePtr OpPC)
The JSON file list parser is used to communicate input to InstallAPI.
@ DType
'dtype' clause, an alias for 'device_type', stored separately for diagnostic purposes.
@ OK_BitField
A bitfield object is a bitfield on a C or C++ record.
bool operator<(DeclarationName LHS, DeclarationName RHS)
Ordering on two declaration names.
@ Asm
Assembly: we accept this only so that we can preprocess it.
@ Result
The result type of a method or function.
LangAS
Defines the address space values used by the address space qualifier of QualType.
const FunctionProtoType * T
@ Success
Template argument deduction was successful.
@ Other
Other implicit parameter.
Diagnostic wrappers for TextAPI types for error reporting.
llvm::PointerType * VoidPtrTy
llvm::IntegerType * Int64Ty
llvm::PointerType * ConstGlobalsPtrTy
void* in the address space for constant globals
CharUnits getIntAlign() const
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
llvm::Type * HalfTy
half, bfloat, float, double
llvm::IntegerType * SizeTy
llvm::IntegerType * Int32Ty
llvm::IntegerType * IntPtrTy
llvm::IntegerType * IntTy
int
llvm::IntegerType * Int16Ty
llvm::PointerType * Int8PtrTy
llvm::PointerType * UnqualPtrTy
llvm::PointerType * AllocaInt8PtrTy
LangAS getASTAllocaAddressSpace() const
EvalResult is a struct with detailed info about an evaluated expression.
APValue Val
Val - This is the value the expression can be folded to.
void clear(SanitizerMask K=SanitizerKind::All)
Disable the sanitizers specified in K.
void set(SanitizerMask K, bool Value)
Enable or disable a certain (single) sanitizer.
bool has(SanitizerMask K) const
Check if a certain (single) sanitizer is enabled.
bool has(XRayInstrMask K) const
#define scalbln(__x, __y)
#define copysign(__x, __y)
#define remquo(__x, __y, __z)
#define nextafter(__x, __y)
#define nexttoward(__x, __y)
#define remainder(__x, __y)
#define fma(__x, __y, __z)