38#include "llvm/ADT/APFloat.h"
39#include "llvm/ADT/APInt.h"
40#include "llvm/ADT/FloatingPointMode.h"
41#include "llvm/ADT/SmallPtrSet.h"
42#include "llvm/ADT/StringExtras.h"
43#include "llvm/Analysis/ValueTracking.h"
44#include "llvm/IR/DataLayout.h"
45#include "llvm/IR/InlineAsm.h"
46#include "llvm/IR/Intrinsics.h"
47#include "llvm/IR/IntrinsicsAArch64.h"
48#include "llvm/IR/IntrinsicsAMDGPU.h"
49#include "llvm/IR/IntrinsicsARM.h"
50#include "llvm/IR/IntrinsicsBPF.h"
51#include "llvm/IR/IntrinsicsDirectX.h"
52#include "llvm/IR/IntrinsicsHexagon.h"
53#include "llvm/IR/IntrinsicsNVPTX.h"
54#include "llvm/IR/IntrinsicsPowerPC.h"
55#include "llvm/IR/IntrinsicsR600.h"
56#include "llvm/IR/IntrinsicsRISCV.h"
57#include "llvm/IR/IntrinsicsS390.h"
58#include "llvm/IR/IntrinsicsWebAssembly.h"
59#include "llvm/IR/IntrinsicsX86.h"
60#include "llvm/IR/MDBuilder.h"
61#include "llvm/IR/MatrixBuilder.h"
62#include "llvm/IR/MemoryModelRelaxationAnnotations.h"
63#include "llvm/Support/AMDGPUAddrSpace.h"
64#include "llvm/Support/ConvertUTF.h"
65#include "llvm/Support/MathExtras.h"
66#include "llvm/Support/ScopedPrinter.h"
67#include "llvm/TargetParser/AArch64TargetParser.h"
68#include "llvm/TargetParser/RISCVISAInfo.h"
69#include "llvm/TargetParser/RISCVTargetParser.h"
70#include "llvm/TargetParser/X86TargetParser.h"
76using namespace CodeGen;
80 Align AlignmentInBytes) {
82 switch (CGF.
getLangOpts().getTrivialAutoVarInit()) {
83 case LangOptions::TrivialAutoVarInitKind::Uninitialized:
86 case LangOptions::TrivialAutoVarInitKind::Zero:
87 Byte = CGF.
Builder.getInt8(0x00);
89 case LangOptions::TrivialAutoVarInitKind::Pattern: {
91 Byte = llvm::dyn_cast<llvm::ConstantInt>(
99 I->addAnnotationMetadata(
"auto-init");
105 Constant *FZeroConst = ConstantFP::getZero(CGF->
FloatTy);
110 FZeroConst = ConstantVector::getSplat(
111 ElementCount::getFixed(VecTy->getNumElements()), FZeroConst);
112 auto *FCompInst = CGF->
Builder.CreateFCmpOLT(Op0, FZeroConst);
113 CMP = CGF->
Builder.CreateIntrinsic(
115 {FCompInst},
nullptr);
117 CMP = CGF->
Builder.CreateFCmpOLT(Op0, FZeroConst);
120 LastInstr = CGF->
Builder.CreateIntrinsic(
121 CGF->
VoidTy, llvm::Intrinsic::dx_discard, {CMP},
nullptr);
126 CGF->
Builder.CreateCondBr(CMP, LT0, End);
128 CGF->
Builder.SetInsertPoint(LT0);
130 CGF->
Builder.CreateIntrinsic(CGF->
VoidTy, llvm::Intrinsic::spv_discard, {},
133 LastInstr = CGF->
Builder.CreateBr(End);
135 CGF->
Builder.SetInsertPoint(End);
137 llvm_unreachable(
"Backend Codegen not supported.");
145 const auto *OutArg1 = dyn_cast<HLSLOutArgExpr>(
E->getArg(1));
146 const auto *OutArg2 = dyn_cast<HLSLOutArgExpr>(
E->getArg(2));
157 Value *LowBits =
nullptr;
158 Value *HighBits =
nullptr;
162 llvm::Type *RetElementTy = CGF->
Int32Ty;
164 RetElementTy = llvm::VectorType::get(
165 CGF->
Int32Ty, ElementCount::getFixed(Op0VecTy->getNumElements()));
166 auto *RetTy = llvm::StructType::get(RetElementTy, RetElementTy);
168 CallInst *CI = CGF->
Builder.CreateIntrinsic(
169 RetTy, Intrinsic::dx_splitdouble, {Op0},
nullptr,
"hlsl.splitdouble");
171 LowBits = CGF->
Builder.CreateExtractValue(CI, 0);
172 HighBits = CGF->
Builder.CreateExtractValue(CI, 1);
177 if (!Op0->
getType()->isVectorTy()) {
178 FixedVectorType *DestTy = FixedVectorType::get(CGF->
Int32Ty, 2);
179 Value *Bitcast = CGF->
Builder.CreateBitCast(Op0, DestTy);
181 LowBits = CGF->
Builder.CreateExtractElement(Bitcast, (uint64_t)0);
182 HighBits = CGF->
Builder.CreateExtractElement(Bitcast, 1);
185 if (
const auto *VecTy =
187 NumElements = VecTy->getNumElements();
189 FixedVectorType *Uint32VecTy =
190 FixedVectorType::get(CGF->
Int32Ty, NumElements * 2);
191 Value *Uint32Vec = CGF->
Builder.CreateBitCast(Op0, Uint32VecTy);
192 if (NumElements == 1) {
193 LowBits = CGF->
Builder.CreateExtractElement(Uint32Vec, (uint64_t)0);
194 HighBits = CGF->
Builder.CreateExtractElement(Uint32Vec, 1);
197 for (
int I = 0,
E = NumElements; I !=
E; ++I) {
198 EvenMask.push_back(I * 2);
199 OddMask.push_back(I * 2 + 1);
201 LowBits = CGF->
Builder.CreateShuffleVector(Uint32Vec, EvenMask);
202 HighBits = CGF->
Builder.CreateShuffleVector(Uint32Vec, OddMask);
216 "asdouble operands types mismatch");
220 llvm::Type *ResultType = CGF.
DoubleTy;
223 N = VTy->getNumElements();
224 ResultType = llvm::FixedVectorType::get(CGF.
DoubleTy, N);
228 return CGF.
Builder.CreateIntrinsic(
229 ResultType, Intrinsic::dx_asdouble,
233 OpLowBits = CGF.
Builder.CreateVectorSplat(1, OpLowBits);
234 OpHighBits = CGF.
Builder.CreateVectorSplat(1, OpHighBits);
238 for (
int i = 0; i < N; i++) {
240 Mask.push_back(i + N);
243 Value *BitVec = CGF.
Builder.CreateShuffleVector(OpLowBits, OpHighBits, Mask);
245 return CGF.
Builder.CreateBitCast(BitVec, ResultType);
252 llvm::Metadata *Ops[] = {llvm::MDString::get(Context,
"x18")};
253 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
254 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
257 llvm::Value *X18 = CGF.
Builder.CreateCall(F, Metadata);
264 unsigned BuiltinID) {
273 static SmallDenseMap<unsigned, StringRef, 64> F128Builtins{
274 {Builtin::BI__builtin___fprintf_chk,
"__fprintf_chkieee128"},
275 {Builtin::BI__builtin___printf_chk,
"__printf_chkieee128"},
276 {Builtin::BI__builtin___snprintf_chk,
"__snprintf_chkieee128"},
277 {Builtin::BI__builtin___sprintf_chk,
"__sprintf_chkieee128"},
278 {Builtin::BI__builtin___vfprintf_chk,
"__vfprintf_chkieee128"},
279 {Builtin::BI__builtin___vprintf_chk,
"__vprintf_chkieee128"},
280 {Builtin::BI__builtin___vsnprintf_chk,
"__vsnprintf_chkieee128"},
281 {Builtin::BI__builtin___vsprintf_chk,
"__vsprintf_chkieee128"},
282 {Builtin::BI__builtin_fprintf,
"__fprintfieee128"},
283 {Builtin::BI__builtin_printf,
"__printfieee128"},
284 {Builtin::BI__builtin_snprintf,
"__snprintfieee128"},
285 {Builtin::BI__builtin_sprintf,
"__sprintfieee128"},
286 {Builtin::BI__builtin_vfprintf,
"__vfprintfieee128"},
287 {Builtin::BI__builtin_vprintf,
"__vprintfieee128"},
288 {Builtin::BI__builtin_vsnprintf,
"__vsnprintfieee128"},
289 {Builtin::BI__builtin_vsprintf,
"__vsprintfieee128"},
290 {Builtin::BI__builtin_fscanf,
"__fscanfieee128"},
291 {Builtin::BI__builtin_scanf,
"__scanfieee128"},
292 {Builtin::BI__builtin_sscanf,
"__sscanfieee128"},
293 {Builtin::BI__builtin_vfscanf,
"__vfscanfieee128"},
294 {Builtin::BI__builtin_vscanf,
"__vscanfieee128"},
295 {Builtin::BI__builtin_vsscanf,
"__vsscanfieee128"},
296 {Builtin::BI__builtin_nexttowardf128,
"__nexttowardieee128"},
302 static SmallDenseMap<unsigned, StringRef, 4> AIXLongDouble64Builtins{
303 {Builtin::BI__builtin_frexpl,
"frexp"},
304 {Builtin::BI__builtin_ldexpl,
"ldexp"},
305 {Builtin::BI__builtin_modfl,
"modf"},
311 if (FD->
hasAttr<AsmLabelAttr>())
317 &
getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad() &&
318 F128Builtins.contains(BuiltinID))
319 Name = F128Builtins[BuiltinID];
322 &llvm::APFloat::IEEEdouble() &&
323 AIXLongDouble64Builtins.contains(BuiltinID))
324 Name = AIXLongDouble64Builtins[BuiltinID];
329 llvm::FunctionType *Ty =
332 return GetOrCreateLLVMFunction(Name, Ty,
D,
false);
338 QualType T, llvm::IntegerType *IntType) {
341 if (
V->getType()->isPointerTy())
342 return CGF.
Builder.CreatePtrToInt(
V, IntType);
344 assert(
V->getType() == IntType);
352 if (ResultType->isPointerTy())
353 return CGF.
Builder.CreateIntToPtr(
V, ResultType);
355 assert(
V->getType() == ResultType);
366 if (Align % Bytes != 0) {
379 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
389 llvm::IntegerType *IntType = llvm::IntegerType::get(
393 llvm::Type *ValueType = Val->getType();
421 llvm::AtomicRMWInst::BinOp Kind,
430 llvm::AtomicRMWInst::BinOp Kind,
432 Instruction::BinaryOps Op,
433 bool Invert =
false) {
442 llvm::IntegerType *IntType = llvm::IntegerType::get(
446 llvm::Type *ValueType = Val->getType();
450 Kind, DestAddr, Val, llvm::AtomicOrdering::SequentiallyConsistent);
455 llvm::ConstantInt::getAllOnesValue(IntType));
479 llvm::IntegerType *IntType = llvm::IntegerType::get(
483 llvm::Type *ValueType = Cmp->getType();
488 DestAddr, Cmp, New, llvm::AtomicOrdering::SequentiallyConsistent,
489 llvm::AtomicOrdering::SequentiallyConsistent);
492 return CGF.
Builder.CreateZExt(CGF.
Builder.CreateExtractValue(Pair, 1),
515 AtomicOrdering SuccessOrdering = AtomicOrdering::SequentiallyConsistent) {
527 auto *RTy = Exchange->getType();
531 if (RTy->isPointerTy()) {
537 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release ?
538 AtomicOrdering::Monotonic :
546 DestAddr, Comparand, Exchange, SuccessOrdering, FailureOrdering);
547 CmpXchg->setVolatile(
true);
550 if (RTy->isPointerTy()) {
571 AtomicOrdering SuccessOrdering) {
572 assert(
E->getNumArgs() == 4);
578 assert(DestPtr->getType()->isPointerTy());
579 assert(!ExchangeHigh->getType()->isPointerTy());
580 assert(!ExchangeLow->getType()->isPointerTy());
583 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release
584 ? AtomicOrdering::Monotonic
589 llvm::Type *Int128Ty = llvm::IntegerType::get(CGF.
getLLVMContext(), 128);
590 Address DestAddr(DestPtr, Int128Ty,
595 ExchangeHigh = CGF.
Builder.CreateZExt(ExchangeHigh, Int128Ty);
596 ExchangeLow = CGF.
Builder.CreateZExt(ExchangeLow, Int128Ty);
598 CGF.
Builder.CreateShl(ExchangeHigh, llvm::ConstantInt::get(Int128Ty, 64));
599 llvm::Value *Exchange = CGF.
Builder.CreateOr(ExchangeHigh, ExchangeLow);
605 SuccessOrdering, FailureOrdering);
611 CXI->setVolatile(
true);
623 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
629 AtomicRMWInst::Add, DestAddr, ConstantInt::get(IntTy, 1), Ordering);
630 return CGF.
Builder.CreateAdd(
Result, ConstantInt::get(IntTy, 1));
635 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
641 AtomicRMWInst::Sub, DestAddr, ConstantInt::get(IntTy, 1), Ordering);
642 return CGF.
Builder.CreateSub(
Result, ConstantInt::get(IntTy, 1));
653 Load->setVolatile(
true);
663 llvm::StoreInst *Store =
665 Store->setVolatile(
true);
674 unsigned ConstrainedIntrinsicID) {
677 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
678 if (CGF.
Builder.getIsFPConstrained()) {
680 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0 });
683 return CGF.
Builder.CreateCall(F, Src0);
691 unsigned ConstrainedIntrinsicID) {
695 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
696 if (CGF.
Builder.getIsFPConstrained()) {
698 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0, Src1 });
701 return CGF.
Builder.CreateCall(F, { Src0, Src1 });
708 llvm::Intrinsic::ID ConstrainedIntrinsicID) {
712 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
713 if (CGF.
Builder.getIsFPConstrained()) {
715 {Src0->getType(), Src1->getType()});
716 return CGF.
Builder.CreateConstrainedFPCall(F, {Src0, Src1});
721 return CGF.
Builder.CreateCall(F, {Src0, Src1});
728 unsigned ConstrainedIntrinsicID) {
733 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
734 if (CGF.
Builder.getIsFPConstrained()) {
736 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0, Src1, Src2 });
739 return CGF.
Builder.CreateCall(F, { Src0, Src1, Src2 });
746 unsigned IntrinsicID,
747 unsigned ConstrainedIntrinsicID,
751 if (CGF.
Builder.getIsFPConstrained())
756 if (CGF.
Builder.getIsFPConstrained())
757 return CGF.
Builder.CreateConstrainedFPCall(F, Args);
759 return CGF.
Builder.CreateCall(F, Args);
768 unsigned IntrinsicID,
769 llvm::StringRef Name =
"") {
770 static_assert(N,
"expect non-empty argument");
772 for (
unsigned I = 0; I < N; ++I)
775 return CGF.
Builder.CreateCall(F, Args, Name);
781 unsigned IntrinsicID) {
786 return CGF.
Builder.CreateCall(F, {Src0, Src1});
792 unsigned IntrinsicID,
793 unsigned ConstrainedIntrinsicID) {
797 if (CGF.
Builder.getIsFPConstrained()) {
798 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
800 {ResultType, Src0->getType()});
801 return CGF.
Builder.CreateConstrainedFPCall(F, {Src0});
805 return CGF.
Builder.CreateCall(F, Src0);
810 llvm::Intrinsic::ID IntrinsicID) {
818 llvm::Value *
Call = CGF.
Builder.CreateCall(F, Src0);
820 llvm::Value *Exp = CGF.
Builder.CreateExtractValue(
Call, 1);
828 llvm::Intrinsic::ID IntrinsicID) {
833 llvm::Function *F = CGF.
CGM.
getIntrinsic(IntrinsicID, {Val->getType()});
834 llvm::Value *
Call = CGF.
Builder.CreateCall(F, Val);
836 llvm::Value *SinResult = CGF.
Builder.CreateExtractValue(
Call, 0);
837 llvm::Value *CosResult = CGF.
Builder.CreateExtractValue(
Call, 1);
843 llvm::StoreInst *StoreSin =
845 llvm::StoreInst *StoreCos =
852 MDNode *
Domain = MDHelper.createAnonymousAliasScopeDomain();
853 MDNode *AliasScope = MDHelper.createAnonymousAliasScope(
Domain);
854 MDNode *AliasScopeList = MDNode::get(
Call->getContext(), AliasScope);
855 StoreSin->setMetadata(LLVMContext::MD_alias_scope, AliasScopeList);
856 StoreCos->setMetadata(LLVMContext::MD_noalias, AliasScopeList);
863 Call->setDoesNotAccessMemory();
872 llvm::Type *Ty =
V->getType();
873 int Width = Ty->getPrimitiveSizeInBits();
874 llvm::Type *IntTy = llvm::IntegerType::get(
C, Width);
876 if (Ty->isPPC_FP128Ty()) {
886 Value *ShiftCst = llvm::ConstantInt::get(IntTy, Width);
891 IntTy = llvm::IntegerType::get(
C, Width);
894 Value *Zero = llvm::Constant::getNullValue(IntTy);
895 return CGF.
Builder.CreateICmpSLT(
V, Zero);
904 auto IsIndirect = [&](
ABIArgInfo const &info) {
905 return info.isIndirect() || info.isIndirectAliased() || info.isInAlloca();
910 return IsIndirect(ArgInfo.info);
915 const CallExpr *
E, llvm::Constant *calleeValue) {
916 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
918 llvm::CallBase *callOrInvoke =
nullptr;
922 nullptr, &callOrInvoke, &FnInfo);
927 bool ConstWithoutErrnoAndExceptions =
931 if (ConstWithoutErrnoAndExceptions && CGF.
CGM.
getLangOpts().MathErrno &&
932 !CGF.
Builder.getIsFPConstrained() &&
Call.isScalar() &&
953 const llvm::Intrinsic::ID IntrinsicID,
954 llvm::Value *
X, llvm::Value *Y,
955 llvm::Value *&Carry) {
957 assert(
X->getType() == Y->getType() &&
958 "Arguments must be the same type. (Did you forget to make sure both "
959 "arguments have the same integer width?)");
962 llvm::Value *Tmp = CGF.
Builder.CreateCall(Callee, {
X, Y});
963 Carry = CGF.
Builder.CreateExtractValue(Tmp, 1);
964 return CGF.
Builder.CreateExtractValue(Tmp, 0);
971 llvm::ConstantRange CR(APInt(32, low), APInt(32, high));
972 Call->addRangeRetAttr(CR);
973 Call->addRetAttr(llvm::Attribute::AttrKind::NoUndef);
978 struct WidthAndSignedness {
984static WidthAndSignedness
996static struct WidthAndSignedness
998 assert(Types.size() > 0 &&
"Empty list of types.");
1002 for (
const auto &
Type : Types) {
1011 for (
const auto &
Type : Types) {
1013 if (Width < MinWidth) {
1022 Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend;
1033 return From == To || (From == 0 && To == 1) || (From == 3 && To == 2);
1038 return ConstantInt::get(ResType, (
Type & 2) ? 0 : -1,
true);
1042CodeGenFunction::evaluateOrEmitBuiltinObjectSize(
const Expr *
E,
unsigned Type,
1043 llvm::IntegerType *ResType,
1044 llvm::Value *EmittedE,
1048 return emitBuiltinObjectSize(
E,
Type, ResType, EmittedE, IsDynamic);
1049 return ConstantInt::get(ResType, ObjectSize,
true);
1063 if ((!FAMDecl || FD == FAMDecl) &&
1065 Ctx, FD, FD->getType(), StrictFlexArraysLevel,
1093 if (FD->getType()->isCountAttributedType())
1105CodeGenFunction::emitFlexibleArrayMemberSize(
const Expr *
E,
unsigned Type,
1106 llvm::IntegerType *ResType) {
1135 const Expr *Idx =
nullptr;
1137 if (
const auto *UO = dyn_cast<UnaryOperator>(
Base);
1138 UO && UO->getOpcode() == UO_AddrOf) {
1140 if (
const auto *ASE = dyn_cast<ArraySubscriptExpr>(SubExpr)) {
1141 Base = ASE->getBase()->IgnoreParenImpCasts();
1144 if (
const auto *IL = dyn_cast<IntegerLiteral>(Idx)) {
1145 int64_t Val = IL->getValue().getSExtValue();
1162 if (
const auto *ME = dyn_cast<MemberExpr>(
Base)) {
1164 const ValueDecl *VD = ME->getMemberDecl();
1166 FAMDecl = dyn_cast<FieldDecl>(VD);
1169 }
else if (
const auto *DRE = dyn_cast<DeclRefExpr>(
Base)) {
1171 QualType Ty = DRE->getDecl()->getType();
1224 if (isa<DeclRefExpr>(
Base))
1248 CountedByInst =
Builder.CreateIntCast(CountedByInst, ResType, IsSigned);
1251 Value *IdxInst =
nullptr;
1259 IdxInst =
Builder.CreateIntCast(IdxInst, ResType, IdxSigned);
1264 Builder.CreateSub(CountedByInst, IdxInst,
"", !IsSigned, IsSigned);
1270 llvm::Constant *ElemSize =
1271 llvm::ConstantInt::get(ResType,
Size.getQuantity(), IsSigned);
1273 Builder.CreateMul(CountedByInst, ElemSize,
"", !IsSigned, IsSigned);
1274 Res =
Builder.CreateIntCast(Res, ResType, IsSigned);
1283 return Builder.CreateSelect(Cmp, Res, ConstantInt::get(ResType, 0, IsSigned));
1296CodeGenFunction::emitBuiltinObjectSize(
const Expr *
E,
unsigned Type,
1297 llvm::IntegerType *ResType,
1298 llvm::Value *EmittedE,
bool IsDynamic) {
1302 auto *Param = dyn_cast<ParmVarDecl>(
D->getDecl());
1303 auto *PS =
D->getDecl()->
getAttr<PassObjectSizeAttr>();
1304 if (Param !=
nullptr && PS !=
nullptr &&
1306 auto Iter = SizeArguments.find(Param);
1307 assert(
Iter != SizeArguments.end());
1310 auto DIter = LocalDeclMap.find(
D);
1311 assert(DIter != LocalDeclMap.end());
1321 if (
Value *
V = emitFlexibleArrayMemberSize(
E,
Type, ResType))
1332 assert(Ptr->
getType()->isPointerTy() &&
1333 "Non-pointer passed to __builtin_object_size?");
1349 enum ActionKind : uint8_t { TestOnly, Complement, Reset,
Set };
1350 enum InterlockingKind : uint8_t {
1359 InterlockingKind Interlocking;
1362 static BitTest decodeBitTestBuiltin(
unsigned BuiltinID);
1367BitTest BitTest::decodeBitTestBuiltin(
unsigned BuiltinID) {
1368 switch (BuiltinID) {
1370 case Builtin::BI_bittest:
1371 return {TestOnly, Unlocked,
false};
1372 case Builtin::BI_bittestandcomplement:
1373 return {Complement, Unlocked,
false};
1374 case Builtin::BI_bittestandreset:
1375 return {Reset, Unlocked,
false};
1376 case Builtin::BI_bittestandset:
1377 return {
Set, Unlocked,
false};
1378 case Builtin::BI_interlockedbittestandreset:
1379 return {Reset, Sequential,
false};
1380 case Builtin::BI_interlockedbittestandset:
1381 return {
Set, Sequential,
false};
1384 case Builtin::BI_bittest64:
1385 return {TestOnly, Unlocked,
true};
1386 case Builtin::BI_bittestandcomplement64:
1387 return {Complement, Unlocked,
true};
1388 case Builtin::BI_bittestandreset64:
1389 return {Reset, Unlocked,
true};
1390 case Builtin::BI_bittestandset64:
1391 return {
Set, Unlocked,
true};
1392 case Builtin::BI_interlockedbittestandreset64:
1393 return {Reset, Sequential,
true};
1394 case Builtin::BI_interlockedbittestandset64:
1395 return {
Set, Sequential,
true};
1398 case Builtin::BI_interlockedbittestandset_acq:
1399 return {
Set, Acquire,
false};
1400 case Builtin::BI_interlockedbittestandset_rel:
1401 return {
Set, Release,
false};
1402 case Builtin::BI_interlockedbittestandset_nf:
1403 return {
Set, NoFence,
false};
1404 case Builtin::BI_interlockedbittestandreset_acq:
1405 return {Reset, Acquire,
false};
1406 case Builtin::BI_interlockedbittestandreset_rel:
1407 return {Reset, Release,
false};
1408 case Builtin::BI_interlockedbittestandreset_nf:
1409 return {Reset, NoFence,
false};
1411 llvm_unreachable(
"expected only bittest intrinsics");
1416 case BitTest::TestOnly:
return '\0';
1417 case BitTest::Complement:
return 'c';
1418 case BitTest::Reset:
return 'r';
1419 case BitTest::Set:
return 's';
1421 llvm_unreachable(
"invalid action");
1429 char SizeSuffix = BT.Is64Bit ?
'q' :
'l';
1433 raw_svector_ostream AsmOS(
Asm);
1434 if (BT.Interlocking != BitTest::Unlocked)
1439 AsmOS << SizeSuffix <<
" $2, ($1)";
1442 std::string Constraints =
"={@ccc},r,r,~{cc},~{memory}";
1444 if (!MachineClobbers.empty()) {
1446 Constraints += MachineClobbers;
1448 llvm::IntegerType *IntType = llvm::IntegerType::get(
1451 llvm::FunctionType *FTy =
1452 llvm::FunctionType::get(CGF.
Int8Ty, {CGF.UnqualPtrTy, IntType},
false);
1454 llvm::InlineAsm *IA =
1455 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
1456 return CGF.
Builder.CreateCall(IA, {BitBase, BitPos});
1459static llvm::AtomicOrdering
1462 case BitTest::Unlocked:
return llvm::AtomicOrdering::NotAtomic;
1463 case BitTest::Sequential:
return llvm::AtomicOrdering::SequentiallyConsistent;
1464 case BitTest::Acquire:
return llvm::AtomicOrdering::Acquire;
1465 case BitTest::Release:
return llvm::AtomicOrdering::Release;
1466 case BitTest::NoFence:
return llvm::AtomicOrdering::Monotonic;
1468 llvm_unreachable(
"invalid interlocking");
1481 BitTest BT = BitTest::decodeBitTestBuiltin(BuiltinID);
1493 BitPos, llvm::ConstantInt::get(BitPos->
getType(), 3),
"bittest.byteidx");
1495 "bittest.byteaddr"),
1499 llvm::ConstantInt::get(CGF.
Int8Ty, 0x7));
1502 Value *Mask =
nullptr;
1503 if (BT.Action != BitTest::TestOnly) {
1504 Mask = CGF.
Builder.CreateShl(llvm::ConstantInt::get(CGF.
Int8Ty, 1), PosLow,
1511 Value *OldByte =
nullptr;
1512 if (Ordering != llvm::AtomicOrdering::NotAtomic) {
1515 llvm::AtomicRMWInst::BinOp RMWOp = llvm::AtomicRMWInst::Or;
1516 if (BT.Action == BitTest::Reset) {
1517 Mask = CGF.
Builder.CreateNot(Mask);
1518 RMWOp = llvm::AtomicRMWInst::And;
1524 Value *NewByte =
nullptr;
1525 switch (BT.Action) {
1526 case BitTest::TestOnly:
1529 case BitTest::Complement:
1530 NewByte = CGF.
Builder.CreateXor(OldByte, Mask);
1532 case BitTest::Reset:
1533 NewByte = CGF.
Builder.CreateAnd(OldByte, CGF.
Builder.CreateNot(Mask));
1536 NewByte = CGF.
Builder.CreateOr(OldByte, Mask);
1545 Value *ShiftedByte = CGF.
Builder.CreateLShr(OldByte, PosLow,
"bittest.shr");
1547 ShiftedByte, llvm::ConstantInt::get(CGF.
Int8Ty, 1),
"bittest.res");
1556 raw_svector_ostream AsmOS(
Asm);
1557 llvm::IntegerType *RetType = CGF.
Int32Ty;
1559 switch (BuiltinID) {
1560 case clang::PPC::BI__builtin_ppc_ldarx:
1564 case clang::PPC::BI__builtin_ppc_lwarx:
1568 case clang::PPC::BI__builtin_ppc_lharx:
1572 case clang::PPC::BI__builtin_ppc_lbarx:
1577 llvm_unreachable(
"Expected only PowerPC load reserve intrinsics");
1580 AsmOS <<
"$0, ${1:y}";
1582 std::string Constraints =
"=r,*Z,~{memory}";
1584 if (!MachineClobbers.empty()) {
1586 Constraints += MachineClobbers;
1590 llvm::FunctionType *FTy = llvm::FunctionType::get(RetType, {PtrType},
false);
1592 llvm::InlineAsm *IA =
1593 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
1594 llvm::CallInst *CI = CGF.
Builder.CreateCall(IA, {Addr});
1596 0, Attribute::get(CGF.
getLLVMContext(), Attribute::ElementType, RetType));
1601enum class MSVCSetJmpKind {
1613 llvm::Value *Arg1 =
nullptr;
1614 llvm::Type *Arg1Ty =
nullptr;
1616 bool IsVarArg =
false;
1617 if (SJKind == MSVCSetJmpKind::_setjmp3) {
1620 Arg1 = llvm::ConstantInt::get(CGF.
IntTy, 0);
1623 Name = SJKind == MSVCSetJmpKind::_setjmp ?
"_setjmp" :
"_setjmpex";
1626 Arg1 = CGF.
Builder.CreateCall(
1629 Arg1 = CGF.
Builder.CreateCall(
1631 llvm::ConstantInt::get(CGF.
Int32Ty, 0));
1635 llvm::Type *ArgTypes[2] = {CGF.
Int8PtrTy, Arg1Ty};
1636 llvm::AttributeList ReturnsTwiceAttr = llvm::AttributeList::get(
1638 llvm::Attribute::ReturnsTwice);
1640 llvm::FunctionType::get(CGF.
IntTy, ArgTypes, IsVarArg), Name,
1641 ReturnsTwiceAttr,
true);
1643 llvm::Value *Buf = CGF.
Builder.CreateBitOrPointerCast(
1645 llvm::Value *Args[] = {Buf, Arg1};
1647 CB->setAttributes(ReturnsTwiceAttr);
1696static std::optional<CodeGenFunction::MSVCIntrin>
1699 switch (BuiltinID) {
1701 return std::nullopt;
1702 case clang::ARM::BI_BitScanForward:
1703 case clang::ARM::BI_BitScanForward64:
1704 return MSVCIntrin::_BitScanForward;
1705 case clang::ARM::BI_BitScanReverse:
1706 case clang::ARM::BI_BitScanReverse64:
1707 return MSVCIntrin::_BitScanReverse;
1708 case clang::ARM::BI_InterlockedAnd64:
1709 return MSVCIntrin::_InterlockedAnd;
1710 case clang::ARM::BI_InterlockedExchange64:
1711 return MSVCIntrin::_InterlockedExchange;
1712 case clang::ARM::BI_InterlockedExchangeAdd64:
1713 return MSVCIntrin::_InterlockedExchangeAdd;
1714 case clang::ARM::BI_InterlockedExchangeSub64:
1715 return MSVCIntrin::_InterlockedExchangeSub;
1716 case clang::ARM::BI_InterlockedOr64:
1717 return MSVCIntrin::_InterlockedOr;
1718 case clang::ARM::BI_InterlockedXor64:
1719 return MSVCIntrin::_InterlockedXor;
1720 case clang::ARM::BI_InterlockedDecrement64:
1721 return MSVCIntrin::_InterlockedDecrement;
1722 case clang::ARM::BI_InterlockedIncrement64:
1723 return MSVCIntrin::_InterlockedIncrement;
1724 case clang::ARM::BI_InterlockedExchangeAdd8_acq:
1725 case clang::ARM::BI_InterlockedExchangeAdd16_acq:
1726 case clang::ARM::BI_InterlockedExchangeAdd_acq:
1727 case clang::ARM::BI_InterlockedExchangeAdd64_acq:
1728 return MSVCIntrin::_InterlockedExchangeAdd_acq;
1729 case clang::ARM::BI_InterlockedExchangeAdd8_rel:
1730 case clang::ARM::BI_InterlockedExchangeAdd16_rel:
1731 case clang::ARM::BI_InterlockedExchangeAdd_rel:
1732 case clang::ARM::BI_InterlockedExchangeAdd64_rel:
1733 return MSVCIntrin::_InterlockedExchangeAdd_rel;
1734 case clang::ARM::BI_InterlockedExchangeAdd8_nf:
1735 case clang::ARM::BI_InterlockedExchangeAdd16_nf:
1736 case clang::ARM::BI_InterlockedExchangeAdd_nf:
1737 case clang::ARM::BI_InterlockedExchangeAdd64_nf:
1738 return MSVCIntrin::_InterlockedExchangeAdd_nf;
1739 case clang::ARM::BI_InterlockedExchange8_acq:
1740 case clang::ARM::BI_InterlockedExchange16_acq:
1741 case clang::ARM::BI_InterlockedExchange_acq:
1742 case clang::ARM::BI_InterlockedExchange64_acq:
1743 case clang::ARM::BI_InterlockedExchangePointer_acq:
1744 return MSVCIntrin::_InterlockedExchange_acq;
1745 case clang::ARM::BI_InterlockedExchange8_rel:
1746 case clang::ARM::BI_InterlockedExchange16_rel:
1747 case clang::ARM::BI_InterlockedExchange_rel:
1748 case clang::ARM::BI_InterlockedExchange64_rel:
1749 case clang::ARM::BI_InterlockedExchangePointer_rel:
1750 return MSVCIntrin::_InterlockedExchange_rel;
1751 case clang::ARM::BI_InterlockedExchange8_nf:
1752 case clang::ARM::BI_InterlockedExchange16_nf:
1753 case clang::ARM::BI_InterlockedExchange_nf:
1754 case clang::ARM::BI_InterlockedExchange64_nf:
1755 case clang::ARM::BI_InterlockedExchangePointer_nf:
1756 return MSVCIntrin::_InterlockedExchange_nf;
1757 case clang::ARM::BI_InterlockedCompareExchange8_acq:
1758 case clang::ARM::BI_InterlockedCompareExchange16_acq:
1759 case clang::ARM::BI_InterlockedCompareExchange_acq:
1760 case clang::ARM::BI_InterlockedCompareExchange64_acq:
1761 case clang::ARM::BI_InterlockedCompareExchangePointer_acq:
1762 return MSVCIntrin::_InterlockedCompareExchange_acq;
1763 case clang::ARM::BI_InterlockedCompareExchange8_rel:
1764 case clang::ARM::BI_InterlockedCompareExchange16_rel:
1765 case clang::ARM::BI_InterlockedCompareExchange_rel:
1766 case clang::ARM::BI_InterlockedCompareExchange64_rel:
1767 case clang::ARM::BI_InterlockedCompareExchangePointer_rel:
1768 return MSVCIntrin::_InterlockedCompareExchange_rel;
1769 case clang::ARM::BI_InterlockedCompareExchange8_nf:
1770 case clang::ARM::BI_InterlockedCompareExchange16_nf:
1771 case clang::ARM::BI_InterlockedCompareExchange_nf:
1772 case clang::ARM::BI_InterlockedCompareExchange64_nf:
1773 return MSVCIntrin::_InterlockedCompareExchange_nf;
1774 case clang::ARM::BI_InterlockedOr8_acq:
1775 case clang::ARM::BI_InterlockedOr16_acq:
1776 case clang::ARM::BI_InterlockedOr_acq:
1777 case clang::ARM::BI_InterlockedOr64_acq:
1778 return MSVCIntrin::_InterlockedOr_acq;
1779 case clang::ARM::BI_InterlockedOr8_rel:
1780 case clang::ARM::BI_InterlockedOr16_rel:
1781 case clang::ARM::BI_InterlockedOr_rel:
1782 case clang::ARM::BI_InterlockedOr64_rel:
1783 return MSVCIntrin::_InterlockedOr_rel;
1784 case clang::ARM::BI_InterlockedOr8_nf:
1785 case clang::ARM::BI_InterlockedOr16_nf:
1786 case clang::ARM::BI_InterlockedOr_nf:
1787 case clang::ARM::BI_InterlockedOr64_nf:
1788 return MSVCIntrin::_InterlockedOr_nf;
1789 case clang::ARM::BI_InterlockedXor8_acq:
1790 case clang::ARM::BI_InterlockedXor16_acq:
1791 case clang::ARM::BI_InterlockedXor_acq:
1792 case clang::ARM::BI_InterlockedXor64_acq:
1793 return MSVCIntrin::_InterlockedXor_acq;
1794 case clang::ARM::BI_InterlockedXor8_rel:
1795 case clang::ARM::BI_InterlockedXor16_rel:
1796 case clang::ARM::BI_InterlockedXor_rel:
1797 case clang::ARM::BI_InterlockedXor64_rel:
1798 return MSVCIntrin::_InterlockedXor_rel;
1799 case clang::ARM::BI_InterlockedXor8_nf:
1800 case clang::ARM::BI_InterlockedXor16_nf:
1801 case clang::ARM::BI_InterlockedXor_nf:
1802 case clang::ARM::BI_InterlockedXor64_nf:
1803 return MSVCIntrin::_InterlockedXor_nf;
1804 case clang::ARM::BI_InterlockedAnd8_acq:
1805 case clang::ARM::BI_InterlockedAnd16_acq:
1806 case clang::ARM::BI_InterlockedAnd_acq:
1807 case clang::ARM::BI_InterlockedAnd64_acq:
1808 return MSVCIntrin::_InterlockedAnd_acq;
1809 case clang::ARM::BI_InterlockedAnd8_rel:
1810 case clang::ARM::BI_InterlockedAnd16_rel:
1811 case clang::ARM::BI_InterlockedAnd_rel:
1812 case clang::ARM::BI_InterlockedAnd64_rel:
1813 return MSVCIntrin::_InterlockedAnd_rel;
1814 case clang::ARM::BI_InterlockedAnd8_nf:
1815 case clang::ARM::BI_InterlockedAnd16_nf:
1816 case clang::ARM::BI_InterlockedAnd_nf:
1817 case clang::ARM::BI_InterlockedAnd64_nf:
1818 return MSVCIntrin::_InterlockedAnd_nf;
1819 case clang::ARM::BI_InterlockedIncrement16_acq:
1820 case clang::ARM::BI_InterlockedIncrement_acq:
1821 case clang::ARM::BI_InterlockedIncrement64_acq:
1822 return MSVCIntrin::_InterlockedIncrement_acq;
1823 case clang::ARM::BI_InterlockedIncrement16_rel:
1824 case clang::ARM::BI_InterlockedIncrement_rel:
1825 case clang::ARM::BI_InterlockedIncrement64_rel:
1826 return MSVCIntrin::_InterlockedIncrement_rel;
1827 case clang::ARM::BI_InterlockedIncrement16_nf:
1828 case clang::ARM::BI_InterlockedIncrement_nf:
1829 case clang::ARM::BI_InterlockedIncrement64_nf:
1830 return MSVCIntrin::_InterlockedIncrement_nf;
1831 case clang::ARM::BI_InterlockedDecrement16_acq:
1832 case clang::ARM::BI_InterlockedDecrement_acq:
1833 case clang::ARM::BI_InterlockedDecrement64_acq:
1834 return MSVCIntrin::_InterlockedDecrement_acq;
1835 case clang::ARM::BI_InterlockedDecrement16_rel:
1836 case clang::ARM::BI_InterlockedDecrement_rel:
1837 case clang::ARM::BI_InterlockedDecrement64_rel:
1838 return MSVCIntrin::_InterlockedDecrement_rel;
1839 case clang::ARM::BI_InterlockedDecrement16_nf:
1840 case clang::ARM::BI_InterlockedDecrement_nf:
1841 case clang::ARM::BI_InterlockedDecrement64_nf:
1842 return MSVCIntrin::_InterlockedDecrement_nf;
1844 llvm_unreachable(
"must return from switch");
1847static std::optional<CodeGenFunction::MSVCIntrin>
1850 switch (BuiltinID) {
1852 return std::nullopt;
1853 case clang::AArch64::BI_BitScanForward:
1854 case clang::AArch64::BI_BitScanForward64:
1855 return MSVCIntrin::_BitScanForward;
1856 case clang::AArch64::BI_BitScanReverse:
1857 case clang::AArch64::BI_BitScanReverse64:
1858 return MSVCIntrin::_BitScanReverse;
1859 case clang::AArch64::BI_InterlockedAnd64:
1860 return MSVCIntrin::_InterlockedAnd;
1861 case clang::AArch64::BI_InterlockedExchange64:
1862 return MSVCIntrin::_InterlockedExchange;
1863 case clang::AArch64::BI_InterlockedExchangeAdd64:
1864 return MSVCIntrin::_InterlockedExchangeAdd;
1865 case clang::AArch64::BI_InterlockedExchangeSub64:
1866 return MSVCIntrin::_InterlockedExchangeSub;
1867 case clang::AArch64::BI_InterlockedOr64:
1868 return MSVCIntrin::_InterlockedOr;
1869 case clang::AArch64::BI_InterlockedXor64:
1870 return MSVCIntrin::_InterlockedXor;
1871 case clang::AArch64::BI_InterlockedDecrement64:
1872 return MSVCIntrin::_InterlockedDecrement;
1873 case clang::AArch64::BI_InterlockedIncrement64:
1874 return MSVCIntrin::_InterlockedIncrement;
1875 case clang::AArch64::BI_InterlockedExchangeAdd8_acq:
1876 case clang::AArch64::BI_InterlockedExchangeAdd16_acq:
1877 case clang::AArch64::BI_InterlockedExchangeAdd_acq:
1878 case clang::AArch64::BI_InterlockedExchangeAdd64_acq:
1879 return MSVCIntrin::_InterlockedExchangeAdd_acq;
1880 case clang::AArch64::BI_InterlockedExchangeAdd8_rel:
1881 case clang::AArch64::BI_InterlockedExchangeAdd16_rel:
1882 case clang::AArch64::BI_InterlockedExchangeAdd_rel:
1883 case clang::AArch64::BI_InterlockedExchangeAdd64_rel:
1884 return MSVCIntrin::_InterlockedExchangeAdd_rel;
1885 case clang::AArch64::BI_InterlockedExchangeAdd8_nf:
1886 case clang::AArch64::BI_InterlockedExchangeAdd16_nf:
1887 case clang::AArch64::BI_InterlockedExchangeAdd_nf:
1888 case clang::AArch64::BI_InterlockedExchangeAdd64_nf:
1889 return MSVCIntrin::_InterlockedExchangeAdd_nf;
1890 case clang::AArch64::BI_InterlockedExchange8_acq:
1891 case clang::AArch64::BI_InterlockedExchange16_acq:
1892 case clang::AArch64::BI_InterlockedExchange_acq:
1893 case clang::AArch64::BI_InterlockedExchange64_acq:
1894 case clang::AArch64::BI_InterlockedExchangePointer_acq:
1895 return MSVCIntrin::_InterlockedExchange_acq;
1896 case clang::AArch64::BI_InterlockedExchange8_rel:
1897 case clang::AArch64::BI_InterlockedExchange16_rel:
1898 case clang::AArch64::BI_InterlockedExchange_rel:
1899 case clang::AArch64::BI_InterlockedExchange64_rel:
1900 case clang::AArch64::BI_InterlockedExchangePointer_rel:
1901 return MSVCIntrin::_InterlockedExchange_rel;
1902 case clang::AArch64::BI_InterlockedExchange8_nf:
1903 case clang::AArch64::BI_InterlockedExchange16_nf:
1904 case clang::AArch64::BI_InterlockedExchange_nf:
1905 case clang::AArch64::BI_InterlockedExchange64_nf:
1906 case clang::AArch64::BI_InterlockedExchangePointer_nf:
1907 return MSVCIntrin::_InterlockedExchange_nf;
1908 case clang::AArch64::BI_InterlockedCompareExchange8_acq:
1909 case clang::AArch64::BI_InterlockedCompareExchange16_acq:
1910 case clang::AArch64::BI_InterlockedCompareExchange_acq:
1911 case clang::AArch64::BI_InterlockedCompareExchange64_acq:
1912 case clang::AArch64::BI_InterlockedCompareExchangePointer_acq:
1913 return MSVCIntrin::_InterlockedCompareExchange_acq;
1914 case clang::AArch64::BI_InterlockedCompareExchange8_rel:
1915 case clang::AArch64::BI_InterlockedCompareExchange16_rel:
1916 case clang::AArch64::BI_InterlockedCompareExchange_rel:
1917 case clang::AArch64::BI_InterlockedCompareExchange64_rel:
1918 case clang::AArch64::BI_InterlockedCompareExchangePointer_rel:
1919 return MSVCIntrin::_InterlockedCompareExchange_rel;
1920 case clang::AArch64::BI_InterlockedCompareExchange8_nf:
1921 case clang::AArch64::BI_InterlockedCompareExchange16_nf:
1922 case clang::AArch64::BI_InterlockedCompareExchange_nf:
1923 case clang::AArch64::BI_InterlockedCompareExchange64_nf:
1924 return MSVCIntrin::_InterlockedCompareExchange_nf;
1925 case clang::AArch64::BI_InterlockedCompareExchange128:
1926 return MSVCIntrin::_InterlockedCompareExchange128;
1927 case clang::AArch64::BI_InterlockedCompareExchange128_acq:
1928 return MSVCIntrin::_InterlockedCompareExchange128_acq;
1929 case clang::AArch64::BI_InterlockedCompareExchange128_nf:
1930 return MSVCIntrin::_InterlockedCompareExchange128_nf;
1931 case clang::AArch64::BI_InterlockedCompareExchange128_rel:
1932 return MSVCIntrin::_InterlockedCompareExchange128_rel;
1933 case clang::AArch64::BI_InterlockedOr8_acq:
1934 case clang::AArch64::BI_InterlockedOr16_acq:
1935 case clang::AArch64::BI_InterlockedOr_acq:
1936 case clang::AArch64::BI_InterlockedOr64_acq:
1937 return MSVCIntrin::_InterlockedOr_acq;
1938 case clang::AArch64::BI_InterlockedOr8_rel:
1939 case clang::AArch64::BI_InterlockedOr16_rel:
1940 case clang::AArch64::BI_InterlockedOr_rel:
1941 case clang::AArch64::BI_InterlockedOr64_rel:
1942 return MSVCIntrin::_InterlockedOr_rel;
1943 case clang::AArch64::BI_InterlockedOr8_nf:
1944 case clang::AArch64::BI_InterlockedOr16_nf:
1945 case clang::AArch64::BI_InterlockedOr_nf:
1946 case clang::AArch64::BI_InterlockedOr64_nf:
1947 return MSVCIntrin::_InterlockedOr_nf;
1948 case clang::AArch64::BI_InterlockedXor8_acq:
1949 case clang::AArch64::BI_InterlockedXor16_acq:
1950 case clang::AArch64::BI_InterlockedXor_acq:
1951 case clang::AArch64::BI_InterlockedXor64_acq:
1952 return MSVCIntrin::_InterlockedXor_acq;
1953 case clang::AArch64::BI_InterlockedXor8_rel:
1954 case clang::AArch64::BI_InterlockedXor16_rel:
1955 case clang::AArch64::BI_InterlockedXor_rel:
1956 case clang::AArch64::BI_InterlockedXor64_rel:
1957 return MSVCIntrin::_InterlockedXor_rel;
1958 case clang::AArch64::BI_InterlockedXor8_nf:
1959 case clang::AArch64::BI_InterlockedXor16_nf:
1960 case clang::AArch64::BI_InterlockedXor_nf:
1961 case clang::AArch64::BI_InterlockedXor64_nf:
1962 return MSVCIntrin::_InterlockedXor_nf;
1963 case clang::AArch64::BI_InterlockedAnd8_acq:
1964 case clang::AArch64::BI_InterlockedAnd16_acq:
1965 case clang::AArch64::BI_InterlockedAnd_acq:
1966 case clang::AArch64::BI_InterlockedAnd64_acq:
1967 return MSVCIntrin::_InterlockedAnd_acq;
1968 case clang::AArch64::BI_InterlockedAnd8_rel:
1969 case clang::AArch64::BI_InterlockedAnd16_rel:
1970 case clang::AArch64::BI_InterlockedAnd_rel:
1971 case clang::AArch64::BI_InterlockedAnd64_rel:
1972 return MSVCIntrin::_InterlockedAnd_rel;
1973 case clang::AArch64::BI_InterlockedAnd8_nf:
1974 case clang::AArch64::BI_InterlockedAnd16_nf:
1975 case clang::AArch64::BI_InterlockedAnd_nf:
1976 case clang::AArch64::BI_InterlockedAnd64_nf:
1977 return MSVCIntrin::_InterlockedAnd_nf;
1978 case clang::AArch64::BI_InterlockedIncrement16_acq:
1979 case clang::AArch64::BI_InterlockedIncrement_acq:
1980 case clang::AArch64::BI_InterlockedIncrement64_acq:
1981 return MSVCIntrin::_InterlockedIncrement_acq;
1982 case clang::AArch64::BI_InterlockedIncrement16_rel:
1983 case clang::AArch64::BI_InterlockedIncrement_rel:
1984 case clang::AArch64::BI_InterlockedIncrement64_rel:
1985 return MSVCIntrin::_InterlockedIncrement_rel;
1986 case clang::AArch64::BI_InterlockedIncrement16_nf:
1987 case clang::AArch64::BI_InterlockedIncrement_nf:
1988 case clang::AArch64::BI_InterlockedIncrement64_nf:
1989 return MSVCIntrin::_InterlockedIncrement_nf;
1990 case clang::AArch64::BI_InterlockedDecrement16_acq:
1991 case clang::AArch64::BI_InterlockedDecrement_acq:
1992 case clang::AArch64::BI_InterlockedDecrement64_acq:
1993 return MSVCIntrin::_InterlockedDecrement_acq;
1994 case clang::AArch64::BI_InterlockedDecrement16_rel:
1995 case clang::AArch64::BI_InterlockedDecrement_rel:
1996 case clang::AArch64::BI_InterlockedDecrement64_rel:
1997 return MSVCIntrin::_InterlockedDecrement_rel;
1998 case clang::AArch64::BI_InterlockedDecrement16_nf:
1999 case clang::AArch64::BI_InterlockedDecrement_nf:
2000 case clang::AArch64::BI_InterlockedDecrement64_nf:
2001 return MSVCIntrin::_InterlockedDecrement_nf;
2003 llvm_unreachable(
"must return from switch");
2006static std::optional<CodeGenFunction::MSVCIntrin>
2009 switch (BuiltinID) {
2011 return std::nullopt;
2012 case clang::X86::BI_BitScanForward:
2013 case clang::X86::BI_BitScanForward64:
2014 return MSVCIntrin::_BitScanForward;
2015 case clang::X86::BI_BitScanReverse:
2016 case clang::X86::BI_BitScanReverse64:
2017 return MSVCIntrin::_BitScanReverse;
2018 case clang::X86::BI_InterlockedAnd64:
2019 return MSVCIntrin::_InterlockedAnd;
2020 case clang::X86::BI_InterlockedCompareExchange128:
2021 return MSVCIntrin::_InterlockedCompareExchange128;
2022 case clang::X86::BI_InterlockedExchange64:
2023 return MSVCIntrin::_InterlockedExchange;
2024 case clang::X86::BI_InterlockedExchangeAdd64:
2025 return MSVCIntrin::_InterlockedExchangeAdd;
2026 case clang::X86::BI_InterlockedExchangeSub64:
2027 return MSVCIntrin::_InterlockedExchangeSub;
2028 case clang::X86::BI_InterlockedOr64:
2029 return MSVCIntrin::_InterlockedOr;
2030 case clang::X86::BI_InterlockedXor64:
2031 return MSVCIntrin::_InterlockedXor;
2032 case clang::X86::BI_InterlockedDecrement64:
2033 return MSVCIntrin::_InterlockedDecrement;
2034 case clang::X86::BI_InterlockedIncrement64:
2035 return MSVCIntrin::_InterlockedIncrement;
2037 llvm_unreachable(
"must return from switch");
2043 switch (BuiltinID) {
2044 case MSVCIntrin::_BitScanForward:
2045 case MSVCIntrin::_BitScanReverse: {
2049 llvm::Type *ArgType = ArgValue->
getType();
2050 llvm::Type *IndexType = IndexAddress.getElementType();
2053 Value *ArgZero = llvm::Constant::getNullValue(ArgType);
2054 Value *ResZero = llvm::Constant::getNullValue(ResultType);
2055 Value *ResOne = llvm::ConstantInt::get(ResultType, 1);
2060 PHINode *
Result =
Builder.CreatePHI(ResultType, 2,
"bitscan_result");
2063 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, ArgZero);
2065 Builder.CreateCondBr(IsZero, End, NotZero);
2068 Builder.SetInsertPoint(NotZero);
2070 if (BuiltinID == MSVCIntrin::_BitScanForward) {
2073 ZeroCount =
Builder.CreateIntCast(ZeroCount, IndexType,
false);
2076 unsigned ArgWidth = cast<llvm::IntegerType>(ArgType)->getBitWidth();
2077 Value *ArgTypeLastIndex = llvm::ConstantInt::get(IndexType, ArgWidth - 1);
2081 ZeroCount =
Builder.CreateIntCast(ZeroCount, IndexType,
false);
2082 Value *Index =
Builder.CreateNSWSub(ArgTypeLastIndex, ZeroCount);
2086 Result->addIncoming(ResOne, NotZero);
2091 case MSVCIntrin::_InterlockedAnd:
2093 case MSVCIntrin::_InterlockedExchange:
2095 case MSVCIntrin::_InterlockedExchangeAdd:
2097 case MSVCIntrin::_InterlockedExchangeSub:
2099 case MSVCIntrin::_InterlockedOr:
2101 case MSVCIntrin::_InterlockedXor:
2103 case MSVCIntrin::_InterlockedExchangeAdd_acq:
2105 AtomicOrdering::Acquire);
2106 case MSVCIntrin::_InterlockedExchangeAdd_rel:
2108 AtomicOrdering::Release);
2109 case MSVCIntrin::_InterlockedExchangeAdd_nf:
2111 AtomicOrdering::Monotonic);
2112 case MSVCIntrin::_InterlockedExchange_acq:
2114 AtomicOrdering::Acquire);
2115 case MSVCIntrin::_InterlockedExchange_rel:
2117 AtomicOrdering::Release);
2118 case MSVCIntrin::_InterlockedExchange_nf:
2120 AtomicOrdering::Monotonic);
2121 case MSVCIntrin::_InterlockedCompareExchange:
2123 case MSVCIntrin::_InterlockedCompareExchange_acq:
2125 case MSVCIntrin::_InterlockedCompareExchange_rel:
2127 case MSVCIntrin::_InterlockedCompareExchange_nf:
2129 case MSVCIntrin::_InterlockedCompareExchange128:
2131 *
this,
E, AtomicOrdering::SequentiallyConsistent);
2132 case MSVCIntrin::_InterlockedCompareExchange128_acq:
2134 case MSVCIntrin::_InterlockedCompareExchange128_rel:
2136 case MSVCIntrin::_InterlockedCompareExchange128_nf:
2138 case MSVCIntrin::_InterlockedOr_acq:
2140 AtomicOrdering::Acquire);
2141 case MSVCIntrin::_InterlockedOr_rel:
2143 AtomicOrdering::Release);
2144 case MSVCIntrin::_InterlockedOr_nf:
2146 AtomicOrdering::Monotonic);
2147 case MSVCIntrin::_InterlockedXor_acq:
2149 AtomicOrdering::Acquire);
2150 case MSVCIntrin::_InterlockedXor_rel:
2152 AtomicOrdering::Release);
2153 case MSVCIntrin::_InterlockedXor_nf:
2155 AtomicOrdering::Monotonic);
2156 case MSVCIntrin::_InterlockedAnd_acq:
2158 AtomicOrdering::Acquire);
2159 case MSVCIntrin::_InterlockedAnd_rel:
2161 AtomicOrdering::Release);
2162 case MSVCIntrin::_InterlockedAnd_nf:
2164 AtomicOrdering::Monotonic);
2165 case MSVCIntrin::_InterlockedIncrement_acq:
2167 case MSVCIntrin::_InterlockedIncrement_rel:
2169 case MSVCIntrin::_InterlockedIncrement_nf:
2171 case MSVCIntrin::_InterlockedDecrement_acq:
2173 case MSVCIntrin::_InterlockedDecrement_rel:
2175 case MSVCIntrin::_InterlockedDecrement_nf:
2178 case MSVCIntrin::_InterlockedDecrement:
2180 case MSVCIntrin::_InterlockedIncrement:
2183 case MSVCIntrin::__fastfail: {
2188 StringRef
Asm, Constraints;
2193 case llvm::Triple::x86:
2194 case llvm::Triple::x86_64:
2196 Constraints =
"{cx}";
2198 case llvm::Triple::thumb:
2200 Constraints =
"{r0}";
2202 case llvm::Triple::aarch64:
2203 Asm =
"brk #0xF003";
2204 Constraints =
"{w0}";
2206 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy, {
Int32Ty},
false);
2207 llvm::InlineAsm *IA =
2208 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
2209 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
2211 llvm::Attribute::NoReturn);
2213 CI->setAttributes(NoReturnAttr);
2217 llvm_unreachable(
"Incorrect MSVC intrinsic!");
2223 CallObjCArcUse(llvm::Value *
object) : object(object) {}
2224 llvm::Value *object;
2233 BuiltinCheckKind Kind) {
2235 "Unsupported builtin check kind");
2241 SanitizerScope SanScope(
this);
2243 ArgValue, llvm::Constant::getNullValue(ArgValue->
getType()));
2244 EmitCheck(std::make_pair(Cond, SanitizerKind::SO_Builtin),
2245 SanitizerHandler::InvalidBuiltin,
2247 llvm::ConstantInt::get(
Builder.getInt8Ty(), Kind)},
2257 SanitizerScope SanScope(
this);
2259 std::make_pair(ArgValue, SanitizerKind::SO_Builtin),
2260 SanitizerHandler::InvalidBuiltin,
2268 return CGF.
Builder.CreateBinaryIntrinsic(
2269 Intrinsic::abs, ArgValue,
2270 ConstantInt::get(CGF.
Builder.getInt1Ty(), HasNSW));
2274 bool SanitizeOverflow) {
2278 if (
const auto *VCI = dyn_cast<llvm::ConstantInt>(ArgValue)) {
2279 if (!VCI->isMinSignedValue())
2280 return EmitAbs(CGF, ArgValue,
true);
2283 CodeGenFunction::SanitizerScope SanScope(&CGF);
2285 Constant *Zero = Constant::getNullValue(ArgValue->
getType());
2286 Value *ResultAndOverflow = CGF.
Builder.CreateBinaryIntrinsic(
2287 Intrinsic::ssub_with_overflow, Zero, ArgValue);
2290 CGF.
Builder.CreateExtractValue(ResultAndOverflow, 1));
2293 if (SanitizeOverflow) {
2294 CGF.
EmitCheck({{NotOverflow, SanitizerKind::SO_SignedIntegerOverflow}},
2295 SanitizerHandler::NegateOverflow,
2300 CGF.
EmitTrapCheck(NotOverflow, SanitizerHandler::SubOverflow);
2302 Value *CmpResult = CGF.
Builder.CreateICmpSLT(ArgValue, Zero,
"abscond");
2303 return CGF.
Builder.CreateSelect(CmpResult,
Result, ArgValue,
"abs");
2308 QualType UnsignedTy =
C.getIntTypeForBitwidth(Size * 8,
false);
2309 return C.getCanonicalType(UnsignedTy);
2319 raw_svector_ostream OS(Name);
2320 OS <<
"__os_log_helper";
2324 for (
const auto &Item : Layout.
Items)
2325 OS <<
"_" <<
int(Item.getSizeByte()) <<
"_"
2326 <<
int(Item.getDescriptorByte());
2329 if (llvm::Function *F =
CGM.
getModule().getFunction(Name))
2339 for (
unsigned int I = 0,
E = Layout.
Items.size(); I <
E; ++I) {
2340 char Size = Layout.
Items[I].getSizeByte();
2347 &Ctx.
Idents.
get(std::string(
"arg") + llvm::to_string(I)), ArgTy,
2349 ArgTys.emplace_back(ArgTy);
2360 llvm::Function *
Fn = llvm::Function::Create(
2361 FuncTy, llvm::GlobalValue::LinkOnceODRLinkage, Name, &
CGM.
getModule());
2362 Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);
2365 Fn->setDoesNotThrow();
2369 Fn->addFnAttr(llvm::Attribute::NoInline);
2387 for (
const auto &Item : Layout.
Items) {
2389 Builder.getInt8(Item.getDescriptorByte()),
2392 Builder.getInt8(Item.getSizeByte()),
2396 if (!
Size.getQuantity())
2413 assert(
E.getNumArgs() >= 2 &&
2414 "__builtin_os_log_format takes at least 2 arguments");
2425 for (
const auto &Item : Layout.
Items) {
2426 int Size = Item.getSizeByte();
2430 llvm::Value *ArgVal;
2434 for (
unsigned I = 0,
E = Item.getMaskType().size(); I <
E; ++I)
2435 Val |= ((
uint64_t)Item.getMaskType()[I]) << I * 8;
2436 ArgVal = llvm::Constant::getIntegerValue(
Int64Ty, llvm::APInt(64, Val));
2437 }
else if (
const Expr *TheExpr = Item.getExpr()) {
2443 auto LifetimeExtendObject = [&](
const Expr *
E) {
2451 if (isa<CallExpr>(
E) || isa<ObjCMessageExpr>(
E))
2456 if (TheExpr->getType()->isObjCRetainableType() &&
2457 getLangOpts().ObjCAutoRefCount && LifetimeExtendObject(TheExpr)) {
2459 "Only scalar can be a ObjC retainable type");
2460 if (!isa<Constant>(ArgVal)) {
2474 pushCleanupAfterFullExpr<CallObjCArcUse>(Cleanup, ArgVal);
2478 ArgVal =
Builder.getInt32(Item.getConstValue().getQuantity());
2481 unsigned ArgValSize =
2485 ArgVal =
Builder.CreateBitOrPointerCast(ArgVal,
IntTy);
2501 unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info,
2502 WidthAndSignedness ResultInfo) {
2503 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2504 Op1Info.Width == Op2Info.Width && Op2Info.Width == ResultInfo.Width &&
2505 !Op1Info.Signed && !Op2Info.Signed && ResultInfo.Signed;
2510 const clang::Expr *Op2, WidthAndSignedness Op2Info,
2512 WidthAndSignedness ResultInfo) {
2514 Builtin::BI__builtin_mul_overflow, Op1Info, Op2Info, ResultInfo) &&
2515 "Cannot specialize this multiply");
2520 llvm::Value *HasOverflow;
2522 CGF, llvm::Intrinsic::umul_with_overflow, V1, V2, HasOverflow);
2527 auto IntMax = llvm::APInt::getSignedMaxValue(ResultInfo.Width);
2528 llvm::Value *IntMaxValue = llvm::ConstantInt::get(
Result->getType(), IntMax);
2530 llvm::Value *IntMaxOverflow = CGF.
Builder.CreateICmpUGT(
Result, IntMaxValue);
2531 HasOverflow = CGF.
Builder.CreateOr(HasOverflow, IntMaxOverflow);
2543 WidthAndSignedness Op1Info,
2544 WidthAndSignedness Op2Info,
2545 WidthAndSignedness ResultInfo) {
2546 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2547 std::max(Op1Info.Width, Op2Info.Width) >= ResultInfo.Width &&
2548 Op1Info.Signed != Op2Info.Signed;
2555 WidthAndSignedness Op1Info,
const clang::Expr *Op2,
2556 WidthAndSignedness Op2Info,
2558 WidthAndSignedness ResultInfo) {
2560 Op2Info, ResultInfo) &&
2561 "Not a mixed-sign multipliction we can specialize");
2564 const clang::Expr *SignedOp = Op1Info.Signed ? Op1 : Op2;
2565 const clang::Expr *UnsignedOp = Op1Info.Signed ? Op2 : Op1;
2568 unsigned SignedOpWidth = Op1Info.Signed ? Op1Info.Width : Op2Info.Width;
2569 unsigned UnsignedOpWidth = Op1Info.Signed ? Op2Info.Width : Op1Info.Width;
2572 if (SignedOpWidth < UnsignedOpWidth)
2574 if (UnsignedOpWidth < SignedOpWidth)
2577 llvm::Type *OpTy =
Signed->getType();
2578 llvm::Value *Zero = llvm::Constant::getNullValue(OpTy);
2581 unsigned OpWidth = std::max(Op1Info.Width, Op2Info.Width);
2584 llvm::Value *IsNegative = CGF.
Builder.CreateICmpSLT(
Signed, Zero);
2585 llvm::Value *AbsOfNegative = CGF.
Builder.CreateSub(Zero,
Signed);
2586 llvm::Value *AbsSigned =
2587 CGF.
Builder.CreateSelect(IsNegative, AbsOfNegative,
Signed);
2590 llvm::Value *UnsignedOverflow;
2591 llvm::Value *UnsignedResult =
2595 llvm::Value *Overflow, *
Result;
2596 if (ResultInfo.Signed) {
2600 llvm::APInt::getSignedMaxValue(ResultInfo.Width).zext(OpWidth);
2601 llvm::Value *MaxResult =
2602 CGF.
Builder.CreateAdd(llvm::ConstantInt::get(OpTy, IntMax),
2603 CGF.
Builder.CreateZExt(IsNegative, OpTy));
2604 llvm::Value *SignedOverflow =
2605 CGF.
Builder.CreateICmpUGT(UnsignedResult, MaxResult);
2606 Overflow = CGF.
Builder.CreateOr(UnsignedOverflow, SignedOverflow);
2609 llvm::Value *NegativeResult = CGF.
Builder.CreateNeg(UnsignedResult);
2610 llvm::Value *SignedResult =
2611 CGF.
Builder.CreateSelect(IsNegative, NegativeResult, UnsignedResult);
2615 llvm::Value *Underflow = CGF.
Builder.CreateAnd(
2616 IsNegative, CGF.
Builder.CreateIsNotNull(UnsignedResult));
2617 Overflow = CGF.
Builder.CreateOr(UnsignedOverflow, Underflow);
2618 if (ResultInfo.Width < OpWidth) {
2620 llvm::APInt::getMaxValue(ResultInfo.Width).zext(OpWidth);
2621 llvm::Value *TruncOverflow = CGF.
Builder.CreateICmpUGT(
2622 UnsignedResult, llvm::ConstantInt::get(OpTy, IntMax));
2623 Overflow = CGF.
Builder.CreateOr(Overflow, TruncOverflow);
2628 IsNegative, CGF.
Builder.CreateNeg(UnsignedResult), UnsignedResult);
2632 assert(Overflow &&
Result &&
"Missing overflow or result");
2643 llvm::SmallPtrSetImpl<const Decl *> &Seen) {
2652 if (!Seen.insert(
Record).second)
2655 assert(
Record->hasDefinition() &&
2656 "Incomplete types should already be diagnosed");
2658 if (
Record->isDynamicClass())
2683 llvm::Type *Ty = Src->getType();
2684 ShiftAmt =
Builder.CreateIntCast(ShiftAmt, Ty,
false);
2687 unsigned IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2694 switch (BuiltinID) {
2695#define MUTATE_LDBL(func) \
2696 case Builtin::BI__builtin_##func##l: \
2697 return Builtin::BI__builtin_##func##f128;
2766 if (CGF.
Builder.getIsFPConstrained() &&
2767 CGF.
Builder.getDefaultConstrainedExcept() != fp::ebIgnore) {
2779 auto UBF = CGF->
CGM.
getModule().getOrInsertFunction(Name, FnTy);
2782 for (
auto &&FormalTy : FnTy->params())
2783 Args.push_back(llvm::PoisonValue::get(FormalTy));
2792 "Should not codegen for consteval builtins");
2799 !
Result.hasSideEffects()) {
2803 if (
Result.Val.isFloat())
2812 if (
getTarget().getTriple().isPPC64() &&
2813 &
getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad())
2820 const unsigned BuiltinIDIfNoAsmLabel =
2821 FD->
hasAttr<AsmLabelAttr>() ? 0 : BuiltinID;
2823 std::optional<bool> ErrnoOverriden;
2827 if (
E->hasStoredFPFeatures()) {
2829 if (OP.hasMathErrnoOverride())
2830 ErrnoOverriden = OP.getMathErrnoOverride();
2839 bool ErrnoOverridenToFalseWithOpt =
2840 ErrnoOverriden.has_value() && !ErrnoOverriden.value() && !OptNone &&
2858 switch (BuiltinID) {
2859 case Builtin::BI__builtin_fma:
2860 case Builtin::BI__builtin_fmaf:
2861 case Builtin::BI__builtin_fmal:
2862 case Builtin::BI__builtin_fmaf16:
2863 case Builtin::BIfma:
2864 case Builtin::BIfmaf:
2865 case Builtin::BIfmal: {
2867 if (Trip.isGNUEnvironment() || Trip.isOSMSVCRT())
2875 bool ConstWithoutErrnoAndExceptions =
2877 bool ConstWithoutExceptions =
2895 bool ConstWithoutErrnoOrExceptions =
2896 ConstWithoutErrnoAndExceptions || ConstWithoutExceptions;
2897 bool GenerateIntrinsics =
2898 (ConstAlways && !OptNone) ||
2900 !(ErrnoOverriden.has_value() && ErrnoOverriden.value()) && !OptNone);
2901 if (!GenerateIntrinsics) {
2902 GenerateIntrinsics =
2903 ConstWithoutErrnoOrExceptions && !ConstWithoutErrnoAndExceptions;
2904 if (!GenerateIntrinsics)
2905 GenerateIntrinsics =
2906 ConstWithoutErrnoOrExceptions &&
2908 !(ErrnoOverriden.has_value() && ErrnoOverriden.value()) && !OptNone);
2909 if (!GenerateIntrinsics)
2910 GenerateIntrinsics =
2911 ConstWithoutErrnoOrExceptions && ErrnoOverridenToFalseWithOpt;
2913 if (GenerateIntrinsics) {
2914 switch (BuiltinIDIfNoAsmLabel) {
2915 case Builtin::BIacos:
2916 case Builtin::BIacosf:
2917 case Builtin::BIacosl:
2918 case Builtin::BI__builtin_acos:
2919 case Builtin::BI__builtin_acosf:
2920 case Builtin::BI__builtin_acosf16:
2921 case Builtin::BI__builtin_acosl:
2922 case Builtin::BI__builtin_acosf128:
2924 *
this,
E, Intrinsic::acos, Intrinsic::experimental_constrained_acos));
2926 case Builtin::BIasin:
2927 case Builtin::BIasinf:
2928 case Builtin::BIasinl:
2929 case Builtin::BI__builtin_asin:
2930 case Builtin::BI__builtin_asinf:
2931 case Builtin::BI__builtin_asinf16:
2932 case Builtin::BI__builtin_asinl:
2933 case Builtin::BI__builtin_asinf128:
2935 *
this,
E, Intrinsic::asin, Intrinsic::experimental_constrained_asin));
2937 case Builtin::BIatan:
2938 case Builtin::BIatanf:
2939 case Builtin::BIatanl:
2940 case Builtin::BI__builtin_atan:
2941 case Builtin::BI__builtin_atanf:
2942 case Builtin::BI__builtin_atanf16:
2943 case Builtin::BI__builtin_atanl:
2944 case Builtin::BI__builtin_atanf128:
2946 *
this,
E, Intrinsic::atan, Intrinsic::experimental_constrained_atan));
2948 case Builtin::BIatan2:
2949 case Builtin::BIatan2f:
2950 case Builtin::BIatan2l:
2951 case Builtin::BI__builtin_atan2:
2952 case Builtin::BI__builtin_atan2f:
2953 case Builtin::BI__builtin_atan2f16:
2954 case Builtin::BI__builtin_atan2l:
2955 case Builtin::BI__builtin_atan2f128:
2957 *
this,
E, Intrinsic::atan2,
2958 Intrinsic::experimental_constrained_atan2));
2960 case Builtin::BIceil:
2961 case Builtin::BIceilf:
2962 case Builtin::BIceill:
2963 case Builtin::BI__builtin_ceil:
2964 case Builtin::BI__builtin_ceilf:
2965 case Builtin::BI__builtin_ceilf16:
2966 case Builtin::BI__builtin_ceill:
2967 case Builtin::BI__builtin_ceilf128:
2970 Intrinsic::experimental_constrained_ceil));
2972 case Builtin::BIcopysign:
2973 case Builtin::BIcopysignf:
2974 case Builtin::BIcopysignl:
2975 case Builtin::BI__builtin_copysign:
2976 case Builtin::BI__builtin_copysignf:
2977 case Builtin::BI__builtin_copysignf16:
2978 case Builtin::BI__builtin_copysignl:
2979 case Builtin::BI__builtin_copysignf128:
2981 emitBuiltinWithOneOverloadedType<2>(*
this,
E, Intrinsic::copysign));
2983 case Builtin::BIcos:
2984 case Builtin::BIcosf:
2985 case Builtin::BIcosl:
2986 case Builtin::BI__builtin_cos:
2987 case Builtin::BI__builtin_cosf:
2988 case Builtin::BI__builtin_cosf16:
2989 case Builtin::BI__builtin_cosl:
2990 case Builtin::BI__builtin_cosf128:
2993 Intrinsic::experimental_constrained_cos));
2995 case Builtin::BIcosh:
2996 case Builtin::BIcoshf:
2997 case Builtin::BIcoshl:
2998 case Builtin::BI__builtin_cosh:
2999 case Builtin::BI__builtin_coshf:
3000 case Builtin::BI__builtin_coshf16:
3001 case Builtin::BI__builtin_coshl:
3002 case Builtin::BI__builtin_coshf128:
3004 *
this,
E, Intrinsic::cosh, Intrinsic::experimental_constrained_cosh));
3006 case Builtin::BIexp:
3007 case Builtin::BIexpf:
3008 case Builtin::BIexpl:
3009 case Builtin::BI__builtin_exp:
3010 case Builtin::BI__builtin_expf:
3011 case Builtin::BI__builtin_expf16:
3012 case Builtin::BI__builtin_expl:
3013 case Builtin::BI__builtin_expf128:
3016 Intrinsic::experimental_constrained_exp));
3018 case Builtin::BIexp2:
3019 case Builtin::BIexp2f:
3020 case Builtin::BIexp2l:
3021 case Builtin::BI__builtin_exp2:
3022 case Builtin::BI__builtin_exp2f:
3023 case Builtin::BI__builtin_exp2f16:
3024 case Builtin::BI__builtin_exp2l:
3025 case Builtin::BI__builtin_exp2f128:
3028 Intrinsic::experimental_constrained_exp2));
3029 case Builtin::BI__builtin_exp10:
3030 case Builtin::BI__builtin_exp10f:
3031 case Builtin::BI__builtin_exp10f16:
3032 case Builtin::BI__builtin_exp10l:
3033 case Builtin::BI__builtin_exp10f128: {
3035 if (
Builder.getIsFPConstrained())
3038 emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::exp10));
3040 case Builtin::BIfabs:
3041 case Builtin::BIfabsf:
3042 case Builtin::BIfabsl:
3043 case Builtin::BI__builtin_fabs:
3044 case Builtin::BI__builtin_fabsf:
3045 case Builtin::BI__builtin_fabsf16:
3046 case Builtin::BI__builtin_fabsl:
3047 case Builtin::BI__builtin_fabsf128:
3049 emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::fabs));
3051 case Builtin::BIfloor:
3052 case Builtin::BIfloorf:
3053 case Builtin::BIfloorl:
3054 case Builtin::BI__builtin_floor:
3055 case Builtin::BI__builtin_floorf:
3056 case Builtin::BI__builtin_floorf16:
3057 case Builtin::BI__builtin_floorl:
3058 case Builtin::BI__builtin_floorf128:
3061 Intrinsic::experimental_constrained_floor));
3063 case Builtin::BIfma:
3064 case Builtin::BIfmaf:
3065 case Builtin::BIfmal:
3066 case Builtin::BI__builtin_fma:
3067 case Builtin::BI__builtin_fmaf:
3068 case Builtin::BI__builtin_fmaf16:
3069 case Builtin::BI__builtin_fmal:
3070 case Builtin::BI__builtin_fmaf128:
3073 Intrinsic::experimental_constrained_fma));
3075 case Builtin::BIfmax:
3076 case Builtin::BIfmaxf:
3077 case Builtin::BIfmaxl:
3078 case Builtin::BI__builtin_fmax:
3079 case Builtin::BI__builtin_fmaxf:
3080 case Builtin::BI__builtin_fmaxf16:
3081 case Builtin::BI__builtin_fmaxl:
3082 case Builtin::BI__builtin_fmaxf128:
3085 Intrinsic::experimental_constrained_maxnum));
3087 case Builtin::BIfmin:
3088 case Builtin::BIfminf:
3089 case Builtin::BIfminl:
3090 case Builtin::BI__builtin_fmin:
3091 case Builtin::BI__builtin_fminf:
3092 case Builtin::BI__builtin_fminf16:
3093 case Builtin::BI__builtin_fminl:
3094 case Builtin::BI__builtin_fminf128:
3097 Intrinsic::experimental_constrained_minnum));
3099 case Builtin::BIfmaximum_num:
3100 case Builtin::BIfmaximum_numf:
3101 case Builtin::BIfmaximum_numl:
3102 case Builtin::BI__builtin_fmaximum_num:
3103 case Builtin::BI__builtin_fmaximum_numf:
3104 case Builtin::BI__builtin_fmaximum_numf16:
3105 case Builtin::BI__builtin_fmaximum_numl:
3106 case Builtin::BI__builtin_fmaximum_numf128:
3108 emitBuiltinWithOneOverloadedType<2>(*
this,
E, Intrinsic::maximumnum));
3110 case Builtin::BIfminimum_num:
3111 case Builtin::BIfminimum_numf:
3112 case Builtin::BIfminimum_numl:
3113 case Builtin::BI__builtin_fminimum_num:
3114 case Builtin::BI__builtin_fminimum_numf:
3115 case Builtin::BI__builtin_fminimum_numf16:
3116 case Builtin::BI__builtin_fminimum_numl:
3117 case Builtin::BI__builtin_fminimum_numf128:
3119 emitBuiltinWithOneOverloadedType<2>(*
this,
E, Intrinsic::minimumnum));
3123 case Builtin::BIfmod:
3124 case Builtin::BIfmodf:
3125 case Builtin::BIfmodl:
3126 case Builtin::BI__builtin_fmod:
3127 case Builtin::BI__builtin_fmodf:
3128 case Builtin::BI__builtin_fmodf16:
3129 case Builtin::BI__builtin_fmodl:
3130 case Builtin::BI__builtin_fmodf128:
3131 case Builtin::BI__builtin_elementwise_fmod: {
3132 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
3138 case Builtin::BIlog:
3139 case Builtin::BIlogf:
3140 case Builtin::BIlogl:
3141 case Builtin::BI__builtin_log:
3142 case Builtin::BI__builtin_logf:
3143 case Builtin::BI__builtin_logf16:
3144 case Builtin::BI__builtin_logl:
3145 case Builtin::BI__builtin_logf128:
3148 Intrinsic::experimental_constrained_log));
3150 case Builtin::BIlog10:
3151 case Builtin::BIlog10f:
3152 case Builtin::BIlog10l:
3153 case Builtin::BI__builtin_log10:
3154 case Builtin::BI__builtin_log10f:
3155 case Builtin::BI__builtin_log10f16:
3156 case Builtin::BI__builtin_log10l:
3157 case Builtin::BI__builtin_log10f128:
3160 Intrinsic::experimental_constrained_log10));
3162 case Builtin::BIlog2:
3163 case Builtin::BIlog2f:
3164 case Builtin::BIlog2l:
3165 case Builtin::BI__builtin_log2:
3166 case Builtin::BI__builtin_log2f:
3167 case Builtin::BI__builtin_log2f16:
3168 case Builtin::BI__builtin_log2l:
3169 case Builtin::BI__builtin_log2f128:
3172 Intrinsic::experimental_constrained_log2));
3174 case Builtin::BInearbyint:
3175 case Builtin::BInearbyintf:
3176 case Builtin::BInearbyintl:
3177 case Builtin::BI__builtin_nearbyint:
3178 case Builtin::BI__builtin_nearbyintf:
3179 case Builtin::BI__builtin_nearbyintl:
3180 case Builtin::BI__builtin_nearbyintf128:
3182 Intrinsic::nearbyint,
3183 Intrinsic::experimental_constrained_nearbyint));
3185 case Builtin::BIpow:
3186 case Builtin::BIpowf:
3187 case Builtin::BIpowl:
3188 case Builtin::BI__builtin_pow:
3189 case Builtin::BI__builtin_powf:
3190 case Builtin::BI__builtin_powf16:
3191 case Builtin::BI__builtin_powl:
3192 case Builtin::BI__builtin_powf128:
3195 Intrinsic::experimental_constrained_pow));
3197 case Builtin::BIrint:
3198 case Builtin::BIrintf:
3199 case Builtin::BIrintl:
3200 case Builtin::BI__builtin_rint:
3201 case Builtin::BI__builtin_rintf:
3202 case Builtin::BI__builtin_rintf16:
3203 case Builtin::BI__builtin_rintl:
3204 case Builtin::BI__builtin_rintf128:
3207 Intrinsic::experimental_constrained_rint));
3209 case Builtin::BIround:
3210 case Builtin::BIroundf:
3211 case Builtin::BIroundl:
3212 case Builtin::BI__builtin_round:
3213 case Builtin::BI__builtin_roundf:
3214 case Builtin::BI__builtin_roundf16:
3215 case Builtin::BI__builtin_roundl:
3216 case Builtin::BI__builtin_roundf128:
3219 Intrinsic::experimental_constrained_round));
3221 case Builtin::BIroundeven:
3222 case Builtin::BIroundevenf:
3223 case Builtin::BIroundevenl:
3224 case Builtin::BI__builtin_roundeven:
3225 case Builtin::BI__builtin_roundevenf:
3226 case Builtin::BI__builtin_roundevenf16:
3227 case Builtin::BI__builtin_roundevenl:
3228 case Builtin::BI__builtin_roundevenf128:
3230 Intrinsic::roundeven,
3231 Intrinsic::experimental_constrained_roundeven));
3233 case Builtin::BIsin:
3234 case Builtin::BIsinf:
3235 case Builtin::BIsinl:
3236 case Builtin::BI__builtin_sin:
3237 case Builtin::BI__builtin_sinf:
3238 case Builtin::BI__builtin_sinf16:
3239 case Builtin::BI__builtin_sinl:
3240 case Builtin::BI__builtin_sinf128:
3243 Intrinsic::experimental_constrained_sin));
3245 case Builtin::BIsinh:
3246 case Builtin::BIsinhf:
3247 case Builtin::BIsinhl:
3248 case Builtin::BI__builtin_sinh:
3249 case Builtin::BI__builtin_sinhf:
3250 case Builtin::BI__builtin_sinhf16:
3251 case Builtin::BI__builtin_sinhl:
3252 case Builtin::BI__builtin_sinhf128:
3254 *
this,
E, Intrinsic::sinh, Intrinsic::experimental_constrained_sinh));
3256 case Builtin::BI__builtin_sincos:
3257 case Builtin::BI__builtin_sincosf:
3258 case Builtin::BI__builtin_sincosf16:
3259 case Builtin::BI__builtin_sincosl:
3260 case Builtin::BI__builtin_sincosf128:
3264 case Builtin::BIsqrt:
3265 case Builtin::BIsqrtf:
3266 case Builtin::BIsqrtl:
3267 case Builtin::BI__builtin_sqrt:
3268 case Builtin::BI__builtin_sqrtf:
3269 case Builtin::BI__builtin_sqrtf16:
3270 case Builtin::BI__builtin_sqrtl:
3271 case Builtin::BI__builtin_sqrtf128:
3272 case Builtin::BI__builtin_elementwise_sqrt: {
3274 *
this,
E, Intrinsic::sqrt, Intrinsic::experimental_constrained_sqrt);
3279 case Builtin::BItan:
3280 case Builtin::BItanf:
3281 case Builtin::BItanl:
3282 case Builtin::BI__builtin_tan:
3283 case Builtin::BI__builtin_tanf:
3284 case Builtin::BI__builtin_tanf16:
3285 case Builtin::BI__builtin_tanl:
3286 case Builtin::BI__builtin_tanf128:
3288 *
this,
E, Intrinsic::tan, Intrinsic::experimental_constrained_tan));
3290 case Builtin::BItanh:
3291 case Builtin::BItanhf:
3292 case Builtin::BItanhl:
3293 case Builtin::BI__builtin_tanh:
3294 case Builtin::BI__builtin_tanhf:
3295 case Builtin::BI__builtin_tanhf16:
3296 case Builtin::BI__builtin_tanhl:
3297 case Builtin::BI__builtin_tanhf128:
3299 *
this,
E, Intrinsic::tanh, Intrinsic::experimental_constrained_tanh));
3301 case Builtin::BItrunc:
3302 case Builtin::BItruncf:
3303 case Builtin::BItruncl:
3304 case Builtin::BI__builtin_trunc:
3305 case Builtin::BI__builtin_truncf:
3306 case Builtin::BI__builtin_truncf16:
3307 case Builtin::BI__builtin_truncl:
3308 case Builtin::BI__builtin_truncf128:
3311 Intrinsic::experimental_constrained_trunc));
3313 case Builtin::BIlround:
3314 case Builtin::BIlroundf:
3315 case Builtin::BIlroundl:
3316 case Builtin::BI__builtin_lround:
3317 case Builtin::BI__builtin_lroundf:
3318 case Builtin::BI__builtin_lroundl:
3319 case Builtin::BI__builtin_lroundf128:
3321 *
this,
E, Intrinsic::lround,
3322 Intrinsic::experimental_constrained_lround));
3324 case Builtin::BIllround:
3325 case Builtin::BIllroundf:
3326 case Builtin::BIllroundl:
3327 case Builtin::BI__builtin_llround:
3328 case Builtin::BI__builtin_llroundf:
3329 case Builtin::BI__builtin_llroundl:
3330 case Builtin::BI__builtin_llroundf128:
3332 *
this,
E, Intrinsic::llround,
3333 Intrinsic::experimental_constrained_llround));
3335 case Builtin::BIlrint:
3336 case Builtin::BIlrintf:
3337 case Builtin::BIlrintl:
3338 case Builtin::BI__builtin_lrint:
3339 case Builtin::BI__builtin_lrintf:
3340 case Builtin::BI__builtin_lrintl:
3341 case Builtin::BI__builtin_lrintf128:
3343 *
this,
E, Intrinsic::lrint,
3344 Intrinsic::experimental_constrained_lrint));
3346 case Builtin::BIllrint:
3347 case Builtin::BIllrintf:
3348 case Builtin::BIllrintl:
3349 case Builtin::BI__builtin_llrint:
3350 case Builtin::BI__builtin_llrintf:
3351 case Builtin::BI__builtin_llrintl:
3352 case Builtin::BI__builtin_llrintf128:
3354 *
this,
E, Intrinsic::llrint,
3355 Intrinsic::experimental_constrained_llrint));
3356 case Builtin::BI__builtin_ldexp:
3357 case Builtin::BI__builtin_ldexpf:
3358 case Builtin::BI__builtin_ldexpl:
3359 case Builtin::BI__builtin_ldexpf16:
3360 case Builtin::BI__builtin_ldexpf128: {
3362 *
this,
E, Intrinsic::ldexp,
3363 Intrinsic::experimental_constrained_ldexp));
3373 Value *Val = A.emitRawPointer(*
this);
3379 SkippedChecks.
set(SanitizerKind::All);
3380 SkippedChecks.
clear(SanitizerKind::Alignment);
3383 if (
auto *CE = dyn_cast<ImplicitCastExpr>(Arg))
3384 if (CE->getCastKind() == CK_BitCast)
3385 Arg = CE->getSubExpr();
3391 switch (BuiltinIDIfNoAsmLabel) {
3393 case Builtin::BI__builtin___CFStringMakeConstantString:
3394 case Builtin::BI__builtin___NSStringMakeConstantString:
3396 case Builtin::BI__builtin_stdarg_start:
3397 case Builtin::BI__builtin_va_start:
3398 case Builtin::BI__va_start:
3399 case Builtin::BI__builtin_va_end:
3403 BuiltinID != Builtin::BI__builtin_va_end);
3405 case Builtin::BI__builtin_va_copy: {
3412 case Builtin::BIabs:
3413 case Builtin::BIlabs:
3414 case Builtin::BIllabs:
3415 case Builtin::BI__builtin_abs:
3416 case Builtin::BI__builtin_labs:
3417 case Builtin::BI__builtin_llabs: {
3418 bool SanitizeOverflow =
SanOpts.
has(SanitizerKind::SignedIntegerOverflow);
3421 switch (
getLangOpts().getSignedOverflowBehavior()) {
3426 if (!SanitizeOverflow) {
3438 case Builtin::BI__builtin_complex: {
3443 case Builtin::BI__builtin_conj:
3444 case Builtin::BI__builtin_conjf:
3445 case Builtin::BI__builtin_conjl:
3446 case Builtin::BIconj:
3447 case Builtin::BIconjf:
3448 case Builtin::BIconjl: {
3450 Value *Real = ComplexVal.first;
3451 Value *Imag = ComplexVal.second;
3452 Imag =
Builder.CreateFNeg(Imag,
"neg");
3455 case Builtin::BI__builtin_creal:
3456 case Builtin::BI__builtin_crealf:
3457 case Builtin::BI__builtin_creall:
3458 case Builtin::BIcreal:
3459 case Builtin::BIcrealf:
3460 case Builtin::BIcreall: {
3465 case Builtin::BI__builtin_preserve_access_index: {
3486 case Builtin::BI__builtin_cimag:
3487 case Builtin::BI__builtin_cimagf:
3488 case Builtin::BI__builtin_cimagl:
3489 case Builtin::BIcimag:
3490 case Builtin::BIcimagf:
3491 case Builtin::BIcimagl: {
3496 case Builtin::BI__builtin_clrsb:
3497 case Builtin::BI__builtin_clrsbl:
3498 case Builtin::BI__builtin_clrsbll: {
3502 llvm::Type *ArgType = ArgValue->
getType();
3506 Value *
Zero = llvm::Constant::getNullValue(ArgType);
3507 Value *IsNeg =
Builder.CreateICmpSLT(ArgValue, Zero,
"isneg");
3509 Value *Tmp =
Builder.CreateSelect(IsNeg, Inverse, ArgValue);
3516 case Builtin::BI__builtin_ctzs:
3517 case Builtin::BI__builtin_ctz:
3518 case Builtin::BI__builtin_ctzl:
3519 case Builtin::BI__builtin_ctzll:
3520 case Builtin::BI__builtin_ctzg: {
3521 bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_ctzg &&
3522 E->getNumArgs() > 1;
3528 llvm::Type *ArgType = ArgValue->
getType();
3535 if (
Result->getType() != ResultType)
3541 Value *
Zero = Constant::getNullValue(ArgType);
3542 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, Zero,
"iszero");
3544 Value *ResultOrFallback =
3545 Builder.CreateSelect(IsZero, FallbackValue,
Result,
"ctzg");
3548 case Builtin::BI__builtin_clzs:
3549 case Builtin::BI__builtin_clz:
3550 case Builtin::BI__builtin_clzl:
3551 case Builtin::BI__builtin_clzll:
3552 case Builtin::BI__builtin_clzg: {
3553 bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_clzg &&
3554 E->getNumArgs() > 1;
3560 llvm::Type *ArgType = ArgValue->
getType();
3567 if (
Result->getType() != ResultType)
3573 Value *
Zero = Constant::getNullValue(ArgType);
3574 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, Zero,
"iszero");
3576 Value *ResultOrFallback =
3577 Builder.CreateSelect(IsZero, FallbackValue,
Result,
"clzg");
3580 case Builtin::BI__builtin_ffs:
3581 case Builtin::BI__builtin_ffsl:
3582 case Builtin::BI__builtin_ffsll: {
3586 llvm::Type *ArgType = ArgValue->
getType();
3591 Builder.CreateAdd(
Builder.CreateCall(F, {ArgValue, Builder.getTrue()}),
3592 llvm::ConstantInt::get(ArgType, 1));
3593 Value *
Zero = llvm::Constant::getNullValue(ArgType);
3594 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, Zero,
"iszero");
3596 if (
Result->getType() != ResultType)
3601 case Builtin::BI__builtin_parity:
3602 case Builtin::BI__builtin_parityl:
3603 case Builtin::BI__builtin_parityll: {
3607 llvm::Type *ArgType = ArgValue->
getType();
3613 if (
Result->getType() != ResultType)
3618 case Builtin::BI__lzcnt16:
3619 case Builtin::BI__lzcnt:
3620 case Builtin::BI__lzcnt64: {
3623 llvm::Type *ArgType = ArgValue->
getType();
3628 if (
Result->getType() != ResultType)
3633 case Builtin::BI__popcnt16:
3634 case Builtin::BI__popcnt:
3635 case Builtin::BI__popcnt64:
3636 case Builtin::BI__builtin_popcount:
3637 case Builtin::BI__builtin_popcountl:
3638 case Builtin::BI__builtin_popcountll:
3639 case Builtin::BI__builtin_popcountg: {
3642 llvm::Type *ArgType = ArgValue->
getType();
3647 if (
Result->getType() != ResultType)
3652 case Builtin::BI__builtin_unpredictable: {
3658 case Builtin::BI__builtin_expect: {
3660 llvm::Type *ArgType = ArgValue->
getType();
3671 Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue},
"expval");
3674 case Builtin::BI__builtin_expect_with_probability: {
3676 llvm::Type *ArgType = ArgValue->
getType();
3679 llvm::APFloat Probability(0.0);
3680 const Expr *ProbArg =
E->getArg(2);
3682 assert(EvalSucceed &&
"probability should be able to evaluate as float");
3684 bool LoseInfo =
false;
3685 Probability.convert(llvm::APFloat::IEEEdouble(),
3686 llvm::RoundingMode::Dynamic, &LoseInfo);
3688 Constant *Confidence = ConstantFP::get(Ty, Probability);
3698 FnExpect, {ArgValue, ExpectedValue, Confidence},
"expval");
3701 case Builtin::BI__builtin_assume_aligned: {
3702 const Expr *Ptr =
E->getArg(0);
3704 Value *OffsetValue =
3708 ConstantInt *AlignmentCI = cast<ConstantInt>(AlignmentValue);
3709 if (AlignmentCI->getValue().ugt(llvm::Value::MaximumAlignment))
3710 AlignmentCI = ConstantInt::get(AlignmentCI->getIntegerType(),
3711 llvm::Value::MaximumAlignment);
3715 AlignmentCI, OffsetValue);
3718 case Builtin::BI__assume:
3719 case Builtin::BI__builtin_assume: {
3725 Builder.CreateCall(FnAssume, ArgValue);
3728 case Builtin::BI__builtin_assume_separate_storage: {
3729 const Expr *Arg0 =
E->getArg(0);
3730 const Expr *Arg1 =
E->getArg(1);
3735 Value *Values[] = {Value0, Value1};
3736 OperandBundleDefT<Value *> OBD(
"separate_storage", Values);
3740 case Builtin::BI__builtin_allow_runtime_check: {
3744 llvm::Value *Allow =
Builder.CreateCall(
3746 llvm::MetadataAsValue::get(Ctx, llvm::MDString::get(Ctx, Kind)));
3749 case Builtin::BI__arithmetic_fence: {
3752 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
3753 llvm::FastMathFlags FMF =
Builder.getFastMathFlags();
3754 bool isArithmeticFenceEnabled =
3755 FMF.allowReassoc() &&
3759 if (isArithmeticFenceEnabled) {
3762 Value *Real =
Builder.CreateArithmeticFence(ComplexVal.first,
3764 Value *Imag =
Builder.CreateArithmeticFence(ComplexVal.second,
3769 Value *Real = ComplexVal.first;
3770 Value *Imag = ComplexVal.second;
3774 if (isArithmeticFenceEnabled)
3779 case Builtin::BI__builtin_bswap16:
3780 case Builtin::BI__builtin_bswap32:
3781 case Builtin::BI__builtin_bswap64:
3782 case Builtin::BI_byteswap_ushort:
3783 case Builtin::BI_byteswap_ulong:
3784 case Builtin::BI_byteswap_uint64: {
3786 emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::bswap));
3788 case Builtin::BI__builtin_bitreverse8:
3789 case Builtin::BI__builtin_bitreverse16:
3790 case Builtin::BI__builtin_bitreverse32:
3791 case Builtin::BI__builtin_bitreverse64: {
3793 emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::bitreverse));
3795 case Builtin::BI__builtin_rotateleft8:
3796 case Builtin::BI__builtin_rotateleft16:
3797 case Builtin::BI__builtin_rotateleft32:
3798 case Builtin::BI__builtin_rotateleft64:
3799 case Builtin::BI_rotl8:
3800 case Builtin::BI_rotl16:
3801 case Builtin::BI_rotl:
3802 case Builtin::BI_lrotl:
3803 case Builtin::BI_rotl64:
3806 case Builtin::BI__builtin_rotateright8:
3807 case Builtin::BI__builtin_rotateright16:
3808 case Builtin::BI__builtin_rotateright32:
3809 case Builtin::BI__builtin_rotateright64:
3810 case Builtin::BI_rotr8:
3811 case Builtin::BI_rotr16:
3812 case Builtin::BI_rotr:
3813 case Builtin::BI_lrotr:
3814 case Builtin::BI_rotr64:
3817 case Builtin::BI__builtin_constant_p: {
3820 const Expr *Arg =
E->getArg(0);
3828 return RValue::get(ConstantInt::get(ResultType, 0));
3833 return RValue::get(ConstantInt::get(ResultType, 0));
3845 if (
Result->getType() != ResultType)
3849 case Builtin::BI__builtin_dynamic_object_size:
3850 case Builtin::BI__builtin_object_size: {
3857 bool IsDynamic = BuiltinID == Builtin::BI__builtin_dynamic_object_size;
3859 nullptr, IsDynamic));
3861 case Builtin::BI__builtin_counted_by_ref: {
3863 llvm::Value *
Result = llvm::ConstantPointerNull::get(
3868 if (
auto *UO = dyn_cast<UnaryOperator>(Arg);
3869 UO && UO->getOpcode() == UO_AddrOf) {
3872 if (
auto *ASE = dyn_cast<ArraySubscriptExpr>(Arg))
3876 if (
const MemberExpr *ME = dyn_cast_if_present<MemberExpr>(Arg)) {
3880 const auto *FAMDecl = cast<FieldDecl>(ME->getMemberDecl());
3884 llvm::report_fatal_error(
"Cannot find the counted_by 'count' field");
3890 case Builtin::BI__builtin_prefetch: {
3894 llvm::ConstantInt::get(
Int32Ty, 0);
3896 llvm::ConstantInt::get(
Int32Ty, 3);
3902 case Builtin::BI__builtin_readcyclecounter: {
3906 case Builtin::BI__builtin_readsteadycounter: {
3910 case Builtin::BI__builtin___clear_cache: {
3916 case Builtin::BI__builtin_trap:
3919 case Builtin::BI__builtin_verbose_trap: {
3920 llvm::DILocation *TrapLocation =
Builder.getCurrentDebugLocation();
3931 case Builtin::BI__debugbreak:
3934 case Builtin::BI__builtin_unreachable: {
3943 case Builtin::BI__builtin_powi:
3944 case Builtin::BI__builtin_powif:
3945 case Builtin::BI__builtin_powil: {
3949 if (
Builder.getIsFPConstrained()) {
3952 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
3959 { Src0->getType(), Src1->getType() });
3962 case Builtin::BI__builtin_frexpl: {
3966 if (&
getTarget().getLongDoubleFormat() == &llvm::APFloat::PPCDoubleDouble())
3970 case Builtin::BI__builtin_frexp:
3971 case Builtin::BI__builtin_frexpf:
3972 case Builtin::BI__builtin_frexpf128:
3973 case Builtin::BI__builtin_frexpf16:
3975 case Builtin::BI__builtin_isgreater:
3976 case Builtin::BI__builtin_isgreaterequal:
3977 case Builtin::BI__builtin_isless:
3978 case Builtin::BI__builtin_islessequal:
3979 case Builtin::BI__builtin_islessgreater:
3980 case Builtin::BI__builtin_isunordered: {
3983 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
3987 switch (BuiltinID) {
3988 default: llvm_unreachable(
"Unknown ordered comparison");
3989 case Builtin::BI__builtin_isgreater:
3990 LHS =
Builder.CreateFCmpOGT(LHS, RHS,
"cmp");
3992 case Builtin::BI__builtin_isgreaterequal:
3993 LHS =
Builder.CreateFCmpOGE(LHS, RHS,
"cmp");
3995 case Builtin::BI__builtin_isless:
3996 LHS =
Builder.CreateFCmpOLT(LHS, RHS,
"cmp");
3998 case Builtin::BI__builtin_islessequal:
3999 LHS =
Builder.CreateFCmpOLE(LHS, RHS,
"cmp");
4001 case Builtin::BI__builtin_islessgreater:
4002 LHS =
Builder.CreateFCmpONE(LHS, RHS,
"cmp");
4004 case Builtin::BI__builtin_isunordered:
4005 LHS =
Builder.CreateFCmpUNO(LHS, RHS,
"cmp");
4012 case Builtin::BI__builtin_isnan: {
4013 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4022 case Builtin::BI__builtin_issignaling: {
4023 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4030 case Builtin::BI__builtin_isinf: {
4031 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4040 case Builtin::BIfinite:
4041 case Builtin::BI__finite:
4042 case Builtin::BIfinitef:
4043 case Builtin::BI__finitef:
4044 case Builtin::BIfinitel:
4045 case Builtin::BI__finitel:
4046 case Builtin::BI__builtin_isfinite: {
4047 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4056 case Builtin::BI__builtin_isnormal: {
4057 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4064 case Builtin::BI__builtin_issubnormal: {
4065 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4068 Builder.CreateZExt(
Builder.createIsFPClass(
V, FPClassTest::fcSubnormal),
4072 case Builtin::BI__builtin_iszero: {
4073 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4080 case Builtin::BI__builtin_isfpclass: {
4085 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4091 case Builtin::BI__builtin_nondeterministic_value: {
4100 case Builtin::BI__builtin_elementwise_abs: {
4105 QT = VecTy->getElementType();
4109 Builder.getFalse(),
nullptr,
"elt.abs");
4111 Result = emitBuiltinWithOneOverloadedType<1>(
4112 *
this,
E, llvm::Intrinsic::fabs,
"elt.abs");
4116 case Builtin::BI__builtin_elementwise_acos:
4117 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4118 *
this,
E, llvm::Intrinsic::acos,
"elt.acos"));
4119 case Builtin::BI__builtin_elementwise_asin:
4120 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4121 *
this,
E, llvm::Intrinsic::asin,
"elt.asin"));
4122 case Builtin::BI__builtin_elementwise_atan:
4123 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4124 *
this,
E, llvm::Intrinsic::atan,
"elt.atan"));
4125 case Builtin::BI__builtin_elementwise_atan2:
4126 return RValue::get(emitBuiltinWithOneOverloadedType<2>(
4127 *
this,
E, llvm::Intrinsic::atan2,
"elt.atan2"));
4128 case Builtin::BI__builtin_elementwise_ceil:
4129 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4130 *
this,
E, llvm::Intrinsic::ceil,
"elt.ceil"));
4131 case Builtin::BI__builtin_elementwise_exp:
4132 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4133 *
this,
E, llvm::Intrinsic::exp,
"elt.exp"));
4134 case Builtin::BI__builtin_elementwise_exp2:
4135 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4136 *
this,
E, llvm::Intrinsic::exp2,
"elt.exp2"));
4137 case Builtin::BI__builtin_elementwise_log:
4138 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4139 *
this,
E, llvm::Intrinsic::log,
"elt.log"));
4140 case Builtin::BI__builtin_elementwise_log2:
4141 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4142 *
this,
E, llvm::Intrinsic::log2,
"elt.log2"));
4143 case Builtin::BI__builtin_elementwise_log10:
4144 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4145 *
this,
E, llvm::Intrinsic::log10,
"elt.log10"));
4146 case Builtin::BI__builtin_elementwise_pow: {
4148 emitBuiltinWithOneOverloadedType<2>(*
this,
E, llvm::Intrinsic::pow));
4150 case Builtin::BI__builtin_elementwise_bitreverse:
4151 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4152 *
this,
E, llvm::Intrinsic::bitreverse,
"elt.bitreverse"));
4153 case Builtin::BI__builtin_elementwise_cos:
4154 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4155 *
this,
E, llvm::Intrinsic::cos,
"elt.cos"));
4156 case Builtin::BI__builtin_elementwise_cosh:
4157 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4158 *
this,
E, llvm::Intrinsic::cosh,
"elt.cosh"));
4159 case Builtin::BI__builtin_elementwise_floor:
4160 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4161 *
this,
E, llvm::Intrinsic::floor,
"elt.floor"));
4162 case Builtin::BI__builtin_elementwise_popcount:
4163 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4164 *
this,
E, llvm::Intrinsic::ctpop,
"elt.ctpop"));
4165 case Builtin::BI__builtin_elementwise_roundeven:
4166 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4167 *
this,
E, llvm::Intrinsic::roundeven,
"elt.roundeven"));
4168 case Builtin::BI__builtin_elementwise_round:
4169 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4170 *
this,
E, llvm::Intrinsic::round,
"elt.round"));
4171 case Builtin::BI__builtin_elementwise_rint:
4172 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4173 *
this,
E, llvm::Intrinsic::rint,
"elt.rint"));
4174 case Builtin::BI__builtin_elementwise_nearbyint:
4175 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4176 *
this,
E, llvm::Intrinsic::nearbyint,
"elt.nearbyint"));
4177 case Builtin::BI__builtin_elementwise_sin:
4178 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4179 *
this,
E, llvm::Intrinsic::sin,
"elt.sin"));
4180 case Builtin::BI__builtin_elementwise_sinh:
4181 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4182 *
this,
E, llvm::Intrinsic::sinh,
"elt.sinh"));
4183 case Builtin::BI__builtin_elementwise_tan:
4184 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4185 *
this,
E, llvm::Intrinsic::tan,
"elt.tan"));
4186 case Builtin::BI__builtin_elementwise_tanh:
4187 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4188 *
this,
E, llvm::Intrinsic::tanh,
"elt.tanh"));
4189 case Builtin::BI__builtin_elementwise_trunc:
4190 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4191 *
this,
E, llvm::Intrinsic::trunc,
"elt.trunc"));
4192 case Builtin::BI__builtin_elementwise_canonicalize:
4193 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4194 *
this,
E, llvm::Intrinsic::canonicalize,
"elt.canonicalize"));
4195 case Builtin::BI__builtin_elementwise_copysign:
4196 return RValue::get(emitBuiltinWithOneOverloadedType<2>(
4197 *
this,
E, llvm::Intrinsic::copysign));
4198 case Builtin::BI__builtin_elementwise_fma:
4200 emitBuiltinWithOneOverloadedType<3>(*
this,
E, llvm::Intrinsic::fma));
4201 case Builtin::BI__builtin_elementwise_add_sat:
4202 case Builtin::BI__builtin_elementwise_sub_sat: {
4206 assert(Op0->
getType()->isIntOrIntVectorTy() &&
"integer type expected");
4209 Ty = VecTy->getElementType();
4212 if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_add_sat)
4213 Opc = IsSigned ? llvm::Intrinsic::sadd_sat : llvm::Intrinsic::uadd_sat;
4215 Opc = IsSigned ? llvm::Intrinsic::ssub_sat : llvm::Intrinsic::usub_sat;
4216 Result =
Builder.CreateBinaryIntrinsic(Opc, Op0, Op1,
nullptr,
"elt.sat");
4220 case Builtin::BI__builtin_elementwise_max: {
4224 if (Op0->
getType()->isIntOrIntVectorTy()) {
4227 Ty = VecTy->getElementType();
4229 ? llvm::Intrinsic::smax
4230 : llvm::Intrinsic::umax,
4231 Op0, Op1,
nullptr,
"elt.max");
4236 case Builtin::BI__builtin_elementwise_min: {
4240 if (Op0->
getType()->isIntOrIntVectorTy()) {
4243 Ty = VecTy->getElementType();
4245 ? llvm::Intrinsic::smin
4246 : llvm::Intrinsic::umin,
4247 Op0, Op1,
nullptr,
"elt.min");
4253 case Builtin::BI__builtin_elementwise_maximum: {
4257 Op1,
nullptr,
"elt.maximum");
4261 case Builtin::BI__builtin_elementwise_minimum: {
4265 Op1,
nullptr,
"elt.minimum");
4269 case Builtin::BI__builtin_reduce_max: {
4270 auto GetIntrinsicID = [
this](
QualType QT) {
4272 QT = VecTy->getElementType();
4277 return llvm::Intrinsic::vector_reduce_smax;
4279 return llvm::Intrinsic::vector_reduce_umax;
4281 return llvm::Intrinsic::vector_reduce_fmax;
4283 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4284 *
this,
E, GetIntrinsicID(
E->getArg(0)->
getType()),
"rdx.min"));
4287 case Builtin::BI__builtin_reduce_min: {
4288 auto GetIntrinsicID = [
this](
QualType QT) {
4290 QT = VecTy->getElementType();
4295 return llvm::Intrinsic::vector_reduce_smin;
4297 return llvm::Intrinsic::vector_reduce_umin;
4299 return llvm::Intrinsic::vector_reduce_fmin;
4302 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4303 *
this,
E, GetIntrinsicID(
E->getArg(0)->
getType()),
"rdx.min"));
4306 case Builtin::BI__builtin_reduce_add:
4307 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4308 *
this,
E, llvm::Intrinsic::vector_reduce_add,
"rdx.add"));
4309 case Builtin::BI__builtin_reduce_mul:
4310 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4311 *
this,
E, llvm::Intrinsic::vector_reduce_mul,
"rdx.mul"));
4312 case Builtin::BI__builtin_reduce_xor:
4313 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4314 *
this,
E, llvm::Intrinsic::vector_reduce_xor,
"rdx.xor"));
4315 case Builtin::BI__builtin_reduce_or:
4316 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4317 *
this,
E, llvm::Intrinsic::vector_reduce_or,
"rdx.or"));
4318 case Builtin::BI__builtin_reduce_and:
4319 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4320 *
this,
E, llvm::Intrinsic::vector_reduce_and,
"rdx.and"));
4321 case Builtin::BI__builtin_reduce_maximum:
4322 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4323 *
this,
E, llvm::Intrinsic::vector_reduce_fmaximum,
"rdx.maximum"));
4324 case Builtin::BI__builtin_reduce_minimum:
4325 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4326 *
this,
E, llvm::Intrinsic::vector_reduce_fminimum,
"rdx.minimum"));
4328 case Builtin::BI__builtin_matrix_transpose: {
4332 Value *
Result = MB.CreateMatrixTranspose(MatValue, MatrixTy->getNumRows(),
4333 MatrixTy->getNumColumns());
4337 case Builtin::BI__builtin_matrix_column_major_load: {
4343 assert(PtrTy &&
"arg0 must be of pointer type");
4353 ResultTy->getNumRows(), ResultTy->getNumColumns(),
"matrix");
4357 case Builtin::BI__builtin_matrix_column_major_store: {
4365 assert(PtrTy &&
"arg1 must be of pointer type");
4374 MatrixTy->getNumRows(), MatrixTy->getNumColumns());
4378 case Builtin::BI__builtin_isinf_sign: {
4380 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4385 AbsArg, ConstantFP::getInfinity(Arg->
getType()),
"isinf");
4391 Value *NegativeOne = ConstantInt::get(
IntTy, -1);
4392 Value *SignResult =
Builder.CreateSelect(IsNeg, NegativeOne, One);
4397 case Builtin::BI__builtin_flt_rounds: {
4402 if (
Result->getType() != ResultType)
4408 case Builtin::BI__builtin_set_flt_rounds: {
4416 case Builtin::BI__builtin_fpclassify: {
4417 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4428 "fpclassify_result");
4432 Value *IsZero =
Builder.CreateFCmpOEQ(
V, Constant::getNullValue(Ty),
4436 Builder.CreateCondBr(IsZero, End, NotZero);
4440 Builder.SetInsertPoint(NotZero);
4444 Builder.CreateCondBr(IsNan, End, NotNan);
4445 Result->addIncoming(NanLiteral, NotZero);
4448 Builder.SetInsertPoint(NotNan);
4451 Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(
V->getType()),
4455 Builder.CreateCondBr(IsInf, End, NotInf);
4456 Result->addIncoming(InfLiteral, NotNan);
4459 Builder.SetInsertPoint(NotInf);
4460 APFloat Smallest = APFloat::getSmallestNormalized(
4463 Builder.CreateFCmpUGE(VAbs, ConstantFP::get(
V->getContext(), Smallest),
4465 Value *NormalResult =
4469 Result->addIncoming(NormalResult, NotInf);
4482 case Builtin::BIalloca:
4483 case Builtin::BI_alloca:
4484 case Builtin::BI__builtin_alloca_uninitialized:
4485 case Builtin::BI__builtin_alloca: {
4489 const Align SuitableAlignmentInBytes =
4493 AllocaInst *AI =
Builder.CreateAlloca(
Builder.getInt8Ty(), Size);
4494 AI->setAlignment(SuitableAlignmentInBytes);
4495 if (BuiltinID != Builtin::BI__builtin_alloca_uninitialized)
4507 case Builtin::BI__builtin_alloca_with_align_uninitialized:
4508 case Builtin::BI__builtin_alloca_with_align: {
4511 auto *AlignmentInBitsCI = cast<ConstantInt>(AlignmentInBitsValue);
4512 unsigned AlignmentInBits = AlignmentInBitsCI->getZExtValue();
4513 const Align AlignmentInBytes =
4515 AllocaInst *AI =
Builder.CreateAlloca(
Builder.getInt8Ty(), Size);
4516 AI->setAlignment(AlignmentInBytes);
4517 if (BuiltinID != Builtin::BI__builtin_alloca_with_align_uninitialized)
4529 case Builtin::BIbzero:
4530 case Builtin::BI__builtin_bzero: {
4539 case Builtin::BIbcopy:
4540 case Builtin::BI__builtin_bcopy: {
4554 case Builtin::BImemcpy:
4555 case Builtin::BI__builtin_memcpy:
4556 case Builtin::BImempcpy:
4557 case Builtin::BI__builtin_mempcpy: {
4561 EmitArgCheck(
TCK_Store, Dest,
E->getArg(0), 0);
4562 EmitArgCheck(
TCK_Load, Src,
E->getArg(1), 1);
4564 if (BuiltinID == Builtin::BImempcpy ||
4565 BuiltinID == Builtin::BI__builtin_mempcpy)
4572 case Builtin::BI__builtin_memcpy_inline: {
4577 EmitArgCheck(
TCK_Store, Dest,
E->getArg(0), 0);
4578 EmitArgCheck(
TCK_Load, Src,
E->getArg(1), 1);
4583 case Builtin::BI__builtin_char_memchr:
4584 BuiltinID = Builtin::BI__builtin_memchr;
4587 case Builtin::BI__builtin___memcpy_chk: {
4594 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
4595 if (
Size.ugt(DstSize))
4599 Value *SizeVal = llvm::ConstantInt::get(
Builder.getContext(), Size);
4604 case Builtin::BI__builtin_objc_memmove_collectable: {
4609 DestAddr, SrcAddr, SizeVal);
4613 case Builtin::BI__builtin___memmove_chk: {
4620 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
4621 if (
Size.ugt(DstSize))
4625 Value *SizeVal = llvm::ConstantInt::get(
Builder.getContext(), Size);
4630 case Builtin::BImemmove:
4631 case Builtin::BI__builtin_memmove: {
4635 EmitArgCheck(
TCK_Store, Dest,
E->getArg(0), 0);
4636 EmitArgCheck(
TCK_Load, Src,
E->getArg(1), 1);
4640 case Builtin::BImemset:
4641 case Builtin::BI__builtin_memset: {
4651 case Builtin::BI__builtin_memset_inline: {
4663 case Builtin::BI__builtin___memset_chk: {
4670 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
4671 if (
Size.ugt(DstSize))
4676 Value *SizeVal = llvm::ConstantInt::get(
Builder.getContext(), Size);
4680 case Builtin::BI__builtin_wmemchr: {
4683 if (!
getTarget().getTriple().isOSMSVCRT())
4691 BasicBlock *Entry =
Builder.GetInsertBlock();
4696 Builder.CreateCondBr(SizeEq0, Exit, CmpEq);
4700 StrPhi->addIncoming(Str, Entry);
4702 SizePhi->addIncoming(Size, Entry);
4706 Value *FoundChr =
Builder.CreateConstInBoundsGEP1_32(WCharTy, StrPhi, 0);
4708 Builder.CreateCondBr(StrEqChr, Exit, Next);
4711 Value *NextStr =
Builder.CreateConstInBoundsGEP1_32(WCharTy, StrPhi, 1);
4713 Value *NextSizeEq0 =
4714 Builder.CreateICmpEQ(NextSize, ConstantInt::get(
SizeTy, 0));
4715 Builder.CreateCondBr(NextSizeEq0, Exit, CmpEq);
4716 StrPhi->addIncoming(NextStr, Next);
4717 SizePhi->addIncoming(NextSize, Next);
4721 Ret->addIncoming(llvm::Constant::getNullValue(Str->
getType()), Entry);
4722 Ret->addIncoming(llvm::Constant::getNullValue(Str->
getType()), Next);
4723 Ret->addIncoming(FoundChr, CmpEq);
4726 case Builtin::BI__builtin_wmemcmp: {
4729 if (!
getTarget().getTriple().isOSMSVCRT())
4738 BasicBlock *Entry =
Builder.GetInsertBlock();
4744 Builder.CreateCondBr(SizeEq0, Exit, CmpGT);
4748 DstPhi->addIncoming(Dst, Entry);
4750 SrcPhi->addIncoming(Src, Entry);
4752 SizePhi->addIncoming(Size, Entry);
4758 Builder.CreateCondBr(DstGtSrc, Exit, CmpLT);
4762 Builder.CreateCondBr(DstLtSrc, Exit, Next);
4765 Value *NextDst =
Builder.CreateConstInBoundsGEP1_32(WCharTy, DstPhi, 1);
4766 Value *NextSrc =
Builder.CreateConstInBoundsGEP1_32(WCharTy, SrcPhi, 1);
4768 Value *NextSizeEq0 =
4769 Builder.CreateICmpEQ(NextSize, ConstantInt::get(
SizeTy, 0));
4770 Builder.CreateCondBr(NextSizeEq0, Exit, CmpGT);
4771 DstPhi->addIncoming(NextDst, Next);
4772 SrcPhi->addIncoming(NextSrc, Next);
4773 SizePhi->addIncoming(NextSize, Next);
4777 Ret->addIncoming(ConstantInt::get(
IntTy, 0), Entry);
4778 Ret->addIncoming(ConstantInt::get(
IntTy, 1), CmpGT);
4779 Ret->addIncoming(ConstantInt::get(
IntTy, -1), CmpLT);
4780 Ret->addIncoming(ConstantInt::get(
IntTy, 0), Next);
4783 case Builtin::BI__builtin_dwarf_cfa: {
4796 llvm::ConstantInt::get(
Int32Ty, Offset)));
4798 case Builtin::BI__builtin_return_address: {
4804 case Builtin::BI_ReturnAddress: {
4808 case Builtin::BI__builtin_frame_address: {
4814 case Builtin::BI__builtin_extract_return_addr: {
4819 case Builtin::BI__builtin_frob_return_addr: {
4824 case Builtin::BI__builtin_dwarf_sp_column: {
4825 llvm::IntegerType *Ty
4834 case Builtin::BI__builtin_init_dwarf_reg_size_table: {
4840 case Builtin::BI__builtin_eh_return: {
4844 llvm::IntegerType *
IntTy = cast<llvm::IntegerType>(
Int->getType());
4845 assert((
IntTy->getBitWidth() == 32 ||
IntTy->getBitWidth() == 64) &&
4846 "LLVM's __builtin_eh_return only supports 32- and 64-bit variants");
4849 : Intrinsic::eh_return_i64);
4858 case Builtin::BI__builtin_unwind_init: {
4863 case Builtin::BI__builtin_extend_pointer: {
4888 case Builtin::BI__builtin_setjmp: {
4892 if (
getTarget().getTriple().getArch() == llvm::Triple::systemz) {
4903 ConstantInt::get(
Int32Ty, 0));
4917 case Builtin::BI__builtin_longjmp: {
4931 case Builtin::BI__builtin_launder: {
4932 const Expr *Arg =
E->getArg(0);
4940 case Builtin::BI__sync_fetch_and_add:
4941 case Builtin::BI__sync_fetch_and_sub:
4942 case Builtin::BI__sync_fetch_and_or:
4943 case Builtin::BI__sync_fetch_and_and:
4944 case Builtin::BI__sync_fetch_and_xor:
4945 case Builtin::BI__sync_fetch_and_nand:
4946 case Builtin::BI__sync_add_and_fetch:
4947 case Builtin::BI__sync_sub_and_fetch:
4948 case Builtin::BI__sync_and_and_fetch:
4949 case Builtin::BI__sync_or_and_fetch:
4950 case Builtin::BI__sync_xor_and_fetch:
4951 case Builtin::BI__sync_nand_and_fetch:
4952 case Builtin::BI__sync_val_compare_and_swap:
4953 case Builtin::BI__sync_bool_compare_and_swap:
4954 case Builtin::BI__sync_lock_test_and_set:
4955 case Builtin::BI__sync_lock_release:
4956 case Builtin::BI__sync_swap:
4957 llvm_unreachable(
"Shouldn't make it through sema");
4958 case Builtin::BI__sync_fetch_and_add_1:
4959 case Builtin::BI__sync_fetch_and_add_2:
4960 case Builtin::BI__sync_fetch_and_add_4:
4961 case Builtin::BI__sync_fetch_and_add_8:
4962 case Builtin::BI__sync_fetch_and_add_16:
4964 case Builtin::BI__sync_fetch_and_sub_1:
4965 case Builtin::BI__sync_fetch_and_sub_2:
4966 case Builtin::BI__sync_fetch_and_sub_4:
4967 case Builtin::BI__sync_fetch_and_sub_8:
4968 case Builtin::BI__sync_fetch_and_sub_16:
4970 case Builtin::BI__sync_fetch_and_or_1:
4971 case Builtin::BI__sync_fetch_and_or_2:
4972 case Builtin::BI__sync_fetch_and_or_4:
4973 case Builtin::BI__sync_fetch_and_or_8:
4974 case Builtin::BI__sync_fetch_and_or_16:
4976 case Builtin::BI__sync_fetch_and_and_1:
4977 case Builtin::BI__sync_fetch_and_and_2:
4978 case Builtin::BI__sync_fetch_and_and_4:
4979 case Builtin::BI__sync_fetch_and_and_8:
4980 case Builtin::BI__sync_fetch_and_and_16:
4982 case Builtin::BI__sync_fetch_and_xor_1:
4983 case Builtin::BI__sync_fetch_and_xor_2:
4984 case Builtin::BI__sync_fetch_and_xor_4:
4985 case Builtin::BI__sync_fetch_and_xor_8:
4986 case Builtin::BI__sync_fetch_and_xor_16:
4988 case Builtin::BI__sync_fetch_and_nand_1:
4989 case Builtin::BI__sync_fetch_and_nand_2:
4990 case Builtin::BI__sync_fetch_and_nand_4:
4991 case Builtin::BI__sync_fetch_and_nand_8:
4992 case Builtin::BI__sync_fetch_and_nand_16:
4996 case Builtin::BI__sync_fetch_and_min:
4998 case Builtin::BI__sync_fetch_and_max:
5000 case Builtin::BI__sync_fetch_and_umin:
5002 case Builtin::BI__sync_fetch_and_umax:
5005 case Builtin::BI__sync_add_and_fetch_1:
5006 case Builtin::BI__sync_add_and_fetch_2:
5007 case Builtin::BI__sync_add_and_fetch_4:
5008 case Builtin::BI__sync_add_and_fetch_8:
5009 case Builtin::BI__sync_add_and_fetch_16:
5011 llvm::Instruction::Add);
5012 case Builtin::BI__sync_sub_and_fetch_1:
5013 case Builtin::BI__sync_sub_and_fetch_2:
5014 case Builtin::BI__sync_sub_and_fetch_4:
5015 case Builtin::BI__sync_sub_and_fetch_8:
5016 case Builtin::BI__sync_sub_and_fetch_16:
5018 llvm::Instruction::Sub);
5019 case Builtin::BI__sync_and_and_fetch_1:
5020 case Builtin::BI__sync_and_and_fetch_2:
5021 case Builtin::BI__sync_and_and_fetch_4:
5022 case Builtin::BI__sync_and_and_fetch_8:
5023 case Builtin::BI__sync_and_and_fetch_16:
5025 llvm::Instruction::And);
5026 case Builtin::BI__sync_or_and_fetch_1:
5027 case Builtin::BI__sync_or_and_fetch_2:
5028 case Builtin::BI__sync_or_and_fetch_4:
5029 case Builtin::BI__sync_or_and_fetch_8:
5030 case Builtin::BI__sync_or_and_fetch_16:
5032 llvm::Instruction::Or);
5033 case Builtin::BI__sync_xor_and_fetch_1:
5034 case Builtin::BI__sync_xor_and_fetch_2:
5035 case Builtin::BI__sync_xor_and_fetch_4:
5036 case Builtin::BI__sync_xor_and_fetch_8:
5037 case Builtin::BI__sync_xor_and_fetch_16:
5039 llvm::Instruction::Xor);
5040 case Builtin::BI__sync_nand_and_fetch_1:
5041 case Builtin::BI__sync_nand_and_fetch_2:
5042 case Builtin::BI__sync_nand_and_fetch_4:
5043 case Builtin::BI__sync_nand_and_fetch_8:
5044 case Builtin::BI__sync_nand_and_fetch_16:
5046 llvm::Instruction::And,
true);
5048 case Builtin::BI__sync_val_compare_and_swap_1:
5049 case Builtin::BI__sync_val_compare_and_swap_2:
5050 case Builtin::BI__sync_val_compare_and_swap_4:
5051 case Builtin::BI__sync_val_compare_and_swap_8:
5052 case Builtin::BI__sync_val_compare_and_swap_16:
5055 case Builtin::BI__sync_bool_compare_and_swap_1:
5056 case Builtin::BI__sync_bool_compare_and_swap_2:
5057 case Builtin::BI__sync_bool_compare_and_swap_4:
5058 case Builtin::BI__sync_bool_compare_and_swap_8:
5059 case Builtin::BI__sync_bool_compare_and_swap_16:
5062 case Builtin::BI__sync_swap_1:
5063 case Builtin::BI__sync_swap_2:
5064 case Builtin::BI__sync_swap_4:
5065 case Builtin::BI__sync_swap_8:
5066 case Builtin::BI__sync_swap_16:
5069 case Builtin::BI__sync_lock_test_and_set_1:
5070 case Builtin::BI__sync_lock_test_and_set_2:
5071 case Builtin::BI__sync_lock_test_and_set_4:
5072 case Builtin::BI__sync_lock_test_and_set_8:
5073 case Builtin::BI__sync_lock_test_and_set_16:
5076 case Builtin::BI__sync_lock_release_1:
5077 case Builtin::BI__sync_lock_release_2:
5078 case Builtin::BI__sync_lock_release_4:
5079 case Builtin::BI__sync_lock_release_8:
5080 case Builtin::BI__sync_lock_release_16: {
5086 llvm::StoreInst *
Store =
5088 Store->setAtomic(llvm::AtomicOrdering::Release);
5092 case Builtin::BI__sync_synchronize: {
5100 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent);
5104 case Builtin::BI__builtin_nontemporal_load:
5106 case Builtin::BI__builtin_nontemporal_store:
5108 case Builtin::BI__c11_atomic_is_lock_free:
5109 case Builtin::BI__atomic_is_lock_free: {
5113 const char *LibCallName =
"__atomic_is_lock_free";
5117 if (BuiltinID == Builtin::BI__atomic_is_lock_free)
5131 case Builtin::BI__atomic_thread_fence:
5132 case Builtin::BI__atomic_signal_fence:
5133 case Builtin::BI__c11_atomic_thread_fence:
5134 case Builtin::BI__c11_atomic_signal_fence: {
5135 llvm::SyncScope::ID SSID;
5136 if (BuiltinID == Builtin::BI__atomic_signal_fence ||
5137 BuiltinID == Builtin::BI__c11_atomic_signal_fence)
5138 SSID = llvm::SyncScope::SingleThread;
5140 SSID = llvm::SyncScope::System;
5142 if (isa<llvm::ConstantInt>(Order)) {
5143 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
5150 Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
5153 Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
5156 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
5159 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
5165 llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB;
5172 Order =
Builder.CreateIntCast(Order,
Builder.getInt32Ty(),
false);
5173 llvm::SwitchInst *SI =
Builder.CreateSwitch(Order, ContBB);
5175 Builder.SetInsertPoint(AcquireBB);
5176 Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
5178 SI->addCase(
Builder.getInt32(1), AcquireBB);
5179 SI->addCase(
Builder.getInt32(2), AcquireBB);
5181 Builder.SetInsertPoint(ReleaseBB);
5182 Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
5184 SI->addCase(
Builder.getInt32(3), ReleaseBB);
5186 Builder.SetInsertPoint(AcqRelBB);
5187 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
5189 SI->addCase(
Builder.getInt32(4), AcqRelBB);
5191 Builder.SetInsertPoint(SeqCstBB);
5192 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
5194 SI->addCase(
Builder.getInt32(5), SeqCstBB);
5196 Builder.SetInsertPoint(ContBB);
5199 case Builtin::BI__scoped_atomic_thread_fence: {
5204 auto Ord = dyn_cast<llvm::ConstantInt>(Order);
5205 auto Scp = dyn_cast<llvm::ConstantInt>(
Scope);
5207 SyncScope SS = ScopeModel->isValid(Scp->getZExtValue())
5208 ? ScopeModel->map(Scp->getZExtValue())
5209 : ScopeModel->map(ScopeModel->getFallBackValue());
5210 switch (Ord->getZExtValue()) {
5217 llvm::AtomicOrdering::Acquire,
5219 llvm::AtomicOrdering::Acquire,
5224 llvm::AtomicOrdering::Release,
5226 llvm::AtomicOrdering::Release,
5230 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease,
5233 llvm::AtomicOrdering::AcquireRelease,
5237 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
5240 llvm::AtomicOrdering::SequentiallyConsistent,
5252 switch (Ord->getZExtValue()) {
5255 ContBB->eraseFromParent();
5259 OrderBBs.emplace_back(
Builder.GetInsertBlock(),
5260 llvm::AtomicOrdering::Acquire);
5263 OrderBBs.emplace_back(
Builder.GetInsertBlock(),
5264 llvm::AtomicOrdering::Release);
5267 OrderBBs.emplace_back(
Builder.GetInsertBlock(),
5268 llvm::AtomicOrdering::AcquireRelease);
5271 OrderBBs.emplace_back(
Builder.GetInsertBlock(),
5272 llvm::AtomicOrdering::SequentiallyConsistent);
5281 Order =
Builder.CreateIntCast(Order,
Builder.getInt32Ty(),
false);
5282 llvm::SwitchInst *SI =
Builder.CreateSwitch(Order, ContBB);
5283 SI->addCase(
Builder.getInt32(1), AcquireBB);
5284 SI->addCase(
Builder.getInt32(2), AcquireBB);
5285 SI->addCase(
Builder.getInt32(3), ReleaseBB);
5286 SI->addCase(
Builder.getInt32(4), AcqRelBB);
5287 SI->addCase(
Builder.getInt32(5), SeqCstBB);
5289 OrderBBs.emplace_back(AcquireBB, llvm::AtomicOrdering::Acquire);
5290 OrderBBs.emplace_back(ReleaseBB, llvm::AtomicOrdering::Release);
5291 OrderBBs.emplace_back(AcqRelBB, llvm::AtomicOrdering::AcquireRelease);
5292 OrderBBs.emplace_back(SeqCstBB,
5293 llvm::AtomicOrdering::SequentiallyConsistent);
5296 for (
auto &[OrderBB, Ordering] : OrderBBs) {
5297 Builder.SetInsertPoint(OrderBB);
5299 SyncScope SS = ScopeModel->isValid(Scp->getZExtValue())
5300 ? ScopeModel->map(Scp->getZExtValue())
5301 : ScopeModel->map(ScopeModel->getFallBackValue());
5307 llvm::DenseMap<unsigned, llvm::BasicBlock *> BBs;
5308 for (
unsigned Scp : ScopeModel->getRuntimeValues())
5312 llvm::SwitchInst *SI =
Builder.CreateSwitch(SC, ContBB);
5313 for (
unsigned Scp : ScopeModel->getRuntimeValues()) {
5315 SI->addCase(
Builder.getInt32(Scp), B);
5326 Builder.SetInsertPoint(ContBB);
5330 case Builtin::BI__builtin_signbit:
5331 case Builtin::BI__builtin_signbitf:
5332 case Builtin::BI__builtin_signbitl: {
5337 case Builtin::BI__warn_memset_zero_len:
5339 case Builtin::BI__annotation: {
5342 for (
const Expr *Arg :
E->arguments()) {
5344 assert(Str->getCharByteWidth() == 2);
5345 StringRef WideBytes = Str->getBytes();
5346 std::string StrUtf8;
5347 if (!convertUTF16ToUTF8String(
5348 ArrayRef(WideBytes.data(), WideBytes.size()), StrUtf8)) {
5352 Strings.push_back(llvm::MDString::get(
getLLVMContext(), StrUtf8));
5362 case Builtin::BI__builtin_annotation: {
5371 StringRef Str = cast<StringLiteral>(AnnotationStrExpr)->getString();
5375 case Builtin::BI__builtin_addcb:
5376 case Builtin::BI__builtin_addcs:
5377 case Builtin::BI__builtin_addc:
5378 case Builtin::BI__builtin_addcl:
5379 case Builtin::BI__builtin_addcll:
5380 case Builtin::BI__builtin_subcb:
5381 case Builtin::BI__builtin_subcs:
5382 case Builtin::BI__builtin_subc:
5383 case Builtin::BI__builtin_subcl:
5384 case Builtin::BI__builtin_subcll: {
5410 llvm::Intrinsic::ID IntrinsicId;
5411 switch (BuiltinID) {
5412 default: llvm_unreachable(
"Unknown multiprecision builtin id.");
5413 case Builtin::BI__builtin_addcb:
5414 case Builtin::BI__builtin_addcs:
5415 case Builtin::BI__builtin_addc:
5416 case Builtin::BI__builtin_addcl:
5417 case Builtin::BI__builtin_addcll:
5418 IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
5420 case Builtin::BI__builtin_subcb:
5421 case Builtin::BI__builtin_subcs:
5422 case Builtin::BI__builtin_subc:
5423 case Builtin::BI__builtin_subcl:
5424 case Builtin::BI__builtin_subcll:
5425 IntrinsicId = llvm::Intrinsic::usub_with_overflow;
5430 llvm::Value *Carry1;
5433 llvm::Value *Carry2;
5435 Sum1, Carryin, Carry2);
5436 llvm::Value *CarryOut =
Builder.CreateZExt(
Builder.CreateOr(Carry1, Carry2),
5442 case Builtin::BI__builtin_add_overflow:
5443 case Builtin::BI__builtin_sub_overflow:
5444 case Builtin::BI__builtin_mul_overflow: {
5452 WidthAndSignedness LeftInfo =
5454 WidthAndSignedness RightInfo =
5456 WidthAndSignedness ResultInfo =
5463 RightInfo, ResultArg, ResultQTy,
5469 *
this, LeftArg, LeftInfo, RightArg, RightInfo, ResultArg, ResultQTy,
5472 WidthAndSignedness EncompassingInfo =
5475 llvm::Type *EncompassingLLVMTy =
5480 llvm::Intrinsic::ID IntrinsicId;
5481 switch (BuiltinID) {
5483 llvm_unreachable(
"Unknown overflow builtin id.");
5484 case Builtin::BI__builtin_add_overflow:
5485 IntrinsicId = EncompassingInfo.Signed
5486 ? llvm::Intrinsic::sadd_with_overflow
5487 : llvm::Intrinsic::uadd_with_overflow;
5489 case Builtin::BI__builtin_sub_overflow:
5490 IntrinsicId = EncompassingInfo.Signed
5491 ? llvm::Intrinsic::ssub_with_overflow
5492 : llvm::Intrinsic::usub_with_overflow;
5494 case Builtin::BI__builtin_mul_overflow:
5495 IntrinsicId = EncompassingInfo.Signed
5496 ? llvm::Intrinsic::smul_with_overflow
5497 : llvm::Intrinsic::umul_with_overflow;
5506 Left =
Builder.CreateIntCast(Left, EncompassingLLVMTy, LeftInfo.Signed);
5507 Right =
Builder.CreateIntCast(Right, EncompassingLLVMTy, RightInfo.Signed);
5510 llvm::Value *Overflow, *
Result;
5513 if (EncompassingInfo.Width > ResultInfo.Width) {
5516 llvm::Value *ResultTrunc =
Builder.CreateTrunc(
Result, ResultLLVMTy);
5520 llvm::Value *ResultTruncExt =
Builder.CreateIntCast(
5521 ResultTrunc, EncompassingLLVMTy, ResultInfo.Signed);
5522 llvm::Value *TruncationOverflow =
5525 Overflow =
Builder.CreateOr(Overflow, TruncationOverflow);
5537 case Builtin::BI__builtin_uadd_overflow:
5538 case Builtin::BI__builtin_uaddl_overflow:
5539 case Builtin::BI__builtin_uaddll_overflow:
5540 case Builtin::BI__builtin_usub_overflow:
5541 case Builtin::BI__builtin_usubl_overflow:
5542 case Builtin::BI__builtin_usubll_overflow:
5543 case Builtin::BI__builtin_umul_overflow:
5544 case Builtin::BI__builtin_umull_overflow:
5545 case Builtin::BI__builtin_umulll_overflow:
5546 case Builtin::BI__builtin_sadd_overflow:
5547 case Builtin::BI__builtin_saddl_overflow:
5548 case Builtin::BI__builtin_saddll_overflow:
5549 case Builtin::BI__builtin_ssub_overflow:
5550 case Builtin::BI__builtin_ssubl_overflow:
5551 case Builtin::BI__builtin_ssubll_overflow:
5552 case Builtin::BI__builtin_smul_overflow:
5553 case Builtin::BI__builtin_smull_overflow:
5554 case Builtin::BI__builtin_smulll_overflow: {
5564 llvm::Intrinsic::ID IntrinsicId;
5565 switch (BuiltinID) {
5566 default: llvm_unreachable(
"Unknown overflow builtin id.");
5567 case Builtin::BI__builtin_uadd_overflow:
5568 case Builtin::BI__builtin_uaddl_overflow:
5569 case Builtin::BI__builtin_uaddll_overflow:
5570 IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
5572 case Builtin::BI__builtin_usub_overflow:
5573 case Builtin::BI__builtin_usubl_overflow:
5574 case Builtin::BI__builtin_usubll_overflow:
5575 IntrinsicId = llvm::Intrinsic::usub_with_overflow;
5577 case Builtin::BI__builtin_umul_overflow:
5578 case Builtin::BI__builtin_umull_overflow:
5579 case Builtin::BI__builtin_umulll_overflow:
5580 IntrinsicId = llvm::Intrinsic::umul_with_overflow;
5582 case Builtin::BI__builtin_sadd_overflow:
5583 case Builtin::BI__builtin_saddl_overflow:
5584 case Builtin::BI__builtin_saddll_overflow:
5585 IntrinsicId = llvm::Intrinsic::sadd_with_overflow;
5587 case Builtin::BI__builtin_ssub_overflow:
5588 case Builtin::BI__builtin_ssubl_overflow:
5589 case Builtin::BI__builtin_ssubll_overflow:
5590 IntrinsicId = llvm::Intrinsic::ssub_with_overflow;
5592 case Builtin::BI__builtin_smul_overflow:
5593 case Builtin::BI__builtin_smull_overflow:
5594 case Builtin::BI__builtin_smulll_overflow:
5595 IntrinsicId = llvm::Intrinsic::smul_with_overflow;
5606 case Builtin::BIaddressof:
5607 case Builtin::BI__addressof:
5608 case Builtin::BI__builtin_addressof:
5610 case Builtin::BI__builtin_function_start:
5613 case Builtin::BI__builtin_operator_new:
5616 case Builtin::BI__builtin_operator_delete:
5621 case Builtin::BI__builtin_is_aligned:
5623 case Builtin::BI__builtin_align_up:
5625 case Builtin::BI__builtin_align_down:
5628 case Builtin::BI__noop:
5631 case Builtin::BI__builtin_call_with_static_chain: {
5633 const Expr *Chain =
E->getArg(1);
5638 case Builtin::BI_InterlockedExchange8:
5639 case Builtin::BI_InterlockedExchange16:
5640 case Builtin::BI_InterlockedExchange:
5641 case Builtin::BI_InterlockedExchangePointer:
5644 case Builtin::BI_InterlockedCompareExchangePointer:
5647 case Builtin::BI_InterlockedCompareExchangePointer_nf:
5650 case Builtin::BI_InterlockedCompareExchange8:
5651 case Builtin::BI_InterlockedCompareExchange16:
5652 case Builtin::BI_InterlockedCompareExchange:
5653 case Builtin::BI_InterlockedCompareExchange64:
5655 case Builtin::BI_InterlockedIncrement16:
5656 case Builtin::BI_InterlockedIncrement:
5659 case Builtin::BI_InterlockedDecrement16:
5660 case Builtin::BI_InterlockedDecrement:
5663 case Builtin::BI_InterlockedAnd8:
5664 case Builtin::BI_InterlockedAnd16:
5665 case Builtin::BI_InterlockedAnd:
5667 case Builtin::BI_InterlockedExchangeAdd8:
5668 case Builtin::BI_InterlockedExchangeAdd16:
5669 case Builtin::BI_InterlockedExchangeAdd:
5672 case Builtin::BI_InterlockedExchangeSub8:
5673 case Builtin::BI_InterlockedExchangeSub16:
5674 case Builtin::BI_InterlockedExchangeSub:
5677 case Builtin::BI_InterlockedOr8:
5678 case Builtin::BI_InterlockedOr16:
5679 case Builtin::BI_InterlockedOr:
5681 case Builtin::BI_InterlockedXor8:
5682 case Builtin::BI_InterlockedXor16:
5683 case Builtin::BI_InterlockedXor:
5686 case Builtin::BI_bittest64:
5687 case Builtin::BI_bittest:
5688 case Builtin::BI_bittestandcomplement64:
5689 case Builtin::BI_bittestandcomplement:
5690 case Builtin::BI_bittestandreset64:
5691 case Builtin::BI_bittestandreset:
5692 case Builtin::BI_bittestandset64:
5693 case Builtin::BI_bittestandset:
5694 case Builtin::BI_interlockedbittestandreset:
5695 case Builtin::BI_interlockedbittestandreset64:
5696 case Builtin::BI_interlockedbittestandset64:
5697 case Builtin::BI_interlockedbittestandset:
5698 case Builtin::BI_interlockedbittestandset_acq:
5699 case Builtin::BI_interlockedbittestandset_rel:
5700 case Builtin::BI_interlockedbittestandset_nf:
5701 case Builtin::BI_interlockedbittestandreset_acq:
5702 case Builtin::BI_interlockedbittestandreset_rel:
5703 case Builtin::BI_interlockedbittestandreset_nf:
5708 case Builtin::BI__iso_volatile_load8:
5709 case Builtin::BI__iso_volatile_load16:
5710 case Builtin::BI__iso_volatile_load32:
5711 case Builtin::BI__iso_volatile_load64:
5713 case Builtin::BI__iso_volatile_store8:
5714 case Builtin::BI__iso_volatile_store16:
5715 case Builtin::BI__iso_volatile_store32:
5716 case Builtin::BI__iso_volatile_store64:
5719 case Builtin::BI__builtin_ptrauth_sign_constant:
5722 case Builtin::BI__builtin_ptrauth_auth:
5723 case Builtin::BI__builtin_ptrauth_auth_and_resign:
5724 case Builtin::BI__builtin_ptrauth_blend_discriminator:
5725 case Builtin::BI__builtin_ptrauth_sign_generic_data:
5726 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
5727 case Builtin::BI__builtin_ptrauth_strip: {
5730 for (
auto argExpr :
E->arguments())
5734 llvm::Type *OrigValueType = Args[0]->getType();
5735 if (OrigValueType->isPointerTy())
5738 switch (BuiltinID) {
5739 case Builtin::BI__builtin_ptrauth_auth_and_resign:
5740 if (Args[4]->getType()->isPointerTy())
5744 case Builtin::BI__builtin_ptrauth_auth:
5745 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
5746 if (Args[2]->getType()->isPointerTy())
5750 case Builtin::BI__builtin_ptrauth_sign_generic_data:
5751 if (Args[1]->getType()->isPointerTy())
5755 case Builtin::BI__builtin_ptrauth_blend_discriminator:
5756 case Builtin::BI__builtin_ptrauth_strip:
5761 auto IntrinsicID = [&]() ->
unsigned {
5762 switch (BuiltinID) {
5763 case Builtin::BI__builtin_ptrauth_auth:
5764 return llvm::Intrinsic::ptrauth_auth;
5765 case Builtin::BI__builtin_ptrauth_auth_and_resign:
5766 return llvm::Intrinsic::ptrauth_resign;
5767 case Builtin::BI__builtin_ptrauth_blend_discriminator:
5768 return llvm::Intrinsic::ptrauth_blend;
5769 case Builtin::BI__builtin_ptrauth_sign_generic_data:
5770 return llvm::Intrinsic::ptrauth_sign_generic;
5771 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
5772 return llvm::Intrinsic::ptrauth_sign;
5773 case Builtin::BI__builtin_ptrauth_strip:
5774 return llvm::Intrinsic::ptrauth_strip;
5776 llvm_unreachable(
"bad ptrauth intrinsic");
5781 if (BuiltinID != Builtin::BI__builtin_ptrauth_sign_generic_data &&
5782 BuiltinID != Builtin::BI__builtin_ptrauth_blend_discriminator &&
5783 OrigValueType->isPointerTy()) {
5789 case Builtin::BI__exception_code:
5790 case Builtin::BI_exception_code:
5792 case Builtin::BI__exception_info:
5793 case Builtin::BI_exception_info:
5795 case Builtin::BI__abnormal_termination:
5796 case Builtin::BI_abnormal_termination:
5798 case Builtin::BI_setjmpex:
5799 if (
getTarget().getTriple().isOSMSVCRT() &&
E->getNumArgs() == 1 &&
5803 case Builtin::BI_setjmp:
5804 if (
getTarget().getTriple().isOSMSVCRT() &&
E->getNumArgs() == 1 &&
5806 if (
getTarget().getTriple().getArch() == llvm::Triple::x86)
5808 else if (
getTarget().getTriple().getArch() == llvm::Triple::aarch64)
5815 case Builtin::BImove:
5816 case Builtin::BImove_if_noexcept:
5817 case Builtin::BIforward:
5818 case Builtin::BIforward_like:
5819 case Builtin::BIas_const:
5821 case Builtin::BI__GetExceptionInfo: {
5822 if (llvm::GlobalVariable *GV =
5828 case Builtin::BI__fastfail:
5831 case Builtin::BI__builtin_coro_id:
5833 case Builtin::BI__builtin_coro_promise:
5835 case Builtin::BI__builtin_coro_resume:
5838 case Builtin::BI__builtin_coro_frame:
5840 case Builtin::BI__builtin_coro_noop:
5842 case Builtin::BI__builtin_coro_free:
5844 case Builtin::BI__builtin_coro_destroy:
5847 case Builtin::BI__builtin_coro_done:
5849 case Builtin::BI__builtin_coro_alloc:
5851 case Builtin::BI__builtin_coro_begin:
5853 case Builtin::BI__builtin_coro_end:
5855 case Builtin::BI__builtin_coro_suspend:
5857 case Builtin::BI__builtin_coro_size:
5859 case Builtin::BI__builtin_coro_align:
5863 case Builtin::BIread_pipe:
5864 case Builtin::BIwrite_pipe: {
5868 Value *PacketSize = OpenCLRT.getPipeElemSize(
E->getArg(0));
5869 Value *PacketAlign = OpenCLRT.getPipeElemAlign(
E->getArg(0));
5872 unsigned GenericAS =
5874 llvm::Type *I8PTy = llvm::PointerType::get(
getLLVMContext(), GenericAS);
5877 if (2U ==
E->getNumArgs()) {
5878 const char *Name = (BuiltinID == Builtin::BIread_pipe) ?
"__read_pipe_2"
5883 llvm::FunctionType *FTy = llvm::FunctionType::get(
5888 {Arg0, ACast, PacketSize, PacketAlign}));
5890 assert(4 ==
E->getNumArgs() &&
5891 "Illegal number of parameters to pipe function");
5892 const char *Name = (BuiltinID == Builtin::BIread_pipe) ?
"__read_pipe_4"
5899 llvm::FunctionType *FTy = llvm::FunctionType::get(
5908 {Arg0, Arg1, Arg2, ACast, PacketSize, PacketAlign}));
5913 case Builtin::BIreserve_read_pipe:
5914 case Builtin::BIreserve_write_pipe:
5915 case Builtin::BIwork_group_reserve_read_pipe:
5916 case Builtin::BIwork_group_reserve_write_pipe:
5917 case Builtin::BIsub_group_reserve_read_pipe:
5918 case Builtin::BIsub_group_reserve_write_pipe: {
5921 if (BuiltinID == Builtin::BIreserve_read_pipe)
5922 Name =
"__reserve_read_pipe";
5923 else if (BuiltinID == Builtin::BIreserve_write_pipe)
5924 Name =
"__reserve_write_pipe";
5925 else if (BuiltinID == Builtin::BIwork_group_reserve_read_pipe)
5926 Name =
"__work_group_reserve_read_pipe";
5927 else if (BuiltinID == Builtin::BIwork_group_reserve_write_pipe)
5928 Name =
"__work_group_reserve_write_pipe";
5929 else if (BuiltinID == Builtin::BIsub_group_reserve_read_pipe)
5930 Name =
"__sub_group_reserve_read_pipe";
5932 Name =
"__sub_group_reserve_write_pipe";
5938 Value *PacketSize = OpenCLRT.getPipeElemSize(
E->getArg(0));
5939 Value *PacketAlign = OpenCLRT.getPipeElemAlign(
E->getArg(0));
5943 llvm::FunctionType *FTy = llvm::FunctionType::get(
5950 {Arg0, Arg1, PacketSize, PacketAlign}));
5954 case Builtin::BIcommit_read_pipe:
5955 case Builtin::BIcommit_write_pipe:
5956 case Builtin::BIwork_group_commit_read_pipe:
5957 case Builtin::BIwork_group_commit_write_pipe:
5958 case Builtin::BIsub_group_commit_read_pipe:
5959 case Builtin::BIsub_group_commit_write_pipe: {
5961 if (BuiltinID == Builtin::BIcommit_read_pipe)
5962 Name =
"__commit_read_pipe";
5963 else if (BuiltinID == Builtin::BIcommit_write_pipe)
5964 Name =
"__commit_write_pipe";
5965 else if (BuiltinID == Builtin::BIwork_group_commit_read_pipe)
5966 Name =
"__work_group_commit_read_pipe";
5967 else if (BuiltinID == Builtin::BIwork_group_commit_write_pipe)
5968 Name =
"__work_group_commit_write_pipe";
5969 else if (BuiltinID == Builtin::BIsub_group_commit_read_pipe)
5970 Name =
"__sub_group_commit_read_pipe";
5972 Name =
"__sub_group_commit_write_pipe";
5977 Value *PacketSize = OpenCLRT.getPipeElemSize(
E->getArg(0));
5978 Value *PacketAlign = OpenCLRT.getPipeElemAlign(
E->getArg(0));
5982 llvm::FunctionType *FTy =
5987 {Arg0, Arg1, PacketSize, PacketAlign}));
5990 case Builtin::BIget_pipe_num_packets:
5991 case Builtin::BIget_pipe_max_packets: {
5992 const char *BaseName;
5994 if (BuiltinID == Builtin::BIget_pipe_num_packets)
5995 BaseName =
"__get_pipe_num_packets";
5997 BaseName =
"__get_pipe_max_packets";
5998 std::string Name = std::string(BaseName) +
5999 std::string(PipeTy->isReadOnly() ?
"_ro" :
"_wo");
6004 Value *PacketSize = OpenCLRT.getPipeElemSize(
E->getArg(0));
6005 Value *PacketAlign = OpenCLRT.getPipeElemAlign(
E->getArg(0));
6007 llvm::FunctionType *FTy = llvm::FunctionType::get(
6011 {Arg0, PacketSize, PacketAlign}));
6015 case Builtin::BIto_global:
6016 case Builtin::BIto_local:
6017 case Builtin::BIto_private: {
6019 auto NewArgT = llvm::PointerType::get(
6022 auto NewRetT = llvm::PointerType::get(
6026 auto FTy = llvm::FunctionType::get(NewRetT, {NewArgT},
false);
6027 llvm::Value *NewArg;
6028 if (Arg0->
getType()->getPointerAddressSpace() !=
6029 NewArgT->getPointerAddressSpace())
6032 NewArg =
Builder.CreateBitOrPointerCast(Arg0, NewArgT);
6033 auto NewName = std::string(
"__") +
E->getDirectCallee()->getName().str();
6048 case Builtin::BIenqueue_kernel: {
6050 unsigned NumArgs =
E->getNumArgs();
6053 llvm::Type *GenericVoidPtrTy =
Builder.getPtrTy(
6065 Name =
"__enqueue_kernel_basic";
6066 llvm::Type *ArgTys[] = {QueueTy,
Int32Ty, RangeTy, GenericVoidPtrTy,
6068 llvm::FunctionType *FTy = llvm::FunctionType::get(
6074 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
6075 llvm::Value *
Block =
6076 Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
6079 {Queue, Flags, Range, Kernel, Block});
6082 assert(NumArgs >= 5 &&
"Invalid enqueue_kernel signature");
6086 auto CreateArrayForSizeVar = [=](
unsigned First)
6087 -> std::tuple<llvm::Value *, llvm::Value *, llvm::Value *> {
6088 llvm::APInt ArraySize(32, NumArgs -
First);
6090 getContext().getSizeType(), ArraySize,
nullptr,
6094 llvm::Value *TmpPtr = Tmp.getPointer();
6099 llvm::Value *Alloca = TmpPtr->stripPointerCasts();
6102 llvm::Value *ElemPtr;
6105 auto *
Zero = llvm::ConstantInt::get(
IntTy, 0);
6106 for (
unsigned I =
First; I < NumArgs; ++I) {
6107 auto *Index = llvm::ConstantInt::get(
IntTy, I -
First);
6119 return std::tie(ElemPtr, TmpSize, Alloca);
6125 Name =
"__enqueue_kernel_varargs";
6129 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
6130 auto *
Block =
Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
6131 llvm::Value *ElemPtr, *TmpSize, *TmpPtr;
6132 std::tie(ElemPtr, TmpSize, TmpPtr) = CreateArrayForSizeVar(4);
6136 llvm::Value *
const Args[] = {Queue, Flags,
6140 llvm::Type *
const ArgTys[] = {
6141 QueueTy,
IntTy, RangeTy, GenericVoidPtrTy,
6142 GenericVoidPtrTy,
IntTy, ElemPtr->getType()};
6144 llvm::FunctionType *FTy = llvm::FunctionType::get(
Int32Ty, ArgTys,
false);
6153 llvm::PointerType *PtrTy = llvm::PointerType::get(
6157 llvm::Value *NumEvents =
6163 llvm::Value *EventWaitList =
nullptr;
6166 EventWaitList = llvm::ConstantPointerNull::get(PtrTy);
6173 EventWaitList =
Builder.CreatePointerCast(EventWaitList, PtrTy);
6175 llvm::Value *EventRet =
nullptr;
6178 EventRet = llvm::ConstantPointerNull::get(PtrTy);
6187 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
6188 llvm::Value *
Block =
6189 Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
6191 std::vector<llvm::Type *> ArgTys = {
6193 PtrTy, PtrTy, GenericVoidPtrTy, GenericVoidPtrTy};
6195 std::vector<llvm::Value *> Args = {Queue, Flags,
Range,
6196 NumEvents, EventWaitList, EventRet,
6201 Name =
"__enqueue_kernel_basic_events";
6202 llvm::FunctionType *FTy = llvm::FunctionType::get(
6210 Args.push_back(ConstantInt::get(
Int32Ty, NumArgs - 7));
6212 Name =
"__enqueue_kernel_events_varargs";
6214 llvm::Value *ElemPtr, *TmpSize, *TmpPtr;
6215 std::tie(ElemPtr, TmpSize, TmpPtr) = CreateArrayForSizeVar(7);
6216 Args.push_back(ElemPtr);
6217 ArgTys.push_back(ElemPtr->getType());
6219 llvm::FunctionType *FTy = llvm::FunctionType::get(
6228 llvm_unreachable(
"Unexpected enqueue_kernel signature");
6232 case Builtin::BIget_kernel_work_group_size: {
6233 llvm::Type *GenericVoidPtrTy =
Builder.getPtrTy(
6238 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
6239 Value *Arg =
Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
6242 llvm::FunctionType::get(
IntTy, {GenericVoidPtrTy, GenericVoidPtrTy},
6244 "__get_kernel_work_group_size_impl"),
6247 case Builtin::BIget_kernel_preferred_work_group_size_multiple: {
6248 llvm::Type *GenericVoidPtrTy =
Builder.getPtrTy(
6253 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
6254 Value *Arg =
Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
6257 llvm::FunctionType::get(
IntTy, {GenericVoidPtrTy, GenericVoidPtrTy},
6259 "__get_kernel_preferred_work_group_size_multiple_impl"),
6262 case Builtin::BIget_kernel_max_sub_group_size_for_ndrange:
6263 case Builtin::BIget_kernel_sub_group_count_for_ndrange: {
6264 llvm::Type *GenericVoidPtrTy =
Builder.getPtrTy(
6271 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
6274 BuiltinID == Builtin::BIget_kernel_max_sub_group_size_for_ndrange
6275 ?
"__get_kernel_max_sub_group_size_for_ndrange_impl"
6276 :
"__get_kernel_sub_group_count_for_ndrange_impl";
6279 llvm::FunctionType::get(
6280 IntTy, {NDRange->getType(), GenericVoidPtrTy, GenericVoidPtrTy},
6283 {NDRange, Kernel, Block}));
6285 case Builtin::BI__builtin_store_half:
6286 case Builtin::BI__builtin_store_halff: {
6293 case Builtin::BI__builtin_load_half: {
6298 case Builtin::BI__builtin_load_halff: {
6303 case Builtin::BI__builtin_printf:
6304 case Builtin::BIprintf:
6305 if (
getTarget().getTriple().isNVPTX() ||
6308 getTarget().getTriple().getVendor() == Triple::VendorType::AMD)) {
6311 if ((
getTarget().getTriple().isAMDGCN() ||
6318 case Builtin::BI__builtin_canonicalize:
6319 case Builtin::BI__builtin_canonicalizef:
6320 case Builtin::BI__builtin_canonicalizef16:
6321 case Builtin::BI__builtin_canonicalizel:
6323 emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::canonicalize));
6325 case Builtin::BI__builtin_thread_pointer: {
6326 if (!
getContext().getTargetInfo().isTLSSupported())
6331 case Builtin::BI__builtin_os_log_format:
6334 case Builtin::BI__xray_customevent: {
6347 auto FTy = F->getFunctionType();
6348 auto Arg0 =
E->getArg(0);
6350 auto Arg0Ty = Arg0->
getType();
6351 auto PTy0 = FTy->getParamType(0);
6352 if (PTy0 != Arg0Val->getType()) {
6353 if (Arg0Ty->isArrayType())
6356 Arg0Val =
Builder.CreatePointerCast(Arg0Val, PTy0);
6359 auto PTy1 = FTy->getParamType(1);
6361 Arg1 =
Builder.CreateTruncOrBitCast(Arg1, PTy1);
6365 case Builtin::BI__xray_typedevent: {
6381 auto FTy = F->getFunctionType();
6383 auto PTy0 = FTy->getParamType(0);
6385 Arg0 =
Builder.CreateTruncOrBitCast(Arg0, PTy0);
6386 auto Arg1 =
E->getArg(1);
6388 auto Arg1Ty = Arg1->
getType();
6389 auto PTy1 = FTy->getParamType(1);
6390 if (PTy1 != Arg1Val->getType()) {
6391 if (Arg1Ty->isArrayType())
6394 Arg1Val =
Builder.CreatePointerCast(Arg1Val, PTy1);
6397 auto PTy2 = FTy->getParamType(2);
6399 Arg2 =
Builder.CreateTruncOrBitCast(Arg2, PTy2);
6403 case Builtin::BI__builtin_ms_va_start:
6404 case Builtin::BI__builtin_ms_va_end:
6407 BuiltinID == Builtin::BI__builtin_ms_va_start));
6409 case Builtin::BI__builtin_ms_va_copy: {
6426 case Builtin::BI__builtin_get_device_side_mangled_name: {
6454 LargestVectorWidth = std::max(LargestVectorWidth, VectorWidth);
6458 Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic;
6460 llvm::Triple::getArchTypePrefix(
getTarget().getTriple().getArch());
6461 if (!Prefix.empty()) {
6462 IntrinsicID = Intrinsic::getIntrinsicForClangBuiltin(Prefix.data(), Name);
6463 if (IntrinsicID == Intrinsic::not_intrinsic && Prefix ==
"spv" &&
6464 getTarget().getTriple().getOS() == llvm::Triple::OSType::AMDHSA)
6465 IntrinsicID = Intrinsic::getIntrinsicForClangBuiltin(
"amdgcn", Name);
6469 if (IntrinsicID == Intrinsic::not_intrinsic)
6470 IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(Prefix.data(), Name);
6473 if (IntrinsicID != Intrinsic::not_intrinsic) {
6478 unsigned ICEArguments = 0;
6484 llvm::FunctionType *FTy = F->getFunctionType();
6486 for (
unsigned i = 0, e =
E->getNumArgs(); i != e; ++i) {
6490 llvm::Type *PTy = FTy->getParamType(i);
6491 if (PTy != ArgValue->
getType()) {
6493 if (
auto *PtrTy = dyn_cast<llvm::PointerType>(PTy)) {
6494 if (PtrTy->getAddressSpace() !=
6495 ArgValue->
getType()->getPointerAddressSpace()) {
6498 PtrTy->getAddressSpace()));
6504 if (PTy->isX86_AMXTy())
6505 ArgValue =
Builder.CreateIntrinsic(Intrinsic::x86_cast_vector_to_tile,
6506 {ArgValue->
getType()}, {ArgValue});
6508 ArgValue =
Builder.CreateBitCast(ArgValue, PTy);
6511 Args.push_back(ArgValue);
6517 llvm::Type *RetTy =
VoidTy;
6521 if (RetTy !=
V->getType()) {
6523 if (
auto *PtrTy = dyn_cast<llvm::PointerType>(RetTy)) {
6524 if (PtrTy->getAddressSpace() !=
V->getType()->getPointerAddressSpace()) {
6527 PtrTy->getAddressSpace()));
6533 if (
V->getType()->isX86_AMXTy())
6534 V =
Builder.CreateIntrinsic(Intrinsic::x86_cast_tile_to_vector, {RetTy},
6540 if (RetTy->isVoidTy())
6560 if (
V->getType()->isVoidTy())
6567 llvm_unreachable(
"No current target builtin returns complex");
6569 llvm_unreachable(
"Bad evaluation kind in EmitBuiltinExpr");
6576 if (
V->getType()->isVoidTy())
6583 llvm_unreachable(
"No current hlsl builtin returns complex");
6585 llvm_unreachable(
"Bad evaluation kind in EmitBuiltinExpr");
6600 llvm::Triple::ArchType Arch) {
6612 case llvm::Triple::arm:
6613 case llvm::Triple::armeb:
6614 case llvm::Triple::thumb:
6615 case llvm::Triple::thumbeb:
6617 case llvm::Triple::aarch64:
6618 case llvm::Triple::aarch64_32:
6619 case llvm::Triple::aarch64_be:
6621 case llvm::Triple::bpfeb:
6622 case llvm::Triple::bpfel:
6624 case llvm::Triple::x86:
6625 case llvm::Triple::x86_64:
6627 case llvm::Triple::ppc:
6628 case llvm::Triple::ppcle:
6629 case llvm::Triple::ppc64:
6630 case llvm::Triple::ppc64le:
6632 case llvm::Triple::r600:
6633 case llvm::Triple::amdgcn:
6635 case llvm::Triple::systemz:
6637 case llvm::Triple::nvptx:
6638 case llvm::Triple::nvptx64:
6640 case llvm::Triple::wasm32:
6641 case llvm::Triple::wasm64:
6643 case llvm::Triple::hexagon:
6645 case llvm::Triple::riscv32:
6646 case llvm::Triple::riscv64:
6648 case llvm::Triple::spirv:
6650 case llvm::Triple::spirv64:
6663 assert(
getContext().getAuxTargetInfo() &&
"Missing aux target info");
6675 bool HasLegalHalfType =
true,
6677 bool AllowBFloatArgsAndRet =
true) {
6678 int IsQuad = TypeFlags.
isQuad();
6683 return llvm::FixedVectorType::get(CGF->
Int8Ty, V1Ty ? 1 : (8 << IsQuad));
6686 return llvm::FixedVectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
6688 if (AllowBFloatArgsAndRet)
6689 return llvm::FixedVectorType::get(CGF->
BFloatTy, V1Ty ? 1 : (4 << IsQuad));
6691 return llvm::FixedVectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
6693 if (HasLegalHalfType)
6694 return llvm::FixedVectorType::get(CGF->
HalfTy, V1Ty ? 1 : (4 << IsQuad));
6696 return llvm::FixedVectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
6698 return llvm::FixedVectorType::get(CGF->
Int32Ty, V1Ty ? 1 : (2 << IsQuad));
6701 return llvm::FixedVectorType::get(CGF->
Int64Ty, V1Ty ? 1 : (1 << IsQuad));
6706 return llvm::FixedVectorType::get(CGF->
Int8Ty, 16);
6708 return llvm::FixedVectorType::get(CGF->
FloatTy, V1Ty ? 1 : (2 << IsQuad));
6710 return llvm::FixedVectorType::get(CGF->
DoubleTy, V1Ty ? 1 : (1 << IsQuad));
6712 llvm_unreachable(
"Unknown vector element type!");
6717 int IsQuad = IntTypeFlags.
isQuad();
6720 return llvm::FixedVectorType::get(CGF->
HalfTy, (4 << IsQuad));
6722 return llvm::FixedVectorType::get(CGF->
FloatTy, (2 << IsQuad));
6724 return llvm::FixedVectorType::get(CGF->
DoubleTy, (1 << IsQuad));
6726 llvm_unreachable(
"Type can't be converted to floating-point!");
6731 const ElementCount &Count) {
6732 Value *SV = llvm::ConstantVector::getSplat(Count,
C);
6733 return Builder.CreateShuffleVector(
V,
V, SV,
"lane");
6737 ElementCount EC = cast<llvm::VectorType>(
V->getType())->getElementCount();
6743 unsigned shift,
bool rightshift) {
6745 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
6746 ai != ae; ++ai, ++j) {
6747 if (F->isConstrainedFPIntrinsic())
6748 if (ai->getType()->isMetadataTy())
6750 if (shift > 0 && shift == j)
6753 Ops[j] =
Builder.CreateBitCast(Ops[j], ai->getType(), name);
6756 if (F->isConstrainedFPIntrinsic())
6757 return Builder.CreateConstrainedFPCall(F, Ops, name);
6759 return Builder.CreateCall(F, Ops, name);
6773 unsigned IID,
bool ExtendLaneArg, llvm::Type *RetTy,
6776 const unsigned ElemCount = Ops[0]->getType()->getPrimitiveSizeInBits() /
6777 RetTy->getPrimitiveSizeInBits();
6778 llvm::Type *Tys[] = {llvm::FixedVectorType::get(RetTy, ElemCount),
6780 if (ExtendLaneArg) {
6781 auto *VT = llvm::FixedVectorType::get(
Int8Ty, 16);
6782 Ops[2] =
Builder.CreateInsertVector(VT, PoisonValue::get(VT), Ops[2],
6789 unsigned IID,
bool ExtendLaneArg, llvm::Type *RetTy,
6792 if (ExtendLaneArg) {
6793 auto *VT = llvm::FixedVectorType::get(
Int8Ty, 16);
6794 Ops[2] =
Builder.CreateInsertVector(VT, PoisonValue::get(VT), Ops[2],
6797 const unsigned ElemCount = Ops[0]->getType()->getPrimitiveSizeInBits() /
6798 RetTy->getPrimitiveSizeInBits();
6799 return EmitFP8NeonCall(IID, {llvm::FixedVectorType::get(RetTy, ElemCount)},
6805 int SV = cast<ConstantInt>(
V)->getSExtValue();
6806 return ConstantInt::get(Ty, neg ? -SV : SV);
6810 llvm::Type *Ty1,
bool Extract,
6814 llvm::Type *Tys[] = {Ty0, Ty1};
6818 Tys[1] = llvm::FixedVectorType::get(
Int8Ty, 8);
6819 Ops[0] =
Builder.CreateExtractVector(Tys[1], Ops[0],
Builder.getInt64(0));
6826 llvm::Type *Ty,
bool usgn,
6828 llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
6830 int ShiftAmt = cast<ConstantInt>(Shift)->getSExtValue();
6831 int EltSize = VTy->getScalarSizeInBits();
6833 Vec =
Builder.CreateBitCast(Vec, Ty);
6837 if (ShiftAmt == EltSize) {
6840 return llvm::ConstantAggregateZero::get(VTy);
6845 Shift = ConstantInt::get(VTy->getElementType(), ShiftAmt);
6851 return Builder.CreateLShr(Vec, Shift, name);
6853 return Builder.CreateAShr(Vec, Shift, name);
6879struct ARMVectorIntrinsicInfo {
6880 const char *NameHint;
6882 unsigned LLVMIntrinsic;
6883 unsigned AltLLVMIntrinsic;
6886 bool operator<(
unsigned RHSBuiltinID)
const {
6887 return BuiltinID < RHSBuiltinID;
6889 bool operator<(
const ARMVectorIntrinsicInfo &TE)
const {
6890 return BuiltinID < TE.BuiltinID;
6895#define NEONMAP0(NameBase) \
6896 { #NameBase, NEON::BI__builtin_neon_ ## NameBase, 0, 0, 0 }
6898#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
6899 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
6900 Intrinsic::LLVMIntrinsic, 0, TypeModifier }
6902#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier) \
6903 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
6904 Intrinsic::LLVMIntrinsic, Intrinsic::AltLLVMIntrinsic, \
6908 NEONMAP1(__a32_vcvt_bf16_f32, arm_neon_vcvtfp2bf, 0),
6915 NEONMAP1(vabs_v, arm_neon_vabs, 0),
6916 NEONMAP1(vabsq_v, arm_neon_vabs, 0),
6920 NEONMAP1(vaesdq_u8, arm_neon_aesd, 0),
6921 NEONMAP1(vaeseq_u8, arm_neon_aese, 0),
6922 NEONMAP1(vaesimcq_u8, arm_neon_aesimc, 0),
6923 NEONMAP1(vaesmcq_u8, arm_neon_aesmc, 0),
6924 NEONMAP1(vbfdot_f32, arm_neon_bfdot, 0),
6925 NEONMAP1(vbfdotq_f32, arm_neon_bfdot, 0),
6926 NEONMAP1(vbfmlalbq_f32, arm_neon_bfmlalb, 0),
6927 NEONMAP1(vbfmlaltq_f32, arm_neon_bfmlalt, 0),
6928 NEONMAP1(vbfmmlaq_f32, arm_neon_bfmmla, 0),
6941 NEONMAP1(vcage_v, arm_neon_vacge, 0),
6942 NEONMAP1(vcageq_v, arm_neon_vacge, 0),
6943 NEONMAP1(vcagt_v, arm_neon_vacgt, 0),
6944 NEONMAP1(vcagtq_v, arm_neon_vacgt, 0),
6945 NEONMAP1(vcale_v, arm_neon_vacge, 0),
6946 NEONMAP1(vcaleq_v, arm_neon_vacge, 0),
6947 NEONMAP1(vcalt_v, arm_neon_vacgt, 0),
6948 NEONMAP1(vcaltq_v, arm_neon_vacgt, 0),
6965 NEONMAP1(vcvt_f16_f32, arm_neon_vcvtfp2hf, 0),
6968 NEONMAP1(vcvt_f32_f16, arm_neon_vcvthf2fp, 0),
6970 NEONMAP1(vcvt_n_f16_s16, arm_neon_vcvtfxs2fp, 0),
6971 NEONMAP1(vcvt_n_f16_u16, arm_neon_vcvtfxu2fp, 0),
6972 NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
6973 NEONMAP1(vcvt_n_s16_f16, arm_neon_vcvtfp2fxs, 0),
6974 NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0),
6975 NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0),
6976 NEONMAP1(vcvt_n_u16_f16, arm_neon_vcvtfp2fxu, 0),
6977 NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0),
6978 NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0),
6985 NEONMAP1(vcvta_s16_f16, arm_neon_vcvtas, 0),
6986 NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0),
6987 NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0),
6988 NEONMAP1(vcvta_u16_f16, arm_neon_vcvtau, 0),
6989 NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0),
6990 NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0),
6991 NEONMAP1(vcvtaq_s16_f16, arm_neon_vcvtas, 0),
6992 NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0),
6993 NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0),
6994 NEONMAP1(vcvtaq_u16_f16, arm_neon_vcvtau, 0),
6995 NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0),
6996 NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0),
6997 NEONMAP1(vcvth_bf16_f32, arm_neon_vcvtbfp2bf, 0),
6998 NEONMAP1(vcvtm_s16_f16, arm_neon_vcvtms, 0),
6999 NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0),
7000 NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0),
7001 NEONMAP1(vcvtm_u16_f16, arm_neon_vcvtmu, 0),
7002 NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0),
7003 NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0),
7004 NEONMAP1(vcvtmq_s16_f16, arm_neon_vcvtms, 0),
7005 NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0),
7006 NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0),
7007 NEONMAP1(vcvtmq_u16_f16, arm_neon_vcvtmu, 0),
7008 NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0),
7009 NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0),
7010 NEONMAP1(vcvtn_s16_f16, arm_neon_vcvtns, 0),
7011 NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0),
7012 NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0),
7013 NEONMAP1(vcvtn_u16_f16, arm_neon_vcvtnu, 0),
7014 NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0),
7015 NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0),
7016 NEONMAP1(vcvtnq_s16_f16, arm_neon_vcvtns, 0),
7017 NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0),
7018 NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0),
7019 NEONMAP1(vcvtnq_u16_f16, arm_neon_vcvtnu, 0),
7020 NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0),
7021 NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0),
7022 NEONMAP1(vcvtp_s16_f16, arm_neon_vcvtps, 0),
7023 NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0),
7024 NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0),
7025 NEONMAP1(vcvtp_u16_f16, arm_neon_vcvtpu, 0),
7026 NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0),
7027 NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0),
7028 NEONMAP1(vcvtpq_s16_f16, arm_neon_vcvtps, 0),
7029 NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0),
7030 NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0),
7031 NEONMAP1(vcvtpq_u16_f16, arm_neon_vcvtpu, 0),
7032 NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0),
7033 NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0),
7037 NEONMAP1(vcvtq_n_f16_s16, arm_neon_vcvtfxs2fp, 0),
7038 NEONMAP1(vcvtq_n_f16_u16, arm_neon_vcvtfxu2fp, 0),
7039 NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
7040 NEONMAP1(vcvtq_n_s16_f16, arm_neon_vcvtfp2fxs, 0),
7041 NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0),
7042 NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0),
7043 NEONMAP1(vcvtq_n_u16_f16, arm_neon_vcvtfp2fxu, 0),
7044 NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0),
7045 NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0),
7052 NEONMAP1(vdot_s32, arm_neon_sdot, 0),
7053 NEONMAP1(vdot_u32, arm_neon_udot, 0),
7054 NEONMAP1(vdotq_s32, arm_neon_sdot, 0),
7055 NEONMAP1(vdotq_u32, arm_neon_udot, 0),
7065 NEONMAP1(vld1_v, arm_neon_vld1, 0),
7066 NEONMAP1(vld1_x2_v, arm_neon_vld1x2, 0),
7067 NEONMAP1(vld1_x3_v, arm_neon_vld1x3, 0),
7068 NEONMAP1(vld1_x4_v, arm_neon_vld1x4, 0),
7070 NEONMAP1(vld1q_v, arm_neon_vld1, 0),
7071 NEONMAP1(vld1q_x2_v, arm_neon_vld1x2, 0),
7072 NEONMAP1(vld1q_x3_v, arm_neon_vld1x3, 0),
7073 NEONMAP1(vld1q_x4_v, arm_neon_vld1x4, 0),
7074 NEONMAP1(vld2_dup_v, arm_neon_vld2dup, 0),
7075 NEONMAP1(vld2_lane_v, arm_neon_vld2lane, 0),
7076 NEONMAP1(vld2_v, arm_neon_vld2, 0),
7077 NEONMAP1(vld2q_dup_v, arm_neon_vld2dup, 0),
7078 NEONMAP1(vld2q_lane_v, arm_neon_vld2lane, 0),
7079 NEONMAP1(vld2q_v, arm_neon_vld2, 0),
7080 NEONMAP1(vld3_dup_v, arm_neon_vld3dup, 0),
7081 NEONMAP1(vld3_lane_v, arm_neon_vld3lane, 0),
7082 NEONMAP1(vld3_v, arm_neon_vld3, 0),
7083 NEONMAP1(vld3q_dup_v, arm_neon_vld3dup, 0),
7084 NEONMAP1(vld3q_lane_v, arm_neon_vld3lane, 0),
7085 NEONMAP1(vld3q_v, arm_neon_vld3, 0),
7086 NEONMAP1(vld4_dup_v, arm_neon_vld4dup, 0),
7087 NEONMAP1(vld4_lane_v, arm_neon_vld4lane, 0),
7088 NEONMAP1(vld4_v, arm_neon_vld4, 0),
7089 NEONMAP1(vld4q_dup_v, arm_neon_vld4dup, 0),
7090 NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0),
7091 NEONMAP1(vld4q_v, arm_neon_vld4, 0),
7100 NEONMAP1(vmmlaq_s32, arm_neon_smmla, 0),
7101 NEONMAP1(vmmlaq_u32, arm_neon_ummla, 0),
7119 NEONMAP2(vqdmlal_v, arm_neon_vqdmull, sadd_sat, 0),
7120 NEONMAP2(vqdmlsl_v, arm_neon_vqdmull, ssub_sat, 0),
7144 NEONMAP1(vqshlu_n_v, arm_neon_vqshiftsu, 0),
7145 NEONMAP1(vqshluq_n_v, arm_neon_vqshiftsu, 0),
7149 NEONMAP2(vrecpe_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
7150 NEONMAP2(vrecpeq_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
7173 NEONMAP2(vrsqrte_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
7174 NEONMAP2(vrsqrteq_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
7178 NEONMAP1(vsha1su0q_u32, arm_neon_sha1su0, 0),
7179 NEONMAP1(vsha1su1q_u32, arm_neon_sha1su1, 0),
7180 NEONMAP1(vsha256h2q_u32, arm_neon_sha256h2, 0),
7181 NEONMAP1(vsha256hq_u32, arm_neon_sha256h, 0),
7182 NEONMAP1(vsha256su0q_u32, arm_neon_sha256su0, 0),
7183 NEONMAP1(vsha256su1q_u32, arm_neon_sha256su1, 0),
7192 NEONMAP1(vst1_v, arm_neon_vst1, 0),
7193 NEONMAP1(vst1_x2_v, arm_neon_vst1x2, 0),
7194 NEONMAP1(vst1_x3_v, arm_neon_vst1x3, 0),
7195 NEONMAP1(vst1_x4_v, arm_neon_vst1x4, 0),
7196 NEONMAP1(vst1q_v, arm_neon_vst1, 0),
7197 NEONMAP1(vst1q_x2_v, arm_neon_vst1x2, 0),
7198 NEONMAP1(vst1q_x3_v, arm_neon_vst1x3, 0),
7199 NEONMAP1(vst1q_x4_v, arm_neon_vst1x4, 0),
7200 NEONMAP1(vst2_lane_v, arm_neon_vst2lane, 0),
7201 NEONMAP1(vst2_v, arm_neon_vst2, 0),
7202 NEONMAP1(vst2q_lane_v, arm_neon_vst2lane, 0),
7203 NEONMAP1(vst2q_v, arm_neon_vst2, 0),
7204 NEONMAP1(vst3_lane_v, arm_neon_vst3lane, 0),
7205 NEONMAP1(vst3_v, arm_neon_vst3, 0),
7206 NEONMAP1(vst3q_lane_v, arm_neon_vst3lane, 0),
7207 NEONMAP1(vst3q_v, arm_neon_vst3, 0),
7208 NEONMAP1(vst4_lane_v, arm_neon_vst4lane, 0),
7209 NEONMAP1(vst4_v, arm_neon_vst4, 0),
7210 NEONMAP1(vst4q_lane_v, arm_neon_vst4lane, 0),
7211 NEONMAP1(vst4q_v, arm_neon_vst4, 0),
7217 NEONMAP1(vusdot_s32, arm_neon_usdot, 0),
7218 NEONMAP1(vusdotq_s32, arm_neon_usdot, 0),
7219 NEONMAP1(vusmmlaq_s32, arm_neon_usmmla, 0),
7231 NEONMAP1(vabs_v, aarch64_neon_abs, 0),
7232 NEONMAP1(vabsq_v, aarch64_neon_abs, 0),
7237 NEONMAP1(vaesdq_u8, aarch64_crypto_aesd, 0),
7238 NEONMAP1(vaeseq_u8, aarch64_crypto_aese, 0),
7239 NEONMAP1(vaesimcq_u8, aarch64_crypto_aesimc, 0),
7240 NEONMAP1(vaesmcq_u8, aarch64_crypto_aesmc, 0),
7249 NEONMAP1(vbfdot_f32, aarch64_neon_bfdot, 0),
7250 NEONMAP1(vbfdotq_f32, aarch64_neon_bfdot, 0),
7251 NEONMAP1(vbfmlalbq_f32, aarch64_neon_bfmlalb, 0),
7252 NEONMAP1(vbfmlaltq_f32, aarch64_neon_bfmlalt, 0),
7253 NEONMAP1(vbfmmlaq_f32, aarch64_neon_bfmmla, 0),
7264 NEONMAP1(vcage_v, aarch64_neon_facge, 0),
7265 NEONMAP1(vcageq_v, aarch64_neon_facge, 0),
7266 NEONMAP1(vcagt_v, aarch64_neon_facgt, 0),
7267 NEONMAP1(vcagtq_v, aarch64_neon_facgt, 0),
7268 NEONMAP1(vcale_v, aarch64_neon_facge, 0),
7269 NEONMAP1(vcaleq_v, aarch64_neon_facge, 0),
7270 NEONMAP1(vcalt_v, aarch64_neon_facgt, 0),
7271 NEONMAP1(vcaltq_v, aarch64_neon_facgt, 0),
7308 NEONMAP1(vcvt_f16_f32, aarch64_neon_vcvtfp2hf, 0),
7311 NEONMAP1(vcvt_f32_f16, aarch64_neon_vcvthf2fp, 0),
7313 NEONMAP1(vcvt_n_f16_s16, aarch64_neon_vcvtfxs2fp, 0),
7314 NEONMAP1(vcvt_n_f16_u16, aarch64_neon_vcvtfxu2fp, 0),
7315 NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
7316 NEONMAP2(vcvt_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
7317 NEONMAP1(vcvt_n_s16_f16, aarch64_neon_vcvtfp2fxs, 0),
7318 NEONMAP1(vcvt_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
7319 NEONMAP1(vcvt_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
7320 NEONMAP1(vcvt_n_u16_f16, aarch64_neon_vcvtfp2fxu, 0),
7321 NEONMAP1(vcvt_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
7322 NEONMAP1(vcvt_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
7328 NEONMAP1(vcvtq_n_f16_s16, aarch64_neon_vcvtfxs2fp, 0),
7329 NEONMAP1(vcvtq_n_f16_u16, aarch64_neon_vcvtfxu2fp, 0),
7330 NEONMAP2(vcvtq_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
7331 NEONMAP2(vcvtq_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
7332 NEONMAP1(vcvtq_n_s16_f16, aarch64_neon_vcvtfp2fxs, 0),
7333 NEONMAP1(vcvtq_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
7334 NEONMAP1(vcvtq_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
7335 NEONMAP1(vcvtq_n_u16_f16, aarch64_neon_vcvtfp2fxu, 0),
7336 NEONMAP1(vcvtq_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
7337 NEONMAP1(vcvtq_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
7339 NEONMAP1(vdot_s32, aarch64_neon_sdot, 0),
7340 NEONMAP1(vdot_u32, aarch64_neon_udot, 0),
7341 NEONMAP1(vdotq_s32, aarch64_neon_sdot, 0),
7342 NEONMAP1(vdotq_u32, aarch64_neon_udot, 0),
7355 NEONMAP1(vfmlal_high_f16, aarch64_neon_fmlal2, 0),
7356 NEONMAP1(vfmlal_low_f16, aarch64_neon_fmlal, 0),
7357 NEONMAP1(vfmlalq_high_f16, aarch64_neon_fmlal2, 0),
7358 NEONMAP1(vfmlalq_low_f16, aarch64_neon_fmlal, 0),
7359 NEONMAP1(vfmlsl_high_f16, aarch64_neon_fmlsl2, 0),
7360 NEONMAP1(vfmlsl_low_f16, aarch64_neon_fmlsl, 0),
7361 NEONMAP1(vfmlslq_high_f16, aarch64_neon_fmlsl2, 0),
7362 NEONMAP1(vfmlslq_low_f16, aarch64_neon_fmlsl, 0),
7367 NEONMAP1(vld1_x2_v, aarch64_neon_ld1x2, 0),
7368 NEONMAP1(vld1_x3_v, aarch64_neon_ld1x3, 0),
7369 NEONMAP1(vld1_x4_v, aarch64_neon_ld1x4, 0),
7370 NEONMAP1(vld1q_x2_v, aarch64_neon_ld1x2, 0),
7371 NEONMAP1(vld1q_x3_v, aarch64_neon_ld1x3, 0),
7372 NEONMAP1(vld1q_x4_v, aarch64_neon_ld1x4, 0),
7373 NEONMAP1(vmmlaq_s32, aarch64_neon_smmla, 0),
7374 NEONMAP1(vmmlaq_u32, aarch64_neon_ummla, 0),
7387 NEONMAP2(vqdmlal_v, aarch64_neon_sqdmull, aarch64_neon_sqadd, 0),
7388 NEONMAP2(vqdmlsl_v, aarch64_neon_sqdmull, aarch64_neon_sqsub, 0),
7389 NEONMAP1(vqdmulh_lane_v, aarch64_neon_sqdmulh_lane, 0),
7390 NEONMAP1(vqdmulh_laneq_v, aarch64_neon_sqdmulh_laneq, 0),
7392 NEONMAP1(vqdmulhq_lane_v, aarch64_neon_sqdmulh_lane, 0),
7393 NEONMAP1(vqdmulhq_laneq_v, aarch64_neon_sqdmulh_laneq, 0),
7408 NEONMAP1(vqrdmulh_lane_v, aarch64_neon_sqrdmulh_lane, 0),
7409 NEONMAP1(vqrdmulh_laneq_v, aarch64_neon_sqrdmulh_laneq, 0),
7411 NEONMAP1(vqrdmulhq_lane_v, aarch64_neon_sqrdmulh_lane, 0),
7412 NEONMAP1(vqrdmulhq_laneq_v, aarch64_neon_sqrdmulh_laneq, 0),
7420 NEONMAP1(vqshlu_n_v, aarch64_neon_sqshlu, 0),
7421 NEONMAP1(vqshluq_n_v, aarch64_neon_sqshlu, 0),
7425 NEONMAP1(vrax1q_u64, aarch64_crypto_rax1, 0),
7426 NEONMAP2(vrecpe_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
7427 NEONMAP2(vrecpeq_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
7454 NEONMAP2(vrsqrte_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
7455 NEONMAP2(vrsqrteq_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
7459 NEONMAP1(vsha1su0q_u32, aarch64_crypto_sha1su0, 0),
7460 NEONMAP1(vsha1su1q_u32, aarch64_crypto_sha1su1, 0),
7461 NEONMAP1(vsha256h2q_u32, aarch64_crypto_sha256h2, 0),
7462 NEONMAP1(vsha256hq_u32, aarch64_crypto_sha256h, 0),
7463 NEONMAP1(vsha256su0q_u32, aarch64_crypto_sha256su0, 0),
7464 NEONMAP1(vsha256su1q_u32, aarch64_crypto_sha256su1, 0),
7465 NEONMAP1(vsha512h2q_u64, aarch64_crypto_sha512h2, 0),
7466 NEONMAP1(vsha512hq_u64, aarch64_crypto_sha512h, 0),
7467 NEONMAP1(vsha512su0q_u64, aarch64_crypto_sha512su0, 0),
7468 NEONMAP1(vsha512su1q_u64, aarch64_crypto_sha512su1, 0),
7477 NEONMAP1(vsm3partw1q_u32, aarch64_crypto_sm3partw1, 0),
7478 NEONMAP1(vsm3partw2q_u32, aarch64_crypto_sm3partw2, 0),
7479 NEONMAP1(vsm3ss1q_u32, aarch64_crypto_sm3ss1, 0),
7480 NEONMAP1(vsm3tt1aq_u32, aarch64_crypto_sm3tt1a, 0),
7481 NEONMAP1(vsm3tt1bq_u32, aarch64_crypto_sm3tt1b, 0),
7482 NEONMAP1(vsm3tt2aq_u32, aarch64_crypto_sm3tt2a, 0),
7483 NEONMAP1(vsm3tt2bq_u32, aarch64_crypto_sm3tt2b, 0),
7484 NEONMAP1(vsm4ekeyq_u32, aarch64_crypto_sm4ekey, 0),
7485 NEONMAP1(vsm4eq_u32, aarch64_crypto_sm4e, 0),
7486 NEONMAP1(vst1_x2_v, aarch64_neon_st1x2, 0),
7487 NEONMAP1(vst1_x3_v, aarch64_neon_st1x3, 0),
7488 NEONMAP1(vst1_x4_v, aarch64_neon_st1x4, 0),
7489 NEONMAP1(vst1q_x2_v, aarch64_neon_st1x2, 0),
7490 NEONMAP1(vst1q_x3_v, aarch64_neon_st1x3, 0),
7491 NEONMAP1(vst1q_x4_v, aarch64_neon_st1x4, 0),
7495 NEONMAP1(vusdot_s32, aarch64_neon_usdot, 0),
7496 NEONMAP1(vusdotq_s32, aarch64_neon_usdot, 0),
7497 NEONMAP1(vusmmlaq_s32, aarch64_neon_usmmla, 0),
7498 NEONMAP1(vxarq_u64, aarch64_crypto_xar, 0),
7555 NEONMAP1(vcvtxd_f32_f64, aarch64_sisd_fcvtxn, 0),
7576 NEONMAP1(vmull_p64, aarch64_neon_pmull64, 0),
7604 NEONMAP1(vqdmulls_s32, aarch64_neon_sqdmulls_scalar, 0),
7685 NEONMAP1(vsha1cq_u32, aarch64_crypto_sha1c, 0),
7686 NEONMAP1(vsha1h_u32, aarch64_crypto_sha1h, 0),
7687 NEONMAP1(vsha1mq_u32, aarch64_crypto_sha1m, 0),
7688 NEONMAP1(vsha1pq_u32, aarch64_crypto_sha1p, 0),
7742 { NEON::BI__builtin_neon_splat_lane_bf16, NEON::BI__builtin_neon_splat_lane_v, },
7743 { NEON::BI__builtin_neon_splat_laneq_bf16, NEON::BI__builtin_neon_splat_laneq_v, },
7744 { NEON::BI__builtin_neon_splatq_lane_bf16, NEON::BI__builtin_neon_splatq_lane_v, },
7745 { NEON::BI__builtin_neon_splatq_laneq_bf16, NEON::BI__builtin_neon_splatq_laneq_v, },
7746 { NEON::BI__builtin_neon_vabd_f16, NEON::BI__builtin_neon_vabd_v, },
7747 { NEON::BI__builtin_neon_vabdq_f16, NEON::BI__builtin_neon_vabdq_v, },
7748 { NEON::BI__builtin_neon_vabs_f16, NEON::BI__builtin_neon_vabs_v, },
7749 { NEON::BI__builtin_neon_vabsq_f16, NEON::BI__builtin_neon_vabsq_v, },
7750 { NEON::BI__builtin_neon_vcage_f16, NEON::BI__builtin_neon_vcage_v, },
7751 { NEON::BI__builtin_neon_vcageq_f16, NEON::BI__builtin_neon_vcageq_v, },
7752 { NEON::BI__builtin_neon_vcagt_f16, NEON::BI__builtin_neon_vcagt_v, },
7753 { NEON::BI__builtin_neon_vcagtq_f16, NEON::BI__builtin_neon_vcagtq_v, },
7754 { NEON::BI__builtin_neon_vcale_f16, NEON::BI__builtin_neon_vcale_v, },
7755 { NEON::BI__builtin_neon_vcaleq_f16, NEON::BI__builtin_neon_vcaleq_v, },
7756 { NEON::BI__builtin_neon_vcalt_f16, NEON::BI__builtin_neon_vcalt_v, },
7757 { NEON::BI__builtin_neon_vcaltq_f16, NEON::BI__builtin_neon_vcaltq_v, },
7758 { NEON::BI__builtin_neon_vceqz_f16, NEON::BI__builtin_neon_vceqz_v, },
7759 { NEON::BI__builtin_neon_vceqzq_f16, NEON::BI__builtin_neon_vceqzq_v, },
7760 { NEON::BI__builtin_neon_vcgez_f16, NEON::BI__builtin_neon_vcgez_v, },
7761 { NEON::BI__builtin_neon_vcgezq_f16, NEON::BI__builtin_neon_vcgezq_v, },
7762 { NEON::BI__builtin_neon_vcgtz_f16, NEON::BI__builtin_neon_vcgtz_v, },
7763 { NEON::BI__builtin_neon_vcgtzq_f16, NEON::BI__builtin_neon_vcgtzq_v, },
7764 { NEON::BI__builtin_neon_vclez_f16, NEON::BI__builtin_neon_vclez_v, },
7765 { NEON::BI__builtin_neon_vclezq_f16, NEON::BI__builtin_neon_vclezq_v, },
7766 { NEON::BI__builtin_neon_vcltz_f16, NEON::BI__builtin_neon_vcltz_v, },
7767 { NEON::BI__builtin_neon_vcltzq_f16, NEON::BI__builtin_neon_vcltzq_v, },
7768 { NEON::BI__builtin_neon_vfma_f16, NEON::BI__builtin_neon_vfma_v, },
7769 { NEON::BI__builtin_neon_vfma_lane_f16, NEON::BI__builtin_neon_vfma_lane_v, },
7770 { NEON::BI__builtin_neon_vfma_laneq_f16, NEON::BI__builtin_neon_vfma_laneq_v, },
7771 { NEON::BI__builtin_neon_vfmaq_f16, NEON::BI__builtin_neon_vfmaq_v, },
7772 { NEON::BI__builtin_neon_vfmaq_lane_f16, NEON::BI__builtin_neon_vfmaq_lane_v, },
7773 { NEON::BI__builtin_neon_vfmaq_laneq_f16, NEON::BI__builtin_neon_vfmaq_laneq_v, },
7774 { NEON::BI__builtin_neon_vld1_bf16_x2, NEON::BI__builtin_neon_vld1_x2_v },
7775 { NEON::BI__builtin_neon_vld1_bf16_x3, NEON::BI__builtin_neon_vld1_x3_v },
7776 { NEON::BI__builtin_neon_vld1_bf16_x4, NEON::BI__builtin_neon_vld1_x4_v },
7777 { NEON::BI__builtin_neon_vld1_bf16, NEON::BI__builtin_neon_vld1_v },
7778 { NEON::BI__builtin_neon_vld1_dup_bf16, NEON::BI__builtin_neon_vld1_dup_v },
7779 { NEON::BI__builtin_neon_vld1_lane_bf16, NEON::BI__builtin_neon_vld1_lane_v },
7780 { NEON::BI__builtin_neon_vld1q_bf16_x2, NEON::BI__builtin_neon_vld1q_x2_v },
7781 { NEON::BI__builtin_neon_vld1q_bf16_x3, NEON::BI__builtin_neon_vld1q_x3_v },
7782 { NEON::BI__builtin_neon_vld1q_bf16_x4, NEON::BI__builtin_neon_vld1q_x4_v },
7783 { NEON::BI__builtin_neon_vld1q_bf16, NEON::BI__builtin_neon_vld1q_v },
7784 { NEON::BI__builtin_neon_vld1q_dup_bf16, NEON::BI__builtin_neon_vld1q_dup_v },
7785 { NEON::BI__builtin_neon_vld1q_lane_bf16, NEON::BI__builtin_neon_vld1q_lane_v },
7786 { NEON::BI__builtin_neon_vld2_bf16, NEON::BI__builtin_neon_vld2_v },
7787 { NEON::BI__builtin_neon_vld2_dup_bf16, NEON::BI__builtin_neon_vld2_dup_v },
7788 { NEON::BI__builtin_neon_vld2_lane_bf16, NEON::BI__builtin_neon_vld2_lane_v },
7789 { NEON::BI__builtin_neon_vld2q_bf16, NEON::BI__builtin_neon_vld2q_v },
7790 { NEON::BI__builtin_neon_vld2q_dup_bf16, NEON::BI__builtin_neon_vld2q_dup_v },
7791 { NEON::BI__builtin_neon_vld2q_lane_bf16, NEON::BI__builtin_neon_vld2q_lane_v },
7792 { NEON::BI__builtin_neon_vld3_bf16, NEON::BI__builtin_neon_vld3_v },
7793 { NEON::BI__builtin_neon_vld3_dup_bf16, NEON::BI__builtin_neon_vld3_dup_v },
7794 { NEON::BI__builtin_neon_vld3_lane_bf16, NEON::BI__builtin_neon_vld3_lane_v },
7795 { NEON::BI__builtin_neon_vld3q_bf16, NEON::BI__builtin_neon_vld3q_v },
7796 { NEON::BI__builtin_neon_vld3q_dup_bf16, NEON::BI__builtin_neon_vld3q_dup_v },
7797 { NEON::BI__builtin_neon_vld3q_lane_bf16, NEON::BI__builtin_neon_vld3q_lane_v },
7798 { NEON::BI__builtin_neon_vld4_bf16, NEON::BI__builtin_neon_vld4_v },
7799 { NEON::BI__builtin_neon_vld4_dup_bf16, NEON::BI__builtin_neon_vld4_dup_v },
7800 { NEON::BI__builtin_neon_vld4_lane_bf16, NEON::BI__builtin_neon_vld4_lane_v },
7801 { NEON::BI__builtin_neon_vld4q_bf16, NEON::BI__builtin_neon_vld4q_v },
7802 { NEON::BI__builtin_neon_vld4q_dup_bf16, NEON::BI__builtin_neon_vld4q_dup_v },
7803 { NEON::BI__builtin_neon_vld4q_lane_bf16, NEON::BI__builtin_neon_vld4q_lane_v },
7804 { NEON::BI__builtin_neon_vmax_f16, NEON::BI__builtin_neon_vmax_v, },
7805 { NEON::BI__builtin_neon_vmaxnm_f16, NEON::BI__builtin_neon_vmaxnm_v, },
7806 { NEON::BI__builtin_neon_vmaxnmq_f16, NEON::BI__builtin_neon_vmaxnmq_v, },
7807 { NEON::BI__builtin_neon_vmaxq_f16, NEON::BI__builtin_neon_vmaxq_v, },
7808 { NEON::BI__builtin_neon_vmin_f16, NEON::BI__builtin_neon_vmin_v, },
7809 { NEON::BI__builtin_neon_vminnm_f16, NEON::BI__builtin_neon_vminnm_v, },
7810 { NEON::BI__builtin_neon_vminnmq_f16, NEON::BI__builtin_neon_vminnmq_v, },
7811 { NEON::BI__builtin_neon_vminq_f16, NEON::BI__builtin_neon_vminq_v, },
7812 { NEON::BI__builtin_neon_vmulx_f16, NEON::BI__builtin_neon_vmulx_v, },
7813 { NEON::BI__builtin_neon_vmulxq_f16, NEON::BI__builtin_neon_vmulxq_v, },
7814 { NEON::BI__builtin_neon_vpadd_f16, NEON::BI__builtin_neon_vpadd_v, },
7815 { NEON::BI__builtin_neon_vpaddq_f16, NEON::BI__builtin_neon_vpaddq_v, },
7816 { NEON::BI__builtin_neon_vpmax_f16, NEON::BI__builtin_neon_vpmax_v, },
7817 { NEON::BI__builtin_neon_vpmaxnm_f16, NEON::BI__builtin_neon_vpmaxnm_v, },
7818 { NEON::BI__builtin_neon_vpmaxnmq_f16, NEON::BI__builtin_neon_vpmaxnmq_v, },
7819 { NEON::BI__builtin_neon_vpmaxq_f16, NEON::BI__builtin_neon_vpmaxq_v, },
7820 { NEON::BI__builtin_neon_vpmin_f16, NEON::BI__builtin_neon_vpmin_v, },
7821 { NEON::BI__builtin_neon_vpminnm_f16, NEON::BI__builtin_neon_vpminnm_v, },
7822 { NEON::BI__builtin_neon_vpminnmq_f16, NEON::BI__builtin_neon_vpminnmq_v, },
7823 { NEON::BI__builtin_neon_vpminq_f16, NEON::BI__builtin_neon_vpminq_v, },
7824 { NEON::BI__builtin_neon_vrecpe_f16, NEON::BI__builtin_neon_vrecpe_v, },
7825 { NEON::BI__builtin_neon_vrecpeq_f16, NEON::BI__builtin_neon_vrecpeq_v, },
7826 { NEON::BI__builtin_neon_vrecps_f16, NEON::BI__builtin_neon_vrecps_v, },
7827 { NEON::BI__builtin_neon_vrecpsq_f16, NEON::BI__builtin_neon_vrecpsq_v, },
7828 { NEON::BI__builtin_neon_vrnd_f16, NEON::BI__builtin_neon_vrnd_v, },
7829 { NEON::BI__builtin_neon_vrnda_f16, NEON::BI__builtin_neon_vrnda_v, },
7830 { NEON::BI__builtin_neon_vrndaq_f16, NEON::BI__builtin_neon_vrndaq_v, },
7831 { NEON::BI__builtin_neon_vrndi_f16, NEON::BI__builtin_neon_vrndi_v, },
7832 { NEON::BI__builtin_neon_vrndiq_f16, NEON::BI__builtin_neon_vrndiq_v, },
7833 { NEON::BI__builtin_neon_vrndm_f16, NEON::BI__builtin_neon_vrndm_v, },
7834 { NEON::BI__builtin_neon_vrndmq_f16, NEON::BI__builtin_neon_vrndmq_v, },
7835 { NEON::BI__builtin_neon_vrndn_f16, NEON::BI__builtin_neon_vrndn_v, },
7836 { NEON::BI__builtin_neon_vrndnq_f16, NEON::BI__builtin_neon_vrndnq_v, },
7837 { NEON::BI__builtin_neon_vrndp_f16, NEON::BI__builtin_neon_vrndp_v, },
7838 { NEON::BI__builtin_neon_vrndpq_f16, NEON::BI__builtin_neon_vrndpq_v, },
7839 { NEON::BI__builtin_neon_vrndq_f16, NEON::BI__builtin_neon_vrndq_v, },
7840 { NEON::BI__builtin_neon_vrndx_f16, NEON::BI__builtin_neon_vrndx_v, },
7841 { NEON::BI__builtin_neon_vrndxq_f16, NEON::BI__builtin_neon_vrndxq_v, },
7842 { NEON::BI__builtin_neon_vrsqrte_f16, NEON::BI__builtin_neon_vrsqrte_v, },
7843 { NEON::BI__builtin_neon_vrsqrteq_f16, NEON::BI__builtin_neon_vrsqrteq_v, },
7844 { NEON::BI__builtin_neon_vrsqrts_f16, NEON::BI__builtin_neon_vrsqrts_v, },
7845 { NEON::BI__builtin_neon_vrsqrtsq_f16, NEON::BI__builtin_neon_vrsqrtsq_v, },
7846 { NEON::BI__builtin_neon_vsqrt_f16, NEON::BI__builtin_neon_vsqrt_v, },
7847 { NEON::BI__builtin_neon_vsqrtq_f16, NEON::BI__builtin_neon_vsqrtq_v, },
7848 { NEON::BI__builtin_neon_vst1_bf16_x2, NEON::BI__builtin_neon_vst1_x2_v },
7849 { NEON::BI__builtin_neon_vst1_bf16_x3, NEON::BI__builtin_neon_vst1_x3_v },
7850 { NEON::BI__builtin_neon_vst1_bf16_x4, NEON::BI__builtin_neon_vst1_x4_v },
7851 { NEON::BI__builtin_neon_vst1_bf16, NEON::BI__builtin_neon_vst1_v },
7852 { NEON::BI__builtin_neon_vst1_lane_bf16, NEON::BI__builtin_neon_vst1_lane_v },
7853 { NEON::BI__builtin_neon_vst1q_bf16_x2, NEON::BI__builtin_neon_vst1q_x2_v },
7854 { NEON::BI__builtin_neon_vst1q_bf16_x3, NEON::BI__builtin_neon_vst1q_x3_v },
7855 { NEON::BI__builtin_neon_vst1q_bf16_x4, NEON::BI__builtin_neon_vst1q_x4_v },
7856 { NEON::BI__builtin_neon_vst1q_bf16, NEON::BI__builtin_neon_vst1q_v },
7857 { NEON::BI__builtin_neon_vst1q_lane_bf16, NEON::BI__builtin_neon_vst1q_lane_v },
7858 { NEON::BI__builtin_neon_vst2_bf16, NEON::BI__builtin_neon_vst2_v },
7859 { NEON::BI__builtin_neon_vst2_lane_bf16, NEON::BI__builtin_neon_vst2_lane_v },
7860 { NEON::BI__builtin_neon_vst2q_bf16, NEON::BI__builtin_neon_vst2q_v },
7861 { NEON::BI__builtin_neon_vst2q_lane_bf16, NEON::BI__builtin_neon_vst2q_lane_v },
7862 { NEON::BI__builtin_neon_vst3_bf16, NEON::BI__builtin_neon_vst3_v },
7863 { NEON::BI__builtin_neon_vst3_lane_bf16, NEON::BI__builtin_neon_vst3_lane_v },
7864 { NEON::BI__builtin_neon_vst3q_bf16, NEON::BI__builtin_neon_vst3q_v },
7865 { NEON::BI__builtin_neon_vst3q_lane_bf16, NEON::BI__builtin_neon_vst3q_lane_v },
7866 { NEON::BI__builtin_neon_vst4_bf16, NEON::BI__builtin_neon_vst4_v },
7867 { NEON::BI__builtin_neon_vst4_lane_bf16, NEON::BI__builtin_neon_vst4_lane_v },
7868 { NEON::BI__builtin_neon_vst4q_bf16, NEON::BI__builtin_neon_vst4q_v },
7869 { NEON::BI__builtin_neon_vst4q_lane_bf16, NEON::BI__builtin_neon_vst4q_lane_v },
7873 { NEON::BI__builtin_neon_vldap1_lane_u64, NEON::BI__builtin_neon_vldap1_lane_s64 },
7874 { NEON::BI__builtin_neon_vldap1_lane_f64, NEON::BI__builtin_neon_vldap1_lane_s64 },
7875 { NEON::BI__builtin_neon_vldap1_lane_p64, NEON::BI__builtin_neon_vldap1_lane_s64 },
7876 { NEON::BI__builtin_neon_vldap1q_lane_u64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
7877 { NEON::BI__builtin_neon_vldap1q_lane_f64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
7878 { NEON::BI__builtin_neon_vldap1q_lane_p64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
7879 { NEON::BI__builtin_neon_vstl1_lane_u64, NEON::BI__builtin_neon_vstl1_lane_s64 },
7880 { NEON::BI__builtin_neon_vstl1_lane_f64, NEON::BI__builtin_neon_vstl1_lane_s64 },
7881 { NEON::BI__builtin_neon_vstl1_lane_p64, NEON::BI__builtin_neon_vstl1_lane_s64 },
7882 { NEON::BI__builtin_neon_vstl1q_lane_u64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
7883 { NEON::BI__builtin_neon_vstl1q_lane_f64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
7884 { NEON::BI__builtin_neon_vstl1q_lane_p64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
7891#define SVEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
7893 #NameBase, SVE::BI__builtin_sve_##NameBase, Intrinsic::LLVMIntrinsic, 0, \
7897#define SVEMAP2(NameBase, TypeModifier) \
7898 { #NameBase, SVE::BI__builtin_sve_##NameBase, 0, 0, TypeModifier }
7900#define GET_SVE_LLVM_INTRINSIC_MAP
7901#include "clang/Basic/arm_sve_builtin_cg.inc"
7902#include "clang/Basic/BuiltinsAArch64NeonSVEBridge_cg.def"
7903#undef GET_SVE_LLVM_INTRINSIC_MAP
7909#define SMEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
7911 #NameBase, SME::BI__builtin_sme_##NameBase, Intrinsic::LLVMIntrinsic, 0, \
7915#define SMEMAP2(NameBase, TypeModifier) \
7916 { #NameBase, SME::BI__builtin_sme_##NameBase, 0, 0, TypeModifier }
7918#define GET_SME_LLVM_INTRINSIC_MAP
7919#include "clang/Basic/arm_sme_builtin_cg.inc"
7920#undef GET_SME_LLVM_INTRINSIC_MAP
7933static const ARMVectorIntrinsicInfo *
7935 unsigned BuiltinID,
bool &MapProvenSorted) {
7938 if (!MapProvenSorted) {
7939 assert(llvm::is_sorted(IntrinsicMap));
7940 MapProvenSorted =
true;
7944 const ARMVectorIntrinsicInfo *Builtin =
7945 llvm::lower_bound(IntrinsicMap, BuiltinID);
7947 if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID)
7955 llvm::Type *ArgType,
7968 Ty = llvm::FixedVectorType::get(
7969 Ty, VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1);
7976 int Elts = VectorSize ? VectorSize / ArgType->getPrimitiveSizeInBits() : 1;
7977 ArgType = llvm::FixedVectorType::get(ArgType, Elts);
7981 Tys.push_back(ArgType);
7984 Tys.push_back(ArgType);
7995 unsigned BuiltinID = SISDInfo.BuiltinID;
7996 unsigned int Int = SISDInfo.LLVMIntrinsic;
7997 unsigned Modifier = SISDInfo.TypeModifier;
7998 const char *
s = SISDInfo.NameHint;
8000 switch (BuiltinID) {
8001 case NEON::BI__builtin_neon_vcled_s64:
8002 case NEON::BI__builtin_neon_vcled_u64:
8003 case NEON::BI__builtin_neon_vcles_f32:
8004 case NEON::BI__builtin_neon_vcled_f64:
8005 case NEON::BI__builtin_neon_vcltd_s64:
8006 case NEON::BI__builtin_neon_vcltd_u64:
8007 case NEON::BI__builtin_neon_vclts_f32:
8008 case NEON::BI__builtin_neon_vcltd_f64:
8009 case NEON::BI__builtin_neon_vcales_f32:
8010 case NEON::BI__builtin_neon_vcaled_f64:
8011 case NEON::BI__builtin_neon_vcalts_f32:
8012 case NEON::BI__builtin_neon_vcaltd_f64:
8016 std::swap(Ops[0], Ops[1]);
8020 assert(Int &&
"Generic code assumes a valid intrinsic");
8023 const Expr *Arg =
E->getArg(0);
8028 ConstantInt *C0 = ConstantInt::get(CGF.
SizeTy, 0);
8029 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
8030 ai != ae; ++ai, ++j) {
8031 llvm::Type *ArgTy = ai->getType();
8032 if (Ops[j]->getType()->getPrimitiveSizeInBits() ==
8033 ArgTy->getPrimitiveSizeInBits())
8036 assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy());
8039 Ops[j] = CGF.
Builder.CreateTruncOrBitCast(
8040 Ops[j], cast<llvm::VectorType>(ArgTy)->getElementType());
8042 CGF.
Builder.CreateInsertElement(PoisonValue::get(ArgTy), Ops[j], C0);
8047 if (ResultType->getPrimitiveSizeInBits().getFixedValue() <
8048 Result->getType()->getPrimitiveSizeInBits().getFixedValue())
8055 unsigned BuiltinID,
unsigned LLVMIntrinsic,
unsigned AltLLVMIntrinsic,
8056 const char *NameHint,
unsigned Modifier,
const CallExpr *
E,
8058 llvm::Triple::ArchType Arch) {
8060 const Expr *Arg =
E->getArg(
E->getNumArgs() - 1);
8061 std::optional<llvm::APSInt> NeonTypeConst =
8068 bool Usgn =
Type.isUnsigned();
8069 bool Quad =
Type.isQuad();
8071 const bool AllowBFloatArgsAndRet =
8074 llvm::FixedVectorType *VTy =
8075 GetNeonType(
this,
Type, HasLegalHalfType,
false, AllowBFloatArgsAndRet);
8076 llvm::Type *Ty = VTy;
8080 auto getAlignmentValue32 = [&](
Address addr) ->
Value* {
8081 return Builder.getInt32(addr.getAlignment().getQuantity());
8084 unsigned Int = LLVMIntrinsic;
8086 Int = AltLLVMIntrinsic;
8088 switch (BuiltinID) {
8090 case NEON::BI__builtin_neon_splat_lane_v:
8091 case NEON::BI__builtin_neon_splat_laneq_v:
8092 case NEON::BI__builtin_neon_splatq_lane_v:
8093 case NEON::BI__builtin_neon_splatq_laneq_v: {
8094 auto NumElements = VTy->getElementCount();
8095 if (BuiltinID == NEON::BI__builtin_neon_splatq_lane_v)
8096 NumElements = NumElements * 2;
8097 if (BuiltinID == NEON::BI__builtin_neon_splat_laneq_v)
8098 NumElements = NumElements.divideCoefficientBy(2);
8100 Ops[0] =
Builder.CreateBitCast(Ops[0], VTy);
8101 return EmitNeonSplat(Ops[0], cast<ConstantInt>(Ops[1]), NumElements);
8103 case NEON::BI__builtin_neon_vpadd_v:
8104 case NEON::BI__builtin_neon_vpaddq_v:
8106 if (VTy->getElementType()->isFloatingPointTy() &&
8107 Int == Intrinsic::aarch64_neon_addp)
8108 Int = Intrinsic::aarch64_neon_faddp;
8110 case NEON::BI__builtin_neon_vabs_v:
8111 case NEON::BI__builtin_neon_vabsq_v:
8112 if (VTy->getElementType()->isFloatingPointTy())
8115 case NEON::BI__builtin_neon_vadd_v:
8116 case NEON::BI__builtin_neon_vaddq_v: {
8117 llvm::Type *VTy = llvm::FixedVectorType::get(
Int8Ty, Quad ? 16 : 8);
8118 Ops[0] =
Builder.CreateBitCast(Ops[0], VTy);
8119 Ops[1] =
Builder.CreateBitCast(Ops[1], VTy);
8120 Ops[0] =
Builder.CreateXor(Ops[0], Ops[1]);
8121 return Builder.CreateBitCast(Ops[0], Ty);
8123 case NEON::BI__builtin_neon_vaddhn_v: {
8124 llvm::FixedVectorType *SrcTy =
8125 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
8128 Ops[0] =
Builder.CreateBitCast(Ops[0], SrcTy);
8129 Ops[1] =
Builder.CreateBitCast(Ops[1], SrcTy);
8130 Ops[0] =
Builder.CreateAdd(Ops[0], Ops[1],
"vaddhn");
8133 Constant *ShiftAmt =
8134 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
8135 Ops[0] =
Builder.CreateLShr(Ops[0], ShiftAmt,
"vaddhn");
8138 return Builder.CreateTrunc(Ops[0], VTy,
"vaddhn");
8140 case NEON::BI__builtin_neon_vcale_v:
8141 case NEON::BI__builtin_neon_vcaleq_v:
8142 case NEON::BI__builtin_neon_vcalt_v:
8143 case NEON::BI__builtin_neon_vcaltq_v:
8144 std::swap(Ops[0], Ops[1]);
8146 case NEON::BI__builtin_neon_vcage_v:
8147 case NEON::BI__builtin_neon_vcageq_v:
8148 case NEON::BI__builtin_neon_vcagt_v:
8149 case NEON::BI__builtin_neon_vcagtq_v: {
8151 switch (VTy->getScalarSizeInBits()) {
8152 default: llvm_unreachable(
"unexpected type");
8163 auto *VecFlt = llvm::FixedVectorType::get(Ty, VTy->getNumElements());
8164 llvm::Type *Tys[] = { VTy, VecFlt };
8168 case NEON::BI__builtin_neon_vceqz_v:
8169 case NEON::BI__builtin_neon_vceqzq_v:
8171 ICmpInst::ICMP_EQ,
"vceqz");
8172 case NEON::BI__builtin_neon_vcgez_v:
8173 case NEON::BI__builtin_neon_vcgezq_v:
8175 ICmpInst::ICMP_SGE,
"vcgez");
8176 case NEON::BI__builtin_neon_vclez_v:
8177 case NEON::BI__builtin_neon_vclezq_v:
8179 ICmpInst::ICMP_SLE,
"vclez");
8180 case NEON::BI__builtin_neon_vcgtz_v:
8181 case NEON::BI__builtin_neon_vcgtzq_v:
8183 ICmpInst::ICMP_SGT,
"vcgtz");
8184 case NEON::BI__builtin_neon_vcltz_v:
8185 case NEON::BI__builtin_neon_vcltzq_v:
8187 ICmpInst::ICMP_SLT,
"vcltz");
8188 case NEON::BI__builtin_neon_vclz_v:
8189 case NEON::BI__builtin_neon_vclzq_v:
8194 case NEON::BI__builtin_neon_vcvt_f32_v:
8195 case NEON::BI__builtin_neon_vcvtq_f32_v:
8196 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
8199 return Usgn ?
Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
8200 :
Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
8201 case NEON::BI__builtin_neon_vcvt_f16_s16:
8202 case NEON::BI__builtin_neon_vcvt_f16_u16:
8203 case NEON::BI__builtin_neon_vcvtq_f16_s16:
8204 case NEON::BI__builtin_neon_vcvtq_f16_u16:
8205 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
8208 return Usgn ?
Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
8209 :
Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
8210 case NEON::BI__builtin_neon_vcvt_n_f16_s16:
8211 case NEON::BI__builtin_neon_vcvt_n_f16_u16:
8212 case NEON::BI__builtin_neon_vcvtq_n_f16_s16:
8213 case NEON::BI__builtin_neon_vcvtq_n_f16_u16: {
8218 case NEON::BI__builtin_neon_vcvt_n_f32_v:
8219 case NEON::BI__builtin_neon_vcvt_n_f64_v:
8220 case NEON::BI__builtin_neon_vcvtq_n_f32_v:
8221 case NEON::BI__builtin_neon_vcvtq_n_f64_v: {
8223 Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
8227 case NEON::BI__builtin_neon_vcvt_n_s16_f16:
8228 case NEON::BI__builtin_neon_vcvt_n_s32_v:
8229 case NEON::BI__builtin_neon_vcvt_n_u16_f16:
8230 case NEON::BI__builtin_neon_vcvt_n_u32_v:
8231 case NEON::BI__builtin_neon_vcvt_n_s64_v:
8232 case NEON::BI__builtin_neon_vcvt_n_u64_v:
8233 case NEON::BI__builtin_neon_vcvtq_n_s16_f16:
8234 case NEON::BI__builtin_neon_vcvtq_n_s32_v:
8235 case NEON::BI__builtin_neon_vcvtq_n_u16_f16:
8236 case NEON::BI__builtin_neon_vcvtq_n_u32_v:
8237 case NEON::BI__builtin_neon_vcvtq_n_s64_v:
8238 case NEON::BI__builtin_neon_vcvtq_n_u64_v: {
8243 case NEON::BI__builtin_neon_vcvt_s32_v:
8244 case NEON::BI__builtin_neon_vcvt_u32_v:
8245 case NEON::BI__builtin_neon_vcvt_s64_v:
8246 case NEON::BI__builtin_neon_vcvt_u64_v:
8247 case NEON::BI__builtin_neon_vcvt_s16_f16:
8248 case NEON::BI__builtin_neon_vcvt_u16_f16:
8249 case NEON::BI__builtin_neon_vcvtq_s32_v:
8250 case NEON::BI__builtin_neon_vcvtq_u32_v:
8251 case NEON::BI__builtin_neon_vcvtq_s64_v:
8252 case NEON::BI__builtin_neon_vcvtq_u64_v:
8253 case NEON::BI__builtin_neon_vcvtq_s16_f16:
8254 case NEON::BI__builtin_neon_vcvtq_u16_f16: {
8256 return Usgn ?
Builder.CreateFPToUI(Ops[0], Ty,
"vcvt")
8257 :
Builder.CreateFPToSI(Ops[0], Ty,
"vcvt");
8259 case NEON::BI__builtin_neon_vcvta_s16_f16:
8260 case NEON::BI__builtin_neon_vcvta_s32_v:
8261 case NEON::BI__builtin_neon_vcvta_s64_v:
8262 case NEON::BI__builtin_neon_vcvta_u16_f16:
8263 case NEON::BI__builtin_neon_vcvta_u32_v:
8264 case NEON::BI__builtin_neon_vcvta_u64_v:
8265 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
8266 case NEON::BI__builtin_neon_vcvtaq_s32_v:
8267 case NEON::BI__builtin_neon_vcvtaq_s64_v:
8268 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
8269 case NEON::BI__builtin_neon_vcvtaq_u32_v:
8270 case NEON::BI__builtin_neon_vcvtaq_u64_v:
8271 case NEON::BI__builtin_neon_vcvtn_s16_f16:
8272 case NEON::BI__builtin_neon_vcvtn_s32_v:
8273 case NEON::BI__builtin_neon_vcvtn_s64_v:
8274 case NEON::BI__builtin_neon_vcvtn_u16_f16:
8275 case NEON::BI__builtin_neon_vcvtn_u32_v:
8276 case NEON::BI__builtin_neon_vcvtn_u64_v:
8277 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
8278 case NEON::BI__builtin_neon_vcvtnq_s32_v:
8279 case NEON::BI__builtin_neon_vcvtnq_s64_v:
8280 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
8281 case NEON::BI__builtin_neon_vcvtnq_u32_v:
8282 case NEON::BI__builtin_neon_vcvtnq_u64_v:
8283 case NEON::BI__builtin_neon_vcvtp_s16_f16:
8284 case NEON::BI__builtin_neon_vcvtp_s32_v:
8285 case NEON::BI__builtin_neon_vcvtp_s64_v:
8286 case NEON::BI__builtin_neon_vcvtp_u16_f16:
8287 case NEON::BI__builtin_neon_vcvtp_u32_v:
8288 case NEON::BI__builtin_neon_vcvtp_u64_v:
8289 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
8290 case NEON::BI__builtin_neon_vcvtpq_s32_v:
8291 case NEON::BI__builtin_neon_vcvtpq_s64_v:
8292 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
8293 case NEON::BI__builtin_neon_vcvtpq_u32_v:
8294 case NEON::BI__builtin_neon_vcvtpq_u64_v:
8295 case NEON::BI__builtin_neon_vcvtm_s16_f16:
8296 case NEON::BI__builtin_neon_vcvtm_s32_v:
8297 case NEON::BI__builtin_neon_vcvtm_s64_v:
8298 case NEON::BI__builtin_neon_vcvtm_u16_f16:
8299 case NEON::BI__builtin_neon_vcvtm_u32_v:
8300 case NEON::BI__builtin_neon_vcvtm_u64_v:
8301 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
8302 case NEON::BI__builtin_neon_vcvtmq_s32_v:
8303 case NEON::BI__builtin_neon_vcvtmq_s64_v:
8304 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
8305 case NEON::BI__builtin_neon_vcvtmq_u32_v:
8306 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
8310 case NEON::BI__builtin_neon_vcvtx_f32_v: {
8311 llvm::Type *Tys[2] = { VTy->getTruncatedElementVectorType(VTy), Ty};
8315 case NEON::BI__builtin_neon_vext_v:
8316 case NEON::BI__builtin_neon_vextq_v: {
8317 int CV = cast<ConstantInt>(Ops[2])->getSExtValue();
8319 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
8320 Indices.push_back(i+CV);
8322 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
8323 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
8324 return Builder.CreateShuffleVector(Ops[0], Ops[1], Indices,
"vext");
8326 case NEON::BI__builtin_neon_vfma_v:
8327 case NEON::BI__builtin_neon_vfmaq_v: {
8328 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
8329 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
8330 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
8334 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
8335 {Ops[1], Ops[2], Ops[0]});
8337 case NEON::BI__builtin_neon_vld1_v:
8338 case NEON::BI__builtin_neon_vld1q_v: {
8340 Ops.push_back(getAlignmentValue32(PtrOp0));
8343 case NEON::BI__builtin_neon_vld1_x2_v:
8344 case NEON::BI__builtin_neon_vld1q_x2_v:
8345 case NEON::BI__builtin_neon_vld1_x3_v:
8346 case NEON::BI__builtin_neon_vld1q_x3_v:
8347 case NEON::BI__builtin_neon_vld1_x4_v:
8348 case NEON::BI__builtin_neon_vld1q_x4_v: {
8351 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld1xN");
8354 case NEON::BI__builtin_neon_vld2_v:
8355 case NEON::BI__builtin_neon_vld2q_v:
8356 case NEON::BI__builtin_neon_vld3_v:
8357 case NEON::BI__builtin_neon_vld3q_v:
8358 case NEON::BI__builtin_neon_vld4_v:
8359 case NEON::BI__builtin_neon_vld4q_v:
8360 case NEON::BI__builtin_neon_vld2_dup_v:
8361 case NEON::BI__builtin_neon_vld2q_dup_v:
8362 case NEON::BI__builtin_neon_vld3_dup_v:
8363 case NEON::BI__builtin_neon_vld3q_dup_v:
8364 case NEON::BI__builtin_neon_vld4_dup_v:
8365 case NEON::BI__builtin_neon_vld4q_dup_v: {
8368 Value *Align = getAlignmentValue32(PtrOp1);
8369 Ops[1] =
Builder.CreateCall(F, {Ops[1], Align}, NameHint);
8372 case NEON::BI__builtin_neon_vld1_dup_v:
8373 case NEON::BI__builtin_neon_vld1q_dup_v: {
8374 Value *
V = PoisonValue::get(Ty);
8377 llvm::Constant *CI = ConstantInt::get(
SizeTy, 0);
8378 Ops[0] =
Builder.CreateInsertElement(
V, Ld, CI);
8381 case NEON::BI__builtin_neon_vld2_lane_v:
8382 case NEON::BI__builtin_neon_vld2q_lane_v:
8383 case NEON::BI__builtin_neon_vld3_lane_v:
8384 case NEON::BI__builtin_neon_vld3q_lane_v:
8385 case NEON::BI__builtin_neon_vld4_lane_v:
8386 case NEON::BI__builtin_neon_vld4q_lane_v: {
8389 for (
unsigned I = 2; I < Ops.size() - 1; ++I)
8390 Ops[I] =
Builder.CreateBitCast(Ops[I], Ty);
8391 Ops.push_back(getAlignmentValue32(PtrOp1));
8395 case NEON::BI__builtin_neon_vmovl_v: {
8396 llvm::FixedVectorType *DTy =
8397 llvm::FixedVectorType::getTruncatedElementVectorType(VTy);
8398 Ops[0] =
Builder.CreateBitCast(Ops[0], DTy);
8400 return Builder.CreateZExt(Ops[0], Ty,
"vmovl");
8401 return Builder.CreateSExt(Ops[0], Ty,
"vmovl");
8403 case NEON::BI__builtin_neon_vmovn_v: {
8404 llvm::FixedVectorType *QTy =
8405 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
8406 Ops[0] =
Builder.CreateBitCast(Ops[0], QTy);
8407 return Builder.CreateTrunc(Ops[0], Ty,
"vmovn");
8409 case NEON::BI__builtin_neon_vmull_v:
8415 Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls;
8418 case NEON::BI__builtin_neon_vpadal_v:
8419 case NEON::BI__builtin_neon_vpadalq_v: {
8421 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
8425 llvm::FixedVectorType::get(EltTy, VTy->getNumElements() * 2);
8426 llvm::Type *Tys[2] = { Ty, NarrowTy };
8429 case NEON::BI__builtin_neon_vpaddl_v:
8430 case NEON::BI__builtin_neon_vpaddlq_v: {
8432 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
8433 llvm::Type *EltTy = llvm::IntegerType::get(
getLLVMContext(), EltBits / 2);
8435 llvm::FixedVectorType::get(EltTy, VTy->getNumElements() * 2);
8436 llvm::Type *Tys[2] = { Ty, NarrowTy };
8439 case NEON::BI__builtin_neon_vqdmlal_v:
8440 case NEON::BI__builtin_neon_vqdmlsl_v: {
8447 case NEON::BI__builtin_neon_vqdmulhq_lane_v:
8448 case NEON::BI__builtin_neon_vqdmulh_lane_v:
8449 case NEON::BI__builtin_neon_vqrdmulhq_lane_v:
8450 case NEON::BI__builtin_neon_vqrdmulh_lane_v: {
8451 auto *RTy = cast<llvm::FixedVectorType>(Ty);
8452 if (BuiltinID == NEON::BI__builtin_neon_vqdmulhq_lane_v ||
8453 BuiltinID == NEON::BI__builtin_neon_vqrdmulhq_lane_v)
8454 RTy = llvm::FixedVectorType::get(RTy->getElementType(),
8455 RTy->getNumElements() * 2);
8456 llvm::Type *Tys[2] = {
8461 case NEON::BI__builtin_neon_vqdmulhq_laneq_v:
8462 case NEON::BI__builtin_neon_vqdmulh_laneq_v:
8463 case NEON::BI__builtin_neon_vqrdmulhq_laneq_v:
8464 case NEON::BI__builtin_neon_vqrdmulh_laneq_v: {
8465 llvm::Type *Tys[2] = {
8470 case NEON::BI__builtin_neon_vqshl_n_v:
8471 case NEON::BI__builtin_neon_vqshlq_n_v:
8474 case NEON::BI__builtin_neon_vqshlu_n_v:
8475 case NEON::BI__builtin_neon_vqshluq_n_v:
8478 case NEON::BI__builtin_neon_vrecpe_v:
8479 case NEON::BI__builtin_neon_vrecpeq_v:
8480 case NEON::BI__builtin_neon_vrsqrte_v:
8481 case NEON::BI__builtin_neon_vrsqrteq_v:
8482 Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic;
8484 case NEON::BI__builtin_neon_vrndi_v:
8485 case NEON::BI__builtin_neon_vrndiq_v:
8487 ? Intrinsic::experimental_constrained_nearbyint
8488 : Intrinsic::nearbyint;
8490 case NEON::BI__builtin_neon_vrshr_n_v:
8491 case NEON::BI__builtin_neon_vrshrq_n_v:
8494 case NEON::BI__builtin_neon_vsha512hq_u64:
8495 case NEON::BI__builtin_neon_vsha512h2q_u64:
8496 case NEON::BI__builtin_neon_vsha512su0q_u64:
8497 case NEON::BI__builtin_neon_vsha512su1q_u64: {
8501 case NEON::BI__builtin_neon_vshl_n_v:
8502 case NEON::BI__builtin_neon_vshlq_n_v:
8504 return Builder.CreateShl(
Builder.CreateBitCast(Ops[0],Ty), Ops[1],
8506 case NEON::BI__builtin_neon_vshll_n_v: {
8507 llvm::FixedVectorType *SrcTy =
8508 llvm::FixedVectorType::getTruncatedElementVectorType(VTy);
8509 Ops[0] =
Builder.CreateBitCast(Ops[0], SrcTy);
8511 Ops[0] =
Builder.CreateZExt(Ops[0], VTy);
8513 Ops[0] =
Builder.CreateSExt(Ops[0], VTy);
8515 return Builder.CreateShl(Ops[0], Ops[1],
"vshll_n");
8517 case NEON::BI__builtin_neon_vshrn_n_v: {
8518 llvm::FixedVectorType *SrcTy =
8519 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
8520 Ops[0] =
Builder.CreateBitCast(Ops[0], SrcTy);
8523 Ops[0] =
Builder.CreateLShr(Ops[0], Ops[1]);
8525 Ops[0] =
Builder.CreateAShr(Ops[0], Ops[1]);
8526 return Builder.CreateTrunc(Ops[0], Ty,
"vshrn_n");
8528 case NEON::BI__builtin_neon_vshr_n_v:
8529 case NEON::BI__builtin_neon_vshrq_n_v:
8531 case NEON::BI__builtin_neon_vst1_v:
8532 case NEON::BI__builtin_neon_vst1q_v:
8533 case NEON::BI__builtin_neon_vst2_v:
8534 case NEON::BI__builtin_neon_vst2q_v:
8535 case NEON::BI__builtin_neon_vst3_v:
8536 case NEON::BI__builtin_neon_vst3q_v:
8537 case NEON::BI__builtin_neon_vst4_v:
8538 case NEON::BI__builtin_neon_vst4q_v:
8539 case NEON::BI__builtin_neon_vst2_lane_v:
8540 case NEON::BI__builtin_neon_vst2q_lane_v:
8541 case NEON::BI__builtin_neon_vst3_lane_v:
8542 case NEON::BI__builtin_neon_vst3q_lane_v:
8543 case NEON::BI__builtin_neon_vst4_lane_v:
8544 case NEON::BI__builtin_neon_vst4q_lane_v: {
8546 Ops.push_back(getAlignmentValue32(PtrOp0));
8549 case NEON::BI__builtin_neon_vsm3partw1q_u32:
8550 case NEON::BI__builtin_neon_vsm3partw2q_u32:
8551 case NEON::BI__builtin_neon_vsm3ss1q_u32:
8552 case NEON::BI__builtin_neon_vsm4ekeyq_u32:
8553 case NEON::BI__builtin_neon_vsm4eq_u32: {
8557 case NEON::BI__builtin_neon_vsm3tt1aq_u32:
8558 case NEON::BI__builtin_neon_vsm3tt1bq_u32:
8559 case NEON::BI__builtin_neon_vsm3tt2aq_u32:
8560 case NEON::BI__builtin_neon_vsm3tt2bq_u32: {
8565 case NEON::BI__builtin_neon_vst1_x2_v:
8566 case NEON::BI__builtin_neon_vst1q_x2_v:
8567 case NEON::BI__builtin_neon_vst1_x3_v:
8568 case NEON::BI__builtin_neon_vst1q_x3_v:
8569 case NEON::BI__builtin_neon_vst1_x4_v:
8570 case NEON::BI__builtin_neon_vst1q_x4_v: {
8573 if (Arch == llvm::Triple::aarch64 || Arch == llvm::Triple::aarch64_be ||
8574 Arch == llvm::Triple::aarch64_32) {
8576 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
8582 case NEON::BI__builtin_neon_vsubhn_v: {
8583 llvm::FixedVectorType *SrcTy =
8584 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
8587 Ops[0] =
Builder.CreateBitCast(Ops[0], SrcTy);
8588 Ops[1] =
Builder.CreateBitCast(Ops[1], SrcTy);
8589 Ops[0] =
Builder.CreateSub(Ops[0], Ops[1],
"vsubhn");
8592 Constant *ShiftAmt =
8593 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
8594 Ops[0] =
Builder.CreateLShr(Ops[0], ShiftAmt,
"vsubhn");
8597 return Builder.CreateTrunc(Ops[0], VTy,
"vsubhn");
8599 case NEON::BI__builtin_neon_vtrn_v:
8600 case NEON::BI__builtin_neon_vtrnq_v: {
8601 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
8602 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
8603 Value *SV =
nullptr;
8605 for (
unsigned vi = 0; vi != 2; ++vi) {
8607 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
8608 Indices.push_back(i+vi);
8609 Indices.push_back(i+e+vi);
8611 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8612 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vtrn");
8617 case NEON::BI__builtin_neon_vtst_v:
8618 case NEON::BI__builtin_neon_vtstq_v: {
8619 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
8620 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
8621 Ops[0] =
Builder.CreateAnd(Ops[0], Ops[1]);
8622 Ops[0] =
Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
8623 ConstantAggregateZero::get(Ty));
8624 return Builder.CreateSExt(Ops[0], Ty,
"vtst");
8626 case NEON::BI__builtin_neon_vuzp_v:
8627 case NEON::BI__builtin_neon_vuzpq_v: {
8628 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
8629 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
8630 Value *SV =
nullptr;
8632 for (
unsigned vi = 0; vi != 2; ++vi) {
8634 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
8635 Indices.push_back(2*i+vi);
8637 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8638 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vuzp");
8643 case NEON::BI__builtin_neon_vxarq_u64: {
8648 case NEON::BI__builtin_neon_vzip_v:
8649 case NEON::BI__builtin_neon_vzipq_v: {
8650 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
8651 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
8652 Value *SV =
nullptr;
8654 for (
unsigned vi = 0; vi != 2; ++vi) {
8656 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
8657 Indices.push_back((i + vi*e) >> 1);
8658 Indices.push_back(((i + vi*e) >> 1)+e);
8660 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8661 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vzip");
8666 case NEON::BI__builtin_neon_vdot_s32:
8667 case NEON::BI__builtin_neon_vdot_u32:
8668 case NEON::BI__builtin_neon_vdotq_s32:
8669 case NEON::BI__builtin_neon_vdotq_u32: {
8671 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8672 llvm::Type *Tys[2] = { Ty, InputTy };
8675 case NEON::BI__builtin_neon_vfmlal_low_f16:
8676 case NEON::BI__builtin_neon_vfmlalq_low_f16: {
8678 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8679 llvm::Type *Tys[2] = { Ty, InputTy };
8682 case NEON::BI__builtin_neon_vfmlsl_low_f16:
8683 case NEON::BI__builtin_neon_vfmlslq_low_f16: {
8685 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8686 llvm::Type *Tys[2] = { Ty, InputTy };
8689 case NEON::BI__builtin_neon_vfmlal_high_f16:
8690 case NEON::BI__builtin_neon_vfmlalq_high_f16: {
8692 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8693 llvm::Type *Tys[2] = { Ty, InputTy };
8696 case NEON::BI__builtin_neon_vfmlsl_high_f16:
8697 case NEON::BI__builtin_neon_vfmlslq_high_f16: {
8699 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8700 llvm::Type *Tys[2] = { Ty, InputTy };
8703 case NEON::BI__builtin_neon_vmmlaq_s32:
8704 case NEON::BI__builtin_neon_vmmlaq_u32: {
8706 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8707 llvm::Type *Tys[2] = { Ty, InputTy };
8710 case NEON::BI__builtin_neon_vusmmlaq_s32: {
8712 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8713 llvm::Type *Tys[2] = { Ty, InputTy };
8716 case NEON::BI__builtin_neon_vusdot_s32:
8717 case NEON::BI__builtin_neon_vusdotq_s32: {
8719 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8720 llvm::Type *Tys[2] = { Ty, InputTy };
8723 case NEON::BI__builtin_neon_vbfdot_f32:
8724 case NEON::BI__builtin_neon_vbfdotq_f32: {
8725 llvm::Type *InputTy =
8726 llvm::FixedVectorType::get(
BFloatTy, Ty->getPrimitiveSizeInBits() / 16);
8727 llvm::Type *Tys[2] = { Ty, InputTy };
8730 case NEON::BI__builtin_neon___a32_vcvt_bf16_f32: {
8731 llvm::Type *Tys[1] = { Ty };
8738 assert(Int &&
"Expected valid intrinsic number");
8751 Value *Op, llvm::Type *Ty,
const CmpInst::Predicate Fp,
8752 const CmpInst::Predicate Ip,
const Twine &Name) {
8753 llvm::Type *OTy = Op->
getType();
8759 if (BitCastInst *BI = dyn_cast<BitCastInst>(Op))
8760 OTy = BI->getOperand(0)->getType();
8762 Op =
Builder.CreateBitCast(Op, OTy);
8763 if (OTy->getScalarType()->isFloatingPointTy()) {
8764 if (Fp == CmpInst::FCMP_OEQ)
8765 Op =
Builder.CreateFCmp(Fp, Op, Constant::getNullValue(OTy));
8767 Op =
Builder.CreateFCmpS(Fp, Op, Constant::getNullValue(OTy));
8769 Op =
Builder.CreateICmp(Ip, Op, Constant::getNullValue(OTy));
8771 return Builder.CreateSExt(Op, Ty, Name);
8776 llvm::Type *ResTy,
unsigned IntID,
8780 TblOps.push_back(ExtOp);
8784 auto *TblTy = cast<llvm::FixedVectorType>(Ops[0]->getType());
8785 for (
unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) {
8786 Indices.push_back(2*i);
8787 Indices.push_back(2*i+1);
8790 int PairPos = 0, End = Ops.size() - 1;
8791 while (PairPos < End) {
8792 TblOps.push_back(CGF.
Builder.CreateShuffleVector(Ops[PairPos],
8793 Ops[PairPos+1], Indices,
8800 if (PairPos == End) {
8801 Value *ZeroTbl = ConstantAggregateZero::get(TblTy);
8802 TblOps.push_back(CGF.
Builder.CreateShuffleVector(Ops[PairPos],
8803 ZeroTbl, Indices, Name));
8807 TblOps.push_back(IndexOp);
8813Value *CodeGenFunction::GetValueForARMHint(
unsigned BuiltinID) {
8815 switch (BuiltinID) {
8818 case clang::ARM::BI__builtin_arm_nop:
8821 case clang::ARM::BI__builtin_arm_yield:
8822 case clang::ARM::BI__yield:
8825 case clang::ARM::BI__builtin_arm_wfe:
8826 case clang::ARM::BI__wfe:
8829 case clang::ARM::BI__builtin_arm_wfi:
8830 case clang::ARM::BI__wfi:
8833 case clang::ARM::BI__builtin_arm_sev:
8834 case clang::ARM::BI__sev:
8837 case clang::ARM::BI__builtin_arm_sevl:
8838 case clang::ARM::BI__sevl:
8857 llvm::Type *ValueType,
bool isExecHi) {
8862 llvm::Value *
Call = Builder.CreateCall(F, {Builder.getInt1(
true)});
8865 Value *Rt2 = Builder.CreateLShr(
Call, 32);
8866 Rt2 = Builder.CreateTrunc(Rt2, CGF.
Int32Ty);
8879 llvm::Type *ValueType,
8881 StringRef SysReg =
"") {
8885 "Unsupported size for register.");
8891 if (SysReg.empty()) {
8893 SysReg = cast<clang::StringLiteral>(SysRegStrExpr)->getString();
8896 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) };
8897 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
8898 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
8902 bool MixedTypes =
RegisterType->isIntegerTy(64) && ValueType->isIntegerTy(32);
8903 assert(!(
RegisterType->isIntegerTy(32) && ValueType->isIntegerTy(64))
8904 &&
"Can't fit 64-bit value in 32-bit register");
8906 if (AccessKind !=
Write) {
8909 AccessKind ==
VolatileRead ? llvm::Intrinsic::read_volatile_register
8910 : llvm::Intrinsic::read_register,
8912 llvm::Value *
Call = Builder.CreateCall(F, Metadata);
8916 return Builder.CreateTrunc(
Call, ValueType);
8918 if (ValueType->isPointerTy())
8920 return Builder.CreateIntToPtr(
Call, ValueType);
8925 llvm::Function *F = CGM.
getIntrinsic(llvm::Intrinsic::write_register, Types);
8930 return Builder.CreateCall(F, { Metadata, ArgValue });
8933 if (ValueType->isPointerTy()) {
8935 ArgValue = Builder.CreatePtrToInt(ArgValue,
RegisterType);
8936 return Builder.CreateCall(F, { Metadata, ArgValue });
8939 return Builder.CreateCall(F, { Metadata, ArgValue });
8945 switch (BuiltinID) {
8947 case NEON::BI__builtin_neon_vget_lane_i8:
8948 case NEON::BI__builtin_neon_vget_lane_i16:
8949 case NEON::BI__builtin_neon_vget_lane_bf16:
8950 case NEON::BI__builtin_neon_vget_lane_i32:
8951 case NEON::BI__builtin_neon_vget_lane_i64:
8952 case NEON::BI__builtin_neon_vget_lane_f32:
8953 case NEON::BI__builtin_neon_vgetq_lane_i8:
8954 case NEON::BI__builtin_neon_vgetq_lane_i16:
8955 case NEON::BI__builtin_neon_vgetq_lane_bf16:
8956 case NEON::BI__builtin_neon_vgetq_lane_i32:
8957 case NEON::BI__builtin_neon_vgetq_lane_i64:
8958 case NEON::BI__builtin_neon_vgetq_lane_f32:
8959 case NEON::BI__builtin_neon_vduph_lane_bf16:
8960 case NEON::BI__builtin_neon_vduph_laneq_bf16:
8961 case NEON::BI__builtin_neon_vset_lane_i8:
8962 case NEON::BI__builtin_neon_vset_lane_i16:
8963 case NEON::BI__builtin_neon_vset_lane_bf16:
8964 case NEON::BI__builtin_neon_vset_lane_i32:
8965 case NEON::BI__builtin_neon_vset_lane_i64:
8966 case NEON::BI__builtin_neon_vset_lane_f32:
8967 case NEON::BI__builtin_neon_vsetq_lane_i8:
8968 case NEON::BI__builtin_neon_vsetq_lane_i16:
8969 case NEON::BI__builtin_neon_vsetq_lane_bf16:
8970 case NEON::BI__builtin_neon_vsetq_lane_i32:
8971 case NEON::BI__builtin_neon_vsetq_lane_i64:
8972 case NEON::BI__builtin_neon_vsetq_lane_f32:
8973 case NEON::BI__builtin_neon_vsha1h_u32:
8974 case NEON::BI__builtin_neon_vsha1cq_u32:
8975 case NEON::BI__builtin_neon_vsha1pq_u32:
8976 case NEON::BI__builtin_neon_vsha1mq_u32:
8977 case NEON::BI__builtin_neon_vcvth_bf16_f32:
8978 case clang::ARM::BI_MoveToCoprocessor:
8979 case clang::ARM::BI_MoveToCoprocessor2:
8988 llvm::Triple::ArchType Arch) {
8989 if (
auto Hint = GetValueForARMHint(BuiltinID))
8992 if (BuiltinID == clang::ARM::BI__emit) {
8994 llvm::FunctionType *FTy =
8995 llvm::FunctionType::get(
VoidTy,
false);
8999 llvm_unreachable(
"Sema will ensure that the parameter is constant");
9002 uint64_t ZExtValue =
Value.zextOrTrunc(IsThumb ? 16 : 32).getZExtValue();
9004 llvm::InlineAsm *Emit =
9005 IsThumb ? InlineAsm::get(FTy,
".inst.n 0x" + utohexstr(ZExtValue),
"",
9007 : InlineAsm::get(FTy,
".inst 0x" + utohexstr(ZExtValue),
"",
9010 return Builder.CreateCall(Emit);
9013 if (BuiltinID == clang::ARM::BI__builtin_arm_dbg) {
9018 if (BuiltinID == clang::ARM::BI__builtin_arm_prefetch) {
9030 if (BuiltinID == clang::ARM::BI__builtin_arm_rbit) {
9033 CGM.
getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
9036 if (BuiltinID == clang::ARM::BI__builtin_arm_clz ||
9037 BuiltinID == clang::ARM::BI__builtin_arm_clz64) {
9041 if (BuiltinID == clang::ARM::BI__builtin_arm_clz64)
9047 if (BuiltinID == clang::ARM::BI__builtin_arm_cls) {
9051 if (BuiltinID == clang::ARM::BI__builtin_arm_cls64) {
9057 if (BuiltinID == clang::ARM::BI__clear_cache) {
9058 assert(
E->getNumArgs() == 2 &&
"__clear_cache takes 2 arguments");
9061 for (
unsigned i = 0; i < 2; i++)
9064 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
9065 StringRef Name = FD->
getName();
9069 if (BuiltinID == clang::ARM::BI__builtin_arm_mcrr ||
9070 BuiltinID == clang::ARM::BI__builtin_arm_mcrr2) {
9073 switch (BuiltinID) {
9074 default: llvm_unreachable(
"unexpected builtin");
9075 case clang::ARM::BI__builtin_arm_mcrr:
9078 case clang::ARM::BI__builtin_arm_mcrr2:
9100 return Builder.CreateCall(F, {Coproc, Opc1, Rt, Rt2, CRm});
9103 if (BuiltinID == clang::ARM::BI__builtin_arm_mrrc ||
9104 BuiltinID == clang::ARM::BI__builtin_arm_mrrc2) {
9107 switch (BuiltinID) {
9108 default: llvm_unreachable(
"unexpected builtin");
9109 case clang::ARM::BI__builtin_arm_mrrc:
9112 case clang::ARM::BI__builtin_arm_mrrc2:
9120 Value *RtAndRt2 =
Builder.CreateCall(F, {Coproc, Opc1, CRm});
9130 Value *ShiftCast = llvm::ConstantInt::get(
Int64Ty, 32);
9131 RtAndRt2 =
Builder.CreateShl(Rt, ShiftCast,
"shl",
true);
9132 RtAndRt2 =
Builder.CreateOr(RtAndRt2, Rt1);
9137 if (BuiltinID == clang::ARM::BI__builtin_arm_ldrexd ||
9138 ((BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
9139 BuiltinID == clang::ARM::BI__builtin_arm_ldaex) &&
9141 BuiltinID == clang::ARM::BI__ldrexd) {
9144 switch (BuiltinID) {
9145 default: llvm_unreachable(
"unexpected builtin");
9146 case clang::ARM::BI__builtin_arm_ldaex:
9149 case clang::ARM::BI__builtin_arm_ldrexd:
9150 case clang::ARM::BI__builtin_arm_ldrex:
9151 case clang::ARM::BI__ldrexd:
9165 Val =
Builder.CreateShl(Val0, ShiftCst,
"shl",
true );
9166 Val =
Builder.CreateOr(Val, Val1);
9170 if (BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
9171 BuiltinID == clang::ARM::BI__builtin_arm_ldaex) {
9180 BuiltinID == clang::ARM::BI__builtin_arm_ldaex ? Intrinsic::arm_ldaex
9181 : Intrinsic::arm_ldrex,
9183 CallInst *Val =
Builder.CreateCall(F, LoadAddr,
"ldrex");
9187 if (RealResTy->isPointerTy())
9188 return Builder.CreateIntToPtr(Val, RealResTy);
9190 llvm::Type *IntResTy = llvm::IntegerType::get(
9192 return Builder.CreateBitCast(
Builder.CreateTruncOrBitCast(Val, IntResTy),
9197 if (BuiltinID == clang::ARM::BI__builtin_arm_strexd ||
9198 ((BuiltinID == clang::ARM::BI__builtin_arm_stlex ||
9199 BuiltinID == clang::ARM::BI__builtin_arm_strex) &&
9202 BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlexd
9203 : Intrinsic::arm_strexd);
9216 return Builder.CreateCall(F, {Arg0, Arg1, StPtr},
"strexd");
9219 if (BuiltinID == clang::ARM::BI__builtin_arm_strex ||
9220 BuiltinID == clang::ARM::BI__builtin_arm_stlex) {
9225 llvm::Type *StoreTy =
9228 if (StoreVal->
getType()->isPointerTy())
9231 llvm::Type *
IntTy = llvm::IntegerType::get(
9239 BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlex
9240 : Intrinsic::arm_strex,
9243 CallInst *CI =
Builder.CreateCall(F, {StoreVal, StoreAddr},
"strex");
9245 1, Attribute::get(
getLLVMContext(), Attribute::ElementType, StoreTy));
9249 if (BuiltinID == clang::ARM::BI__builtin_arm_clrex) {
9255 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
9256 switch (BuiltinID) {
9257 case clang::ARM::BI__builtin_arm_crc32b:
9258 CRCIntrinsicID = Intrinsic::arm_crc32b;
break;
9259 case clang::ARM::BI__builtin_arm_crc32cb:
9260 CRCIntrinsicID = Intrinsic::arm_crc32cb;
break;
9261 case clang::ARM::BI__builtin_arm_crc32h:
9262 CRCIntrinsicID = Intrinsic::arm_crc32h;
break;
9263 case clang::ARM::BI__builtin_arm_crc32ch:
9264 CRCIntrinsicID = Intrinsic::arm_crc32ch;
break;
9265 case clang::ARM::BI__builtin_arm_crc32w:
9266 case clang::ARM::BI__builtin_arm_crc32d:
9267 CRCIntrinsicID = Intrinsic::arm_crc32w;
break;
9268 case clang::ARM::BI__builtin_arm_crc32cw:
9269 case clang::ARM::BI__builtin_arm_crc32cd:
9270 CRCIntrinsicID = Intrinsic::arm_crc32cw;
break;
9273 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
9279 if (BuiltinID == clang::ARM::BI__builtin_arm_crc32d ||
9280 BuiltinID == clang::ARM::BI__builtin_arm_crc32cd) {
9288 return Builder.CreateCall(F, {Res, Arg1b});
9293 return Builder.CreateCall(F, {Arg0, Arg1});
9297 if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
9298 BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
9299 BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
9300 BuiltinID == clang::ARM::BI__builtin_arm_wsr ||
9301 BuiltinID == clang::ARM::BI__builtin_arm_wsr64 ||
9302 BuiltinID == clang::ARM::BI__builtin_arm_wsrp) {
9305 if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
9306 BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
9307 BuiltinID == clang::ARM::BI__builtin_arm_rsrp)
9310 bool IsPointerBuiltin = BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
9311 BuiltinID == clang::ARM::BI__builtin_arm_wsrp;
9313 bool Is64Bit = BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
9314 BuiltinID == clang::ARM::BI__builtin_arm_wsr64;
9316 llvm::Type *ValueType;
9318 if (IsPointerBuiltin) {
9321 }
else if (Is64Bit) {
9331 if (BuiltinID == ARM::BI__builtin_sponentry) {
9350 return P.first == BuiltinID;
9353 BuiltinID = It->second;
9357 unsigned ICEArguments = 0;
9362 auto getAlignmentValue32 = [&](
Address addr) ->
Value* {
9363 return Builder.getInt32(addr.getAlignment().getQuantity());
9370 unsigned NumArgs =
E->getNumArgs() - (HasExtraArg ? 1 : 0);
9371 for (
unsigned i = 0, e = NumArgs; i != e; i++) {
9373 switch (BuiltinID) {
9374 case NEON::BI__builtin_neon_vld1_v:
9375 case NEON::BI__builtin_neon_vld1q_v:
9376 case NEON::BI__builtin_neon_vld1q_lane_v:
9377 case NEON::BI__builtin_neon_vld1_lane_v:
9378 case NEON::BI__builtin_neon_vld1_dup_v:
9379 case NEON::BI__builtin_neon_vld1q_dup_v:
9380 case NEON::BI__builtin_neon_vst1_v:
9381 case NEON::BI__builtin_neon_vst1q_v:
9382 case NEON::BI__builtin_neon_vst1q_lane_v:
9383 case NEON::BI__builtin_neon_vst1_lane_v:
9384 case NEON::BI__builtin_neon_vst2_v:
9385 case NEON::BI__builtin_neon_vst2q_v:
9386 case NEON::BI__builtin_neon_vst2_lane_v:
9387 case NEON::BI__builtin_neon_vst2q_lane_v:
9388 case NEON::BI__builtin_neon_vst3_v:
9389 case NEON::BI__builtin_neon_vst3q_v:
9390 case NEON::BI__builtin_neon_vst3_lane_v:
9391 case NEON::BI__builtin_neon_vst3q_lane_v:
9392 case NEON::BI__builtin_neon_vst4_v:
9393 case NEON::BI__builtin_neon_vst4q_v:
9394 case NEON::BI__builtin_neon_vst4_lane_v:
9395 case NEON::BI__builtin_neon_vst4q_lane_v:
9404 switch (BuiltinID) {
9405 case NEON::BI__builtin_neon_vld2_v:
9406 case NEON::BI__builtin_neon_vld2q_v:
9407 case NEON::BI__builtin_neon_vld3_v:
9408 case NEON::BI__builtin_neon_vld3q_v:
9409 case NEON::BI__builtin_neon_vld4_v:
9410 case NEON::BI__builtin_neon_vld4q_v:
9411 case NEON::BI__builtin_neon_vld2_lane_v:
9412 case NEON::BI__builtin_neon_vld2q_lane_v:
9413 case NEON::BI__builtin_neon_vld3_lane_v:
9414 case NEON::BI__builtin_neon_vld3q_lane_v:
9415 case NEON::BI__builtin_neon_vld4_lane_v:
9416 case NEON::BI__builtin_neon_vld4q_lane_v:
9417 case NEON::BI__builtin_neon_vld2_dup_v:
9418 case NEON::BI__builtin_neon_vld2q_dup_v:
9419 case NEON::BI__builtin_neon_vld3_dup_v:
9420 case NEON::BI__builtin_neon_vld3q_dup_v:
9421 case NEON::BI__builtin_neon_vld4_dup_v:
9422 case NEON::BI__builtin_neon_vld4q_dup_v:
9434 switch (BuiltinID) {
9437 case NEON::BI__builtin_neon_vget_lane_i8:
9438 case NEON::BI__builtin_neon_vget_lane_i16:
9439 case NEON::BI__builtin_neon_vget_lane_i32:
9440 case NEON::BI__builtin_neon_vget_lane_i64:
9441 case NEON::BI__builtin_neon_vget_lane_bf16:
9442 case NEON::BI__builtin_neon_vget_lane_f32:
9443 case NEON::BI__builtin_neon_vgetq_lane_i8:
9444 case NEON::BI__builtin_neon_vgetq_lane_i16:
9445 case NEON::BI__builtin_neon_vgetq_lane_i32:
9446 case NEON::BI__builtin_neon_vgetq_lane_i64:
9447 case NEON::BI__builtin_neon_vgetq_lane_bf16:
9448 case NEON::BI__builtin_neon_vgetq_lane_f32:
9449 case NEON::BI__builtin_neon_vduph_lane_bf16:
9450 case NEON::BI__builtin_neon_vduph_laneq_bf16:
9451 return Builder.CreateExtractElement(Ops[0], Ops[1],
"vget_lane");
9453 case NEON::BI__builtin_neon_vrndns_f32: {
9455 llvm::Type *Tys[] = {Arg->
getType()};
9457 return Builder.CreateCall(F, {Arg},
"vrndn"); }
9459 case NEON::BI__builtin_neon_vset_lane_i8:
9460 case NEON::BI__builtin_neon_vset_lane_i16:
9461 case NEON::BI__builtin_neon_vset_lane_i32:
9462 case NEON::BI__builtin_neon_vset_lane_i64:
9463 case NEON::BI__builtin_neon_vset_lane_bf16:
9464 case NEON::BI__builtin_neon_vset_lane_f32:
9465 case NEON::BI__builtin_neon_vsetq_lane_i8:
9466 case NEON::BI__builtin_neon_vsetq_lane_i16:
9467 case NEON::BI__builtin_neon_vsetq_lane_i32:
9468 case NEON::BI__builtin_neon_vsetq_lane_i64:
9469 case NEON::BI__builtin_neon_vsetq_lane_bf16:
9470 case NEON::BI__builtin_neon_vsetq_lane_f32:
9471 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
9473 case NEON::BI__builtin_neon_vsha1h_u32:
9476 case NEON::BI__builtin_neon_vsha1cq_u32:
9479 case NEON::BI__builtin_neon_vsha1pq_u32:
9482 case NEON::BI__builtin_neon_vsha1mq_u32:
9486 case NEON::BI__builtin_neon_vcvth_bf16_f32: {
9493 case clang::ARM::BI_MoveToCoprocessor:
9494 case clang::ARM::BI_MoveToCoprocessor2: {
9496 ? Intrinsic::arm_mcr
9497 : Intrinsic::arm_mcr2);
9498 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0],
9499 Ops[3], Ops[4], Ops[5]});
9504 assert(HasExtraArg);
9505 const Expr *Arg =
E->getArg(
E->getNumArgs()-1);
9506 std::optional<llvm::APSInt>
Result =
9511 if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f ||
9512 BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_d) {
9515 if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f)
9521 bool usgn =
Result->getZExtValue() == 1;
9522 unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr;
9526 return Builder.CreateCall(F, Ops,
"vcvtr");
9531 bool usgn =
Type.isUnsigned();
9532 bool rightShift =
false;
9534 llvm::FixedVectorType *VTy =
9537 llvm::Type *Ty = VTy;
9548 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
9549 Builtin->NameHint, Builtin->TypeModifier,
E, Ops, PtrOp0, PtrOp1, Arch);
9552 switch (BuiltinID) {
9553 default:
return nullptr;
9554 case NEON::BI__builtin_neon_vld1q_lane_v:
9557 if (VTy->getElementType()->isIntegerTy(64)) {
9559 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
9560 int Lane = cast<ConstantInt>(Ops[2])->getZExtValue();
9561 Value *SV = llvm::ConstantVector::get(ConstantInt::get(
Int32Ty, 1-Lane));
9562 Ops[1] =
Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
9564 Ty = llvm::FixedVectorType::get(VTy->getElementType(), 1);
9567 Value *Align = getAlignmentValue32(PtrOp0);
9570 int Indices[] = {1 - Lane, Lane};
9571 return Builder.CreateShuffleVector(Ops[1], Ld, Indices,
"vld1q_lane");
9574 case NEON::BI__builtin_neon_vld1_lane_v: {
9575 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
9578 return Builder.CreateInsertElement(Ops[1], Ld, Ops[2],
"vld1_lane");
9580 case NEON::BI__builtin_neon_vqrshrn_n_v:
9582 usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns;
9585 case NEON::BI__builtin_neon_vqrshrun_n_v:
9587 Ops,
"vqrshrun_n", 1,
true);
9588 case NEON::BI__builtin_neon_vqshrn_n_v:
9589 Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns;
9592 case NEON::BI__builtin_neon_vqshrun_n_v:
9594 Ops,
"vqshrun_n", 1,
true);
9595 case NEON::BI__builtin_neon_vrecpe_v:
9596 case NEON::BI__builtin_neon_vrecpeq_v:
9599 case NEON::BI__builtin_neon_vrshrn_n_v:
9601 Ops,
"vrshrn_n", 1,
true);
9602 case NEON::BI__builtin_neon_vrsra_n_v:
9603 case NEON::BI__builtin_neon_vrsraq_n_v:
9604 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
9605 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
9607 Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
9609 return Builder.CreateAdd(Ops[0], Ops[1],
"vrsra_n");
9610 case NEON::BI__builtin_neon_vsri_n_v:
9611 case NEON::BI__builtin_neon_vsriq_n_v:
9614 case NEON::BI__builtin_neon_vsli_n_v:
9615 case NEON::BI__builtin_neon_vsliq_n_v:
9619 case NEON::BI__builtin_neon_vsra_n_v:
9620 case NEON::BI__builtin_neon_vsraq_n_v:
9621 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
9623 return Builder.CreateAdd(Ops[0], Ops[1]);
9624 case NEON::BI__builtin_neon_vst1q_lane_v:
9627 if (VTy->getElementType()->isIntegerTy(64)) {
9628 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
9629 Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2]));
9630 Ops[1] =
Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
9631 Ops[2] = getAlignmentValue32(PtrOp0);
9632 llvm::Type *Tys[] = {
Int8PtrTy, Ops[1]->getType()};
9637 case NEON::BI__builtin_neon_vst1_lane_v: {
9638 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
9639 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2]);
9643 case NEON::BI__builtin_neon_vtbl1_v:
9646 case NEON::BI__builtin_neon_vtbl2_v:
9649 case NEON::BI__builtin_neon_vtbl3_v:
9652 case NEON::BI__builtin_neon_vtbl4_v:
9655 case NEON::BI__builtin_neon_vtbx1_v:
9658 case NEON::BI__builtin_neon_vtbx2_v:
9661 case NEON::BI__builtin_neon_vtbx3_v:
9664 case NEON::BI__builtin_neon_vtbx4_v:
9670template<
typename Integer>
9679 return Unsigned ? Builder.CreateZExt(
V,
T) : Builder.CreateSExt(
V,
T);
9689 unsigned LaneBits = cast<llvm::VectorType>(
V->getType())
9691 ->getPrimitiveSizeInBits();
9692 if (Shift == LaneBits) {
9697 return llvm::Constant::getNullValue(
V->getType());
9701 return Unsigned ? Builder.CreateLShr(
V, Shift) : Builder.CreateAShr(
V, Shift);
9708 unsigned Elements = 128 /
V->getType()->getPrimitiveSizeInBits();
9709 return Builder.CreateVectorSplat(Elements,
V);
9715 llvm::Type *DestType) {
9728 V->getType()->getScalarSizeInBits() != DestType->getScalarSizeInBits()) {
9729 return Builder.CreateCall(
9731 {DestType, V->getType()}),
9734 return Builder.CreateBitCast(
V, DestType);
9742 unsigned InputElements =
9743 cast<llvm::FixedVectorType>(
V->getType())->getNumElements();
9744 for (
unsigned i = 0; i < InputElements; i += 2)
9745 Indices.push_back(i + Odd);
9746 return Builder.CreateShuffleVector(
V, Indices);
9752 assert(V0->getType() == V1->getType() &&
"Can't zip different vector types");
9754 unsigned InputElements =
9755 cast<llvm::FixedVectorType>(V0->getType())->getNumElements();
9756 for (
unsigned i = 0; i < InputElements; i++) {
9757 Indices.push_back(i);
9758 Indices.push_back(i + InputElements);
9760 return Builder.CreateShuffleVector(V0, V1, Indices);
9763template<
unsigned HighBit,
unsigned OtherBits>
9767 llvm::Type *
T = cast<llvm::VectorType>(VT)->getElementType();
9768 unsigned LaneBits =
T->getPrimitiveSizeInBits();
9769 uint32_t
Value = HighBit << (LaneBits - 1);
9771 Value |= (1UL << (LaneBits - 1)) - 1;
9772 llvm::Value *Lane = llvm::ConstantInt::get(
T,
Value);
9778 unsigned ReverseWidth) {
9782 unsigned LaneSize =
V->getType()->getScalarSizeInBits();
9783 unsigned Elements = 128 / LaneSize;
9784 unsigned Mask = ReverseWidth / LaneSize - 1;
9785 for (
unsigned i = 0; i < Elements; i++)
9786 Indices.push_back(i ^ Mask);
9787 return Builder.CreateShuffleVector(
V, Indices);
9793 llvm::Triple::ArchType Arch) {
9794 enum class CustomCodeGen { VLD24, VST24 } CustomCodeGenType;
9795 Intrinsic::ID IRIntr;
9796 unsigned NumVectors;
9799 switch (BuiltinID) {
9800 #include "clang/Basic/arm_mve_builtin_cg.inc"
9811 switch (CustomCodeGenType) {
9813 case CustomCodeGen::VLD24: {
9819 assert(MvecLType->isStructTy() &&
9820 "Return type for vld[24]q should be a struct");
9821 assert(MvecLType->getStructNumElements() == 1 &&
9822 "Return-type struct for vld[24]q should have one element");
9823 auto MvecLTypeInner = MvecLType->getStructElementType(0);
9824 assert(MvecLTypeInner->isArrayTy() &&
9825 "Return-type struct for vld[24]q should contain an array");
9826 assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
9827 "Array member of return-type struct vld[24]q has wrong length");
9828 auto VecLType = MvecLTypeInner->getArrayElementType();
9830 Tys.push_back(VecLType);
9832 auto Addr =
E->getArg(0);
9838 Value *MvecOut = PoisonValue::get(MvecLType);
9839 for (
unsigned i = 0; i < NumVectors; ++i) {
9840 Value *Vec =
Builder.CreateExtractValue(LoadResult, i);
9841 MvecOut =
Builder.CreateInsertValue(MvecOut, Vec, {0, i});
9850 case CustomCodeGen::VST24: {
9854 auto Addr =
E->getArg(0);
9858 auto MvecCType =
E->getArg(1)->
getType();
9860 assert(MvecLType->isStructTy() &&
"Data type for vst2q should be a struct");
9861 assert(MvecLType->getStructNumElements() == 1 &&
9862 "Data-type struct for vst2q should have one element");
9863 auto MvecLTypeInner = MvecLType->getStructElementType(0);
9864 assert(MvecLTypeInner->isArrayTy() &&
9865 "Data-type struct for vst2q should contain an array");
9866 assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
9867 "Array member of return-type struct vld[24]q has wrong length");
9868 auto VecLType = MvecLTypeInner->getArrayElementType();
9870 Tys.push_back(VecLType);
9875 for (
unsigned i = 0; i < NumVectors; i++)
9876 Ops.push_back(
Builder.CreateExtractValue(Mvec, {0, i}));
9879 Value *ToReturn =
nullptr;
9880 for (
unsigned i = 0; i < NumVectors; i++) {
9881 Ops.push_back(llvm::ConstantInt::get(
Int32Ty, i));
9882 ToReturn =
Builder.CreateCall(F, Ops);
9888 llvm_unreachable(
"unknown custom codegen type.");
9894 llvm::Triple::ArchType Arch) {
9895 switch (BuiltinID) {
9898#include "clang/Basic/arm_cde_builtin_cg.inc"
9905 llvm::Triple::ArchType Arch) {
9906 unsigned int Int = 0;
9907 const char *
s =
nullptr;
9909 switch (BuiltinID) {
9912 case NEON::BI__builtin_neon_vtbl1_v:
9913 case NEON::BI__builtin_neon_vqtbl1_v:
9914 case NEON::BI__builtin_neon_vqtbl1q_v:
9915 case NEON::BI__builtin_neon_vtbl2_v:
9916 case NEON::BI__builtin_neon_vqtbl2_v:
9917 case NEON::BI__builtin_neon_vqtbl2q_v:
9918 case NEON::BI__builtin_neon_vtbl3_v:
9919 case NEON::BI__builtin_neon_vqtbl3_v:
9920 case NEON::BI__builtin_neon_vqtbl3q_v:
9921 case NEON::BI__builtin_neon_vtbl4_v:
9922 case NEON::BI__builtin_neon_vqtbl4_v:
9923 case NEON::BI__builtin_neon_vqtbl4q_v:
9925 case NEON::BI__builtin_neon_vtbx1_v:
9926 case NEON::BI__builtin_neon_vqtbx1_v:
9927 case NEON::BI__builtin_neon_vqtbx1q_v:
9928 case NEON::BI__builtin_neon_vtbx2_v:
9929 case NEON::BI__builtin_neon_vqtbx2_v:
9930 case NEON::BI__builtin_neon_vqtbx2q_v:
9931 case NEON::BI__builtin_neon_vtbx3_v:
9932 case NEON::BI__builtin_neon_vqtbx3_v:
9933 case NEON::BI__builtin_neon_vqtbx3q_v:
9934 case NEON::BI__builtin_neon_vtbx4_v:
9935 case NEON::BI__builtin_neon_vqtbx4_v:
9936 case NEON::BI__builtin_neon_vqtbx4q_v:
9940 assert(
E->getNumArgs() >= 3);
9943 const Expr *Arg =
E->getArg(
E->getNumArgs() - 1);
9944 std::optional<llvm::APSInt>
Result =
9959 switch (BuiltinID) {
9960 case NEON::BI__builtin_neon_vtbl1_v: {
9962 Ty, Intrinsic::aarch64_neon_tbl1,
"vtbl1");
9964 case NEON::BI__builtin_neon_vtbl2_v: {
9966 Ty, Intrinsic::aarch64_neon_tbl1,
"vtbl1");
9968 case NEON::BI__builtin_neon_vtbl3_v: {
9970 Ty, Intrinsic::aarch64_neon_tbl2,
"vtbl2");
9972 case NEON::BI__builtin_neon_vtbl4_v: {
9974 Ty, Intrinsic::aarch64_neon_tbl2,
"vtbl2");
9976 case NEON::BI__builtin_neon_vtbx1_v: {
9979 Intrinsic::aarch64_neon_tbl1,
"vtbl1");
9981 llvm::Constant *EightV = ConstantInt::get(Ty, 8);
9982 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV);
9983 CmpRes = Builder.CreateSExt(CmpRes, Ty);
9985 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
9986 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
9987 return Builder.CreateOr(EltsFromInput, EltsFromTbl,
"vtbx");
9989 case NEON::BI__builtin_neon_vtbx2_v: {
9991 Ty, Intrinsic::aarch64_neon_tbx1,
"vtbx1");
9993 case NEON::BI__builtin_neon_vtbx3_v: {
9996 Intrinsic::aarch64_neon_tbl2,
"vtbl2");
9998 llvm::Constant *TwentyFourV = ConstantInt::get(Ty, 24);
9999 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4],
10001 CmpRes = Builder.CreateSExt(CmpRes, Ty);
10003 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
10004 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
10005 return Builder.CreateOr(EltsFromInput, EltsFromTbl,
"vtbx");
10007 case NEON::BI__builtin_neon_vtbx4_v: {
10009 Ty, Intrinsic::aarch64_neon_tbx2,
"vtbx2");
10011 case NEON::BI__builtin_neon_vqtbl1_v:
10012 case NEON::BI__builtin_neon_vqtbl1q_v:
10013 Int = Intrinsic::aarch64_neon_tbl1;
s =
"vtbl1";
break;
10014 case NEON::BI__builtin_neon_vqtbl2_v:
10015 case NEON::BI__builtin_neon_vqtbl2q_v: {
10016 Int = Intrinsic::aarch64_neon_tbl2;
s =
"vtbl2";
break;
10017 case NEON::BI__builtin_neon_vqtbl3_v:
10018 case NEON::BI__builtin_neon_vqtbl3q_v:
10019 Int = Intrinsic::aarch64_neon_tbl3;
s =
"vtbl3";
break;
10020 case NEON::BI__builtin_neon_vqtbl4_v:
10021 case NEON::BI__builtin_neon_vqtbl4q_v:
10022 Int = Intrinsic::aarch64_neon_tbl4;
s =
"vtbl4";
break;
10023 case NEON::BI__builtin_neon_vqtbx1_v:
10024 case NEON::BI__builtin_neon_vqtbx1q_v:
10025 Int = Intrinsic::aarch64_neon_tbx1;
s =
"vtbx1";
break;
10026 case NEON::BI__builtin_neon_vqtbx2_v:
10027 case NEON::BI__builtin_neon_vqtbx2q_v:
10028 Int = Intrinsic::aarch64_neon_tbx2;
s =
"vtbx2";
break;
10029 case NEON::BI__builtin_neon_vqtbx3_v:
10030 case NEON::BI__builtin_neon_vqtbx3q_v:
10031 Int = Intrinsic::aarch64_neon_tbx3;
s =
"vtbx3";
break;
10032 case NEON::BI__builtin_neon_vqtbx4_v:
10033 case NEON::BI__builtin_neon_vqtbx4q_v:
10034 Int = Intrinsic::aarch64_neon_tbx4;
s =
"vtbx4";
break;
10046 auto *VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
10048 Value *
V = PoisonValue::get(VTy);
10049 llvm::Constant *CI = ConstantInt::get(
SizeTy, 0);
10050 Op =
Builder.CreateInsertElement(
V, Op, CI);
10059 case SVETypeFlags::MemEltTyDefault:
10061 case SVETypeFlags::MemEltTyInt8:
10063 case SVETypeFlags::MemEltTyInt16:
10065 case SVETypeFlags::MemEltTyInt32:
10067 case SVETypeFlags::MemEltTyInt64:
10070 llvm_unreachable(
"Unknown MemEltType");
10076 llvm_unreachable(
"Invalid SVETypeFlag!");
10078 case SVETypeFlags::EltTyInt8:
10080 case SVETypeFlags::EltTyInt16:
10082 case SVETypeFlags::EltTyInt32:
10084 case SVETypeFlags::EltTyInt64:
10086 case SVETypeFlags::EltTyInt128:
10087 return Builder.getInt128Ty();
10089 case SVETypeFlags::EltTyFloat16:
10091 case SVETypeFlags::EltTyFloat32:
10093 case SVETypeFlags::EltTyFloat64:
10094 return Builder.getDoubleTy();
10096 case SVETypeFlags::EltTyBFloat16:
10097 return Builder.getBFloatTy();
10099 case SVETypeFlags::EltTyBool8:
10100 case SVETypeFlags::EltTyBool16:
10101 case SVETypeFlags::EltTyBool32:
10102 case SVETypeFlags::EltTyBool64:
10109llvm::ScalableVectorType *
10112 default: llvm_unreachable(
"Unhandled SVETypeFlag!");
10114 case SVETypeFlags::EltTyInt8:
10115 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 16);
10116 case SVETypeFlags::EltTyInt16:
10117 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 8);
10118 case SVETypeFlags::EltTyInt32:
10119 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 4);
10120 case SVETypeFlags::EltTyInt64:
10121 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 2);
10123 case SVETypeFlags::EltTyBFloat16:
10124 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 8);
10125 case SVETypeFlags::EltTyFloat16:
10126 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 8);
10127 case SVETypeFlags::EltTyFloat32:
10128 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 4);
10129 case SVETypeFlags::EltTyFloat64:
10130 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 2);
10132 case SVETypeFlags::EltTyBool8:
10133 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 16);
10134 case SVETypeFlags::EltTyBool16:
10135 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 8);
10136 case SVETypeFlags::EltTyBool32:
10137 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 4);
10138 case SVETypeFlags::EltTyBool64:
10139 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 2);
10144llvm::ScalableVectorType *
10148 llvm_unreachable(
"Invalid SVETypeFlag!");
10150 case SVETypeFlags::EltTyInt8:
10151 return llvm::ScalableVectorType::get(
Builder.getInt8Ty(), 16);
10152 case SVETypeFlags::EltTyInt16:
10153 return llvm::ScalableVectorType::get(
Builder.getInt16Ty(), 8);
10154 case SVETypeFlags::EltTyInt32:
10155 return llvm::ScalableVectorType::get(
Builder.getInt32Ty(), 4);
10156 case SVETypeFlags::EltTyInt64:
10157 return llvm::ScalableVectorType::get(
Builder.getInt64Ty(), 2);
10159 case SVETypeFlags::EltTyMFloat8:
10160 return llvm::ScalableVectorType::get(
Builder.getInt8Ty(), 16);
10161 case SVETypeFlags::EltTyFloat16:
10162 return llvm::ScalableVectorType::get(
Builder.getHalfTy(), 8);
10163 case SVETypeFlags::EltTyBFloat16:
10164 return llvm::ScalableVectorType::get(
Builder.getBFloatTy(), 8);
10165 case SVETypeFlags::EltTyFloat32:
10166 return llvm::ScalableVectorType::get(
Builder.getFloatTy(), 4);
10167 case SVETypeFlags::EltTyFloat64:
10168 return llvm::ScalableVectorType::get(
Builder.getDoubleTy(), 2);
10170 case SVETypeFlags::EltTyBool8:
10171 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 16);
10172 case SVETypeFlags::EltTyBool16:
10173 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 8);
10174 case SVETypeFlags::EltTyBool32:
10175 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 4);
10176 case SVETypeFlags::EltTyBool64:
10177 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 2);
10192 return llvm::ScalableVectorType::get(EltTy, NumElts);
10198 llvm::ScalableVectorType *VTy) {
10200 if (isa<TargetExtType>(Pred->
getType()) &&
10201 cast<TargetExtType>(Pred->
getType())->getName() ==
"aarch64.svcount")
10204 auto *RTy = llvm::VectorType::get(IntegerType::get(
getLLVMContext(), 1), VTy);
10209 llvm::Type *IntrinsicTy;
10210 switch (VTy->getMinNumElements()) {
10212 llvm_unreachable(
"unsupported element count!");
10217 IntID = Intrinsic::aarch64_sve_convert_from_svbool;
10221 IntID = Intrinsic::aarch64_sve_convert_to_svbool;
10222 IntrinsicTy = Pred->
getType();
10228 assert(
C->getType() == RTy &&
"Unexpected return type!");
10233 llvm::StructType *Ty) {
10234 if (PredTuple->
getType() == Ty)
10237 Value *
Ret = llvm::PoisonValue::get(Ty);
10238 for (
unsigned I = 0; I < Ty->getNumElements(); ++I) {
10239 Value *Pred =
Builder.CreateExtractValue(PredTuple, I);
10241 Pred, cast<llvm::ScalableVectorType>(Ty->getTypeAtIndex(I)));
10242 Ret =
Builder.CreateInsertValue(Ret, Pred, I);
10252 auto *OverloadedTy =
10256 if (Ops[1]->getType()->isVectorTy())
10276 Ops[0], cast<llvm::ScalableVectorType>(F->getArg(0)->getType()));
10281 if (Ops.size() == 2) {
10282 assert(Ops[1]->getType()->isVectorTy() &&
"Scalar base requires an offset");
10283 Ops.push_back(ConstantInt::get(
Int64Ty, 0));
10288 if (!TypeFlags.
isByteIndexed() && Ops[1]->getType()->isVectorTy()) {
10289 unsigned BytesPerElt =
10290 OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
10291 Ops[2] =
Builder.CreateShl(Ops[2], Log2_32(BytesPerElt));
10306 auto *OverloadedTy =
10311 Ops.insert(Ops.begin(), Ops.pop_back_val());
10314 if (Ops[2]->getType()->isVectorTy())
10329 if (Ops.size() == 3) {
10330 assert(Ops[1]->getType()->isVectorTy() &&
"Scalar base requires an offset");
10331 Ops.push_back(ConstantInt::get(
Int64Ty, 0));
10336 Ops[0] =
Builder.CreateTrunc(Ops[0], OverloadedTy);
10346 Ops[1], cast<llvm::ScalableVectorType>(F->getArg(1)->getType()));
10350 if (!TypeFlags.
isByteIndexed() && Ops[2]->getType()->isVectorTy()) {
10351 unsigned BytesPerElt =
10352 OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
10353 Ops[3] =
Builder.CreateShl(Ops[3], Log2_32(BytesPerElt));
10356 return Builder.CreateCall(F, Ops);
10364 auto *OverloadedTy = dyn_cast<llvm::ScalableVectorType>(Ops[1]->getType());
10366 OverloadedTy = cast<llvm::ScalableVectorType>(Ops[2]->getType());
10372 if (Ops[1]->getType()->isVectorTy()) {
10373 if (Ops.size() == 3) {
10375 Ops.push_back(ConstantInt::get(
Int64Ty, 0));
10378 std::swap(Ops[2], Ops[3]);
10382 unsigned BytesPerElt = MemEltTy->getPrimitiveSizeInBits() / 8;
10383 if (BytesPerElt > 1)
10384 Ops[2] =
Builder.CreateShl(Ops[2], Log2_32(BytesPerElt));
10389 return Builder.CreateCall(F, Ops);
10395 llvm::ScalableVectorType *VTy =
getSVEType(TypeFlags);
10397 Value *BasePtr = Ops[1];
10400 if (Ops.size() > 2)
10404 return Builder.CreateCall(F, {Predicate, BasePtr});
10410 llvm::ScalableVectorType *VTy =
getSVEType(TypeFlags);
10414 case Intrinsic::aarch64_sve_st2:
10415 case Intrinsic::aarch64_sve_st1_pn_x2:
10416 case Intrinsic::aarch64_sve_stnt1_pn_x2:
10417 case Intrinsic::aarch64_sve_st2q:
10420 case Intrinsic::aarch64_sve_st3:
10421 case Intrinsic::aarch64_sve_st3q:
10424 case Intrinsic::aarch64_sve_st4:
10425 case Intrinsic::aarch64_sve_st1_pn_x4:
10426 case Intrinsic::aarch64_sve_stnt1_pn_x4:
10427 case Intrinsic::aarch64_sve_st4q:
10431 llvm_unreachable(
"unknown intrinsic!");
10435 Value *BasePtr = Ops[1];
10438 if (Ops.size() > (2 + N))
10444 for (
unsigned I = Ops.size() - N; I < Ops.size(); ++I)
10445 Operands.push_back(Ops[I]);
10446 Operands.append({Predicate, BasePtr});
10449 return Builder.CreateCall(F, Operands);
10457 unsigned BuiltinID) {
10469 llvm::ScalableVectorType *Ty =
getSVEType(TypeFlags);
10475 llvm::Type *OverloadedTy =
getSVEType(TypeFlags);
10482 unsigned BuiltinID) {
10485 auto *MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
10488 Value *BasePtr = Ops[1];
10491 if (Ops.size() > 3)
10494 Value *PrfOp = Ops.back();
10497 return Builder.CreateCall(F, {Predicate, BasePtr, PrfOp});
10501 llvm::Type *ReturnTy,
10503 unsigned IntrinsicID,
10504 bool IsZExtReturn) {
10511 auto VectorTy = cast<llvm::ScalableVectorType>(ReturnTy);
10512 llvm::ScalableVectorType *MemoryTy =
nullptr;
10513 llvm::ScalableVectorType *PredTy =
nullptr;
10514 bool IsQuadLoad =
false;
10515 switch (IntrinsicID) {
10516 case Intrinsic::aarch64_sve_ld1uwq:
10517 case Intrinsic::aarch64_sve_ld1udq:
10518 MemoryTy = llvm::ScalableVectorType::get(MemEltTy, 1);
10519 PredTy = llvm::ScalableVectorType::get(
10524 MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
10530 Value *BasePtr = Ops[1];
10533 if (Ops.size() > 2)
10538 cast<llvm::Instruction>(
Builder.CreateCall(F, {Predicate, BasePtr}));
10545 return IsZExtReturn ?
Builder.CreateZExt(Load, VectorTy)
10546 :
Builder.CreateSExt(Load, VectorTy);
10551 unsigned IntrinsicID) {
10558 auto VectorTy = cast<llvm::ScalableVectorType>(Ops.back()->getType());
10559 auto MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
10561 auto PredTy = MemoryTy;
10562 auto AddrMemoryTy = MemoryTy;
10563 bool IsQuadStore =
false;
10565 switch (IntrinsicID) {
10566 case Intrinsic::aarch64_sve_st1wq:
10567 case Intrinsic::aarch64_sve_st1dq:
10568 AddrMemoryTy = llvm::ScalableVectorType::get(MemEltTy, 1);
10570 llvm::ScalableVectorType::get(IntegerType::get(
getLLVMContext(), 1), 1);
10571 IsQuadStore =
true;
10577 Value *BasePtr = Ops[1];
10580 if (Ops.size() == 4)
10585 IsQuadStore ? Ops.back() :
Builder.CreateTrunc(Ops.back(), MemoryTy);
10590 cast<llvm::Instruction>(
Builder.CreateCall(F, {Val, Predicate, BasePtr}));
10603 NewOps.push_back(Ops[2]);
10605 llvm::Value *BasePtr = Ops[3];
10606 llvm::Value *RealSlice = Ops[1];
10609 if (Ops.size() == 5) {
10612 llvm::Value *StreamingVectorLengthCall =
10613 Builder.CreateCall(StreamingVectorLength);
10614 llvm::Value *Mulvl =
10615 Builder.CreateMul(StreamingVectorLengthCall, Ops[4],
"mulvl");
10619 RealSlice =
Builder.CreateAdd(RealSlice, Ops[4]);
10622 NewOps.push_back(BasePtr);
10623 NewOps.push_back(Ops[0]);
10624 NewOps.push_back(RealSlice);
10626 return Builder.CreateCall(F, NewOps);
10638 return Builder.CreateCall(F, Ops);
10645 if (Ops.size() == 0)
10646 Ops.push_back(llvm::ConstantInt::get(
Int32Ty, 255));
10648 return Builder.CreateCall(F, Ops);
10654 if (Ops.size() == 2)
10655 Ops.push_back(
Builder.getInt32(0));
10659 return Builder.CreateCall(F, Ops);
10665 return Builder.CreateVectorSplat(
10666 cast<llvm::VectorType>(Ty)->getElementCount(), Scalar);
10670 if (
auto *Ty =
Scalar->getType(); Ty->isVectorTy()) {
10672 auto *VecTy = cast<llvm::VectorType>(Ty);
10673 ElementCount EC = VecTy->getElementCount();
10674 assert(EC.isScalar() && VecTy->getElementType() ==
Int8Ty &&
10675 "Only <1 x i8> expected");
10690 if (
auto *StructTy = dyn_cast<StructType>(Ty)) {
10691 Value *Tuple = llvm::PoisonValue::get(Ty);
10693 for (
unsigned I = 0; I < StructTy->getNumElements(); ++I) {
10695 Value *Out =
Builder.CreateBitCast(In, StructTy->getTypeAtIndex(I));
10696 Tuple =
Builder.CreateInsertValue(Tuple, Out, I);
10702 return Builder.CreateBitCast(Val, Ty);
10707 auto *SplatZero = Constant::getNullValue(Ty);
10708 Ops.insert(Ops.begin(), SplatZero);
10713 auto *SplatUndef = UndefValue::get(Ty);
10714 Ops.insert(Ops.begin(), SplatUndef);
10719 llvm::Type *ResultType,
10724 llvm::Type *DefaultType =
getSVEType(TypeFlags);
10727 return {DefaultType, Ops[1]->getType()};
10733 return {Ops[0]->getType(), Ops.back()->getType()};
10735 if (TypeFlags.
isReductionQV() && !ResultType->isScalableTy() &&
10736 ResultType->isVectorTy())
10737 return {ResultType, Ops[1]->getType()};
10740 return {DefaultType};
10746 "Expects TypleFlags.isTupleSet() or TypeFlags.isTupleGet()");
10747 unsigned Idx = cast<ConstantInt>(Ops[1])->getZExtValue();
10750 return Builder.CreateInsertValue(Ops[0], Ops[2], Idx);
10751 return Builder.CreateExtractValue(Ops[0], Idx);
10757 assert(TypeFlags.
isTupleCreate() &&
"Expects TypleFlag isTupleCreate");
10759 Value *Tuple = llvm::PoisonValue::get(Ty);
10760 for (
unsigned Idx = 0; Idx < Ops.size(); Idx++)
10761 Tuple =
Builder.CreateInsertValue(Tuple, Ops[Idx], Idx);
10770 unsigned ICEArguments = 0;
10779 for (
unsigned i = 0, e =
E->getNumArgs(); i != e; i++) {
10780 bool IsICE = ICEArguments & (1 << i);
10786 std::optional<llvm::APSInt>
Result =
10788 assert(
Result &&
"Expected argument to be a constant");
10798 if (isa<StructType>(Arg->getType()) && !IsTupleGetOrSet) {
10799 for (
unsigned I = 0; I < Arg->getType()->getStructNumElements(); ++I)
10800 Ops.push_back(
Builder.CreateExtractValue(Arg, I));
10805 Ops.push_back(Arg);
10812 if (BuiltinID >= SVE::BI__builtin_sve_reinterpret_s8_s8 &&
10813 BuiltinID <= SVE::BI__builtin_sve_reinterpret_f64_f64_x4) {
10828 else if (TypeFlags.
isStore())
10846 else if (TypeFlags.
isUndef())
10847 return UndefValue::get(Ty);
10848 else if (Builtin->LLVMIntrinsic != 0) {
10852 Ops.pop_back_val());
10853 if (TypeFlags.
getMergeType() == SVETypeFlags::MergeZeroExp)
10856 if (TypeFlags.
getMergeType() == SVETypeFlags::MergeAnyExp)
10862 Ops.push_back(
Builder.getInt32( 31));
10864 Ops.insert(&Ops[1],
Builder.getInt32( 31));
10867 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
10868 if (
auto PredTy = dyn_cast<llvm::VectorType>(Ops[i]->getType()))
10869 if (PredTy->getElementType()->isIntegerTy(1))
10879 std::swap(Ops[1], Ops[2]);
10881 std::swap(Ops[1], Ops[2]);
10884 std::swap(Ops[1], Ops[2]);
10887 std::swap(Ops[1], Ops[3]);
10890 if (TypeFlags.
getMergeType() == SVETypeFlags::MergeZero) {
10891 llvm::Type *OpndTy = Ops[1]->getType();
10892 auto *SplatZero = Constant::getNullValue(OpndTy);
10893 Ops[1] =
Builder.CreateSelect(Ops[0], Ops[1], SplatZero);
10900 if (
Call->getType() == Ty)
10904 if (
auto PredTy = dyn_cast<llvm::ScalableVectorType>(Ty))
10906 if (
auto PredTupleTy = dyn_cast<llvm::StructType>(Ty))
10909 llvm_unreachable(
"unsupported element count!");
10912 switch (BuiltinID) {
10916 case SVE::BI__builtin_sve_svreinterpret_b: {
10920 CGM.
getIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool, SVCountTy);
10921 return Builder.CreateCall(CastFromSVCountF, Ops[0]);
10923 case SVE::BI__builtin_sve_svreinterpret_c: {
10927 CGM.
getIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool, SVCountTy);
10928 return Builder.CreateCall(CastToSVCountF, Ops[0]);
10931 case SVE::BI__builtin_sve_svpsel_lane_b8:
10932 case SVE::BI__builtin_sve_svpsel_lane_b16:
10933 case SVE::BI__builtin_sve_svpsel_lane_b32:
10934 case SVE::BI__builtin_sve_svpsel_lane_b64:
10935 case SVE::BI__builtin_sve_svpsel_lane_c8:
10936 case SVE::BI__builtin_sve_svpsel_lane_c16:
10937 case SVE::BI__builtin_sve_svpsel_lane_c32:
10938 case SVE::BI__builtin_sve_svpsel_lane_c64: {
10939 bool IsSVCount = isa<TargetExtType>(Ops[0]->getType());
10940 assert(((!IsSVCount || cast<TargetExtType>(Ops[0]->getType())->
getName() ==
10941 "aarch64.svcount")) &&
10942 "Unexpected TargetExtType");
10946 CGM.
getIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool, SVCountTy);
10948 CGM.
getIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool, SVCountTy);
10952 llvm::Value *Ops0 =
10953 IsSVCount ?
Builder.CreateCall(CastFromSVCountF, Ops[0]) : Ops[0];
10955 llvm::Value *PSel =
Builder.CreateCall(F, {Ops0, Ops1, Ops[2]});
10956 return IsSVCount ?
Builder.CreateCall(CastToSVCountF, PSel) : PSel;
10958 case SVE::BI__builtin_sve_svmov_b_z: {
10961 llvm::Type* OverloadedTy =
getSVEType(TypeFlags);
10963 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[1]});
10966 case SVE::BI__builtin_sve_svnot_b_z: {
10969 llvm::Type* OverloadedTy =
getSVEType(TypeFlags);
10971 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[0]});
10974 case SVE::BI__builtin_sve_svmovlb_u16:
10975 case SVE::BI__builtin_sve_svmovlb_u32:
10976 case SVE::BI__builtin_sve_svmovlb_u64:
10977 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllb);
10979 case SVE::BI__builtin_sve_svmovlb_s16:
10980 case SVE::BI__builtin_sve_svmovlb_s32:
10981 case SVE::BI__builtin_sve_svmovlb_s64:
10982 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllb);
10984 case SVE::BI__builtin_sve_svmovlt_u16:
10985 case SVE::BI__builtin_sve_svmovlt_u32:
10986 case SVE::BI__builtin_sve_svmovlt_u64:
10987 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllt);
10989 case SVE::BI__builtin_sve_svmovlt_s16:
10990 case SVE::BI__builtin_sve_svmovlt_s32:
10991 case SVE::BI__builtin_sve_svmovlt_s64:
10992 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllt);
10994 case SVE::BI__builtin_sve_svpmullt_u16:
10995 case SVE::BI__builtin_sve_svpmullt_u64:
10996 case SVE::BI__builtin_sve_svpmullt_n_u16:
10997 case SVE::BI__builtin_sve_svpmullt_n_u64:
10998 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullt_pair);
11000 case SVE::BI__builtin_sve_svpmullb_u16:
11001 case SVE::BI__builtin_sve_svpmullb_u64:
11002 case SVE::BI__builtin_sve_svpmullb_n_u16:
11003 case SVE::BI__builtin_sve_svpmullb_n_u64:
11004 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullb_pair);
11006 case SVE::BI__builtin_sve_svdup_n_b8:
11007 case SVE::BI__builtin_sve_svdup_n_b16:
11008 case SVE::BI__builtin_sve_svdup_n_b32:
11009 case SVE::BI__builtin_sve_svdup_n_b64: {
11011 Builder.CreateICmpNE(Ops[0], Constant::getNullValue(Ops[0]->getType()));
11012 llvm::ScalableVectorType *OverloadedTy =
getSVEType(TypeFlags);
11017 case SVE::BI__builtin_sve_svdupq_n_b8:
11018 case SVE::BI__builtin_sve_svdupq_n_b16:
11019 case SVE::BI__builtin_sve_svdupq_n_b32:
11020 case SVE::BI__builtin_sve_svdupq_n_b64:
11021 case SVE::BI__builtin_sve_svdupq_n_u8:
11022 case SVE::BI__builtin_sve_svdupq_n_s8:
11023 case SVE::BI__builtin_sve_svdupq_n_u64:
11024 case SVE::BI__builtin_sve_svdupq_n_f64:
11025 case SVE::BI__builtin_sve_svdupq_n_s64:
11026 case SVE::BI__builtin_sve_svdupq_n_u16:
11027 case SVE::BI__builtin_sve_svdupq_n_f16:
11028 case SVE::BI__builtin_sve_svdupq_n_bf16:
11029 case SVE::BI__builtin_sve_svdupq_n_s16:
11030 case SVE::BI__builtin_sve_svdupq_n_u32:
11031 case SVE::BI__builtin_sve_svdupq_n_f32:
11032 case SVE::BI__builtin_sve_svdupq_n_s32: {
11035 unsigned NumOpnds = Ops.size();
11038 cast<llvm::VectorType>(Ty)->getElementType()->isIntegerTy(1);
11043 llvm::Type *EltTy = Ops[0]->getType();
11048 for (
unsigned I = 0; I < NumOpnds; ++I)
11049 VecOps.push_back(
Builder.CreateZExt(Ops[I], EltTy));
11054 OverloadedTy, PoisonValue::get(OverloadedTy), Vec,
Builder.getInt64(0));
11069 : Intrinsic::aarch64_sve_cmpne_wide,
11076 case SVE::BI__builtin_sve_svpfalse_b:
11077 return ConstantInt::getFalse(Ty);
11079 case SVE::BI__builtin_sve_svpfalse_c: {
11080 auto SVBoolTy = ScalableVectorType::get(
Builder.getInt1Ty(), 16);
11083 return Builder.CreateCall(CastToSVCountF, ConstantInt::getFalse(SVBoolTy));
11086 case SVE::BI__builtin_sve_svlen_bf16:
11087 case SVE::BI__builtin_sve_svlen_f16:
11088 case SVE::BI__builtin_sve_svlen_f32:
11089 case SVE::BI__builtin_sve_svlen_f64:
11090 case SVE::BI__builtin_sve_svlen_s8:
11091 case SVE::BI__builtin_sve_svlen_s16:
11092 case SVE::BI__builtin_sve_svlen_s32:
11093 case SVE::BI__builtin_sve_svlen_s64:
11094 case SVE::BI__builtin_sve_svlen_u8:
11095 case SVE::BI__builtin_sve_svlen_u16:
11096 case SVE::BI__builtin_sve_svlen_u32:
11097 case SVE::BI__builtin_sve_svlen_u64: {
11099 auto VTy = cast<llvm::VectorType>(
getSVEType(TF));
11101 llvm::ConstantInt::get(Ty, VTy->getElementCount().getKnownMinValue());
11107 case SVE::BI__builtin_sve_svtbl2_u8:
11108 case SVE::BI__builtin_sve_svtbl2_s8:
11109 case SVE::BI__builtin_sve_svtbl2_u16:
11110 case SVE::BI__builtin_sve_svtbl2_s16:
11111 case SVE::BI__builtin_sve_svtbl2_u32:
11112 case SVE::BI__builtin_sve_svtbl2_s32:
11113 case SVE::BI__builtin_sve_svtbl2_u64:
11114 case SVE::BI__builtin_sve_svtbl2_s64:
11115 case SVE::BI__builtin_sve_svtbl2_f16:
11116 case SVE::BI__builtin_sve_svtbl2_bf16:
11117 case SVE::BI__builtin_sve_svtbl2_f32:
11118 case SVE::BI__builtin_sve_svtbl2_f64: {
11120 auto VTy = cast<llvm::ScalableVectorType>(
getSVEType(TF));
11122 return Builder.CreateCall(F, Ops);
11125 case SVE::BI__builtin_sve_svset_neonq_s8:
11126 case SVE::BI__builtin_sve_svset_neonq_s16:
11127 case SVE::BI__builtin_sve_svset_neonq_s32:
11128 case SVE::BI__builtin_sve_svset_neonq_s64:
11129 case SVE::BI__builtin_sve_svset_neonq_u8:
11130 case SVE::BI__builtin_sve_svset_neonq_u16:
11131 case SVE::BI__builtin_sve_svset_neonq_u32:
11132 case SVE::BI__builtin_sve_svset_neonq_u64:
11133 case SVE::BI__builtin_sve_svset_neonq_f16:
11134 case SVE::BI__builtin_sve_svset_neonq_f32:
11135 case SVE::BI__builtin_sve_svset_neonq_f64:
11136 case SVE::BI__builtin_sve_svset_neonq_bf16: {
11137 return Builder.CreateInsertVector(Ty, Ops[0], Ops[1],
Builder.getInt64(0));
11140 case SVE::BI__builtin_sve_svget_neonq_s8:
11141 case SVE::BI__builtin_sve_svget_neonq_s16:
11142 case SVE::BI__builtin_sve_svget_neonq_s32:
11143 case SVE::BI__builtin_sve_svget_neonq_s64:
11144 case SVE::BI__builtin_sve_svget_neonq_u8:
11145 case SVE::BI__builtin_sve_svget_neonq_u16:
11146 case SVE::BI__builtin_sve_svget_neonq_u32:
11147 case SVE::BI__builtin_sve_svget_neonq_u64:
11148 case SVE::BI__builtin_sve_svget_neonq_f16:
11149 case SVE::BI__builtin_sve_svget_neonq_f32:
11150 case SVE::BI__builtin_sve_svget_neonq_f64:
11151 case SVE::BI__builtin_sve_svget_neonq_bf16: {
11152 return Builder.CreateExtractVector(Ty, Ops[0],
Builder.getInt64(0));
11155 case SVE::BI__builtin_sve_svdup_neonq_s8:
11156 case SVE::BI__builtin_sve_svdup_neonq_s16:
11157 case SVE::BI__builtin_sve_svdup_neonq_s32:
11158 case SVE::BI__builtin_sve_svdup_neonq_s64:
11159 case SVE::BI__builtin_sve_svdup_neonq_u8:
11160 case SVE::BI__builtin_sve_svdup_neonq_u16:
11161 case SVE::BI__builtin_sve_svdup_neonq_u32:
11162 case SVE::BI__builtin_sve_svdup_neonq_u64:
11163 case SVE::BI__builtin_sve_svdup_neonq_f16:
11164 case SVE::BI__builtin_sve_svdup_neonq_f32:
11165 case SVE::BI__builtin_sve_svdup_neonq_f64:
11166 case SVE::BI__builtin_sve_svdup_neonq_bf16: {
11169 return Builder.CreateIntrinsic(Intrinsic::aarch64_sve_dupq_lane, {Ty},
11181 switch (BuiltinID) {
11184 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x1:
11187 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x2:
11188 case SME::BI__builtin_sme_svsudot_za32_s8_vg1x2:
11191 case SME::BI__builtin_sme_svsudot_za32_s8_vg1x4:
11192 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x4:
11198 for (
unsigned I = 0; I < MultiVec; ++I)
11199 std::swap(Ops[I + 1], Ops[I + 1 + MultiVec]);
11212 return EmitSMELd1St1(TypeFlags, Ops, Builtin->LLVMIntrinsic);
11215 else if (BuiltinID == SME::BI__builtin_sme_svzero_mask_za ||
11216 BuiltinID == SME::BI__builtin_sme_svzero_za)
11217 return EmitSMEZero(TypeFlags, Ops, Builtin->LLVMIntrinsic);
11218 else if (BuiltinID == SME::BI__builtin_sme_svldr_vnum_za ||
11219 BuiltinID == SME::BI__builtin_sme_svstr_vnum_za ||
11220 BuiltinID == SME::BI__builtin_sme_svldr_za ||
11221 BuiltinID == SME::BI__builtin_sme_svstr_za)
11222 return EmitSMELdrStr(TypeFlags, Ops, Builtin->LLVMIntrinsic);
11227 Ops.pop_back_val());
11232 if (Builtin->LLVMIntrinsic == 0)
11235 if (BuiltinID == SME::BI__builtin_sme___arm_in_streaming_mode) {
11238 const auto *FD = cast<FunctionDecl>(
CurFuncDecl);
11240 unsigned SMEAttrs = FPT->getAArch64SMEAttributes();
11243 return ConstantInt::getBool(
Builder.getContext(), IsStreaming);
11249 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
11250 if (
auto PredTy = dyn_cast<llvm::VectorType>(Ops[i]->getType()))
11251 if (PredTy->getElementType()->isIntegerTy(1))
11259 return Builder.CreateCall(F, Ops);
11264 llvm::Triple::ArchType Arch) {
11273 if (BuiltinID == Builtin::BI__builtin_cpu_supports)
11274 return EmitAArch64CpuSupports(
E);
11276 unsigned HintID =
static_cast<unsigned>(-1);
11277 switch (BuiltinID) {
11279 case clang::AArch64::BI__builtin_arm_nop:
11282 case clang::AArch64::BI__builtin_arm_yield:
11283 case clang::AArch64::BI__yield:
11286 case clang::AArch64::BI__builtin_arm_wfe:
11287 case clang::AArch64::BI__wfe:
11290 case clang::AArch64::BI__builtin_arm_wfi:
11291 case clang::AArch64::BI__wfi:
11294 case clang::AArch64::BI__builtin_arm_sev:
11295 case clang::AArch64::BI__sev:
11298 case clang::AArch64::BI__builtin_arm_sevl:
11299 case clang::AArch64::BI__sevl:
11304 if (HintID !=
static_cast<unsigned>(-1)) {
11306 return Builder.CreateCall(F, llvm::ConstantInt::get(
Int32Ty, HintID));
11309 if (BuiltinID == clang::AArch64::BI__builtin_arm_trap) {
11315 if (BuiltinID == clang::AArch64::BI__builtin_arm_get_sme_state) {
11320 "__arm_sme_state"));
11322 "aarch64_pstate_sm_compatible");
11323 CI->setAttributes(Attrs);
11324 CI->setCallingConv(
11325 llvm::CallingConv::
11326 AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2);
11333 if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit) {
11335 "rbit of unusual size!");
11338 CGM.
getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
11340 if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit64) {
11342 "rbit of unusual size!");
11345 CGM.
getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
11348 if (BuiltinID == clang::AArch64::BI__builtin_arm_clz ||
11349 BuiltinID == clang::AArch64::BI__builtin_arm_clz64) {
11353 if (BuiltinID == clang::AArch64::BI__builtin_arm_clz64)
11358 if (BuiltinID == clang::AArch64::BI__builtin_arm_cls) {
11363 if (BuiltinID == clang::AArch64::BI__builtin_arm_cls64) {
11369 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint32zf ||
11370 BuiltinID == clang::AArch64::BI__builtin_arm_rint32z) {
11372 llvm::Type *Ty = Arg->getType();
11377 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint64zf ||
11378 BuiltinID == clang::AArch64::BI__builtin_arm_rint64z) {
11380 llvm::Type *Ty = Arg->getType();
11385 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint32xf ||
11386 BuiltinID == clang::AArch64::BI__builtin_arm_rint32x) {
11388 llvm::Type *Ty = Arg->getType();
11393 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint64xf ||
11394 BuiltinID == clang::AArch64::BI__builtin_arm_rint64x) {
11396 llvm::Type *Ty = Arg->getType();
11401 if (BuiltinID == clang::AArch64::BI__builtin_arm_jcvt) {
11403 "__jcvt of unusual size!");
11409 if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b ||
11410 BuiltinID == clang::AArch64::BI__builtin_arm_st64b ||
11411 BuiltinID == clang::AArch64::BI__builtin_arm_st64bv ||
11412 BuiltinID == clang::AArch64::BI__builtin_arm_st64bv0) {
11416 if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b) {
11420 llvm::Value *Val =
Builder.CreateCall(F, MemAddr);
11421 llvm::Value *ToRet;
11422 for (
size_t i = 0; i < 8; i++) {
11423 llvm::Value *ValOffsetPtr =
11434 Args.push_back(MemAddr);
11435 for (
size_t i = 0; i < 8; i++) {
11436 llvm::Value *ValOffsetPtr =
11443 auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_st64b
11444 ? Intrinsic::aarch64_st64b
11445 : BuiltinID == clang::AArch64::BI__builtin_arm_st64bv
11446 ? Intrinsic::aarch64_st64bv
11447 : Intrinsic::aarch64_st64bv0);
11449 return Builder.CreateCall(F, Args);
11453 if (BuiltinID == clang::AArch64::BI__builtin_arm_rndr ||
11454 BuiltinID == clang::AArch64::BI__builtin_arm_rndrrs) {
11456 auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_rndr
11457 ? Intrinsic::aarch64_rndr
11458 : Intrinsic::aarch64_rndrrs);
11460 llvm::Value *Val =
Builder.CreateCall(F);
11461 Value *RandomValue =
Builder.CreateExtractValue(Val, 0);
11470 if (BuiltinID == clang::AArch64::BI__clear_cache) {
11471 assert(
E->getNumArgs() == 2 &&
"__clear_cache takes 2 arguments");
11474 for (
unsigned i = 0; i < 2; i++)
11477 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
11478 StringRef Name = FD->
getName();
11482 if ((BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
11483 BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) &&
11487 ? Intrinsic::aarch64_ldaxp
11488 : Intrinsic::aarch64_ldxp);
11495 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
11496 Val0 =
Builder.CreateZExt(Val0, Int128Ty);
11497 Val1 =
Builder.CreateZExt(Val1, Int128Ty);
11499 Value *ShiftCst = llvm::ConstantInt::get(Int128Ty, 64);
11500 Val =
Builder.CreateShl(Val0, ShiftCst,
"shl",
true );
11501 Val =
Builder.CreateOr(Val, Val1);
11503 }
else if (BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
11504 BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) {
11509 llvm::Type *
IntTy =
11514 ? Intrinsic::aarch64_ldaxr
11515 : Intrinsic::aarch64_ldxr,
11517 CallInst *Val =
Builder.CreateCall(F, LoadAddr,
"ldxr");
11521 if (RealResTy->isPointerTy())
11522 return Builder.CreateIntToPtr(Val, RealResTy);
11524 llvm::Type *IntResTy = llvm::IntegerType::get(
11526 return Builder.CreateBitCast(
Builder.CreateTruncOrBitCast(Val, IntResTy),
11530 if ((BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
11531 BuiltinID == clang::AArch64::BI__builtin_arm_stlex) &&
11535 ? Intrinsic::aarch64_stlxp
11536 : Intrinsic::aarch64_stxp);
11548 return Builder.CreateCall(F, {Arg0, Arg1, StPtr},
"stxp");
11551 if (BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
11552 BuiltinID == clang::AArch64::BI__builtin_arm_stlex) {
11557 llvm::Type *StoreTy =
11560 if (StoreVal->
getType()->isPointerTy())
11563 llvm::Type *
IntTy = llvm::IntegerType::get(
11572 ? Intrinsic::aarch64_stlxr
11573 : Intrinsic::aarch64_stxr,
11575 CallInst *CI =
Builder.CreateCall(F, {StoreVal, StoreAddr},
"stxr");
11577 1, Attribute::get(
getLLVMContext(), Attribute::ElementType, StoreTy));
11581 if (BuiltinID == clang::AArch64::BI__getReg) {
11584 llvm_unreachable(
"Sema will ensure that the parameter is constant");
11590 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, Reg)};
11591 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
11592 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
11594 llvm::Function *F =
11596 return Builder.CreateCall(F, Metadata);
11599 if (BuiltinID == clang::AArch64::BI__break) {
11602 llvm_unreachable(
"Sema will ensure that the parameter is constant");
11604 llvm::Function *F =
CGM.
getIntrinsic(llvm::Intrinsic::aarch64_break);
11608 if (BuiltinID == clang::AArch64::BI__builtin_arm_clrex) {
11610 return Builder.CreateCall(F);
11613 if (BuiltinID == clang::AArch64::BI_ReadWriteBarrier)
11614 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
11615 llvm::SyncScope::SingleThread);
11618 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
11619 switch (BuiltinID) {
11620 case clang::AArch64::BI__builtin_arm_crc32b:
11621 CRCIntrinsicID = Intrinsic::aarch64_crc32b;
break;
11622 case clang::AArch64::BI__builtin_arm_crc32cb:
11623 CRCIntrinsicID = Intrinsic::aarch64_crc32cb;
break;
11624 case clang::AArch64::BI__builtin_arm_crc32h:
11625 CRCIntrinsicID = Intrinsic::aarch64_crc32h;
break;
11626 case clang::AArch64::BI__builtin_arm_crc32ch:
11627 CRCIntrinsicID = Intrinsic::aarch64_crc32ch;
break;
11628 case clang::AArch64::BI__builtin_arm_crc32w:
11629 CRCIntrinsicID = Intrinsic::aarch64_crc32w;
break;
11630 case clang::AArch64::BI__builtin_arm_crc32cw:
11631 CRCIntrinsicID = Intrinsic::aarch64_crc32cw;
break;
11632 case clang::AArch64::BI__builtin_arm_crc32d:
11633 CRCIntrinsicID = Intrinsic::aarch64_crc32x;
break;
11634 case clang::AArch64::BI__builtin_arm_crc32cd:
11635 CRCIntrinsicID = Intrinsic::aarch64_crc32cx;
break;
11638 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
11643 llvm::Type *DataTy = F->getFunctionType()->getParamType(1);
11644 Arg1 =
Builder.CreateZExtOrBitCast(Arg1, DataTy);
11646 return Builder.CreateCall(F, {Arg0, Arg1});
11650 if (BuiltinID == AArch64::BI__builtin_arm_mops_memset_tag) {
11657 CGM.
getIntrinsic(Intrinsic::aarch64_mops_memset_tag), {Dst, Val, Size});
11661 Intrinsic::ID MTEIntrinsicID = Intrinsic::not_intrinsic;
11662 switch (BuiltinID) {
11663 case clang::AArch64::BI__builtin_arm_irg:
11664 MTEIntrinsicID = Intrinsic::aarch64_irg;
break;
11665 case clang::AArch64::BI__builtin_arm_addg:
11666 MTEIntrinsicID = Intrinsic::aarch64_addg;
break;
11667 case clang::AArch64::BI__builtin_arm_gmi:
11668 MTEIntrinsicID = Intrinsic::aarch64_gmi;
break;
11669 case clang::AArch64::BI__builtin_arm_ldg:
11670 MTEIntrinsicID = Intrinsic::aarch64_ldg;
break;
11671 case clang::AArch64::BI__builtin_arm_stg:
11672 MTEIntrinsicID = Intrinsic::aarch64_stg;
break;
11673 case clang::AArch64::BI__builtin_arm_subp:
11674 MTEIntrinsicID = Intrinsic::aarch64_subp;
break;
11677 if (MTEIntrinsicID != Intrinsic::not_intrinsic) {
11678 if (MTEIntrinsicID == Intrinsic::aarch64_irg) {
11686 if (MTEIntrinsicID == Intrinsic::aarch64_addg) {
11692 {Pointer, TagOffset});
11694 if (MTEIntrinsicID == Intrinsic::aarch64_gmi) {
11705 if (MTEIntrinsicID == Intrinsic::aarch64_ldg) {
11708 {TagAddress, TagAddress});
11713 if (MTEIntrinsicID == Intrinsic::aarch64_stg) {
11716 {TagAddress, TagAddress});
11718 if (MTEIntrinsicID == Intrinsic::aarch64_subp) {
11726 if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
11727 BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
11728 BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
11729 BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
11730 BuiltinID == clang::AArch64::BI__builtin_arm_wsr ||
11731 BuiltinID == clang::AArch64::BI__builtin_arm_wsr64 ||
11732 BuiltinID == clang::AArch64::BI__builtin_arm_wsr128 ||
11733 BuiltinID == clang::AArch64::BI__builtin_arm_wsrp) {
11736 if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
11737 BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
11738 BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
11739 BuiltinID == clang::AArch64::BI__builtin_arm_rsrp)
11742 bool IsPointerBuiltin = BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
11743 BuiltinID == clang::AArch64::BI__builtin_arm_wsrp;
11745 bool Is32Bit = BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
11746 BuiltinID == clang::AArch64::BI__builtin_arm_wsr;
11748 bool Is128Bit = BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
11749 BuiltinID == clang::AArch64::BI__builtin_arm_wsr128;
11751 llvm::Type *ValueType;
11755 }
else if (Is128Bit) {
11756 llvm::Type *Int128Ty =
11758 ValueType = Int128Ty;
11760 }
else if (IsPointerBuiltin) {
11770 if (BuiltinID == clang::AArch64::BI_ReadStatusReg ||
11771 BuiltinID == clang::AArch64::BI_WriteStatusReg) {
11777 std::string SysRegStr;
11778 llvm::raw_string_ostream(SysRegStr) <<
11779 ((1 << 1) | ((SysReg >> 14) & 1)) <<
":" <<
11780 ((SysReg >> 11) & 7) <<
":" <<
11781 ((SysReg >> 7) & 15) <<
":" <<
11782 ((SysReg >> 3) & 15) <<
":" <<
11785 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysRegStr) };
11786 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
11787 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
11792 if (BuiltinID == clang::AArch64::BI_ReadStatusReg) {
11793 llvm::Function *F =
CGM.
getIntrinsic(llvm::Intrinsic::read_register, Types);
11795 return Builder.CreateCall(F, Metadata);
11798 llvm::Function *F =
CGM.
getIntrinsic(llvm::Intrinsic::write_register, Types);
11801 return Builder.CreateCall(F, { Metadata, ArgValue });
11804 if (BuiltinID == clang::AArch64::BI_AddressOfReturnAddress) {
11805 llvm::Function *F =
11807 return Builder.CreateCall(F);
11810 if (BuiltinID == clang::AArch64::BI__builtin_sponentry) {
11812 return Builder.CreateCall(F);
11815 if (BuiltinID == clang::AArch64::BI__mulh ||
11816 BuiltinID == clang::AArch64::BI__umulh) {
11818 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
11820 bool IsSigned = BuiltinID == clang::AArch64::BI__mulh;
11826 Value *MulResult, *HigherBits;
11828 MulResult =
Builder.CreateNSWMul(LHS, RHS);
11829 HigherBits =
Builder.CreateAShr(MulResult, 64);
11831 MulResult =
Builder.CreateNUWMul(LHS, RHS);
11832 HigherBits =
Builder.CreateLShr(MulResult, 64);
11834 HigherBits =
Builder.CreateIntCast(HigherBits, ResType, IsSigned);
11839 if (BuiltinID == AArch64::BI__writex18byte ||
11840 BuiltinID == AArch64::BI__writex18word ||
11841 BuiltinID == AArch64::BI__writex18dword ||
11842 BuiltinID == AArch64::BI__writex18qword) {
11858 if (BuiltinID == AArch64::BI__readx18byte ||
11859 BuiltinID == AArch64::BI__readx18word ||
11860 BuiltinID == AArch64::BI__readx18dword ||
11861 BuiltinID == AArch64::BI__readx18qword) {
11876 if (BuiltinID == AArch64::BI__addx18byte ||
11877 BuiltinID == AArch64::BI__addx18word ||
11878 BuiltinID == AArch64::BI__addx18dword ||
11879 BuiltinID == AArch64::BI__addx18qword ||
11880 BuiltinID == AArch64::BI__incx18byte ||
11881 BuiltinID == AArch64::BI__incx18word ||
11882 BuiltinID == AArch64::BI__incx18dword ||
11883 BuiltinID == AArch64::BI__incx18qword) {
11886 switch (BuiltinID) {
11887 case AArch64::BI__incx18byte:
11889 isIncrement =
true;
11891 case AArch64::BI__incx18word:
11893 isIncrement =
true;
11895 case AArch64::BI__incx18dword:
11897 isIncrement =
true;
11899 case AArch64::BI__incx18qword:
11901 isIncrement =
true;
11905 isIncrement =
false;
11930 if (BuiltinID == AArch64::BI_CopyDoubleFromInt64 ||
11931 BuiltinID == AArch64::BI_CopyFloatFromInt32 ||
11932 BuiltinID == AArch64::BI_CopyInt32FromFloat ||
11933 BuiltinID == AArch64::BI_CopyInt64FromDouble) {
11936 return Builder.CreateBitCast(Arg, RetTy);
11939 if (BuiltinID == AArch64::BI_CountLeadingOnes ||
11940 BuiltinID == AArch64::BI_CountLeadingOnes64 ||
11941 BuiltinID == AArch64::BI_CountLeadingZeros ||
11942 BuiltinID == AArch64::BI_CountLeadingZeros64) {
11944 llvm::Type *ArgType = Arg->
getType();
11946 if (BuiltinID == AArch64::BI_CountLeadingOnes ||
11947 BuiltinID == AArch64::BI_CountLeadingOnes64)
11948 Arg =
Builder.CreateXor(Arg, Constant::getAllOnesValue(ArgType));
11953 if (BuiltinID == AArch64::BI_CountLeadingOnes64 ||
11954 BuiltinID == AArch64::BI_CountLeadingZeros64)
11959 if (BuiltinID == AArch64::BI_CountLeadingSigns ||
11960 BuiltinID == AArch64::BI_CountLeadingSigns64) {
11963 Function *F = (BuiltinID == AArch64::BI_CountLeadingSigns)
11968 if (BuiltinID == AArch64::BI_CountLeadingSigns64)
11973 if (BuiltinID == AArch64::BI_CountOneBits ||
11974 BuiltinID == AArch64::BI_CountOneBits64) {
11976 llvm::Type *ArgType = ArgValue->
getType();
11980 if (BuiltinID == AArch64::BI_CountOneBits64)
11985 if (BuiltinID == AArch64::BI__prefetch) {
11994 if (BuiltinID == AArch64::BI__hlt) {
12000 return ConstantInt::get(
Builder.getInt32Ty(), 0);
12003 if (BuiltinID == NEON::BI__builtin_neon_vcvth_bf16_f32)
12004 return Builder.CreateFPTrunc(
12011 if (std::optional<MSVCIntrin> MsvcIntId =
12017 return P.first == BuiltinID;
12020 BuiltinID = It->second;
12024 unsigned ICEArguments = 0;
12031 for (
unsigned i = 0, e =
E->getNumArgs() - 1; i != e; i++) {
12033 switch (BuiltinID) {
12034 case NEON::BI__builtin_neon_vld1_v:
12035 case NEON::BI__builtin_neon_vld1q_v:
12036 case NEON::BI__builtin_neon_vld1_dup_v:
12037 case NEON::BI__builtin_neon_vld1q_dup_v:
12038 case NEON::BI__builtin_neon_vld1_lane_v:
12039 case NEON::BI__builtin_neon_vld1q_lane_v:
12040 case NEON::BI__builtin_neon_vst1_v:
12041 case NEON::BI__builtin_neon_vst1q_v:
12042 case NEON::BI__builtin_neon_vst1_lane_v:
12043 case NEON::BI__builtin_neon_vst1q_lane_v:
12044 case NEON::BI__builtin_neon_vldap1_lane_s64:
12045 case NEON::BI__builtin_neon_vldap1q_lane_s64:
12046 case NEON::BI__builtin_neon_vstl1_lane_s64:
12047 case NEON::BI__builtin_neon_vstl1q_lane_s64:
12065 assert(
Result &&
"SISD intrinsic should have been handled");
12069 const Expr *Arg =
E->getArg(
E->getNumArgs()-1);
12071 if (std::optional<llvm::APSInt>
Result =
12076 bool usgn =
Type.isUnsigned();
12077 bool quad =
Type.isQuad();
12080 switch (BuiltinID) {
12082 case NEON::BI__builtin_neon_vabsh_f16:
12085 case NEON::BI__builtin_neon_vaddq_p128: {
12088 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
12089 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
12090 Ops[0] =
Builder.CreateXor(Ops[0], Ops[1]);
12091 llvm::Type *Int128Ty = llvm::Type::getIntNTy(
getLLVMContext(), 128);
12092 return Builder.CreateBitCast(Ops[0], Int128Ty);
12094 case NEON::BI__builtin_neon_vldrq_p128: {
12095 llvm::Type *Int128Ty = llvm::Type::getIntNTy(
getLLVMContext(), 128);
12100 case NEON::BI__builtin_neon_vstrq_p128: {
12101 Value *Ptr = Ops[0];
12104 case NEON::BI__builtin_neon_vcvts_f32_u32:
12105 case NEON::BI__builtin_neon_vcvtd_f64_u64:
12108 case NEON::BI__builtin_neon_vcvts_f32_s32:
12109 case NEON::BI__builtin_neon_vcvtd_f64_s64: {
12111 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
12114 Ops[0] =
Builder.CreateBitCast(Ops[0], InTy);
12116 return Builder.CreateUIToFP(Ops[0], FTy);
12117 return Builder.CreateSIToFP(Ops[0], FTy);
12119 case NEON::BI__builtin_neon_vcvth_f16_u16:
12120 case NEON::BI__builtin_neon_vcvth_f16_u32:
12121 case NEON::BI__builtin_neon_vcvth_f16_u64:
12124 case NEON::BI__builtin_neon_vcvth_f16_s16:
12125 case NEON::BI__builtin_neon_vcvth_f16_s32:
12126 case NEON::BI__builtin_neon_vcvth_f16_s64: {
12128 llvm::Type *FTy =
HalfTy;
12130 if (Ops[0]->getType()->getPrimitiveSizeInBits() == 64)
12132 else if (Ops[0]->getType()->getPrimitiveSizeInBits() == 32)
12136 Ops[0] =
Builder.CreateBitCast(Ops[0], InTy);
12138 return Builder.CreateUIToFP(Ops[0], FTy);
12139 return Builder.CreateSIToFP(Ops[0], FTy);
12141 case NEON::BI__builtin_neon_vcvtah_u16_f16:
12142 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
12143 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
12144 case NEON::BI__builtin_neon_vcvtph_u16_f16:
12145 case NEON::BI__builtin_neon_vcvth_u16_f16:
12146 case NEON::BI__builtin_neon_vcvtah_s16_f16:
12147 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
12148 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
12149 case NEON::BI__builtin_neon_vcvtph_s16_f16:
12150 case NEON::BI__builtin_neon_vcvth_s16_f16: {
12153 llvm::Type* FTy =
HalfTy;
12154 llvm::Type *Tys[2] = {InTy, FTy};
12156 switch (BuiltinID) {
12157 default: llvm_unreachable(
"missing builtin ID in switch!");
12158 case NEON::BI__builtin_neon_vcvtah_u16_f16:
12159 Int = Intrinsic::aarch64_neon_fcvtau;
break;
12160 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
12161 Int = Intrinsic::aarch64_neon_fcvtmu;
break;
12162 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
12163 Int = Intrinsic::aarch64_neon_fcvtnu;
break;
12164 case NEON::BI__builtin_neon_vcvtph_u16_f16:
12165 Int = Intrinsic::aarch64_neon_fcvtpu;
break;
12166 case NEON::BI__builtin_neon_vcvth_u16_f16:
12167 Int = Intrinsic::aarch64_neon_fcvtzu;
break;
12168 case NEON::BI__builtin_neon_vcvtah_s16_f16:
12169 Int = Intrinsic::aarch64_neon_fcvtas;
break;
12170 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
12171 Int = Intrinsic::aarch64_neon_fcvtms;
break;
12172 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
12173 Int = Intrinsic::aarch64_neon_fcvtns;
break;
12174 case NEON::BI__builtin_neon_vcvtph_s16_f16:
12175 Int = Intrinsic::aarch64_neon_fcvtps;
break;
12176 case NEON::BI__builtin_neon_vcvth_s16_f16:
12177 Int = Intrinsic::aarch64_neon_fcvtzs;
break;
12182 case NEON::BI__builtin_neon_vcaleh_f16:
12183 case NEON::BI__builtin_neon_vcalth_f16:
12184 case NEON::BI__builtin_neon_vcageh_f16:
12185 case NEON::BI__builtin_neon_vcagth_f16: {
12188 llvm::Type* FTy =
HalfTy;
12189 llvm::Type *Tys[2] = {InTy, FTy};
12191 switch (BuiltinID) {
12192 default: llvm_unreachable(
"missing builtin ID in switch!");
12193 case NEON::BI__builtin_neon_vcageh_f16:
12194 Int = Intrinsic::aarch64_neon_facge;
break;
12195 case NEON::BI__builtin_neon_vcagth_f16:
12196 Int = Intrinsic::aarch64_neon_facgt;
break;
12197 case NEON::BI__builtin_neon_vcaleh_f16:
12198 Int = Intrinsic::aarch64_neon_facge; std::swap(Ops[0], Ops[1]);
break;
12199 case NEON::BI__builtin_neon_vcalth_f16:
12200 Int = Intrinsic::aarch64_neon_facgt; std::swap(Ops[0], Ops[1]);
break;
12205 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
12206 case NEON::BI__builtin_neon_vcvth_n_u16_f16: {
12209 llvm::Type* FTy =
HalfTy;
12210 llvm::Type *Tys[2] = {InTy, FTy};
12212 switch (BuiltinID) {
12213 default: llvm_unreachable(
"missing builtin ID in switch!");
12214 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
12215 Int = Intrinsic::aarch64_neon_vcvtfp2fxs;
break;
12216 case NEON::BI__builtin_neon_vcvth_n_u16_f16:
12217 Int = Intrinsic::aarch64_neon_vcvtfp2fxu;
break;
12222 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
12223 case NEON::BI__builtin_neon_vcvth_n_f16_u16: {
12225 llvm::Type* FTy =
HalfTy;
12227 llvm::Type *Tys[2] = {FTy, InTy};
12229 switch (BuiltinID) {
12230 default: llvm_unreachable(
"missing builtin ID in switch!");
12231 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
12232 Int = Intrinsic::aarch64_neon_vcvtfxs2fp;
12233 Ops[0] =
Builder.CreateSExt(Ops[0], InTy,
"sext");
12235 case NEON::BI__builtin_neon_vcvth_n_f16_u16:
12236 Int = Intrinsic::aarch64_neon_vcvtfxu2fp;
12237 Ops[0] =
Builder.CreateZExt(Ops[0], InTy);
12242 case NEON::BI__builtin_neon_vpaddd_s64: {
12243 auto *Ty = llvm::FixedVectorType::get(
Int64Ty, 2);
12246 Vec =
Builder.CreateBitCast(Vec, Ty,
"v2i64");
12247 llvm::Value *Idx0 = llvm::ConstantInt::get(
SizeTy, 0);
12248 llvm::Value *Idx1 = llvm::ConstantInt::get(
SizeTy, 1);
12249 Value *Op0 =
Builder.CreateExtractElement(Vec, Idx0,
"lane0");
12250 Value *Op1 =
Builder.CreateExtractElement(Vec, Idx1,
"lane1");
12252 return Builder.CreateAdd(Op0, Op1,
"vpaddd");
12254 case NEON::BI__builtin_neon_vpaddd_f64: {
12255 auto *Ty = llvm::FixedVectorType::get(
DoubleTy, 2);
12258 Vec =
Builder.CreateBitCast(Vec, Ty,
"v2f64");
12259 llvm::Value *Idx0 = llvm::ConstantInt::get(
SizeTy, 0);
12260 llvm::Value *Idx1 = llvm::ConstantInt::get(
SizeTy, 1);
12261 Value *Op0 =
Builder.CreateExtractElement(Vec, Idx0,
"lane0");
12262 Value *Op1 =
Builder.CreateExtractElement(Vec, Idx1,
"lane1");
12264 return Builder.CreateFAdd(Op0, Op1,
"vpaddd");
12266 case NEON::BI__builtin_neon_vpadds_f32: {
12267 auto *Ty = llvm::FixedVectorType::get(
FloatTy, 2);
12270 Vec =
Builder.CreateBitCast(Vec, Ty,
"v2f32");
12271 llvm::Value *Idx0 = llvm::ConstantInt::get(
SizeTy, 0);
12272 llvm::Value *Idx1 = llvm::ConstantInt::get(
SizeTy, 1);
12273 Value *Op0 =
Builder.CreateExtractElement(Vec, Idx0,
"lane0");
12274 Value *Op1 =
Builder.CreateExtractElement(Vec, Idx1,
"lane1");
12276 return Builder.CreateFAdd(Op0, Op1,
"vpaddd");
12278 case NEON::BI__builtin_neon_vceqzd_s64:
12279 case NEON::BI__builtin_neon_vceqzd_f64:
12280 case NEON::BI__builtin_neon_vceqzs_f32:
12281 case NEON::BI__builtin_neon_vceqzh_f16:
12285 ICmpInst::FCMP_OEQ, ICmpInst::ICMP_EQ,
"vceqz");
12286 case NEON::BI__builtin_neon_vcgezd_s64:
12287 case NEON::BI__builtin_neon_vcgezd_f64:
12288 case NEON::BI__builtin_neon_vcgezs_f32:
12289 case NEON::BI__builtin_neon_vcgezh_f16:
12293 ICmpInst::FCMP_OGE, ICmpInst::ICMP_SGE,
"vcgez");
12294 case NEON::BI__builtin_neon_vclezd_s64:
12295 case NEON::BI__builtin_neon_vclezd_f64:
12296 case NEON::BI__builtin_neon_vclezs_f32:
12297 case NEON::BI__builtin_neon_vclezh_f16:
12301 ICmpInst::FCMP_OLE, ICmpInst::ICMP_SLE,
"vclez");
12302 case NEON::BI__builtin_neon_vcgtzd_s64:
12303 case NEON::BI__builtin_neon_vcgtzd_f64:
12304 case NEON::BI__builtin_neon_vcgtzs_f32:
12305 case NEON::BI__builtin_neon_vcgtzh_f16:
12309 ICmpInst::FCMP_OGT, ICmpInst::ICMP_SGT,
"vcgtz");
12310 case NEON::BI__builtin_neon_vcltzd_s64:
12311 case NEON::BI__builtin_neon_vcltzd_f64:
12312 case NEON::BI__builtin_neon_vcltzs_f32:
12313 case NEON::BI__builtin_neon_vcltzh_f16:
12317 ICmpInst::FCMP_OLT, ICmpInst::ICMP_SLT,
"vcltz");
12319 case NEON::BI__builtin_neon_vceqzd_u64: {
12323 Builder.CreateICmpEQ(Ops[0], llvm::Constant::getNullValue(
Int64Ty));
12326 case NEON::BI__builtin_neon_vceqd_f64:
12327 case NEON::BI__builtin_neon_vcled_f64:
12328 case NEON::BI__builtin_neon_vcltd_f64:
12329 case NEON::BI__builtin_neon_vcged_f64:
12330 case NEON::BI__builtin_neon_vcgtd_f64: {
12331 llvm::CmpInst::Predicate
P;
12332 switch (BuiltinID) {
12333 default: llvm_unreachable(
"missing builtin ID in switch!");
12334 case NEON::BI__builtin_neon_vceqd_f64:
P = llvm::FCmpInst::FCMP_OEQ;
break;
12335 case NEON::BI__builtin_neon_vcled_f64:
P = llvm::FCmpInst::FCMP_OLE;
break;
12336 case NEON::BI__builtin_neon_vcltd_f64:
P = llvm::FCmpInst::FCMP_OLT;
break;
12337 case NEON::BI__builtin_neon_vcged_f64:
P = llvm::FCmpInst::FCMP_OGE;
break;
12338 case NEON::BI__builtin_neon_vcgtd_f64:
P = llvm::FCmpInst::FCMP_OGT;
break;
12343 if (
P == llvm::FCmpInst::FCMP_OEQ)
12344 Ops[0] =
Builder.CreateFCmp(
P, Ops[0], Ops[1]);
12346 Ops[0] =
Builder.CreateFCmpS(
P, Ops[0], Ops[1]);
12349 case NEON::BI__builtin_neon_vceqs_f32:
12350 case NEON::BI__builtin_neon_vcles_f32:
12351 case NEON::BI__builtin_neon_vclts_f32:
12352 case NEON::BI__builtin_neon_vcges_f32:
12353 case NEON::BI__builtin_neon_vcgts_f32: {
12354 llvm::CmpInst::Predicate
P;
12355 switch (BuiltinID) {
12356 default: llvm_unreachable(
"missing builtin ID in switch!");
12357 case NEON::BI__builtin_neon_vceqs_f32:
P = llvm::FCmpInst::FCMP_OEQ;
break;
12358 case NEON::BI__builtin_neon_vcles_f32:
P = llvm::FCmpInst::FCMP_OLE;
break;
12359 case NEON::BI__builtin_neon_vclts_f32:
P = llvm::FCmpInst::FCMP_OLT;
break;
12360 case NEON::BI__builtin_neon_vcges_f32:
P = llvm::FCmpInst::FCMP_OGE;
break;
12361 case NEON::BI__builtin_neon_vcgts_f32:
P = llvm::FCmpInst::FCMP_OGT;
break;
12366 if (
P == llvm::FCmpInst::FCMP_OEQ)
12367 Ops[0] =
Builder.CreateFCmp(
P, Ops[0], Ops[1]);
12369 Ops[0] =
Builder.CreateFCmpS(
P, Ops[0], Ops[1]);
12372 case NEON::BI__builtin_neon_vceqh_f16:
12373 case NEON::BI__builtin_neon_vcleh_f16:
12374 case NEON::BI__builtin_neon_vclth_f16:
12375 case NEON::BI__builtin_neon_vcgeh_f16:
12376 case NEON::BI__builtin_neon_vcgth_f16: {
12377 llvm::CmpInst::Predicate
P;
12378 switch (BuiltinID) {
12379 default: llvm_unreachable(
"missing builtin ID in switch!");
12380 case NEON::BI__builtin_neon_vceqh_f16:
P = llvm::FCmpInst::FCMP_OEQ;
break;
12381 case NEON::BI__builtin_neon_vcleh_f16:
P = llvm::FCmpInst::FCMP_OLE;
break;
12382 case NEON::BI__builtin_neon_vclth_f16:
P = llvm::FCmpInst::FCMP_OLT;
break;
12383 case NEON::BI__builtin_neon_vcgeh_f16:
P = llvm::FCmpInst::FCMP_OGE;
break;
12384 case NEON::BI__builtin_neon_vcgth_f16:
P = llvm::FCmpInst::FCMP_OGT;
break;
12389 if (
P == llvm::FCmpInst::FCMP_OEQ)
12390 Ops[0] =
Builder.CreateFCmp(
P, Ops[0], Ops[1]);
12392 Ops[0] =
Builder.CreateFCmpS(
P, Ops[0], Ops[1]);
12395 case NEON::BI__builtin_neon_vceqd_s64:
12396 case NEON::BI__builtin_neon_vceqd_u64:
12397 case NEON::BI__builtin_neon_vcgtd_s64:
12398 case NEON::BI__builtin_neon_vcgtd_u64:
12399 case NEON::BI__builtin_neon_vcltd_s64:
12400 case NEON::BI__builtin_neon_vcltd_u64:
12401 case NEON::BI__builtin_neon_vcged_u64:
12402 case NEON::BI__builtin_neon_vcged_s64:
12403 case NEON::BI__builtin_neon_vcled_u64:
12404 case NEON::BI__builtin_neon_vcled_s64: {
12405 llvm::CmpInst::Predicate
P;
12406 switch (BuiltinID) {
12407 default: llvm_unreachable(
"missing builtin ID in switch!");
12408 case NEON::BI__builtin_neon_vceqd_s64:
12409 case NEON::BI__builtin_neon_vceqd_u64:
P = llvm::ICmpInst::ICMP_EQ;
break;
12410 case NEON::BI__builtin_neon_vcgtd_s64:
P = llvm::ICmpInst::ICMP_SGT;
break;
12411 case NEON::BI__builtin_neon_vcgtd_u64:
P = llvm::ICmpInst::ICMP_UGT;
break;
12412 case NEON::BI__builtin_neon_vcltd_s64:
P = llvm::ICmpInst::ICMP_SLT;
break;
12413 case NEON::BI__builtin_neon_vcltd_u64:
P = llvm::ICmpInst::ICMP_ULT;
break;
12414 case NEON::BI__builtin_neon_vcged_u64:
P = llvm::ICmpInst::ICMP_UGE;
break;
12415 case NEON::BI__builtin_neon_vcged_s64:
P = llvm::ICmpInst::ICMP_SGE;
break;
12416 case NEON::BI__builtin_neon_vcled_u64:
P = llvm::ICmpInst::ICMP_ULE;
break;
12417 case NEON::BI__builtin_neon_vcled_s64:
P = llvm::ICmpInst::ICMP_SLE;
break;
12422 Ops[0] =
Builder.CreateICmp(
P, Ops[0], Ops[1]);
12425 case NEON::BI__builtin_neon_vtstd_s64:
12426 case NEON::BI__builtin_neon_vtstd_u64: {
12430 Ops[0] =
Builder.CreateAnd(Ops[0], Ops[1]);
12431 Ops[0] =
Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
12432 llvm::Constant::getNullValue(
Int64Ty));
12435 case NEON::BI__builtin_neon_vset_lane_i8:
12436 case NEON::BI__builtin_neon_vset_lane_i16:
12437 case NEON::BI__builtin_neon_vset_lane_i32:
12438 case NEON::BI__builtin_neon_vset_lane_i64:
12439 case NEON::BI__builtin_neon_vset_lane_bf16:
12440 case NEON::BI__builtin_neon_vset_lane_f32:
12441 case NEON::BI__builtin_neon_vsetq_lane_i8:
12442 case NEON::BI__builtin_neon_vsetq_lane_i16:
12443 case NEON::BI__builtin_neon_vsetq_lane_i32:
12444 case NEON::BI__builtin_neon_vsetq_lane_i64:
12445 case NEON::BI__builtin_neon_vsetq_lane_bf16:
12446 case NEON::BI__builtin_neon_vsetq_lane_f32:
12448 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
12449 case NEON::BI__builtin_neon_vset_lane_f64:
12452 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(
DoubleTy, 1));
12454 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
12455 case NEON::BI__builtin_neon_vsetq_lane_f64:
12458 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(
DoubleTy, 2));
12460 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
12462 case NEON::BI__builtin_neon_vget_lane_i8:
12463 case NEON::BI__builtin_neon_vdupb_lane_i8:
12465 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int8Ty, 8));
12468 case NEON::BI__builtin_neon_vgetq_lane_i8:
12469 case NEON::BI__builtin_neon_vdupb_laneq_i8:
12471 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int8Ty, 16));
12474 case NEON::BI__builtin_neon_vget_lane_i16:
12475 case NEON::BI__builtin_neon_vduph_lane_i16:
12477 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int16Ty, 4));
12480 case NEON::BI__builtin_neon_vgetq_lane_i16:
12481 case NEON::BI__builtin_neon_vduph_laneq_i16:
12483 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int16Ty, 8));
12486 case NEON::BI__builtin_neon_vget_lane_i32:
12487 case NEON::BI__builtin_neon_vdups_lane_i32:
12489 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int32Ty, 2));
12492 case NEON::BI__builtin_neon_vdups_lane_f32:
12494 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
FloatTy, 2));
12497 case NEON::BI__builtin_neon_vgetq_lane_i32:
12498 case NEON::BI__builtin_neon_vdups_laneq_i32:
12500 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int32Ty, 4));
12503 case NEON::BI__builtin_neon_vget_lane_i64:
12504 case NEON::BI__builtin_neon_vdupd_lane_i64:
12506 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int64Ty, 1));
12509 case NEON::BI__builtin_neon_vdupd_lane_f64:
12511 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
DoubleTy, 1));
12514 case NEON::BI__builtin_neon_vgetq_lane_i64:
12515 case NEON::BI__builtin_neon_vdupd_laneq_i64:
12517 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int64Ty, 2));
12520 case NEON::BI__builtin_neon_vget_lane_f32:
12522 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
FloatTy, 2));
12525 case NEON::BI__builtin_neon_vget_lane_f64:
12527 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
DoubleTy, 1));
12530 case NEON::BI__builtin_neon_vgetq_lane_f32:
12531 case NEON::BI__builtin_neon_vdups_laneq_f32:
12533 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
FloatTy, 4));
12536 case NEON::BI__builtin_neon_vgetq_lane_f64:
12537 case NEON::BI__builtin_neon_vdupd_laneq_f64:
12539 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
DoubleTy, 2));
12542 case NEON::BI__builtin_neon_vaddh_f16:
12544 return Builder.CreateFAdd(Ops[0], Ops[1],
"vaddh");
12545 case NEON::BI__builtin_neon_vsubh_f16:
12547 return Builder.CreateFSub(Ops[0], Ops[1],
"vsubh");
12548 case NEON::BI__builtin_neon_vmulh_f16:
12550 return Builder.CreateFMul(Ops[0], Ops[1],
"vmulh");
12551 case NEON::BI__builtin_neon_vdivh_f16:
12553 return Builder.CreateFDiv(Ops[0], Ops[1],
"vdivh");
12554 case NEON::BI__builtin_neon_vfmah_f16:
12557 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma,
HalfTy,
12559 case NEON::BI__builtin_neon_vfmsh_f16: {
12564 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma,
HalfTy,
12567 case NEON::BI__builtin_neon_vaddd_s64:
12568 case NEON::BI__builtin_neon_vaddd_u64:
12570 case NEON::BI__builtin_neon_vsubd_s64:
12571 case NEON::BI__builtin_neon_vsubd_u64:
12573 case NEON::BI__builtin_neon_vqdmlalh_s16:
12574 case NEON::BI__builtin_neon_vqdmlslh_s16: {
12578 auto *VTy = llvm::FixedVectorType::get(
Int32Ty, 4);
12580 ProductOps,
"vqdmlXl");
12581 Constant *CI = ConstantInt::get(
SizeTy, 0);
12582 Ops[1] =
Builder.CreateExtractElement(Ops[1], CI,
"lane0");
12584 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlalh_s16
12585 ? Intrinsic::aarch64_neon_sqadd
12586 : Intrinsic::aarch64_neon_sqsub;
12589 case NEON::BI__builtin_neon_vqshlud_n_s64: {
12595 case NEON::BI__builtin_neon_vqshld_n_u64:
12596 case NEON::BI__builtin_neon_vqshld_n_s64: {
12597 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vqshld_n_u64
12598 ? Intrinsic::aarch64_neon_uqshl
12599 : Intrinsic::aarch64_neon_sqshl;
12604 case NEON::BI__builtin_neon_vrshrd_n_u64:
12605 case NEON::BI__builtin_neon_vrshrd_n_s64: {
12606 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrshrd_n_u64
12607 ? Intrinsic::aarch64_neon_urshl
12608 : Intrinsic::aarch64_neon_srshl;
12610 int SV = cast<ConstantInt>(Ops[1])->getSExtValue();
12611 Ops[1] = ConstantInt::get(
Int64Ty, -SV);
12614 case NEON::BI__builtin_neon_vrsrad_n_u64:
12615 case NEON::BI__builtin_neon_vrsrad_n_s64: {
12616 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrsrad_n_u64
12617 ? Intrinsic::aarch64_neon_urshl
12618 : Intrinsic::aarch64_neon_srshl;
12622 {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)});
12625 case NEON::BI__builtin_neon_vshld_n_s64:
12626 case NEON::BI__builtin_neon_vshld_n_u64: {
12627 llvm::ConstantInt *Amt = cast<ConstantInt>(
EmitScalarExpr(
E->getArg(1)));
12629 Ops[0], ConstantInt::get(
Int64Ty, Amt->getZExtValue()),
"shld_n");
12631 case NEON::BI__builtin_neon_vshrd_n_s64: {
12632 llvm::ConstantInt *Amt = cast<ConstantInt>(
EmitScalarExpr(
E->getArg(1)));
12634 Ops[0], ConstantInt::get(
Int64Ty, std::min(
static_cast<uint64_t>(63),
12635 Amt->getZExtValue())),
12638 case NEON::BI__builtin_neon_vshrd_n_u64: {
12639 llvm::ConstantInt *Amt = cast<ConstantInt>(
EmitScalarExpr(
E->getArg(1)));
12640 uint64_t ShiftAmt = Amt->getZExtValue();
12642 if (ShiftAmt == 64)
12643 return ConstantInt::get(
Int64Ty, 0);
12644 return Builder.CreateLShr(Ops[0], ConstantInt::get(
Int64Ty, ShiftAmt),
12647 case NEON::BI__builtin_neon_vsrad_n_s64: {
12648 llvm::ConstantInt *Amt = cast<ConstantInt>(
EmitScalarExpr(
E->getArg(2)));
12650 Ops[1], ConstantInt::get(
Int64Ty, std::min(
static_cast<uint64_t>(63),
12651 Amt->getZExtValue())),
12653 return Builder.CreateAdd(Ops[0], Ops[1]);
12655 case NEON::BI__builtin_neon_vsrad_n_u64: {
12656 llvm::ConstantInt *Amt = cast<ConstantInt>(
EmitScalarExpr(
E->getArg(2)));
12657 uint64_t ShiftAmt = Amt->getZExtValue();
12660 if (ShiftAmt == 64)
12662 Ops[1] =
Builder.CreateLShr(Ops[1], ConstantInt::get(
Int64Ty, ShiftAmt),
12664 return Builder.CreateAdd(Ops[0], Ops[1]);
12666 case NEON::BI__builtin_neon_vqdmlalh_lane_s16:
12667 case NEON::BI__builtin_neon_vqdmlalh_laneq_s16:
12668 case NEON::BI__builtin_neon_vqdmlslh_lane_s16:
12669 case NEON::BI__builtin_neon_vqdmlslh_laneq_s16: {
12675 auto *VTy = llvm::FixedVectorType::get(
Int32Ty, 4);
12677 ProductOps,
"vqdmlXl");
12678 Constant *CI = ConstantInt::get(
SizeTy, 0);
12679 Ops[1] =
Builder.CreateExtractElement(Ops[1], CI,
"lane0");
12682 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlalh_lane_s16 ||
12683 BuiltinID == NEON::BI__builtin_neon_vqdmlalh_laneq_s16)
12684 ? Intrinsic::aarch64_neon_sqadd
12685 : Intrinsic::aarch64_neon_sqsub;
12688 case NEON::BI__builtin_neon_vqdmlals_s32:
12689 case NEON::BI__builtin_neon_vqdmlsls_s32: {
12691 ProductOps.push_back(Ops[1]);
12695 ProductOps,
"vqdmlXl");
12697 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlals_s32
12698 ? Intrinsic::aarch64_neon_sqadd
12699 : Intrinsic::aarch64_neon_sqsub;
12702 case NEON::BI__builtin_neon_vqdmlals_lane_s32:
12703 case NEON::BI__builtin_neon_vqdmlals_laneq_s32:
12704 case NEON::BI__builtin_neon_vqdmlsls_lane_s32:
12705 case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: {
12709 ProductOps.push_back(Ops[1]);
12710 ProductOps.push_back(Ops[2]);
12713 ProductOps,
"vqdmlXl");
12716 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlals_lane_s32 ||
12717 BuiltinID == NEON::BI__builtin_neon_vqdmlals_laneq_s32)
12718 ? Intrinsic::aarch64_neon_sqadd
12719 : Intrinsic::aarch64_neon_sqsub;
12722 case NEON::BI__builtin_neon_vget_lane_bf16:
12723 case NEON::BI__builtin_neon_vduph_lane_bf16:
12724 case NEON::BI__builtin_neon_vduph_lane_f16: {
12728 case NEON::BI__builtin_neon_vgetq_lane_bf16:
12729 case NEON::BI__builtin_neon_vduph_laneq_bf16:
12730 case NEON::BI__builtin_neon_vduph_laneq_f16: {
12734 case NEON::BI__builtin_neon_vcvt_bf16_f32: {
12735 llvm::Type *V4F32 = FixedVectorType::get(
Builder.getFloatTy(), 4);
12736 llvm::Type *V4BF16 = FixedVectorType::get(
Builder.getBFloatTy(), 4);
12737 return Builder.CreateFPTrunc(
Builder.CreateBitCast(Ops[0], V4F32), V4BF16);
12739 case NEON::BI__builtin_neon_vcvtq_low_bf16_f32: {
12741 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
12742 llvm::Type *V4F32 = FixedVectorType::get(
Builder.getFloatTy(), 4);
12743 llvm::Type *V4BF16 = FixedVectorType::get(
Builder.getBFloatTy(), 4);
12744 llvm::Value *Trunc =
12745 Builder.CreateFPTrunc(
Builder.CreateBitCast(Ops[0], V4F32), V4BF16);
12746 return Builder.CreateShuffleVector(
12747 Trunc, ConstantAggregateZero::get(V4BF16), ConcatMask);
12749 case NEON::BI__builtin_neon_vcvtq_high_bf16_f32: {
12751 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
12753 std::iota(LoMask.begin(), LoMask.end(), 0);
12754 llvm::Type *V4F32 = FixedVectorType::get(
Builder.getFloatTy(), 4);
12755 llvm::Type *V4BF16 = FixedVectorType::get(
Builder.getBFloatTy(), 4);
12756 llvm::Type *V8BF16 = FixedVectorType::get(
Builder.getBFloatTy(), 8);
12757 llvm::Value *Inactive =
Builder.CreateShuffleVector(
12758 Builder.CreateBitCast(Ops[0], V8BF16), LoMask);
12759 llvm::Value *Trunc =
12760 Builder.CreateFPTrunc(
Builder.CreateBitCast(Ops[1], V4F32), V4BF16);
12761 return Builder.CreateShuffleVector(Inactive, Trunc, ConcatMask);
12764 case clang::AArch64::BI_InterlockedAdd:
12765 case clang::AArch64::BI_InterlockedAdd64: {
12768 AtomicRMWInst *RMWI =
12770 llvm::AtomicOrdering::SequentiallyConsistent);
12771 return Builder.CreateAdd(RMWI, Val);
12776 llvm::Type *Ty = VTy;
12787 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
12788 Builtin->NameHint, Builtin->TypeModifier,
E, Ops,
12795 bool ExtractLow =
false;
12796 bool ExtendLaneArg =
false;
12797 switch (BuiltinID) {
12798 default:
return nullptr;
12799 case NEON::BI__builtin_neon_vbsl_v:
12800 case NEON::BI__builtin_neon_vbslq_v: {
12801 llvm::Type *BitTy = llvm::VectorType::getInteger(VTy);
12802 Ops[0] =
Builder.CreateBitCast(Ops[0], BitTy,
"vbsl");
12803 Ops[1] =
Builder.CreateBitCast(Ops[1], BitTy,
"vbsl");
12804 Ops[2] =
Builder.CreateBitCast(Ops[2], BitTy,
"vbsl");
12806 Ops[1] =
Builder.CreateAnd(Ops[0], Ops[1],
"vbsl");
12807 Ops[2] =
Builder.CreateAnd(
Builder.CreateNot(Ops[0]), Ops[2],
"vbsl");
12808 Ops[0] =
Builder.CreateOr(Ops[1], Ops[2],
"vbsl");
12809 return Builder.CreateBitCast(Ops[0], Ty);
12811 case NEON::BI__builtin_neon_vfma_lane_v:
12812 case NEON::BI__builtin_neon_vfmaq_lane_v: {
12815 Value *Addend = Ops[0];
12816 Value *Multiplicand = Ops[1];
12817 Value *LaneSource = Ops[2];
12818 Ops[0] = Multiplicand;
12819 Ops[1] = LaneSource;
12823 auto *SourceTy = BuiltinID == NEON::BI__builtin_neon_vfmaq_lane_v
12824 ? llvm::FixedVectorType::get(VTy->getElementType(),
12825 VTy->getNumElements() / 2)
12827 llvm::Constant *cst = cast<Constant>(Ops[3]);
12828 Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(), cst);
12829 Ops[1] =
Builder.CreateBitCast(Ops[1], SourceTy);
12830 Ops[1] =
Builder.CreateShuffleVector(Ops[1], Ops[1], SV,
"lane");
12833 Int =
Builder.getIsFPConstrained() ? Intrinsic::experimental_constrained_fma
12837 case NEON::BI__builtin_neon_vfma_laneq_v: {
12838 auto *VTy = cast<llvm::FixedVectorType>(Ty);
12840 if (VTy && VTy->getElementType() ==
DoubleTy) {
12843 llvm::FixedVectorType *VTy =
12845 Ops[2] =
Builder.CreateBitCast(Ops[2], VTy);
12846 Ops[2] =
Builder.CreateExtractElement(Ops[2], Ops[3],
"extract");
12849 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma,
12850 DoubleTy, {Ops[1], Ops[2], Ops[0]});
12853 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
12854 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
12856 auto *STy = llvm::FixedVectorType::get(VTy->getElementType(),
12857 VTy->getNumElements() * 2);
12858 Ops[2] =
Builder.CreateBitCast(Ops[2], STy);
12859 Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(),
12860 cast<ConstantInt>(Ops[3]));
12861 Ops[2] =
Builder.CreateShuffleVector(Ops[2], Ops[2], SV,
"lane");
12864 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
12865 {Ops[2], Ops[1], Ops[0]});
12867 case NEON::BI__builtin_neon_vfmaq_laneq_v: {
12868 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
12869 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
12871 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
12874 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
12875 {Ops[2], Ops[1], Ops[0]});
12877 case NEON::BI__builtin_neon_vfmah_lane_f16:
12878 case NEON::BI__builtin_neon_vfmas_lane_f32:
12879 case NEON::BI__builtin_neon_vfmah_laneq_f16:
12880 case NEON::BI__builtin_neon_vfmas_laneq_f32:
12881 case NEON::BI__builtin_neon_vfmad_lane_f64:
12882 case NEON::BI__builtin_neon_vfmad_laneq_f64: {
12885 Ops[2] =
Builder.CreateExtractElement(Ops[2], Ops[3],
"extract");
12887 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
12888 {Ops[1], Ops[2], Ops[0]});
12890 case NEON::BI__builtin_neon_vmull_v:
12892 Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull;
12893 if (
Type.isPoly())
Int = Intrinsic::aarch64_neon_pmull;
12895 case NEON::BI__builtin_neon_vmax_v:
12896 case NEON::BI__builtin_neon_vmaxq_v:
12898 Int = usgn ? Intrinsic::aarch64_neon_umax : Intrinsic::aarch64_neon_smax;
12899 if (Ty->isFPOrFPVectorTy())
Int = Intrinsic::aarch64_neon_fmax;
12901 case NEON::BI__builtin_neon_vmaxh_f16: {
12903 Int = Intrinsic::aarch64_neon_fmax;
12906 case NEON::BI__builtin_neon_vmin_v:
12907 case NEON::BI__builtin_neon_vminq_v:
12909 Int = usgn ? Intrinsic::aarch64_neon_umin : Intrinsic::aarch64_neon_smin;
12910 if (Ty->isFPOrFPVectorTy())
Int = Intrinsic::aarch64_neon_fmin;
12912 case NEON::BI__builtin_neon_vminh_f16: {
12914 Int = Intrinsic::aarch64_neon_fmin;
12917 case NEON::BI__builtin_neon_vabd_v:
12918 case NEON::BI__builtin_neon_vabdq_v:
12920 Int = usgn ? Intrinsic::aarch64_neon_uabd : Intrinsic::aarch64_neon_sabd;
12921 if (Ty->isFPOrFPVectorTy())
Int = Intrinsic::aarch64_neon_fabd;
12923 case NEON::BI__builtin_neon_vpadal_v:
12924 case NEON::BI__builtin_neon_vpadalq_v: {
12925 unsigned ArgElts = VTy->getNumElements();
12926 llvm::IntegerType *EltTy = cast<IntegerType>(VTy->getElementType());
12927 unsigned BitWidth = EltTy->getBitWidth();
12928 auto *ArgTy = llvm::FixedVectorType::get(
12929 llvm::IntegerType::get(
getLLVMContext(), BitWidth / 2), 2 * ArgElts);
12930 llvm::Type* Tys[2] = { VTy, ArgTy };
12931 Int = usgn ? Intrinsic::aarch64_neon_uaddlp : Intrinsic::aarch64_neon_saddlp;
12933 TmpOps.push_back(Ops[1]);
12936 llvm::Value *addend =
Builder.CreateBitCast(Ops[0], tmp->getType());
12937 return Builder.CreateAdd(tmp, addend);
12939 case NEON::BI__builtin_neon_vpmin_v:
12940 case NEON::BI__builtin_neon_vpminq_v:
12942 Int = usgn ? Intrinsic::aarch64_neon_uminp : Intrinsic::aarch64_neon_sminp;
12943 if (Ty->isFPOrFPVectorTy())
Int = Intrinsic::aarch64_neon_fminp;
12945 case NEON::BI__builtin_neon_vpmax_v:
12946 case NEON::BI__builtin_neon_vpmaxq_v:
12948 Int = usgn ? Intrinsic::aarch64_neon_umaxp : Intrinsic::aarch64_neon_smaxp;
12949 if (Ty->isFPOrFPVectorTy())
Int = Intrinsic::aarch64_neon_fmaxp;
12951 case NEON::BI__builtin_neon_vminnm_v:
12952 case NEON::BI__builtin_neon_vminnmq_v:
12953 Int = Intrinsic::aarch64_neon_fminnm;
12955 case NEON::BI__builtin_neon_vminnmh_f16:
12957 Int = Intrinsic::aarch64_neon_fminnm;
12959 case NEON::BI__builtin_neon_vmaxnm_v:
12960 case NEON::BI__builtin_neon_vmaxnmq_v:
12961 Int = Intrinsic::aarch64_neon_fmaxnm;
12963 case NEON::BI__builtin_neon_vmaxnmh_f16:
12965 Int = Intrinsic::aarch64_neon_fmaxnm;
12967 case NEON::BI__builtin_neon_vrecpss_f32: {
12972 case NEON::BI__builtin_neon_vrecpsd_f64:
12976 case NEON::BI__builtin_neon_vrecpsh_f16:
12980 case NEON::BI__builtin_neon_vqshrun_n_v:
12981 Int = Intrinsic::aarch64_neon_sqshrun;
12983 case NEON::BI__builtin_neon_vqrshrun_n_v:
12984 Int = Intrinsic::aarch64_neon_sqrshrun;
12986 case NEON::BI__builtin_neon_vqshrn_n_v:
12987 Int = usgn ? Intrinsic::aarch64_neon_uqshrn : Intrinsic::aarch64_neon_sqshrn;
12989 case NEON::BI__builtin_neon_vrshrn_n_v:
12990 Int = Intrinsic::aarch64_neon_rshrn;
12992 case NEON::BI__builtin_neon_vqrshrn_n_v:
12993 Int = usgn ? Intrinsic::aarch64_neon_uqrshrn : Intrinsic::aarch64_neon_sqrshrn;
12995 case NEON::BI__builtin_neon_vrndah_f16: {
12998 ? Intrinsic::experimental_constrained_round
12999 : Intrinsic::round;
13002 case NEON::BI__builtin_neon_vrnda_v:
13003 case NEON::BI__builtin_neon_vrndaq_v: {
13005 ? Intrinsic::experimental_constrained_round
13006 : Intrinsic::round;
13009 case NEON::BI__builtin_neon_vrndih_f16: {
13012 ? Intrinsic::experimental_constrained_nearbyint
13013 : Intrinsic::nearbyint;
13016 case NEON::BI__builtin_neon_vrndmh_f16: {
13019 ? Intrinsic::experimental_constrained_floor
13020 : Intrinsic::floor;
13023 case NEON::BI__builtin_neon_vrndm_v:
13024 case NEON::BI__builtin_neon_vrndmq_v: {
13026 ? Intrinsic::experimental_constrained_floor
13027 : Intrinsic::floor;
13030 case NEON::BI__builtin_neon_vrndnh_f16: {
13033 ? Intrinsic::experimental_constrained_roundeven
13034 : Intrinsic::roundeven;
13037 case NEON::BI__builtin_neon_vrndn_v:
13038 case NEON::BI__builtin_neon_vrndnq_v: {
13040 ? Intrinsic::experimental_constrained_roundeven
13041 : Intrinsic::roundeven;
13044 case NEON::BI__builtin_neon_vrndns_f32: {
13047 ? Intrinsic::experimental_constrained_roundeven
13048 : Intrinsic::roundeven;
13051 case NEON::BI__builtin_neon_vrndph_f16: {
13054 ? Intrinsic::experimental_constrained_ceil
13058 case NEON::BI__builtin_neon_vrndp_v:
13059 case NEON::BI__builtin_neon_vrndpq_v: {
13061 ? Intrinsic::experimental_constrained_ceil
13065 case NEON::BI__builtin_neon_vrndxh_f16: {
13068 ? Intrinsic::experimental_constrained_rint
13072 case NEON::BI__builtin_neon_vrndx_v:
13073 case NEON::BI__builtin_neon_vrndxq_v: {
13075 ? Intrinsic::experimental_constrained_rint
13079 case NEON::BI__builtin_neon_vrndh_f16: {
13082 ? Intrinsic::experimental_constrained_trunc
13083 : Intrinsic::trunc;
13086 case NEON::BI__builtin_neon_vrnd32x_f32:
13087 case NEON::BI__builtin_neon_vrnd32xq_f32:
13088 case NEON::BI__builtin_neon_vrnd32x_f64:
13089 case NEON::BI__builtin_neon_vrnd32xq_f64: {
13091 Int = Intrinsic::aarch64_neon_frint32x;
13094 case NEON::BI__builtin_neon_vrnd32z_f32:
13095 case NEON::BI__builtin_neon_vrnd32zq_f32:
13096 case NEON::BI__builtin_neon_vrnd32z_f64:
13097 case NEON::BI__builtin_neon_vrnd32zq_f64: {
13099 Int = Intrinsic::aarch64_neon_frint32z;
13102 case NEON::BI__builtin_neon_vrnd64x_f32:
13103 case NEON::BI__builtin_neon_vrnd64xq_f32:
13104 case NEON::BI__builtin_neon_vrnd64x_f64:
13105 case NEON::BI__builtin_neon_vrnd64xq_f64: {
13107 Int = Intrinsic::aarch64_neon_frint64x;
13110 case NEON::BI__builtin_neon_vrnd64z_f32:
13111 case NEON::BI__builtin_neon_vrnd64zq_f32:
13112 case NEON::BI__builtin_neon_vrnd64z_f64:
13113 case NEON::BI__builtin_neon_vrnd64zq_f64: {
13115 Int = Intrinsic::aarch64_neon_frint64z;
13118 case NEON::BI__builtin_neon_vrnd_v:
13119 case NEON::BI__builtin_neon_vrndq_v: {
13121 ? Intrinsic::experimental_constrained_trunc
13122 : Intrinsic::trunc;
13125 case NEON::BI__builtin_neon_vcvt_f64_v:
13126 case NEON::BI__builtin_neon_vcvtq_f64_v:
13127 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
13129 return usgn ?
Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
13130 :
Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
13131 case NEON::BI__builtin_neon_vcvt_f64_f32: {
13133 "unexpected vcvt_f64_f32 builtin");
13137 return Builder.CreateFPExt(Ops[0], Ty,
"vcvt");
13139 case NEON::BI__builtin_neon_vcvt_f32_f64: {
13141 "unexpected vcvt_f32_f64 builtin");
13145 return Builder.CreateFPTrunc(Ops[0], Ty,
"vcvt");
13147 case NEON::BI__builtin_neon_vcvt_s32_v:
13148 case NEON::BI__builtin_neon_vcvt_u32_v:
13149 case NEON::BI__builtin_neon_vcvt_s64_v:
13150 case NEON::BI__builtin_neon_vcvt_u64_v:
13151 case NEON::BI__builtin_neon_vcvt_s16_f16:
13152 case NEON::BI__builtin_neon_vcvt_u16_f16:
13153 case NEON::BI__builtin_neon_vcvtq_s32_v:
13154 case NEON::BI__builtin_neon_vcvtq_u32_v:
13155 case NEON::BI__builtin_neon_vcvtq_s64_v:
13156 case NEON::BI__builtin_neon_vcvtq_u64_v:
13157 case NEON::BI__builtin_neon_vcvtq_s16_f16:
13158 case NEON::BI__builtin_neon_vcvtq_u16_f16: {
13160 usgn ? Intrinsic::aarch64_neon_fcvtzu : Intrinsic::aarch64_neon_fcvtzs;
13164 case NEON::BI__builtin_neon_vcvta_s16_f16:
13165 case NEON::BI__builtin_neon_vcvta_u16_f16:
13166 case NEON::BI__builtin_neon_vcvta_s32_v:
13167 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
13168 case NEON::BI__builtin_neon_vcvtaq_s32_v:
13169 case NEON::BI__builtin_neon_vcvta_u32_v:
13170 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
13171 case NEON::BI__builtin_neon_vcvtaq_u32_v:
13172 case NEON::BI__builtin_neon_vcvta_s64_v:
13173 case NEON::BI__builtin_neon_vcvtaq_s64_v:
13174 case NEON::BI__builtin_neon_vcvta_u64_v:
13175 case NEON::BI__builtin_neon_vcvtaq_u64_v: {
13176 Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas;
13180 case NEON::BI__builtin_neon_vcvtm_s16_f16:
13181 case NEON::BI__builtin_neon_vcvtm_s32_v:
13182 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
13183 case NEON::BI__builtin_neon_vcvtmq_s32_v:
13184 case NEON::BI__builtin_neon_vcvtm_u16_f16:
13185 case NEON::BI__builtin_neon_vcvtm_u32_v:
13186 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
13187 case NEON::BI__builtin_neon_vcvtmq_u32_v:
13188 case NEON::BI__builtin_neon_vcvtm_s64_v:
13189 case NEON::BI__builtin_neon_vcvtmq_s64_v:
13190 case NEON::BI__builtin_neon_vcvtm_u64_v:
13191 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
13192 Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms;
13196 case NEON::BI__builtin_neon_vcvtn_s16_f16:
13197 case NEON::BI__builtin_neon_vcvtn_s32_v:
13198 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
13199 case NEON::BI__builtin_neon_vcvtnq_s32_v:
13200 case NEON::BI__builtin_neon_vcvtn_u16_f16:
13201 case NEON::BI__builtin_neon_vcvtn_u32_v:
13202 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
13203 case NEON::BI__builtin_neon_vcvtnq_u32_v:
13204 case NEON::BI__builtin_neon_vcvtn_s64_v:
13205 case NEON::BI__builtin_neon_vcvtnq_s64_v:
13206 case NEON::BI__builtin_neon_vcvtn_u64_v:
13207 case NEON::BI__builtin_neon_vcvtnq_u64_v: {
13208 Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns;
13212 case NEON::BI__builtin_neon_vcvtp_s16_f16:
13213 case NEON::BI__builtin_neon_vcvtp_s32_v:
13214 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
13215 case NEON::BI__builtin_neon_vcvtpq_s32_v:
13216 case NEON::BI__builtin_neon_vcvtp_u16_f16:
13217 case NEON::BI__builtin_neon_vcvtp_u32_v:
13218 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
13219 case NEON::BI__builtin_neon_vcvtpq_u32_v:
13220 case NEON::BI__builtin_neon_vcvtp_s64_v:
13221 case NEON::BI__builtin_neon_vcvtpq_s64_v:
13222 case NEON::BI__builtin_neon_vcvtp_u64_v:
13223 case NEON::BI__builtin_neon_vcvtpq_u64_v: {
13224 Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps;
13228 case NEON::BI__builtin_neon_vmulx_v:
13229 case NEON::BI__builtin_neon_vmulxq_v: {
13230 Int = Intrinsic::aarch64_neon_fmulx;
13233 case NEON::BI__builtin_neon_vmulxh_lane_f16:
13234 case NEON::BI__builtin_neon_vmulxh_laneq_f16: {
13238 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2],
"extract");
13240 Int = Intrinsic::aarch64_neon_fmulx;
13243 case NEON::BI__builtin_neon_vmul_lane_v:
13244 case NEON::BI__builtin_neon_vmul_laneq_v: {
13247 if (BuiltinID == NEON::BI__builtin_neon_vmul_laneq_v)
13250 llvm::FixedVectorType *VTy =
13252 Ops[1] =
Builder.CreateBitCast(Ops[1], VTy);
13253 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2],
"extract");
13257 case NEON::BI__builtin_neon_vnegd_s64:
13259 case NEON::BI__builtin_neon_vnegh_f16:
13261 case NEON::BI__builtin_neon_vpmaxnm_v:
13262 case NEON::BI__builtin_neon_vpmaxnmq_v: {
13263 Int = Intrinsic::aarch64_neon_fmaxnmp;
13266 case NEON::BI__builtin_neon_vpminnm_v:
13267 case NEON::BI__builtin_neon_vpminnmq_v: {
13268 Int = Intrinsic::aarch64_neon_fminnmp;
13271 case NEON::BI__builtin_neon_vsqrth_f16: {
13274 ? Intrinsic::experimental_constrained_sqrt
13278 case NEON::BI__builtin_neon_vsqrt_v:
13279 case NEON::BI__builtin_neon_vsqrtq_v: {
13281 ? Intrinsic::experimental_constrained_sqrt
13283 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
13286 case NEON::BI__builtin_neon_vrbit_v:
13287 case NEON::BI__builtin_neon_vrbitq_v: {
13288 Int = Intrinsic::bitreverse;
13291 case NEON::BI__builtin_neon_vaddv_u8:
13295 case NEON::BI__builtin_neon_vaddv_s8: {
13296 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
13298 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
13299 llvm::Type *Tys[2] = { Ty, VTy };
13304 case NEON::BI__builtin_neon_vaddv_u16:
13307 case NEON::BI__builtin_neon_vaddv_s16: {
13308 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
13310 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
13311 llvm::Type *Tys[2] = { Ty, VTy };
13316 case NEON::BI__builtin_neon_vaddvq_u8:
13319 case NEON::BI__builtin_neon_vaddvq_s8: {
13320 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
13322 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
13323 llvm::Type *Tys[2] = { Ty, VTy };
13328 case NEON::BI__builtin_neon_vaddvq_u16:
13331 case NEON::BI__builtin_neon_vaddvq_s16: {
13332 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
13334 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
13335 llvm::Type *Tys[2] = { Ty, VTy };
13340 case NEON::BI__builtin_neon_vmaxv_u8: {
13341 Int = Intrinsic::aarch64_neon_umaxv;
13343 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
13344 llvm::Type *Tys[2] = { Ty, VTy };
13349 case NEON::BI__builtin_neon_vmaxv_u16: {
13350 Int = Intrinsic::aarch64_neon_umaxv;
13352 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
13353 llvm::Type *Tys[2] = { Ty, VTy };
13358 case NEON::BI__builtin_neon_vmaxvq_u8: {
13359 Int = Intrinsic::aarch64_neon_umaxv;
13361 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
13362 llvm::Type *Tys[2] = { Ty, VTy };
13367 case NEON::BI__builtin_neon_vmaxvq_u16: {
13368 Int = Intrinsic::aarch64_neon_umaxv;
13370 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
13371 llvm::Type *Tys[2] = { Ty, VTy };
13376 case NEON::BI__builtin_neon_vmaxv_s8: {
13377 Int = Intrinsic::aarch64_neon_smaxv;
13379 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
13380 llvm::Type *Tys[2] = { Ty, VTy };
13385 case NEON::BI__builtin_neon_vmaxv_s16: {
13386 Int = Intrinsic::aarch64_neon_smaxv;
13388 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
13389 llvm::Type *Tys[2] = { Ty, VTy };
13394 case NEON::BI__builtin_neon_vmaxvq_s8: {
13395 Int = Intrinsic::aarch64_neon_smaxv;
13397 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
13398 llvm::Type *Tys[2] = { Ty, VTy };
13403 case NEON::BI__builtin_neon_vmaxvq_s16: {
13404 Int = Intrinsic::aarch64_neon_smaxv;
13406 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
13407 llvm::Type *Tys[2] = { Ty, VTy };
13412 case NEON::BI__builtin_neon_vmaxv_f16: {
13413 Int = Intrinsic::aarch64_neon_fmaxv;
13415 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
13416 llvm::Type *Tys[2] = { Ty, VTy };
13421 case NEON::BI__builtin_neon_vmaxvq_f16: {
13422 Int = Intrinsic::aarch64_neon_fmaxv;
13424 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
13425 llvm::Type *Tys[2] = { Ty, VTy };
13430 case NEON::BI__builtin_neon_vminv_u8: {
13431 Int = Intrinsic::aarch64_neon_uminv;
13433 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
13434 llvm::Type *Tys[2] = { Ty, VTy };
13439 case NEON::BI__builtin_neon_vminv_u16: {
13440 Int = Intrinsic::aarch64_neon_uminv;
13442 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
13443 llvm::Type *Tys[2] = { Ty, VTy };
13448 case NEON::BI__builtin_neon_vminvq_u8: {
13449 Int = Intrinsic::aarch64_neon_uminv;
13451 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
13452 llvm::Type *Tys[2] = { Ty, VTy };
13457 case NEON::BI__builtin_neon_vminvq_u16: {
13458 Int = Intrinsic::aarch64_neon_uminv;
13460 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
13461 llvm::Type *Tys[2] = { Ty, VTy };
13466 case NEON::BI__builtin_neon_vminv_s8: {
13467 Int = Intrinsic::aarch64_neon_sminv;
13469 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
13470 llvm::Type *Tys[2] = { Ty, VTy };
13475 case NEON::BI__builtin_neon_vminv_s16: {
13476 Int = Intrinsic::aarch64_neon_sminv;
13478 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
13479 llvm::Type *Tys[2] = { Ty, VTy };
13484 case NEON::BI__builtin_neon_vminvq_s8: {
13485 Int = Intrinsic::aarch64_neon_sminv;
13487 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
13488 llvm::Type *Tys[2] = { Ty, VTy };
13493 case NEON::BI__builtin_neon_vminvq_s16: {
13494 Int = Intrinsic::aarch64_neon_sminv;
13496 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
13497 llvm::Type *Tys[2] = { Ty, VTy };
13502 case NEON::BI__builtin_neon_vminv_f16: {
13503 Int = Intrinsic::aarch64_neon_fminv;
13505 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
13506 llvm::Type *Tys[2] = { Ty, VTy };
13511 case NEON::BI__builtin_neon_vminvq_f16: {
13512 Int = Intrinsic::aarch64_neon_fminv;
13514 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
13515 llvm::Type *Tys[2] = { Ty, VTy };
13520 case NEON::BI__builtin_neon_vmaxnmv_f16: {
13521 Int = Intrinsic::aarch64_neon_fmaxnmv;
13523 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
13524 llvm::Type *Tys[2] = { Ty, VTy };
13529 case NEON::BI__builtin_neon_vmaxnmvq_f16: {
13530 Int = Intrinsic::aarch64_neon_fmaxnmv;
13532 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
13533 llvm::Type *Tys[2] = { Ty, VTy };
13538 case NEON::BI__builtin_neon_vminnmv_f16: {
13539 Int = Intrinsic::aarch64_neon_fminnmv;
13541 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
13542 llvm::Type *Tys[2] = { Ty, VTy };
13547 case NEON::BI__builtin_neon_vminnmvq_f16: {
13548 Int = Intrinsic::aarch64_neon_fminnmv;
13550 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
13551 llvm::Type *Tys[2] = { Ty, VTy };
13556 case NEON::BI__builtin_neon_vmul_n_f64: {
13559 return Builder.CreateFMul(Ops[0], RHS);
13561 case NEON::BI__builtin_neon_vaddlv_u8: {
13562 Int = Intrinsic::aarch64_neon_uaddlv;
13564 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
13565 llvm::Type *Tys[2] = { Ty, VTy };
13570 case NEON::BI__builtin_neon_vaddlv_u16: {
13571 Int = Intrinsic::aarch64_neon_uaddlv;
13573 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
13574 llvm::Type *Tys[2] = { Ty, VTy };
13578 case NEON::BI__builtin_neon_vaddlvq_u8: {
13579 Int = Intrinsic::aarch64_neon_uaddlv;
13581 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
13582 llvm::Type *Tys[2] = { Ty, VTy };
13587 case NEON::BI__builtin_neon_vaddlvq_u16: {
13588 Int = Intrinsic::aarch64_neon_uaddlv;
13590 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
13591 llvm::Type *Tys[2] = { Ty, VTy };
13595 case NEON::BI__builtin_neon_vaddlv_s8: {
13596 Int = Intrinsic::aarch64_neon_saddlv;
13598 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
13599 llvm::Type *Tys[2] = { Ty, VTy };
13604 case NEON::BI__builtin_neon_vaddlv_s16: {
13605 Int = Intrinsic::aarch64_neon_saddlv;
13607 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
13608 llvm::Type *Tys[2] = { Ty, VTy };
13612 case NEON::BI__builtin_neon_vaddlvq_s8: {
13613 Int = Intrinsic::aarch64_neon_saddlv;
13615 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
13616 llvm::Type *Tys[2] = { Ty, VTy };
13621 case NEON::BI__builtin_neon_vaddlvq_s16: {
13622 Int = Intrinsic::aarch64_neon_saddlv;
13624 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
13625 llvm::Type *Tys[2] = { Ty, VTy };
13629 case NEON::BI__builtin_neon_vsri_n_v:
13630 case NEON::BI__builtin_neon_vsriq_n_v: {
13631 Int = Intrinsic::aarch64_neon_vsri;
13635 case NEON::BI__builtin_neon_vsli_n_v:
13636 case NEON::BI__builtin_neon_vsliq_n_v: {
13637 Int = Intrinsic::aarch64_neon_vsli;
13641 case NEON::BI__builtin_neon_vsra_n_v:
13642 case NEON::BI__builtin_neon_vsraq_n_v:
13643 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
13645 return Builder.CreateAdd(Ops[0], Ops[1]);
13646 case NEON::BI__builtin_neon_vrsra_n_v:
13647 case NEON::BI__builtin_neon_vrsraq_n_v: {
13648 Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl;
13650 TmpOps.push_back(Ops[1]);
13651 TmpOps.push_back(Ops[2]);
13653 llvm::Value *tmp =
EmitNeonCall(F, TmpOps,
"vrshr_n", 1,
true);
13654 Ops[0] =
Builder.CreateBitCast(Ops[0], VTy);
13655 return Builder.CreateAdd(Ops[0], tmp);
13657 case NEON::BI__builtin_neon_vld1_v:
13658 case NEON::BI__builtin_neon_vld1q_v: {
13661 case NEON::BI__builtin_neon_vst1_v:
13662 case NEON::BI__builtin_neon_vst1q_v:
13663 Ops[1] =
Builder.CreateBitCast(Ops[1], VTy);
13665 case NEON::BI__builtin_neon_vld1_lane_v:
13666 case NEON::BI__builtin_neon_vld1q_lane_v: {
13667 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13670 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vld1_lane");
13672 case NEON::BI__builtin_neon_vldap1_lane_s64:
13673 case NEON::BI__builtin_neon_vldap1q_lane_s64: {
13674 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13676 VTy->getElementType(), Ops[0], PtrOp0.
getAlignment());
13677 LI->setAtomic(llvm::AtomicOrdering::Acquire);
13679 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vldap1_lane");
13681 case NEON::BI__builtin_neon_vld1_dup_v:
13682 case NEON::BI__builtin_neon_vld1q_dup_v: {
13683 Value *
V = PoisonValue::get(Ty);
13686 llvm::Constant *CI = ConstantInt::get(
Int32Ty, 0);
13687 Ops[0] =
Builder.CreateInsertElement(
V, Ops[0], CI);
13690 case NEON::BI__builtin_neon_vst1_lane_v:
13691 case NEON::BI__builtin_neon_vst1q_lane_v:
13692 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13693 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2]);
13695 case NEON::BI__builtin_neon_vstl1_lane_s64:
13696 case NEON::BI__builtin_neon_vstl1q_lane_s64: {
13697 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13698 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2]);
13699 llvm::StoreInst *SI =
13701 SI->setAtomic(llvm::AtomicOrdering::Release);
13704 case NEON::BI__builtin_neon_vld2_v:
13705 case NEON::BI__builtin_neon_vld2q_v: {
13708 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld2");
13711 case NEON::BI__builtin_neon_vld3_v:
13712 case NEON::BI__builtin_neon_vld3q_v: {
13715 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld3");
13718 case NEON::BI__builtin_neon_vld4_v:
13719 case NEON::BI__builtin_neon_vld4q_v: {
13722 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld4");
13725 case NEON::BI__builtin_neon_vld2_dup_v:
13726 case NEON::BI__builtin_neon_vld2q_dup_v: {
13729 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld2");
13732 case NEON::BI__builtin_neon_vld3_dup_v:
13733 case NEON::BI__builtin_neon_vld3q_dup_v: {
13736 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld3");
13739 case NEON::BI__builtin_neon_vld4_dup_v:
13740 case NEON::BI__builtin_neon_vld4q_dup_v: {
13743 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld4");
13746 case NEON::BI__builtin_neon_vld2_lane_v:
13747 case NEON::BI__builtin_neon_vld2q_lane_v: {
13748 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
13750 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
13751 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13752 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13757 case NEON::BI__builtin_neon_vld3_lane_v:
13758 case NEON::BI__builtin_neon_vld3q_lane_v: {
13759 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
13761 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
13762 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13763 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13764 Ops[3] =
Builder.CreateBitCast(Ops[3], Ty);
13769 case NEON::BI__builtin_neon_vld4_lane_v:
13770 case NEON::BI__builtin_neon_vld4q_lane_v: {
13771 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
13773 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
13774 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13775 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13776 Ops[3] =
Builder.CreateBitCast(Ops[3], Ty);
13777 Ops[4] =
Builder.CreateBitCast(Ops[4], Ty);
13782 case NEON::BI__builtin_neon_vst2_v:
13783 case NEON::BI__builtin_neon_vst2q_v: {
13784 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13785 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() };
13789 case NEON::BI__builtin_neon_vst2_lane_v:
13790 case NEON::BI__builtin_neon_vst2q_lane_v: {
13791 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13793 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
13797 case NEON::BI__builtin_neon_vst3_v:
13798 case NEON::BI__builtin_neon_vst3q_v: {
13799 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13800 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
13804 case NEON::BI__builtin_neon_vst3_lane_v:
13805 case NEON::BI__builtin_neon_vst3q_lane_v: {
13806 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13808 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
13812 case NEON::BI__builtin_neon_vst4_v:
13813 case NEON::BI__builtin_neon_vst4q_v: {
13814 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13815 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
13819 case NEON::BI__builtin_neon_vst4_lane_v:
13820 case NEON::BI__builtin_neon_vst4q_lane_v: {
13821 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13823 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() };
13827 case NEON::BI__builtin_neon_vtrn_v:
13828 case NEON::BI__builtin_neon_vtrnq_v: {
13829 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13830 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13831 Value *SV =
nullptr;
13833 for (
unsigned vi = 0; vi != 2; ++vi) {
13835 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
13836 Indices.push_back(i+vi);
13837 Indices.push_back(i+e+vi);
13839 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
13840 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vtrn");
13845 case NEON::BI__builtin_neon_vuzp_v:
13846 case NEON::BI__builtin_neon_vuzpq_v: {
13847 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13848 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13849 Value *SV =
nullptr;
13851 for (
unsigned vi = 0; vi != 2; ++vi) {
13853 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
13854 Indices.push_back(2*i+vi);
13856 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
13857 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vuzp");
13862 case NEON::BI__builtin_neon_vzip_v:
13863 case NEON::BI__builtin_neon_vzipq_v: {
13864 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13865 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13866 Value *SV =
nullptr;
13868 for (
unsigned vi = 0; vi != 2; ++vi) {
13870 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
13871 Indices.push_back((i + vi*e) >> 1);
13872 Indices.push_back(((i + vi*e) >> 1)+e);
13874 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
13875 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vzip");
13880 case NEON::BI__builtin_neon_vqtbl1q_v: {
13884 case NEON::BI__builtin_neon_vqtbl2q_v: {
13888 case NEON::BI__builtin_neon_vqtbl3q_v: {
13892 case NEON::BI__builtin_neon_vqtbl4q_v: {
13896 case NEON::BI__builtin_neon_vqtbx1q_v: {
13900 case NEON::BI__builtin_neon_vqtbx2q_v: {
13904 case NEON::BI__builtin_neon_vqtbx3q_v: {
13908 case NEON::BI__builtin_neon_vqtbx4q_v: {
13912 case NEON::BI__builtin_neon_vsqadd_v:
13913 case NEON::BI__builtin_neon_vsqaddq_v: {
13914 Int = Intrinsic::aarch64_neon_usqadd;
13917 case NEON::BI__builtin_neon_vuqadd_v:
13918 case NEON::BI__builtin_neon_vuqaddq_v: {
13919 Int = Intrinsic::aarch64_neon_suqadd;
13923 case NEON::BI__builtin_neon_vluti2_laneq_bf16:
13924 case NEON::BI__builtin_neon_vluti2_laneq_f16:
13925 case NEON::BI__builtin_neon_vluti2_laneq_p16:
13926 case NEON::BI__builtin_neon_vluti2_laneq_p8:
13927 case NEON::BI__builtin_neon_vluti2_laneq_s16:
13928 case NEON::BI__builtin_neon_vluti2_laneq_s8:
13929 case NEON::BI__builtin_neon_vluti2_laneq_u16:
13930 case NEON::BI__builtin_neon_vluti2_laneq_u8: {
13931 Int = Intrinsic::aarch64_neon_vluti2_laneq;
13932 llvm::Type *Tys[2];
13938 case NEON::BI__builtin_neon_vluti2q_laneq_bf16:
13939 case NEON::BI__builtin_neon_vluti2q_laneq_f16:
13940 case NEON::BI__builtin_neon_vluti2q_laneq_p16:
13941 case NEON::BI__builtin_neon_vluti2q_laneq_p8:
13942 case NEON::BI__builtin_neon_vluti2q_laneq_s16:
13943 case NEON::BI__builtin_neon_vluti2q_laneq_s8:
13944 case NEON::BI__builtin_neon_vluti2q_laneq_u16:
13945 case NEON::BI__builtin_neon_vluti2q_laneq_u8: {
13946 Int = Intrinsic::aarch64_neon_vluti2_laneq;
13947 llvm::Type *Tys[2];
13953 case NEON::BI__builtin_neon_vluti2_lane_bf16:
13954 case NEON::BI__builtin_neon_vluti2_lane_f16:
13955 case NEON::BI__builtin_neon_vluti2_lane_p16:
13956 case NEON::BI__builtin_neon_vluti2_lane_p8:
13957 case NEON::BI__builtin_neon_vluti2_lane_s16:
13958 case NEON::BI__builtin_neon_vluti2_lane_s8:
13959 case NEON::BI__builtin_neon_vluti2_lane_u16:
13960 case NEON::BI__builtin_neon_vluti2_lane_u8: {
13961 Int = Intrinsic::aarch64_neon_vluti2_lane;
13962 llvm::Type *Tys[2];
13968 case NEON::BI__builtin_neon_vluti2q_lane_bf16:
13969 case NEON::BI__builtin_neon_vluti2q_lane_f16:
13970 case NEON::BI__builtin_neon_vluti2q_lane_p16:
13971 case NEON::BI__builtin_neon_vluti2q_lane_p8:
13972 case NEON::BI__builtin_neon_vluti2q_lane_s16:
13973 case NEON::BI__builtin_neon_vluti2q_lane_s8:
13974 case NEON::BI__builtin_neon_vluti2q_lane_u16:
13975 case NEON::BI__builtin_neon_vluti2q_lane_u8: {
13976 Int = Intrinsic::aarch64_neon_vluti2_lane;
13977 llvm::Type *Tys[2];
13983 case NEON::BI__builtin_neon_vluti4q_lane_p8:
13984 case NEON::BI__builtin_neon_vluti4q_lane_s8:
13985 case NEON::BI__builtin_neon_vluti4q_lane_u8: {
13986 Int = Intrinsic::aarch64_neon_vluti4q_lane;
13989 case NEON::BI__builtin_neon_vluti4q_laneq_p8:
13990 case NEON::BI__builtin_neon_vluti4q_laneq_s8:
13991 case NEON::BI__builtin_neon_vluti4q_laneq_u8: {
13992 Int = Intrinsic::aarch64_neon_vluti4q_laneq;
13995 case NEON::BI__builtin_neon_vluti4q_lane_bf16_x2:
13996 case NEON::BI__builtin_neon_vluti4q_lane_f16_x2:
13997 case NEON::BI__builtin_neon_vluti4q_lane_p16_x2:
13998 case NEON::BI__builtin_neon_vluti4q_lane_s16_x2:
13999 case NEON::BI__builtin_neon_vluti4q_lane_u16_x2: {
14000 Int = Intrinsic::aarch64_neon_vluti4q_lane_x2;
14003 case NEON::BI__builtin_neon_vluti4q_laneq_bf16_x2:
14004 case NEON::BI__builtin_neon_vluti4q_laneq_f16_x2:
14005 case NEON::BI__builtin_neon_vluti4q_laneq_p16_x2:
14006 case NEON::BI__builtin_neon_vluti4q_laneq_s16_x2:
14007 case NEON::BI__builtin_neon_vluti4q_laneq_u16_x2: {
14008 Int = Intrinsic::aarch64_neon_vluti4q_laneq_x2;
14011 case NEON::BI__builtin_neon_vcvt1_low_bf16_mf8_fpm:
14014 case NEON::BI__builtin_neon_vcvt1_bf16_mf8_fpm:
14015 case NEON::BI__builtin_neon_vcvt1_high_bf16_mf8_fpm:
14017 llvm::FixedVectorType::get(
BFloatTy, 8),
14018 Ops[0]->getType(), ExtractLow, Ops,
E,
"vbfcvt1");
14019 case NEON::BI__builtin_neon_vcvt2_low_bf16_mf8_fpm:
14022 case NEON::BI__builtin_neon_vcvt2_bf16_mf8_fpm:
14023 case NEON::BI__builtin_neon_vcvt2_high_bf16_mf8_fpm:
14025 llvm::FixedVectorType::get(
BFloatTy, 8),
14026 Ops[0]->getType(), ExtractLow, Ops,
E,
"vbfcvt2");
14027 case NEON::BI__builtin_neon_vcvt1_low_f16_mf8_fpm:
14030 case NEON::BI__builtin_neon_vcvt1_f16_mf8_fpm:
14031 case NEON::BI__builtin_neon_vcvt1_high_f16_mf8_fpm:
14033 llvm::FixedVectorType::get(
HalfTy, 8),
14034 Ops[0]->getType(), ExtractLow, Ops,
E,
"vbfcvt1");
14035 case NEON::BI__builtin_neon_vcvt2_low_f16_mf8_fpm:
14038 case NEON::BI__builtin_neon_vcvt2_f16_mf8_fpm:
14039 case NEON::BI__builtin_neon_vcvt2_high_f16_mf8_fpm:
14041 llvm::FixedVectorType::get(
HalfTy, 8),
14042 Ops[0]->getType(), ExtractLow, Ops,
E,
"vbfcvt2");
14043 case NEON::BI__builtin_neon_vcvt_mf8_f32_fpm:
14045 llvm::FixedVectorType::get(
Int8Ty, 8),
14046 Ops[0]->getType(),
false, Ops,
E,
"vfcvtn");
14047 case NEON::BI__builtin_neon_vcvt_mf8_f16_fpm:
14049 llvm::FixedVectorType::get(
Int8Ty, 8),
14050 llvm::FixedVectorType::get(
HalfTy, 4),
false, Ops,
14052 case NEON::BI__builtin_neon_vcvtq_mf8_f16_fpm:
14054 llvm::FixedVectorType::get(
Int8Ty, 16),
14055 llvm::FixedVectorType::get(
HalfTy, 8),
false, Ops,
14057 case NEON::BI__builtin_neon_vcvt_high_mf8_f32_fpm: {
14058 llvm::Type *Ty = llvm::FixedVectorType::get(
Int8Ty, 16);
14059 Ops[0] =
Builder.CreateInsertVector(Ty, PoisonValue::get(Ty), Ops[0],
14062 Ops[1]->getType(),
false, Ops,
E,
"vfcvtn2");
14065 case NEON::BI__builtin_neon_vdot_f16_mf8_fpm:
14066 case NEON::BI__builtin_neon_vdotq_f16_mf8_fpm:
14069 case NEON::BI__builtin_neon_vdot_lane_f16_mf8_fpm:
14070 case NEON::BI__builtin_neon_vdotq_lane_f16_mf8_fpm:
14071 ExtendLaneArg =
true;
14073 case NEON::BI__builtin_neon_vdot_laneq_f16_mf8_fpm:
14074 case NEON::BI__builtin_neon_vdotq_laneq_f16_mf8_fpm:
14076 ExtendLaneArg,
HalfTy, Ops,
E,
"fdot2_lane");
14077 case NEON::BI__builtin_neon_vdot_f32_mf8_fpm:
14078 case NEON::BI__builtin_neon_vdotq_f32_mf8_fpm:
14081 case NEON::BI__builtin_neon_vdot_lane_f32_mf8_fpm:
14082 case NEON::BI__builtin_neon_vdotq_lane_f32_mf8_fpm:
14083 ExtendLaneArg =
true;
14085 case NEON::BI__builtin_neon_vdot_laneq_f32_mf8_fpm:
14086 case NEON::BI__builtin_neon_vdotq_laneq_f32_mf8_fpm:
14088 ExtendLaneArg,
FloatTy, Ops,
E,
"fdot4_lane");
14090 case NEON::BI__builtin_neon_vmlalbq_f16_mf8_fpm:
14092 {llvm::FixedVectorType::get(
HalfTy, 8)}, Ops,
E,
14094 case NEON::BI__builtin_neon_vmlaltq_f16_mf8_fpm:
14096 {llvm::FixedVectorType::get(
HalfTy, 8)}, Ops,
E,
14098 case NEON::BI__builtin_neon_vmlallbbq_f32_mf8_fpm:
14100 {llvm::FixedVectorType::get(
FloatTy, 4)}, Ops,
E,
14102 case NEON::BI__builtin_neon_vmlallbtq_f32_mf8_fpm:
14104 {llvm::FixedVectorType::get(
FloatTy, 4)}, Ops,
E,
14106 case NEON::BI__builtin_neon_vmlalltbq_f32_mf8_fpm:
14108 {llvm::FixedVectorType::get(
FloatTy, 4)}, Ops,
E,
14110 case NEON::BI__builtin_neon_vmlallttq_f32_mf8_fpm:
14112 {llvm::FixedVectorType::get(
FloatTy, 4)}, Ops,
E,
14114 case NEON::BI__builtin_neon_vmlalbq_lane_f16_mf8_fpm:
14115 ExtendLaneArg =
true;
14117 case NEON::BI__builtin_neon_vmlalbq_laneq_f16_mf8_fpm:
14119 ExtendLaneArg,
HalfTy, Ops,
E,
"vmlal_lane");
14120 case NEON::BI__builtin_neon_vmlaltq_lane_f16_mf8_fpm:
14121 ExtendLaneArg =
true;
14123 case NEON::BI__builtin_neon_vmlaltq_laneq_f16_mf8_fpm:
14125 ExtendLaneArg,
HalfTy, Ops,
E,
"vmlal_lane");
14126 case NEON::BI__builtin_neon_vmlallbbq_lane_f32_mf8_fpm:
14127 ExtendLaneArg =
true;
14129 case NEON::BI__builtin_neon_vmlallbbq_laneq_f32_mf8_fpm:
14131 ExtendLaneArg,
FloatTy, Ops,
E,
"vmlall_lane");
14132 case NEON::BI__builtin_neon_vmlallbtq_lane_f32_mf8_fpm:
14133 ExtendLaneArg =
true;
14135 case NEON::BI__builtin_neon_vmlallbtq_laneq_f32_mf8_fpm:
14137 ExtendLaneArg,
FloatTy, Ops,
E,
"vmlall_lane");
14138 case NEON::BI__builtin_neon_vmlalltbq_lane_f32_mf8_fpm:
14139 ExtendLaneArg =
true;
14141 case NEON::BI__builtin_neon_vmlalltbq_laneq_f32_mf8_fpm:
14143 ExtendLaneArg,
FloatTy, Ops,
E,
"vmlall_lane");
14144 case NEON::BI__builtin_neon_vmlallttq_lane_f32_mf8_fpm:
14145 ExtendLaneArg =
true;
14147 case NEON::BI__builtin_neon_vmlallttq_laneq_f32_mf8_fpm:
14149 ExtendLaneArg,
FloatTy, Ops,
E,
"vmlall_lane");
14150 case NEON::BI__builtin_neon_vamin_f16:
14151 case NEON::BI__builtin_neon_vaminq_f16:
14152 case NEON::BI__builtin_neon_vamin_f32:
14153 case NEON::BI__builtin_neon_vaminq_f32:
14154 case NEON::BI__builtin_neon_vaminq_f64: {
14155 Int = Intrinsic::aarch64_neon_famin;
14158 case NEON::BI__builtin_neon_vamax_f16:
14159 case NEON::BI__builtin_neon_vamaxq_f16:
14160 case NEON::BI__builtin_neon_vamax_f32:
14161 case NEON::BI__builtin_neon_vamaxq_f32:
14162 case NEON::BI__builtin_neon_vamaxq_f64: {
14163 Int = Intrinsic::aarch64_neon_famax;
14166 case NEON::BI__builtin_neon_vscale_f16:
14167 case NEON::BI__builtin_neon_vscaleq_f16:
14168 case NEON::BI__builtin_neon_vscale_f32:
14169 case NEON::BI__builtin_neon_vscaleq_f32:
14170 case NEON::BI__builtin_neon_vscaleq_f64: {
14171 Int = Intrinsic::aarch64_neon_fp8_fscale;
14179 assert((BuiltinID == BPF::BI__builtin_preserve_field_info ||
14180 BuiltinID == BPF::BI__builtin_btf_type_id ||
14181 BuiltinID == BPF::BI__builtin_preserve_type_info ||
14182 BuiltinID == BPF::BI__builtin_preserve_enum_value) &&
14183 "unexpected BPF builtin");
14190 switch (BuiltinID) {
14192 llvm_unreachable(
"Unexpected BPF builtin");
14193 case BPF::BI__builtin_preserve_field_info: {
14194 const Expr *Arg =
E->getArg(0);
14199 "using __builtin_preserve_field_info() without -g");
14212 Value *InfoKind = ConstantInt::get(
Int64Ty,
C->getSExtValue());
14215 llvm::Function *FnGetFieldInfo = llvm::Intrinsic::getOrInsertDeclaration(
14216 &
CGM.
getModule(), llvm::Intrinsic::bpf_preserve_field_info,
14217 {FieldAddr->getType()});
14218 return Builder.CreateCall(FnGetFieldInfo, {FieldAddr, InfoKind});
14220 case BPF::BI__builtin_btf_type_id:
14221 case BPF::BI__builtin_preserve_type_info: {
14227 const Expr *Arg0 =
E->getArg(0);
14232 Value *FlagValue = ConstantInt::get(
Int64Ty, Flag->getSExtValue());
14233 Value *SeqNumVal = ConstantInt::get(
Int32Ty, BuiltinSeqNum++);
14235 llvm::Function *FnDecl;
14236 if (BuiltinID == BPF::BI__builtin_btf_type_id)
14237 FnDecl = llvm::Intrinsic::getOrInsertDeclaration(
14238 &
CGM.
getModule(), llvm::Intrinsic::bpf_btf_type_id, {});
14240 FnDecl = llvm::Intrinsic::getOrInsertDeclaration(
14241 &
CGM.
getModule(), llvm::Intrinsic::bpf_preserve_type_info, {});
14242 CallInst *
Fn =
Builder.CreateCall(FnDecl, {SeqNumVal, FlagValue});
14243 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
14246 case BPF::BI__builtin_preserve_enum_value: {
14252 const Expr *Arg0 =
E->getArg(0);
14257 const auto *UO = cast<UnaryOperator>(Arg0->
IgnoreParens());
14258 const auto *CE = cast<CStyleCastExpr>(UO->getSubExpr());
14259 const auto *DR = cast<DeclRefExpr>(CE->getSubExpr());
14260 const auto *Enumerator = cast<EnumConstantDecl>(DR->getDecl());
14262 auto InitVal = Enumerator->getInitVal();
14263 std::string InitValStr;
14264 if (InitVal.isNegative() || InitVal >
uint64_t(INT64_MAX))
14265 InitValStr = std::to_string(InitVal.getSExtValue());
14267 InitValStr = std::to_string(InitVal.getZExtValue());
14268 std::string EnumStr = Enumerator->getNameAsString() +
":" + InitValStr;
14269 Value *EnumStrVal =
Builder.CreateGlobalString(EnumStr);
14272 Value *FlagValue = ConstantInt::get(
Int64Ty, Flag->getSExtValue());
14273 Value *SeqNumVal = ConstantInt::get(
Int32Ty, BuiltinSeqNum++);
14275 llvm::Function *IntrinsicFn = llvm::Intrinsic::getOrInsertDeclaration(
14276 &
CGM.
getModule(), llvm::Intrinsic::bpf_preserve_enum_value, {});
14278 Builder.CreateCall(IntrinsicFn, {SeqNumVal, EnumStrVal, FlagValue});
14279 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
14287 assert((Ops.size() & (Ops.size() - 1)) == 0 &&
14288 "Not a power-of-two sized vector!");
14289 bool AllConstants =
true;
14290 for (
unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i)
14291 AllConstants &= isa<Constant>(Ops[i]);
14294 if (AllConstants) {
14296 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
14297 CstOps.push_back(cast<Constant>(Ops[i]));
14298 return llvm::ConstantVector::get(CstOps);
14303 llvm::FixedVectorType::get(Ops[0]->getType(), Ops.size()));
14305 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
14313 unsigned NumElts) {
14315 auto *MaskTy = llvm::FixedVectorType::get(
14317 cast<IntegerType>(Mask->
getType())->getBitWidth());
14318 Value *MaskVec = CGF.
Builder.CreateBitCast(Mask, MaskTy);
14324 for (
unsigned i = 0; i != NumElts; ++i)
14326 MaskVec = CGF.
Builder.CreateShuffleVector(
14327 MaskVec, MaskVec,
ArrayRef(Indices, NumElts),
"extract");
14334 Value *Ptr = Ops[0];
14338 cast<llvm::FixedVectorType>(Ops[1]->getType())->getNumElements());
14340 return CGF.
Builder.CreateMaskedStore(Ops[1], Ptr, Alignment, MaskVec);
14345 llvm::Type *Ty = Ops[1]->getType();
14346 Value *Ptr = Ops[0];
14349 CGF, Ops[2], cast<llvm::FixedVectorType>(Ty)->getNumElements());
14351 return CGF.
Builder.CreateMaskedLoad(Ty, Ptr, Alignment, MaskVec, Ops[1]);
14356 auto *ResultTy = cast<llvm::VectorType>(Ops[1]->getType());
14357 Value *Ptr = Ops[0];
14360 CGF, Ops[2], cast<FixedVectorType>(ResultTy)->getNumElements());
14362 llvm::Function *F = CGF.
CGM.
getIntrinsic(Intrinsic::masked_expandload,
14364 return CGF.
Builder.CreateCall(F, { Ptr, MaskVec, Ops[1] });
14370 auto *ResultTy = cast<llvm::FixedVectorType>(Ops[1]->getType());
14374 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
14375 : Intrinsic::x86_avx512_mask_expand;
14377 return CGF.
Builder.CreateCall(F, { Ops[0], Ops[1], MaskVec });
14382 auto *ResultTy = cast<llvm::FixedVectorType>(Ops[1]->getType());
14383 Value *Ptr = Ops[0];
14387 llvm::Function *F = CGF.
CGM.
getIntrinsic(Intrinsic::masked_compressstore,
14389 return CGF.
Builder.CreateCall(F, { Ops[1], Ptr, MaskVec });
14394 bool InvertLHS =
false) {
14395 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
14400 LHS = CGF.
Builder.CreateNot(LHS);
14402 return CGF.
Builder.CreateBitCast(CGF.
Builder.CreateBinOp(Opc, LHS, RHS),
14403 Ops[0]->getType());
14407 Value *Amt,
bool IsRight) {
14408 llvm::Type *Ty = Op0->
getType();
14414 unsigned NumElts = cast<llvm::FixedVectorType>(Ty)->getNumElements();
14415 Amt = CGF.
Builder.CreateIntCast(Amt, Ty->getScalarType(),
false);
14416 Amt = CGF.
Builder.CreateVectorSplat(NumElts, Amt);
14419 unsigned IID = IsRight ? Intrinsic::fshr : Intrinsic::fshl;
14421 return CGF.
Builder.CreateCall(F, {Op0, Op1, Amt});
14426 Value *Op0 = Ops[0];
14427 Value *Op1 = Ops[1];
14428 llvm::Type *Ty = Op0->
getType();
14429 uint64_t Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
14431 CmpInst::Predicate Pred;
14434 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
14437 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
14440 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
14443 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
14446 Pred = ICmpInst::ICMP_EQ;
14449 Pred = ICmpInst::ICMP_NE;
14452 return llvm::Constant::getNullValue(Ty);
14454 return llvm::Constant::getAllOnesValue(Ty);
14456 llvm_unreachable(
"Unexpected XOP vpcom/vpcomu predicate");
14468 if (
const auto *
C = dyn_cast<Constant>(Mask))
14469 if (
C->isAllOnesValue())
14473 CGF, Mask, cast<llvm::FixedVectorType>(Op0->
getType())->getNumElements());
14475 return CGF.
Builder.CreateSelect(Mask, Op0, Op1);
14481 if (
const auto *
C = dyn_cast<Constant>(Mask))
14482 if (
C->isAllOnesValue())
14485 auto *MaskTy = llvm::FixedVectorType::get(
14486 CGF.
Builder.getInt1Ty(), Mask->
getType()->getIntegerBitWidth());
14487 Mask = CGF.
Builder.CreateBitCast(Mask, MaskTy);
14488 Mask = CGF.
Builder.CreateExtractElement(Mask, (uint64_t)0);
14489 return CGF.
Builder.CreateSelect(Mask, Op0, Op1);
14493 unsigned NumElts,
Value *MaskIn) {
14495 const auto *
C = dyn_cast<Constant>(MaskIn);
14496 if (!
C || !
C->isAllOnesValue())
14502 for (
unsigned i = 0; i != NumElts; ++i)
14504 for (
unsigned i = NumElts; i != 8; ++i)
14505 Indices[i] = i % NumElts + NumElts;
14506 Cmp = CGF.
Builder.CreateShuffleVector(
14507 Cmp, llvm::Constant::getNullValue(Cmp->getType()), Indices);
14510 return CGF.
Builder.CreateBitCast(Cmp,
14512 std::max(NumElts, 8U)));
14517 assert((Ops.size() == 2 || Ops.size() == 4) &&
14518 "Unexpected number of arguments");
14520 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
14524 Cmp = Constant::getNullValue(
14525 llvm::FixedVectorType::get(CGF.
Builder.getInt1Ty(), NumElts));
14526 }
else if (CC == 7) {
14527 Cmp = Constant::getAllOnesValue(
14528 llvm::FixedVectorType::get(CGF.
Builder.getInt1Ty(), NumElts));
14530 ICmpInst::Predicate Pred;
14532 default: llvm_unreachable(
"Unknown condition code");
14533 case 0: Pred = ICmpInst::ICMP_EQ;
break;
14534 case 1: Pred =
Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
break;
14535 case 2: Pred =
Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
break;
14536 case 4: Pred = ICmpInst::ICMP_NE;
break;
14537 case 5: Pred =
Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
break;
14538 case 6: Pred =
Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
break;
14540 Cmp = CGF.
Builder.CreateICmp(Pred, Ops[0], Ops[1]);
14543 Value *MaskIn =
nullptr;
14544 if (Ops.size() == 4)
14551 Value *Zero = Constant::getNullValue(In->getType());
14557 unsigned Rnd = cast<llvm::ConstantInt>(Ops[3])->getZExtValue();
14558 llvm::Type *Ty = Ops[1]->getType();
14562 Intrinsic::ID IID = IsSigned ? Intrinsic::x86_avx512_sitofp_round
14563 : Intrinsic::x86_avx512_uitofp_round;
14565 Res = CGF.
Builder.CreateCall(F, { Ops[0], Ops[3] });
14567 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
14568 Res = IsSigned ? CGF.
Builder.CreateSIToFP(Ops[0], Ty)
14569 : CGF.
Builder.CreateUIToFP(Ops[0], Ty);
14580 bool Subtract =
false;
14581 Intrinsic::ID IID = Intrinsic::not_intrinsic;
14582 switch (BuiltinID) {
14584 case clang::X86::BI__builtin_ia32_vfmsubph512_mask3:
14587 case clang::X86::BI__builtin_ia32_vfmaddph512_mask:
14588 case clang::X86::BI__builtin_ia32_vfmaddph512_maskz:
14589 case clang::X86::BI__builtin_ia32_vfmaddph512_mask3:
14590 IID = llvm::Intrinsic::x86_avx512fp16_vfmadd_ph_512;
14592 case clang::X86::BI__builtin_ia32_vfmsubaddph512_mask3:
14595 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask:
14596 case clang::X86::BI__builtin_ia32_vfmaddsubph512_maskz:
14597 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask3:
14598 IID = llvm::Intrinsic::x86_avx512fp16_vfmaddsub_ph_512;
14600 case clang::X86::BI__builtin_ia32_vfmsubps512_mask3:
14603 case clang::X86::BI__builtin_ia32_vfmaddps512_mask:
14604 case clang::X86::BI__builtin_ia32_vfmaddps512_maskz:
14605 case clang::X86::BI__builtin_ia32_vfmaddps512_mask3:
14606 IID = llvm::Intrinsic::x86_avx512_vfmadd_ps_512;
break;
14607 case clang::X86::BI__builtin_ia32_vfmsubpd512_mask3:
14610 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask:
14611 case clang::X86::BI__builtin_ia32_vfmaddpd512_maskz:
14612 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask3:
14613 IID = llvm::Intrinsic::x86_avx512_vfmadd_pd_512;
break;
14614 case clang::X86::BI__builtin_ia32_vfmsubaddps512_mask3:
14617 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask:
14618 case clang::X86::BI__builtin_ia32_vfmaddsubps512_maskz:
14619 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask3:
14620 IID = llvm::Intrinsic::x86_avx512_vfmaddsub_ps_512;
14622 case clang::X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
14625 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask:
14626 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
14627 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
14628 IID = llvm::Intrinsic::x86_avx512_vfmaddsub_pd_512;
14630 case clang::X86::BI__builtin_ia32_vfmsubph256_round_mask3:
14633 case clang::X86::BI__builtin_ia32_vfmaddph256_round_mask:
14634 case clang::X86::BI__builtin_ia32_vfmaddph256_round_maskz:
14635 case clang::X86::BI__builtin_ia32_vfmaddph256_round_mask3:
14636 IID = llvm::Intrinsic::x86_avx10_vfmaddph256;
14638 case clang::X86::BI__builtin_ia32_vfmsubaddph256_round_mask3:
14641 case clang::X86::BI__builtin_ia32_vfmaddsubph256_round_mask:
14642 case clang::X86::BI__builtin_ia32_vfmaddsubph256_round_maskz:
14643 case clang::X86::BI__builtin_ia32_vfmaddsubph256_round_mask3:
14644 IID = llvm::Intrinsic::x86_avx10_vfmaddsubph256;
14646 case clang::X86::BI__builtin_ia32_vfmsubps256_round_mask3:
14649 case clang::X86::BI__builtin_ia32_vfmaddps256_round_mask:
14650 case clang::X86::BI__builtin_ia32_vfmaddps256_round_maskz:
14651 case clang::X86::BI__builtin_ia32_vfmaddps256_round_mask3:
14652 IID = llvm::Intrinsic::x86_avx10_vfmaddps256;
14654 case clang::X86::BI__builtin_ia32_vfmsubpd256_round_mask3:
14657 case clang::X86::BI__builtin_ia32_vfmaddpd256_round_mask:
14658 case clang::X86::BI__builtin_ia32_vfmaddpd256_round_maskz:
14659 case clang::X86::BI__builtin_ia32_vfmaddpd256_round_mask3:
14660 IID = llvm::Intrinsic::x86_avx10_vfmaddpd256;
14662 case clang::X86::BI__builtin_ia32_vfmsubaddps256_round_mask3:
14665 case clang::X86::BI__builtin_ia32_vfmaddsubps256_round_mask:
14666 case clang::X86::BI__builtin_ia32_vfmaddsubps256_round_maskz:
14667 case clang::X86::BI__builtin_ia32_vfmaddsubps256_round_mask3:
14668 IID = llvm::Intrinsic::x86_avx10_vfmaddsubps256;
14670 case clang::X86::BI__builtin_ia32_vfmsubaddpd256_round_mask3:
14673 case clang::X86::BI__builtin_ia32_vfmaddsubpd256_round_mask:
14674 case clang::X86::BI__builtin_ia32_vfmaddsubpd256_round_maskz:
14675 case clang::X86::BI__builtin_ia32_vfmaddsubpd256_round_mask3:
14676 IID = llvm::Intrinsic::x86_avx10_vfmaddsubpd256;
14690 if (IID != Intrinsic::not_intrinsic &&
14691 (cast<llvm::ConstantInt>(Ops.back())->getZExtValue() != (uint64_t)4 ||
14694 Res = CGF.
Builder.CreateCall(Intr, {A, B,
C, Ops.back() });
14696 llvm::Type *Ty = A->
getType();
14698 if (CGF.
Builder.getIsFPConstrained()) {
14699 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
14700 FMA = CGF.
CGM.
getIntrinsic(Intrinsic::experimental_constrained_fma, Ty);
14701 Res = CGF.
Builder.CreateConstrainedFPCall(FMA, {A, B,
C});
14704 Res = CGF.
Builder.CreateCall(FMA, {A, B,
C});
14709 Value *MaskFalseVal =
nullptr;
14710 switch (BuiltinID) {
14711 case clang::X86::BI__builtin_ia32_vfmaddph512_mask:
14712 case clang::X86::BI__builtin_ia32_vfmaddps512_mask:
14713 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask:
14714 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask:
14715 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask:
14716 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask:
14717 case clang::X86::BI__builtin_ia32_vfmaddph256_round_mask:
14718 case clang::X86::BI__builtin_ia32_vfmaddps256_round_mask:
14719 case clang::X86::BI__builtin_ia32_vfmaddpd256_round_mask:
14720 case clang::X86::BI__builtin_ia32_vfmaddsubph256_round_mask:
14721 case clang::X86::BI__builtin_ia32_vfmaddsubps256_round_mask:
14722 case clang::X86::BI__builtin_ia32_vfmaddsubpd256_round_mask:
14723 MaskFalseVal = Ops[0];
14725 case clang::X86::BI__builtin_ia32_vfmaddph512_maskz:
14726 case clang::X86::BI__builtin_ia32_vfmaddps512_maskz:
14727 case clang::X86::BI__builtin_ia32_vfmaddpd512_maskz:
14728 case clang::X86::BI__builtin_ia32_vfmaddsubph512_maskz:
14729 case clang::X86::BI__builtin_ia32_vfmaddsubps512_maskz:
14730 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
14731 case clang::X86::BI__builtin_ia32_vfmaddph256_round_maskz:
14732 case clang::X86::BI__builtin_ia32_vfmaddps256_round_maskz:
14733 case clang::X86::BI__builtin_ia32_vfmaddpd256_round_maskz:
14734 case clang::X86::BI__builtin_ia32_vfmaddsubph256_round_maskz:
14735 case clang::X86::BI__builtin_ia32_vfmaddsubps256_round_maskz:
14736 case clang::X86::BI__builtin_ia32_vfmaddsubpd256_round_maskz:
14737 MaskFalseVal = Constant::getNullValue(Ops[0]->getType());
14739 case clang::X86::BI__builtin_ia32_vfmsubph512_mask3:
14740 case clang::X86::BI__builtin_ia32_vfmaddph512_mask3:
14741 case clang::X86::BI__builtin_ia32_vfmsubps512_mask3:
14742 case clang::X86::BI__builtin_ia32_vfmaddps512_mask3:
14743 case clang::X86::BI__builtin_ia32_vfmsubpd512_mask3:
14744 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask3:
14745 case clang::X86::BI__builtin_ia32_vfmsubaddph512_mask3:
14746 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask3:
14747 case clang::X86::BI__builtin_ia32_vfmsubaddps512_mask3:
14748 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask3:
14749 case clang::X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
14750 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
14751 case clang::X86::BI__builtin_ia32_vfmsubph256_round_mask3:
14752 case clang::X86::BI__builtin_ia32_vfmaddph256_round_mask3:
14753 case clang::X86::BI__builtin_ia32_vfmsubps256_round_mask3:
14754 case clang::X86::BI__builtin_ia32_vfmaddps256_round_mask3:
14755 case clang::X86::BI__builtin_ia32_vfmsubpd256_round_mask3:
14756 case clang::X86::BI__builtin_ia32_vfmaddpd256_round_mask3:
14757 case clang::X86::BI__builtin_ia32_vfmsubaddph256_round_mask3:
14758 case clang::X86::BI__builtin_ia32_vfmaddsubph256_round_mask3:
14759 case clang::X86::BI__builtin_ia32_vfmsubaddps256_round_mask3:
14760 case clang::X86::BI__builtin_ia32_vfmaddsubps256_round_mask3:
14761 case clang::X86::BI__builtin_ia32_vfmsubaddpd256_round_mask3:
14762 case clang::X86::BI__builtin_ia32_vfmaddsubpd256_round_mask3:
14763 MaskFalseVal = Ops[2];
14775 bool ZeroMask =
false,
unsigned PTIdx = 0,
14776 bool NegAcc =
false) {
14778 if (Ops.size() > 4)
14779 Rnd = cast<llvm::ConstantInt>(Ops[4])->getZExtValue();
14782 Ops[2] = CGF.
Builder.CreateFNeg(Ops[2]);
14784 Ops[0] = CGF.
Builder.CreateExtractElement(Ops[0], (uint64_t)0);
14785 Ops[1] = CGF.
Builder.CreateExtractElement(Ops[1], (uint64_t)0);
14786 Ops[2] = CGF.
Builder.CreateExtractElement(Ops[2], (uint64_t)0);
14791 switch (Ops[0]->getType()->getPrimitiveSizeInBits()) {
14793 IID = Intrinsic::x86_avx512fp16_vfmadd_f16;
14796 IID = Intrinsic::x86_avx512_vfmadd_f32;
14799 IID = Intrinsic::x86_avx512_vfmadd_f64;
14802 llvm_unreachable(
"Unexpected size");
14805 {Ops[0], Ops[1], Ops[2], Ops[4]});
14806 }
else if (CGF.
Builder.getIsFPConstrained()) {
14807 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
14809 Intrinsic::experimental_constrained_fma, Ops[0]->getType());
14810 Res = CGF.
Builder.CreateConstrainedFPCall(FMA, Ops.slice(0, 3));
14813 Res = CGF.
Builder.CreateCall(FMA, Ops.slice(0, 3));
14816 if (Ops.size() > 3) {
14817 Value *PassThru = ZeroMask ? Constant::getNullValue(Res->
getType())
14823 if (NegAcc && PTIdx == 2)
14824 PassThru = CGF.
Builder.CreateExtractElement(Upper, (uint64_t)0);
14828 return CGF.
Builder.CreateInsertElement(Upper, Res, (uint64_t)0);
14833 llvm::Type *Ty = Ops[0]->getType();
14835 Ty = llvm::FixedVectorType::get(CGF.
Int64Ty,
14836 Ty->getPrimitiveSizeInBits() / 64);
14842 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
14843 LHS = CGF.
Builder.CreateShl(LHS, ShiftAmt);
14844 LHS = CGF.
Builder.CreateAShr(LHS, ShiftAmt);
14845 RHS = CGF.
Builder.CreateShl(RHS, ShiftAmt);
14846 RHS = CGF.
Builder.CreateAShr(RHS, ShiftAmt);
14849 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
14850 LHS = CGF.
Builder.CreateAnd(LHS, Mask);
14851 RHS = CGF.
Builder.CreateAnd(RHS, Mask);
14854 return CGF.
Builder.CreateMul(LHS, RHS);
14862 llvm::Type *Ty = Ops[0]->getType();
14864 unsigned VecWidth = Ty->getPrimitiveSizeInBits();
14865 unsigned EltWidth = Ty->getScalarSizeInBits();
14867 if (VecWidth == 128 && EltWidth == 32)
14868 IID = Intrinsic::x86_avx512_pternlog_d_128;
14869 else if (VecWidth == 256 && EltWidth == 32)
14870 IID = Intrinsic::x86_avx512_pternlog_d_256;
14871 else if (VecWidth == 512 && EltWidth == 32)
14872 IID = Intrinsic::x86_avx512_pternlog_d_512;
14873 else if (VecWidth == 128 && EltWidth == 64)
14874 IID = Intrinsic::x86_avx512_pternlog_q_128;
14875 else if (VecWidth == 256 && EltWidth == 64)
14876 IID = Intrinsic::x86_avx512_pternlog_q_256;
14877 else if (VecWidth == 512 && EltWidth == 64)
14878 IID = Intrinsic::x86_avx512_pternlog_q_512;
14880 llvm_unreachable(
"Unexpected intrinsic");
14884 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty) : Ops[0];
14889 llvm::Type *DstTy) {
14890 unsigned NumberOfElements =
14891 cast<llvm::FixedVectorType>(DstTy)->getNumElements();
14893 return CGF.
Builder.CreateSExt(Mask, DstTy,
"vpmovm2");
14898 StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
14899 return EmitX86CpuIs(CPUStr);
14905 llvm::Type *DstTy) {
14906 assert((Ops.size() == 1 || Ops.size() == 3 || Ops.size() == 4) &&
14907 "Unknown cvtph2ps intrinsic");
14910 if (Ops.size() == 4 && cast<llvm::ConstantInt>(Ops[3])->getZExtValue() != 4) {
14913 return CGF.
Builder.CreateCall(F, {Ops[0], Ops[1], Ops[2], Ops[3]});
14916 unsigned NumDstElts = cast<llvm::FixedVectorType>(DstTy)->getNumElements();
14917 Value *Src = Ops[0];
14921 cast<llvm::FixedVectorType>(Src->
getType())->getNumElements()) {
14922 assert(NumDstElts == 4 &&
"Unexpected vector size");
14927 auto *HalfTy = llvm::FixedVectorType::get(
14929 Src = CGF.
Builder.CreateBitCast(Src, HalfTy);
14932 Value *Res = CGF.
Builder.CreateFPExt(Src, DstTy,
"cvtph2ps");
14934 if (Ops.size() >= 3)
14939Value *CodeGenFunction::EmitX86CpuIs(StringRef CPUStr) {
14950 llvm::ArrayType::get(
Int32Ty, 1));
14954 cast<llvm::GlobalValue>(CpuModel)->setDSOLocal(
true);
14960 std::tie(Index,
Value) = StringSwitch<std::pair<unsigned, unsigned>>(CPUStr)
14962 .Case(STRING, {0u,
static_cast<unsigned>(llvm::X86::ENUM)})
14964 .Case(
ALIAS, {1u,
static_cast<unsigned>(llvm::X86::ENUM)})
14966 .Case(STR, {1u,
static_cast<unsigned>(llvm::X86::ENUM)})
14968 .Case(
ALIAS, {2u,
static_cast<unsigned>(llvm::X86::ENUM)})
14970 .Case(STR, {2u,
static_cast<unsigned>(llvm::X86::ENUM)})
14971#include
"llvm/TargetParser/X86TargetParser.def"
14973 assert(
Value != 0 &&
"Invalid CPUStr passed to CpuIs");
14976 llvm::Value *Idxs[] = {ConstantInt::get(
Int32Ty, 0),
14977 ConstantInt::get(
Int32Ty, Index)};
14983 return Builder.CreateICmpEQ(CpuValue,
14989 StringRef FeatureStr = cast<StringLiteral>(FeatureExpr)->getString();
14990 if (!
getContext().getTargetInfo().validateCpuSupports(FeatureStr))
14992 return EmitX86CpuSupports(FeatureStr);
14996 return EmitX86CpuSupports(llvm::X86::getCpuSupportsMask(FeatureStrs));
15000CodeGenFunction::EmitX86CpuSupports(std::array<uint32_t, 4> FeatureMask) {
15002 if (FeatureMask[0] != 0) {
15010 llvm::ArrayType::get(
Int32Ty, 1));
15014 cast<llvm::GlobalValue>(CpuModel)->setDSOLocal(
true);
15031 llvm::Type *ATy = llvm::ArrayType::get(
Int32Ty, 3);
15032 llvm::Constant *CpuFeatures2 =
15034 cast<llvm::GlobalValue>(CpuFeatures2)->setDSOLocal(
true);
15035 for (
int i = 1; i != 4; ++i) {
15036 const uint32_t M = FeatureMask[i];
15053Value *CodeGenFunction::EmitAArch64CpuInit() {
15054 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy,
false);
15055 llvm::FunctionCallee
Func =
15057 cast<llvm::GlobalValue>(
Func.getCallee())->setDSOLocal(
true);
15058 cast<llvm::GlobalValue>(
Func.getCallee())
15059 ->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
15064 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy, {
VoidPtrTy},
false);
15065 llvm::FunctionCallee
Func =
15067 auto *CalleeGV = cast<llvm::GlobalValue>(
Func.getCallee());
15068 CalleeGV->setDSOLocal(
true);
15069 CalleeGV->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
15073Value *CodeGenFunction::EmitX86CpuInit() {
15074 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy,
15076 llvm::FunctionCallee
Func =
15078 cast<llvm::GlobalValue>(
Func.getCallee())->setDSOLocal(
true);
15079 cast<llvm::GlobalValue>(
Func.getCallee())
15080 ->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
15084Value *CodeGenFunction::EmitAArch64CpuSupports(
const CallExpr *
E) {
15086 StringRef ArgStr = cast<StringLiteral>(ArgExpr)->getString();
15088 ArgStr.split(Features,
"+");
15089 for (
auto &Feature : Features) {
15090 Feature = Feature.trim();
15091 if (!llvm::AArch64::parseFMVExtension(Feature))
15093 if (Feature !=
"default")
15094 Features.push_back(Feature);
15096 return EmitAArch64CpuSupports(Features);
15101 uint64_t FeaturesMask = llvm::AArch64::getCpuSupportsMask(FeaturesStrs);
15103 if (FeaturesMask != 0) {
15108 llvm::Type *STy = llvm::StructType::get(
Int64Ty);
15109 llvm::Constant *AArch64CPUFeatures =
15111 cast<llvm::GlobalValue>(AArch64CPUFeatures)->setDSOLocal(
true);
15113 STy, AArch64CPUFeatures,
15128 StringRef FeatureStr = cast<StringLiteral>(FeatureExpr)->getString();
15129 if (!
getContext().getTargetInfo().validateCpuSupports(FeatureStr))
15137 llvm::Type *Int32Ty = Builder.getInt32Ty();
15138 llvm::Type *Int64Ty = Builder.getInt64Ty();
15139 llvm::ArrayType *ArrayOfInt64Ty =
15140 llvm::ArrayType::get(Int64Ty, llvm::RISCVISAInfo::FeatureBitSize);
15141 llvm::Type *StructTy = llvm::StructType::get(Int32Ty, ArrayOfInt64Ty);
15142 llvm::Constant *RISCVFeaturesBits =
15144 cast<llvm::GlobalValue>(RISCVFeaturesBits)->setDSOLocal(
true);
15145 Value *IndexVal = llvm::ConstantInt::get(Int32Ty, Index);
15146 llvm::Value *GEPIndices[] = {Builder.getInt32(0), Builder.getInt32(1),
15149 Builder.CreateInBoundsGEP(StructTy, RISCVFeaturesBits, GEPIndices);
15150 Value *FeaturesBit =
15152 return FeaturesBit;
15156 const unsigned RISCVFeatureLength = llvm::RISCVISAInfo::FeatureBitSize;
15157 uint64_t RequireBitMasks[RISCVFeatureLength] = {0};
15159 for (
auto Feat : FeaturesStrs) {
15160 auto [GroupID, BitPos] = RISCVISAInfo::getRISCVFeaturesBitsInfo(Feat);
15167 RequireBitMasks[GroupID] |= (1ULL << BitPos);
15171 for (
unsigned Idx = 0; Idx < RISCVFeatureLength; Idx++) {
15172 if (RequireBitMasks[Idx] == 0)
15182 assert(
Result &&
"Should have value here.");
15189 if (BuiltinID == Builtin::BI__builtin_cpu_is)
15190 return EmitX86CpuIs(
E);
15191 if (BuiltinID == Builtin::BI__builtin_cpu_supports)
15192 return EmitX86CpuSupports(
E);
15193 if (BuiltinID == Builtin::BI__builtin_cpu_init)
15194 return EmitX86CpuInit();
15202 bool IsMaskFCmp =
false;
15203 bool IsConjFMA =
false;
15206 unsigned ICEArguments = 0;
15211 for (
unsigned i = 0, e =
E->getNumArgs(); i != e; i++) {
15221 auto getCmpIntrinsicCall = [
this, &Ops](Intrinsic::ID
ID,
unsigned Imm) {
15222 Ops.push_back(llvm::ConstantInt::get(
Int8Ty, Imm));
15224 return Builder.CreateCall(F, Ops);
15232 auto getVectorFCmpIR = [
this, &Ops,
E](CmpInst::Predicate Pred,
15233 bool IsSignaling) {
15234 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
15237 Cmp =
Builder.CreateFCmpS(Pred, Ops[0], Ops[1]);
15239 Cmp =
Builder.CreateFCmp(Pred, Ops[0], Ops[1]);
15240 llvm::VectorType *FPVecTy = cast<llvm::VectorType>(Ops[0]->getType());
15241 llvm::VectorType *IntVecTy = llvm::VectorType::getInteger(FPVecTy);
15243 return Builder.CreateBitCast(Sext, FPVecTy);
15246 switch (BuiltinID) {
15247 default:
return nullptr;
15248 case X86::BI_mm_prefetch: {
15250 ConstantInt *
C = cast<ConstantInt>(Ops[1]);
15251 Value *RW = ConstantInt::get(
Int32Ty, (
C->getZExtValue() >> 2) & 0x1);
15252 Value *Locality = ConstantInt::get(
Int32Ty,
C->getZExtValue() & 0x3);
15257 case X86::BI_mm_clflush: {
15261 case X86::BI_mm_lfence: {
15264 case X86::BI_mm_mfence: {
15267 case X86::BI_mm_sfence: {
15270 case X86::BI_mm_pause: {
15273 case X86::BI__rdtsc: {
15276 case X86::BI__builtin_ia32_rdtscp: {
15282 case X86::BI__builtin_ia32_lzcnt_u16:
15283 case X86::BI__builtin_ia32_lzcnt_u32:
15284 case X86::BI__builtin_ia32_lzcnt_u64: {
15288 case X86::BI__builtin_ia32_tzcnt_u16:
15289 case X86::BI__builtin_ia32_tzcnt_u32:
15290 case X86::BI__builtin_ia32_tzcnt_u64: {
15294 case X86::BI__builtin_ia32_undef128:
15295 case X86::BI__builtin_ia32_undef256:
15296 case X86::BI__builtin_ia32_undef512:
15303 case X86::BI__builtin_ia32_vec_ext_v4hi:
15304 case X86::BI__builtin_ia32_vec_ext_v16qi:
15305 case X86::BI__builtin_ia32_vec_ext_v8hi:
15306 case X86::BI__builtin_ia32_vec_ext_v4si:
15307 case X86::BI__builtin_ia32_vec_ext_v4sf:
15308 case X86::BI__builtin_ia32_vec_ext_v2di:
15309 case X86::BI__builtin_ia32_vec_ext_v32qi:
15310 case X86::BI__builtin_ia32_vec_ext_v16hi:
15311 case X86::BI__builtin_ia32_vec_ext_v8si:
15312 case X86::BI__builtin_ia32_vec_ext_v4di: {
15314 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15315 uint64_t Index = cast<ConstantInt>(Ops[1])->getZExtValue();
15316 Index &= NumElts - 1;
15319 return Builder.CreateExtractElement(Ops[0], Index);
15321 case X86::BI__builtin_ia32_vec_set_v4hi:
15322 case X86::BI__builtin_ia32_vec_set_v16qi:
15323 case X86::BI__builtin_ia32_vec_set_v8hi:
15324 case X86::BI__builtin_ia32_vec_set_v4si:
15325 case X86::BI__builtin_ia32_vec_set_v2di:
15326 case X86::BI__builtin_ia32_vec_set_v32qi:
15327 case X86::BI__builtin_ia32_vec_set_v16hi:
15328 case X86::BI__builtin_ia32_vec_set_v8si:
15329 case X86::BI__builtin_ia32_vec_set_v4di: {
15331 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15332 unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue();
15333 Index &= NumElts - 1;
15336 return Builder.CreateInsertElement(Ops[0], Ops[1], Index);
15338 case X86::BI_mm_setcsr:
15339 case X86::BI__builtin_ia32_ldmxcsr: {
15345 case X86::BI_mm_getcsr:
15346 case X86::BI__builtin_ia32_stmxcsr: {
15352 case X86::BI__builtin_ia32_xsave:
15353 case X86::BI__builtin_ia32_xsave64:
15354 case X86::BI__builtin_ia32_xrstor:
15355 case X86::BI__builtin_ia32_xrstor64:
15356 case X86::BI__builtin_ia32_xsaveopt:
15357 case X86::BI__builtin_ia32_xsaveopt64:
15358 case X86::BI__builtin_ia32_xrstors:
15359 case X86::BI__builtin_ia32_xrstors64:
15360 case X86::BI__builtin_ia32_xsavec:
15361 case X86::BI__builtin_ia32_xsavec64:
15362 case X86::BI__builtin_ia32_xsaves:
15363 case X86::BI__builtin_ia32_xsaves64:
15364 case X86::BI__builtin_ia32_xsetbv:
15365 case X86::BI_xsetbv: {
15367#define INTRINSIC_X86_XSAVE_ID(NAME) \
15368 case X86::BI__builtin_ia32_##NAME: \
15369 ID = Intrinsic::x86_##NAME; \
15371 switch (BuiltinID) {
15372 default: llvm_unreachable(
"Unsupported intrinsic!");
15386 case X86::BI_xsetbv:
15387 ID = Intrinsic::x86_xsetbv;
15390#undef INTRINSIC_X86_XSAVE_ID
15395 Ops.push_back(Mlo);
15398 case X86::BI__builtin_ia32_xgetbv:
15399 case X86::BI_xgetbv:
15401 case X86::BI__builtin_ia32_storedqudi128_mask:
15402 case X86::BI__builtin_ia32_storedqusi128_mask:
15403 case X86::BI__builtin_ia32_storedquhi128_mask:
15404 case X86::BI__builtin_ia32_storedquqi128_mask:
15405 case X86::BI__builtin_ia32_storeupd128_mask:
15406 case X86::BI__builtin_ia32_storeups128_mask:
15407 case X86::BI__builtin_ia32_storedqudi256_mask:
15408 case X86::BI__builtin_ia32_storedqusi256_mask:
15409 case X86::BI__builtin_ia32_storedquhi256_mask:
15410 case X86::BI__builtin_ia32_storedquqi256_mask:
15411 case X86::BI__builtin_ia32_storeupd256_mask:
15412 case X86::BI__builtin_ia32_storeups256_mask:
15413 case X86::BI__builtin_ia32_storedqudi512_mask:
15414 case X86::BI__builtin_ia32_storedqusi512_mask:
15415 case X86::BI__builtin_ia32_storedquhi512_mask:
15416 case X86::BI__builtin_ia32_storedquqi512_mask:
15417 case X86::BI__builtin_ia32_storeupd512_mask:
15418 case X86::BI__builtin_ia32_storeups512_mask:
15421 case X86::BI__builtin_ia32_storesbf16128_mask:
15422 case X86::BI__builtin_ia32_storesh128_mask:
15423 case X86::BI__builtin_ia32_storess128_mask:
15424 case X86::BI__builtin_ia32_storesd128_mask:
15427 case X86::BI__builtin_ia32_cvtmask2b128:
15428 case X86::BI__builtin_ia32_cvtmask2b256:
15429 case X86::BI__builtin_ia32_cvtmask2b512:
15430 case X86::BI__builtin_ia32_cvtmask2w128:
15431 case X86::BI__builtin_ia32_cvtmask2w256:
15432 case X86::BI__builtin_ia32_cvtmask2w512:
15433 case X86::BI__builtin_ia32_cvtmask2d128:
15434 case X86::BI__builtin_ia32_cvtmask2d256:
15435 case X86::BI__builtin_ia32_cvtmask2d512:
15436 case X86::BI__builtin_ia32_cvtmask2q128:
15437 case X86::BI__builtin_ia32_cvtmask2q256:
15438 case X86::BI__builtin_ia32_cvtmask2q512:
15441 case X86::BI__builtin_ia32_cvtb2mask128:
15442 case X86::BI__builtin_ia32_cvtb2mask256:
15443 case X86::BI__builtin_ia32_cvtb2mask512:
15444 case X86::BI__builtin_ia32_cvtw2mask128:
15445 case X86::BI__builtin_ia32_cvtw2mask256:
15446 case X86::BI__builtin_ia32_cvtw2mask512:
15447 case X86::BI__builtin_ia32_cvtd2mask128:
15448 case X86::BI__builtin_ia32_cvtd2mask256:
15449 case X86::BI__builtin_ia32_cvtd2mask512:
15450 case X86::BI__builtin_ia32_cvtq2mask128:
15451 case X86::BI__builtin_ia32_cvtq2mask256:
15452 case X86::BI__builtin_ia32_cvtq2mask512:
15455 case X86::BI__builtin_ia32_cvtdq2ps512_mask:
15456 case X86::BI__builtin_ia32_cvtqq2ps512_mask:
15457 case X86::BI__builtin_ia32_cvtqq2pd512_mask:
15458 case X86::BI__builtin_ia32_vcvtw2ph512_mask:
15459 case X86::BI__builtin_ia32_vcvtdq2ph512_mask:
15460 case X86::BI__builtin_ia32_vcvtqq2ph512_mask:
15461 case X86::BI__builtin_ia32_vcvtdq2ph256_round_mask:
15462 case X86::BI__builtin_ia32_vcvtdq2ps256_round_mask:
15463 case X86::BI__builtin_ia32_vcvtqq2pd256_round_mask:
15464 case X86::BI__builtin_ia32_vcvtqq2ph256_round_mask:
15465 case X86::BI__builtin_ia32_vcvtqq2ps256_round_mask:
15466 case X86::BI__builtin_ia32_vcvtw2ph256_round_mask:
15468 case X86::BI__builtin_ia32_cvtudq2ps512_mask:
15469 case X86::BI__builtin_ia32_cvtuqq2ps512_mask:
15470 case X86::BI__builtin_ia32_cvtuqq2pd512_mask:
15471 case X86::BI__builtin_ia32_vcvtuw2ph512_mask:
15472 case X86::BI__builtin_ia32_vcvtudq2ph512_mask:
15473 case X86::BI__builtin_ia32_vcvtuqq2ph512_mask:
15474 case X86::BI__builtin_ia32_vcvtudq2ph256_round_mask:
15475 case X86::BI__builtin_ia32_vcvtudq2ps256_round_mask:
15476 case X86::BI__builtin_ia32_vcvtuqq2pd256_round_mask:
15477 case X86::BI__builtin_ia32_vcvtuqq2ph256_round_mask:
15478 case X86::BI__builtin_ia32_vcvtuqq2ps256_round_mask:
15479 case X86::BI__builtin_ia32_vcvtuw2ph256_round_mask:
15482 case X86::BI__builtin_ia32_vfmaddss3:
15483 case X86::BI__builtin_ia32_vfmaddsd3:
15484 case X86::BI__builtin_ia32_vfmaddsh3_mask:
15485 case X86::BI__builtin_ia32_vfmaddss3_mask:
15486 case X86::BI__builtin_ia32_vfmaddsd3_mask:
15488 case X86::BI__builtin_ia32_vfmaddss:
15489 case X86::BI__builtin_ia32_vfmaddsd:
15491 Constant::getNullValue(Ops[0]->getType()));
15492 case X86::BI__builtin_ia32_vfmaddsh3_maskz:
15493 case X86::BI__builtin_ia32_vfmaddss3_maskz:
15494 case X86::BI__builtin_ia32_vfmaddsd3_maskz:
15496 case X86::BI__builtin_ia32_vfmaddsh3_mask3:
15497 case X86::BI__builtin_ia32_vfmaddss3_mask3:
15498 case X86::BI__builtin_ia32_vfmaddsd3_mask3:
15500 case X86::BI__builtin_ia32_vfmsubsh3_mask3:
15501 case X86::BI__builtin_ia32_vfmsubss3_mask3:
15502 case X86::BI__builtin_ia32_vfmsubsd3_mask3:
15505 case X86::BI__builtin_ia32_vfmaddph:
15506 case X86::BI__builtin_ia32_vfmaddps:
15507 case X86::BI__builtin_ia32_vfmaddpd:
15508 case X86::BI__builtin_ia32_vfmaddph256:
15509 case X86::BI__builtin_ia32_vfmaddps256:
15510 case X86::BI__builtin_ia32_vfmaddpd256:
15511 case X86::BI__builtin_ia32_vfmaddph512_mask:
15512 case X86::BI__builtin_ia32_vfmaddph512_maskz:
15513 case X86::BI__builtin_ia32_vfmaddph512_mask3:
15514 case X86::BI__builtin_ia32_vfmaddnepbh128:
15515 case X86::BI__builtin_ia32_vfmaddnepbh256:
15516 case X86::BI__builtin_ia32_vfmaddnepbh512:
15517 case X86::BI__builtin_ia32_vfmaddps512_mask:
15518 case X86::BI__builtin_ia32_vfmaddps512_maskz:
15519 case X86::BI__builtin_ia32_vfmaddps512_mask3:
15520 case X86::BI__builtin_ia32_vfmsubps512_mask3:
15521 case X86::BI__builtin_ia32_vfmaddpd512_mask:
15522 case X86::BI__builtin_ia32_vfmaddpd512_maskz:
15523 case X86::BI__builtin_ia32_vfmaddpd512_mask3:
15524 case X86::BI__builtin_ia32_vfmsubpd512_mask3:
15525 case X86::BI__builtin_ia32_vfmsubph512_mask3:
15526 case X86::BI__builtin_ia32_vfmaddph256_round_mask:
15527 case X86::BI__builtin_ia32_vfmaddph256_round_maskz:
15528 case X86::BI__builtin_ia32_vfmaddph256_round_mask3:
15529 case X86::BI__builtin_ia32_vfmaddps256_round_mask:
15530 case X86::BI__builtin_ia32_vfmaddps256_round_maskz:
15531 case X86::BI__builtin_ia32_vfmaddps256_round_mask3:
15532 case X86::BI__builtin_ia32_vfmsubps256_round_mask3:
15533 case X86::BI__builtin_ia32_vfmaddpd256_round_mask:
15534 case X86::BI__builtin_ia32_vfmaddpd256_round_maskz:
15535 case X86::BI__builtin_ia32_vfmaddpd256_round_mask3:
15536 case X86::BI__builtin_ia32_vfmsubpd256_round_mask3:
15537 case X86::BI__builtin_ia32_vfmsubph256_round_mask3:
15539 case X86::BI__builtin_ia32_vfmaddsubph512_mask:
15540 case X86::BI__builtin_ia32_vfmaddsubph512_maskz:
15541 case X86::BI__builtin_ia32_vfmaddsubph512_mask3:
15542 case X86::BI__builtin_ia32_vfmsubaddph512_mask3:
15543 case X86::BI__builtin_ia32_vfmaddsubps512_mask:
15544 case X86::BI__builtin_ia32_vfmaddsubps512_maskz:
15545 case X86::BI__builtin_ia32_vfmaddsubps512_mask3:
15546 case X86::BI__builtin_ia32_vfmsubaddps512_mask3:
15547 case X86::BI__builtin_ia32_vfmaddsubpd512_mask:
15548 case X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
15549 case X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
15550 case X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
15551 case X86::BI__builtin_ia32_vfmaddsubph256_round_mask:
15552 case X86::BI__builtin_ia32_vfmaddsubph256_round_maskz:
15553 case X86::BI__builtin_ia32_vfmaddsubph256_round_mask3:
15554 case X86::BI__builtin_ia32_vfmsubaddph256_round_mask3:
15555 case X86::BI__builtin_ia32_vfmaddsubps256_round_mask:
15556 case X86::BI__builtin_ia32_vfmaddsubps256_round_maskz:
15557 case X86::BI__builtin_ia32_vfmaddsubps256_round_mask3:
15558 case X86::BI__builtin_ia32_vfmsubaddps256_round_mask3:
15559 case X86::BI__builtin_ia32_vfmaddsubpd256_round_mask:
15560 case X86::BI__builtin_ia32_vfmaddsubpd256_round_maskz:
15561 case X86::BI__builtin_ia32_vfmaddsubpd256_round_mask3:
15562 case X86::BI__builtin_ia32_vfmsubaddpd256_round_mask3:
15565 case X86::BI__builtin_ia32_movdqa32store128_mask:
15566 case X86::BI__builtin_ia32_movdqa64store128_mask:
15567 case X86::BI__builtin_ia32_storeaps128_mask:
15568 case X86::BI__builtin_ia32_storeapd128_mask:
15569 case X86::BI__builtin_ia32_movdqa32store256_mask:
15570 case X86::BI__builtin_ia32_movdqa64store256_mask:
15571 case X86::BI__builtin_ia32_storeaps256_mask:
15572 case X86::BI__builtin_ia32_storeapd256_mask:
15573 case X86::BI__builtin_ia32_movdqa32store512_mask:
15574 case X86::BI__builtin_ia32_movdqa64store512_mask:
15575 case X86::BI__builtin_ia32_storeaps512_mask:
15576 case X86::BI__builtin_ia32_storeapd512_mask:
15581 case X86::BI__builtin_ia32_loadups128_mask:
15582 case X86::BI__builtin_ia32_loadups256_mask:
15583 case X86::BI__builtin_ia32_loadups512_mask:
15584 case X86::BI__builtin_ia32_loadupd128_mask:
15585 case X86::BI__builtin_ia32_loadupd256_mask:
15586 case X86::BI__builtin_ia32_loadupd512_mask:
15587 case X86::BI__builtin_ia32_loaddquqi128_mask:
15588 case X86::BI__builtin_ia32_loaddquqi256_mask:
15589 case X86::BI__builtin_ia32_loaddquqi512_mask:
15590 case X86::BI__builtin_ia32_loaddquhi128_mask:
15591 case X86::BI__builtin_ia32_loaddquhi256_mask:
15592 case X86::BI__builtin_ia32_loaddquhi512_mask:
15593 case X86::BI__builtin_ia32_loaddqusi128_mask:
15594 case X86::BI__builtin_ia32_loaddqusi256_mask:
15595 case X86::BI__builtin_ia32_loaddqusi512_mask:
15596 case X86::BI__builtin_ia32_loaddqudi128_mask:
15597 case X86::BI__builtin_ia32_loaddqudi256_mask:
15598 case X86::BI__builtin_ia32_loaddqudi512_mask:
15601 case X86::BI__builtin_ia32_loadsbf16128_mask:
15602 case X86::BI__builtin_ia32_loadsh128_mask:
15603 case X86::BI__builtin_ia32_loadss128_mask:
15604 case X86::BI__builtin_ia32_loadsd128_mask:
15607 case X86::BI__builtin_ia32_loadaps128_mask:
15608 case X86::BI__builtin_ia32_loadaps256_mask:
15609 case X86::BI__builtin_ia32_loadaps512_mask:
15610 case X86::BI__builtin_ia32_loadapd128_mask:
15611 case X86::BI__builtin_ia32_loadapd256_mask:
15612 case X86::BI__builtin_ia32_loadapd512_mask:
15613 case X86::BI__builtin_ia32_movdqa32load128_mask:
15614 case X86::BI__builtin_ia32_movdqa32load256_mask:
15615 case X86::BI__builtin_ia32_movdqa32load512_mask:
15616 case X86::BI__builtin_ia32_movdqa64load128_mask:
15617 case X86::BI__builtin_ia32_movdqa64load256_mask:
15618 case X86::BI__builtin_ia32_movdqa64load512_mask:
15623 case X86::BI__builtin_ia32_expandloaddf128_mask:
15624 case X86::BI__builtin_ia32_expandloaddf256_mask:
15625 case X86::BI__builtin_ia32_expandloaddf512_mask:
15626 case X86::BI__builtin_ia32_expandloadsf128_mask:
15627 case X86::BI__builtin_ia32_expandloadsf256_mask:
15628 case X86::BI__builtin_ia32_expandloadsf512_mask:
15629 case X86::BI__builtin_ia32_expandloaddi128_mask:
15630 case X86::BI__builtin_ia32_expandloaddi256_mask:
15631 case X86::BI__builtin_ia32_expandloaddi512_mask:
15632 case X86::BI__builtin_ia32_expandloadsi128_mask:
15633 case X86::BI__builtin_ia32_expandloadsi256_mask:
15634 case X86::BI__builtin_ia32_expandloadsi512_mask:
15635 case X86::BI__builtin_ia32_expandloadhi128_mask:
15636 case X86::BI__builtin_ia32_expandloadhi256_mask:
15637 case X86::BI__builtin_ia32_expandloadhi512_mask:
15638 case X86::BI__builtin_ia32_expandloadqi128_mask:
15639 case X86::BI__builtin_ia32_expandloadqi256_mask:
15640 case X86::BI__builtin_ia32_expandloadqi512_mask:
15643 case X86::BI__builtin_ia32_compressstoredf128_mask:
15644 case X86::BI__builtin_ia32_compressstoredf256_mask:
15645 case X86::BI__builtin_ia32_compressstoredf512_mask:
15646 case X86::BI__builtin_ia32_compressstoresf128_mask:
15647 case X86::BI__builtin_ia32_compressstoresf256_mask:
15648 case X86::BI__builtin_ia32_compressstoresf512_mask:
15649 case X86::BI__builtin_ia32_compressstoredi128_mask:
15650 case X86::BI__builtin_ia32_compressstoredi256_mask:
15651 case X86::BI__builtin_ia32_compressstoredi512_mask:
15652 case X86::BI__builtin_ia32_compressstoresi128_mask:
15653 case X86::BI__builtin_ia32_compressstoresi256_mask:
15654 case X86::BI__builtin_ia32_compressstoresi512_mask:
15655 case X86::BI__builtin_ia32_compressstorehi128_mask:
15656 case X86::BI__builtin_ia32_compressstorehi256_mask:
15657 case X86::BI__builtin_ia32_compressstorehi512_mask:
15658 case X86::BI__builtin_ia32_compressstoreqi128_mask:
15659 case X86::BI__builtin_ia32_compressstoreqi256_mask:
15660 case X86::BI__builtin_ia32_compressstoreqi512_mask:
15663 case X86::BI__builtin_ia32_expanddf128_mask:
15664 case X86::BI__builtin_ia32_expanddf256_mask:
15665 case X86::BI__builtin_ia32_expanddf512_mask:
15666 case X86::BI__builtin_ia32_expandsf128_mask:
15667 case X86::BI__builtin_ia32_expandsf256_mask:
15668 case X86::BI__builtin_ia32_expandsf512_mask:
15669 case X86::BI__builtin_ia32_expanddi128_mask:
15670 case X86::BI__builtin_ia32_expanddi256_mask:
15671 case X86::BI__builtin_ia32_expanddi512_mask:
15672 case X86::BI__builtin_ia32_expandsi128_mask:
15673 case X86::BI__builtin_ia32_expandsi256_mask:
15674 case X86::BI__builtin_ia32_expandsi512_mask:
15675 case X86::BI__builtin_ia32_expandhi128_mask:
15676 case X86::BI__builtin_ia32_expandhi256_mask:
15677 case X86::BI__builtin_ia32_expandhi512_mask:
15678 case X86::BI__builtin_ia32_expandqi128_mask:
15679 case X86::BI__builtin_ia32_expandqi256_mask:
15680 case X86::BI__builtin_ia32_expandqi512_mask:
15683 case X86::BI__builtin_ia32_compressdf128_mask:
15684 case X86::BI__builtin_ia32_compressdf256_mask:
15685 case X86::BI__builtin_ia32_compressdf512_mask:
15686 case X86::BI__builtin_ia32_compresssf128_mask:
15687 case X86::BI__builtin_ia32_compresssf256_mask:
15688 case X86::BI__builtin_ia32_compresssf512_mask:
15689 case X86::BI__builtin_ia32_compressdi128_mask:
15690 case X86::BI__builtin_ia32_compressdi256_mask:
15691 case X86::BI__builtin_ia32_compressdi512_mask:
15692 case X86::BI__builtin_ia32_compresssi128_mask:
15693 case X86::BI__builtin_ia32_compresssi256_mask:
15694 case X86::BI__builtin_ia32_compresssi512_mask:
15695 case X86::BI__builtin_ia32_compresshi128_mask:
15696 case X86::BI__builtin_ia32_compresshi256_mask:
15697 case X86::BI__builtin_ia32_compresshi512_mask:
15698 case X86::BI__builtin_ia32_compressqi128_mask:
15699 case X86::BI__builtin_ia32_compressqi256_mask:
15700 case X86::BI__builtin_ia32_compressqi512_mask:
15703 case X86::BI__builtin_ia32_gather3div2df:
15704 case X86::BI__builtin_ia32_gather3div2di:
15705 case X86::BI__builtin_ia32_gather3div4df:
15706 case X86::BI__builtin_ia32_gather3div4di:
15707 case X86::BI__builtin_ia32_gather3div4sf:
15708 case X86::BI__builtin_ia32_gather3div4si:
15709 case X86::BI__builtin_ia32_gather3div8sf:
15710 case X86::BI__builtin_ia32_gather3div8si:
15711 case X86::BI__builtin_ia32_gather3siv2df:
15712 case X86::BI__builtin_ia32_gather3siv2di:
15713 case X86::BI__builtin_ia32_gather3siv4df:
15714 case X86::BI__builtin_ia32_gather3siv4di:
15715 case X86::BI__builtin_ia32_gather3siv4sf:
15716 case X86::BI__builtin_ia32_gather3siv4si:
15717 case X86::BI__builtin_ia32_gather3siv8sf:
15718 case X86::BI__builtin_ia32_gather3siv8si:
15719 case X86::BI__builtin_ia32_gathersiv8df:
15720 case X86::BI__builtin_ia32_gathersiv16sf:
15721 case X86::BI__builtin_ia32_gatherdiv8df:
15722 case X86::BI__builtin_ia32_gatherdiv16sf:
15723 case X86::BI__builtin_ia32_gathersiv8di:
15724 case X86::BI__builtin_ia32_gathersiv16si:
15725 case X86::BI__builtin_ia32_gatherdiv8di:
15726 case X86::BI__builtin_ia32_gatherdiv16si: {
15728 switch (BuiltinID) {
15729 default: llvm_unreachable(
"Unexpected builtin");
15730 case X86::BI__builtin_ia32_gather3div2df:
15731 IID = Intrinsic::x86_avx512_mask_gather3div2_df;
15733 case X86::BI__builtin_ia32_gather3div2di:
15734 IID = Intrinsic::x86_avx512_mask_gather3div2_di;
15736 case X86::BI__builtin_ia32_gather3div4df:
15737 IID = Intrinsic::x86_avx512_mask_gather3div4_df;
15739 case X86::BI__builtin_ia32_gather3div4di:
15740 IID = Intrinsic::x86_avx512_mask_gather3div4_di;
15742 case X86::BI__builtin_ia32_gather3div4sf:
15743 IID = Intrinsic::x86_avx512_mask_gather3div4_sf;
15745 case X86::BI__builtin_ia32_gather3div4si:
15746 IID = Intrinsic::x86_avx512_mask_gather3div4_si;
15748 case X86::BI__builtin_ia32_gather3div8sf:
15749 IID = Intrinsic::x86_avx512_mask_gather3div8_sf;
15751 case X86::BI__builtin_ia32_gather3div8si:
15752 IID = Intrinsic::x86_avx512_mask_gather3div8_si;
15754 case X86::BI__builtin_ia32_gather3siv2df:
15755 IID = Intrinsic::x86_avx512_mask_gather3siv2_df;
15757 case X86::BI__builtin_ia32_gather3siv2di:
15758 IID = Intrinsic::x86_avx512_mask_gather3siv2_di;
15760 case X86::BI__builtin_ia32_gather3siv4df:
15761 IID = Intrinsic::x86_avx512_mask_gather3siv4_df;
15763 case X86::BI__builtin_ia32_gather3siv4di:
15764 IID = Intrinsic::x86_avx512_mask_gather3siv4_di;
15766 case X86::BI__builtin_ia32_gather3siv4sf:
15767 IID = Intrinsic::x86_avx512_mask_gather3siv4_sf;
15769 case X86::BI__builtin_ia32_gather3siv4si:
15770 IID = Intrinsic::x86_avx512_mask_gather3siv4_si;
15772 case X86::BI__builtin_ia32_gather3siv8sf:
15773 IID = Intrinsic::x86_avx512_mask_gather3siv8_sf;
15775 case X86::BI__builtin_ia32_gather3siv8si:
15776 IID = Intrinsic::x86_avx512_mask_gather3siv8_si;
15778 case X86::BI__builtin_ia32_gathersiv8df:
15779 IID = Intrinsic::x86_avx512_mask_gather_dpd_512;
15781 case X86::BI__builtin_ia32_gathersiv16sf:
15782 IID = Intrinsic::x86_avx512_mask_gather_dps_512;
15784 case X86::BI__builtin_ia32_gatherdiv8df:
15785 IID = Intrinsic::x86_avx512_mask_gather_qpd_512;
15787 case X86::BI__builtin_ia32_gatherdiv16sf:
15788 IID = Intrinsic::x86_avx512_mask_gather_qps_512;
15790 case X86::BI__builtin_ia32_gathersiv8di:
15791 IID = Intrinsic::x86_avx512_mask_gather_dpq_512;
15793 case X86::BI__builtin_ia32_gathersiv16si:
15794 IID = Intrinsic::x86_avx512_mask_gather_dpi_512;
15796 case X86::BI__builtin_ia32_gatherdiv8di:
15797 IID = Intrinsic::x86_avx512_mask_gather_qpq_512;
15799 case X86::BI__builtin_ia32_gatherdiv16si:
15800 IID = Intrinsic::x86_avx512_mask_gather_qpi_512;
15804 unsigned MinElts = std::min(
15805 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(),
15806 cast<llvm::FixedVectorType>(Ops[2]->getType())->getNumElements());
15809 return Builder.CreateCall(Intr, Ops);
15812 case X86::BI__builtin_ia32_scattersiv8df:
15813 case X86::BI__builtin_ia32_scattersiv16sf:
15814 case X86::BI__builtin_ia32_scatterdiv8df:
15815 case X86::BI__builtin_ia32_scatterdiv16sf:
15816 case X86::BI__builtin_ia32_scattersiv8di:
15817 case X86::BI__builtin_ia32_scattersiv16si:
15818 case X86::BI__builtin_ia32_scatterdiv8di:
15819 case X86::BI__builtin_ia32_scatterdiv16si:
15820 case X86::BI__builtin_ia32_scatterdiv2df:
15821 case X86::BI__builtin_ia32_scatterdiv2di:
15822 case X86::BI__builtin_ia32_scatterdiv4df:
15823 case X86::BI__builtin_ia32_scatterdiv4di:
15824 case X86::BI__builtin_ia32_scatterdiv4sf:
15825 case X86::BI__builtin_ia32_scatterdiv4si:
15826 case X86::BI__builtin_ia32_scatterdiv8sf:
15827 case X86::BI__builtin_ia32_scatterdiv8si:
15828 case X86::BI__builtin_ia32_scattersiv2df:
15829 case X86::BI__builtin_ia32_scattersiv2di:
15830 case X86::BI__builtin_ia32_scattersiv4df:
15831 case X86::BI__builtin_ia32_scattersiv4di:
15832 case X86::BI__builtin_ia32_scattersiv4sf:
15833 case X86::BI__builtin_ia32_scattersiv4si:
15834 case X86::BI__builtin_ia32_scattersiv8sf:
15835 case X86::BI__builtin_ia32_scattersiv8si: {
15837 switch (BuiltinID) {
15838 default: llvm_unreachable(
"Unexpected builtin");
15839 case X86::BI__builtin_ia32_scattersiv8df:
15840 IID = Intrinsic::x86_avx512_mask_scatter_dpd_512;
15842 case X86::BI__builtin_ia32_scattersiv16sf:
15843 IID = Intrinsic::x86_avx512_mask_scatter_dps_512;
15845 case X86::BI__builtin_ia32_scatterdiv8df:
15846 IID = Intrinsic::x86_avx512_mask_scatter_qpd_512;
15848 case X86::BI__builtin_ia32_scatterdiv16sf:
15849 IID = Intrinsic::x86_avx512_mask_scatter_qps_512;
15851 case X86::BI__builtin_ia32_scattersiv8di:
15852 IID = Intrinsic::x86_avx512_mask_scatter_dpq_512;
15854 case X86::BI__builtin_ia32_scattersiv16si:
15855 IID = Intrinsic::x86_avx512_mask_scatter_dpi_512;
15857 case X86::BI__builtin_ia32_scatterdiv8di:
15858 IID = Intrinsic::x86_avx512_mask_scatter_qpq_512;
15860 case X86::BI__builtin_ia32_scatterdiv16si:
15861 IID = Intrinsic::x86_avx512_mask_scatter_qpi_512;
15863 case X86::BI__builtin_ia32_scatterdiv2df:
15864 IID = Intrinsic::x86_avx512_mask_scatterdiv2_df;
15866 case X86::BI__builtin_ia32_scatterdiv2di:
15867 IID = Intrinsic::x86_avx512_mask_scatterdiv2_di;
15869 case X86::BI__builtin_ia32_scatterdiv4df:
15870 IID = Intrinsic::x86_avx512_mask_scatterdiv4_df;
15872 case X86::BI__builtin_ia32_scatterdiv4di:
15873 IID = Intrinsic::x86_avx512_mask_scatterdiv4_di;
15875 case X86::BI__builtin_ia32_scatterdiv4sf:
15876 IID = Intrinsic::x86_avx512_mask_scatterdiv4_sf;
15878 case X86::BI__builtin_ia32_scatterdiv4si:
15879 IID = Intrinsic::x86_avx512_mask_scatterdiv4_si;
15881 case X86::BI__builtin_ia32_scatterdiv8sf:
15882 IID = Intrinsic::x86_avx512_mask_scatterdiv8_sf;
15884 case X86::BI__builtin_ia32_scatterdiv8si:
15885 IID = Intrinsic::x86_avx512_mask_scatterdiv8_si;
15887 case X86::BI__builtin_ia32_scattersiv2df:
15888 IID = Intrinsic::x86_avx512_mask_scattersiv2_df;
15890 case X86::BI__builtin_ia32_scattersiv2di:
15891 IID = Intrinsic::x86_avx512_mask_scattersiv2_di;
15893 case X86::BI__builtin_ia32_scattersiv4df:
15894 IID = Intrinsic::x86_avx512_mask_scattersiv4_df;
15896 case X86::BI__builtin_ia32_scattersiv4di:
15897 IID = Intrinsic::x86_avx512_mask_scattersiv4_di;
15899 case X86::BI__builtin_ia32_scattersiv4sf:
15900 IID = Intrinsic::x86_avx512_mask_scattersiv4_sf;
15902 case X86::BI__builtin_ia32_scattersiv4si:
15903 IID = Intrinsic::x86_avx512_mask_scattersiv4_si;
15905 case X86::BI__builtin_ia32_scattersiv8sf:
15906 IID = Intrinsic::x86_avx512_mask_scattersiv8_sf;
15908 case X86::BI__builtin_ia32_scattersiv8si:
15909 IID = Intrinsic::x86_avx512_mask_scattersiv8_si;
15913 unsigned MinElts = std::min(
15914 cast<llvm::FixedVectorType>(Ops[2]->getType())->getNumElements(),
15915 cast<llvm::FixedVectorType>(Ops[3]->getType())->getNumElements());
15918 return Builder.CreateCall(Intr, Ops);
15921 case X86::BI__builtin_ia32_vextractf128_pd256:
15922 case X86::BI__builtin_ia32_vextractf128_ps256:
15923 case X86::BI__builtin_ia32_vextractf128_si256:
15924 case X86::BI__builtin_ia32_extract128i256:
15925 case X86::BI__builtin_ia32_extractf64x4_mask:
15926 case X86::BI__builtin_ia32_extractf32x4_mask:
15927 case X86::BI__builtin_ia32_extracti64x4_mask:
15928 case X86::BI__builtin_ia32_extracti32x4_mask:
15929 case X86::BI__builtin_ia32_extractf32x8_mask:
15930 case X86::BI__builtin_ia32_extracti32x8_mask:
15931 case X86::BI__builtin_ia32_extractf32x4_256_mask:
15932 case X86::BI__builtin_ia32_extracti32x4_256_mask:
15933 case X86::BI__builtin_ia32_extractf64x2_256_mask:
15934 case X86::BI__builtin_ia32_extracti64x2_256_mask:
15935 case X86::BI__builtin_ia32_extractf64x2_512_mask:
15936 case X86::BI__builtin_ia32_extracti64x2_512_mask: {
15938 unsigned NumElts = DstTy->getNumElements();
15939 unsigned SrcNumElts =
15940 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15941 unsigned SubVectors = SrcNumElts / NumElts;
15942 unsigned Index = cast<ConstantInt>(Ops[1])->getZExtValue();
15943 assert(llvm::isPowerOf2_32(SubVectors) &&
"Expected power of 2 subvectors");
15944 Index &= SubVectors - 1;
15948 for (
unsigned i = 0; i != NumElts; ++i)
15949 Indices[i] = i + Index;
15954 if (Ops.size() == 4)
15959 case X86::BI__builtin_ia32_vinsertf128_pd256:
15960 case X86::BI__builtin_ia32_vinsertf128_ps256:
15961 case X86::BI__builtin_ia32_vinsertf128_si256:
15962 case X86::BI__builtin_ia32_insert128i256:
15963 case X86::BI__builtin_ia32_insertf64x4:
15964 case X86::BI__builtin_ia32_insertf32x4:
15965 case X86::BI__builtin_ia32_inserti64x4:
15966 case X86::BI__builtin_ia32_inserti32x4:
15967 case X86::BI__builtin_ia32_insertf32x8:
15968 case X86::BI__builtin_ia32_inserti32x8:
15969 case X86::BI__builtin_ia32_insertf32x4_256:
15970 case X86::BI__builtin_ia32_inserti32x4_256:
15971 case X86::BI__builtin_ia32_insertf64x2_256:
15972 case X86::BI__builtin_ia32_inserti64x2_256:
15973 case X86::BI__builtin_ia32_insertf64x2_512:
15974 case X86::BI__builtin_ia32_inserti64x2_512: {
15975 unsigned DstNumElts =
15976 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15977 unsigned SrcNumElts =
15978 cast<llvm::FixedVectorType>(Ops[1]->getType())->getNumElements();
15979 unsigned SubVectors = DstNumElts / SrcNumElts;
15980 unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue();
15981 assert(llvm::isPowerOf2_32(SubVectors) &&
"Expected power of 2 subvectors");
15982 Index &= SubVectors - 1;
15983 Index *= SrcNumElts;
15986 for (
unsigned i = 0; i != DstNumElts; ++i)
15987 Indices[i] = (i >= SrcNumElts) ? SrcNumElts + (i % SrcNumElts) : i;
15990 Ops[1],
ArrayRef(Indices, DstNumElts),
"widen");
15992 for (
unsigned i = 0; i != DstNumElts; ++i) {
15993 if (i >= Index && i < (Index + SrcNumElts))
15994 Indices[i] = (i - Index) + DstNumElts;
15999 return Builder.CreateShuffleVector(Ops[0], Op1,
16000 ArrayRef(Indices, DstNumElts),
"insert");
16002 case X86::BI__builtin_ia32_pmovqd512_mask:
16003 case X86::BI__builtin_ia32_pmovwb512_mask: {
16004 Value *Res =
Builder.CreateTrunc(Ops[0], Ops[1]->getType());
16007 case X86::BI__builtin_ia32_pmovdb512_mask:
16008 case X86::BI__builtin_ia32_pmovdw512_mask:
16009 case X86::BI__builtin_ia32_pmovqw512_mask: {
16010 if (
const auto *
C = dyn_cast<Constant>(Ops[2]))
16011 if (
C->isAllOnesValue())
16012 return Builder.CreateTrunc(Ops[0], Ops[1]->getType());
16015 switch (BuiltinID) {
16016 default: llvm_unreachable(
"Unsupported intrinsic!");
16017 case X86::BI__builtin_ia32_pmovdb512_mask:
16018 IID = Intrinsic::x86_avx512_mask_pmov_db_512;
16020 case X86::BI__builtin_ia32_pmovdw512_mask:
16021 IID = Intrinsic::x86_avx512_mask_pmov_dw_512;
16023 case X86::BI__builtin_ia32_pmovqw512_mask:
16024 IID = Intrinsic::x86_avx512_mask_pmov_qw_512;
16029 return Builder.CreateCall(Intr, Ops);
16031 case X86::BI__builtin_ia32_pblendw128:
16032 case X86::BI__builtin_ia32_blendpd:
16033 case X86::BI__builtin_ia32_blendps:
16034 case X86::BI__builtin_ia32_blendpd256:
16035 case X86::BI__builtin_ia32_blendps256:
16036 case X86::BI__builtin_ia32_pblendw256:
16037 case X86::BI__builtin_ia32_pblendd128:
16038 case X86::BI__builtin_ia32_pblendd256: {
16040 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16041 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
16046 for (
unsigned i = 0; i != NumElts; ++i)
16047 Indices[i] = ((Imm >> (i % 8)) & 0x1) ? NumElts + i : i;
16049 return Builder.CreateShuffleVector(Ops[0], Ops[1],
16050 ArrayRef(Indices, NumElts),
"blend");
16052 case X86::BI__builtin_ia32_pshuflw:
16053 case X86::BI__builtin_ia32_pshuflw256:
16054 case X86::BI__builtin_ia32_pshuflw512: {
16055 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
16056 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
16057 unsigned NumElts = Ty->getNumElements();
16060 Imm = (Imm & 0xff) * 0x01010101;
16063 for (
unsigned l = 0; l != NumElts; l += 8) {
16064 for (
unsigned i = 0; i != 4; ++i) {
16065 Indices[l + i] = l + (Imm & 3);
16068 for (
unsigned i = 4; i != 8; ++i)
16069 Indices[l + i] = l + i;
16072 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
16075 case X86::BI__builtin_ia32_pshufhw:
16076 case X86::BI__builtin_ia32_pshufhw256:
16077 case X86::BI__builtin_ia32_pshufhw512: {
16078 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
16079 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
16080 unsigned NumElts = Ty->getNumElements();
16083 Imm = (Imm & 0xff) * 0x01010101;
16086 for (
unsigned l = 0; l != NumElts; l += 8) {
16087 for (
unsigned i = 0; i != 4; ++i)
16088 Indices[l + i] = l + i;
16089 for (
unsigned i = 4; i != 8; ++i) {
16090 Indices[l + i] = l + 4 + (Imm & 3);
16095 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
16098 case X86::BI__builtin_ia32_pshufd:
16099 case X86::BI__builtin_ia32_pshufd256:
16100 case X86::BI__builtin_ia32_pshufd512:
16101 case X86::BI__builtin_ia32_vpermilpd:
16102 case X86::BI__builtin_ia32_vpermilps:
16103 case X86::BI__builtin_ia32_vpermilpd256:
16104 case X86::BI__builtin_ia32_vpermilps256:
16105 case X86::BI__builtin_ia32_vpermilpd512:
16106 case X86::BI__builtin_ia32_vpermilps512: {
16107 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
16108 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
16109 unsigned NumElts = Ty->getNumElements();
16110 unsigned NumLanes = Ty->getPrimitiveSizeInBits() / 128;
16111 unsigned NumLaneElts = NumElts / NumLanes;
16114 Imm = (Imm & 0xff) * 0x01010101;
16117 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
16118 for (
unsigned i = 0; i != NumLaneElts; ++i) {
16119 Indices[i + l] = (Imm % NumLaneElts) + l;
16120 Imm /= NumLaneElts;
16124 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
16127 case X86::BI__builtin_ia32_shufpd:
16128 case X86::BI__builtin_ia32_shufpd256:
16129 case X86::BI__builtin_ia32_shufpd512:
16130 case X86::BI__builtin_ia32_shufps:
16131 case X86::BI__builtin_ia32_shufps256:
16132 case X86::BI__builtin_ia32_shufps512: {
16133 uint32_t Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
16134 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
16135 unsigned NumElts = Ty->getNumElements();
16136 unsigned NumLanes = Ty->getPrimitiveSizeInBits() / 128;
16137 unsigned NumLaneElts = NumElts / NumLanes;
16140 Imm = (Imm & 0xff) * 0x01010101;
16143 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
16144 for (
unsigned i = 0; i != NumLaneElts; ++i) {
16145 unsigned Index = Imm % NumLaneElts;
16146 Imm /= NumLaneElts;
16147 if (i >= (NumLaneElts / 2))
16149 Indices[l + i] = l + Index;
16153 return Builder.CreateShuffleVector(Ops[0], Ops[1],
16154 ArrayRef(Indices, NumElts),
"shufp");
16156 case X86::BI__builtin_ia32_permdi256:
16157 case X86::BI__builtin_ia32_permdf256:
16158 case X86::BI__builtin_ia32_permdi512:
16159 case X86::BI__builtin_ia32_permdf512: {
16160 unsigned Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
16161 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
16162 unsigned NumElts = Ty->getNumElements();
16166 for (
unsigned l = 0; l != NumElts; l += 4)
16167 for (
unsigned i = 0; i != 4; ++i)
16168 Indices[l + i] = l + ((Imm >> (2 * i)) & 0x3);
16170 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
16173 case X86::BI__builtin_ia32_palignr128:
16174 case X86::BI__builtin_ia32_palignr256:
16175 case X86::BI__builtin_ia32_palignr512: {
16176 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff;
16179 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16180 assert(NumElts % 16 == 0);
16184 if (ShiftVal >= 32)
16189 if (ShiftVal > 16) {
16192 Ops[0] = llvm::Constant::getNullValue(Ops[0]->getType());
16197 for (
unsigned l = 0; l != NumElts; l += 16) {
16198 for (
unsigned i = 0; i != 16; ++i) {
16199 unsigned Idx = ShiftVal + i;
16201 Idx += NumElts - 16;
16202 Indices[l + i] = Idx + l;
16206 return Builder.CreateShuffleVector(Ops[1], Ops[0],
16207 ArrayRef(Indices, NumElts),
"palignr");
16209 case X86::BI__builtin_ia32_alignd128:
16210 case X86::BI__builtin_ia32_alignd256:
16211 case X86::BI__builtin_ia32_alignd512:
16212 case X86::BI__builtin_ia32_alignq128:
16213 case X86::BI__builtin_ia32_alignq256:
16214 case X86::BI__builtin_ia32_alignq512: {
16216 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16217 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff;
16220 ShiftVal &= NumElts - 1;
16223 for (
unsigned i = 0; i != NumElts; ++i)
16224 Indices[i] = i + ShiftVal;
16226 return Builder.CreateShuffleVector(Ops[1], Ops[0],
16227 ArrayRef(Indices, NumElts),
"valign");
16229 case X86::BI__builtin_ia32_shuf_f32x4_256:
16230 case X86::BI__builtin_ia32_shuf_f64x2_256:
16231 case X86::BI__builtin_ia32_shuf_i32x4_256:
16232 case X86::BI__builtin_ia32_shuf_i64x2_256:
16233 case X86::BI__builtin_ia32_shuf_f32x4:
16234 case X86::BI__builtin_ia32_shuf_f64x2:
16235 case X86::BI__builtin_ia32_shuf_i32x4:
16236 case X86::BI__builtin_ia32_shuf_i64x2: {
16237 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
16238 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
16239 unsigned NumElts = Ty->getNumElements();
16240 unsigned NumLanes = Ty->getPrimitiveSizeInBits() == 512 ? 4 : 2;
16241 unsigned NumLaneElts = NumElts / NumLanes;
16244 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
16245 unsigned Index = (Imm % NumLanes) * NumLaneElts;
16247 if (l >= (NumElts / 2))
16249 for (
unsigned i = 0; i != NumLaneElts; ++i) {
16250 Indices[l + i] = Index + i;
16254 return Builder.CreateShuffleVector(Ops[0], Ops[1],
16255 ArrayRef(Indices, NumElts),
"shuf");
16258 case X86::BI__builtin_ia32_vperm2f128_pd256:
16259 case X86::BI__builtin_ia32_vperm2f128_ps256:
16260 case X86::BI__builtin_ia32_vperm2f128_si256:
16261 case X86::BI__builtin_ia32_permti256: {
16262 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
16264 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16273 for (
unsigned l = 0; l != 2; ++l) {
16275 if (Imm & (1 << ((l * 4) + 3)))
16276 OutOps[l] = llvm::ConstantAggregateZero::get(Ops[0]->getType());
16277 else if (Imm & (1 << ((l * 4) + 1)))
16278 OutOps[l] = Ops[1];
16280 OutOps[l] = Ops[0];
16282 for (
unsigned i = 0; i != NumElts/2; ++i) {
16284 unsigned Idx = (l * NumElts) + i;
16287 if (Imm & (1 << (l * 4)))
16289 Indices[(l * (NumElts/2)) + i] = Idx;
16293 return Builder.CreateShuffleVector(OutOps[0], OutOps[1],
16294 ArrayRef(Indices, NumElts),
"vperm");
16297 case X86::BI__builtin_ia32_pslldqi128_byteshift:
16298 case X86::BI__builtin_ia32_pslldqi256_byteshift:
16299 case X86::BI__builtin_ia32_pslldqi512_byteshift: {
16300 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
16301 auto *ResultType = cast<llvm::FixedVectorType>(Ops[0]->getType());
16303 unsigned NumElts = ResultType->getNumElements() * 8;
16306 if (ShiftVal >= 16)
16307 return llvm::Constant::getNullValue(ResultType);
16311 for (
unsigned l = 0; l != NumElts; l += 16) {
16312 for (
unsigned i = 0; i != 16; ++i) {
16313 unsigned Idx = NumElts + i - ShiftVal;
16314 if (Idx < NumElts) Idx -= NumElts - 16;
16315 Indices[l + i] = Idx + l;
16319 auto *VecTy = llvm::FixedVectorType::get(
Int8Ty, NumElts);
16321 Value *
Zero = llvm::Constant::getNullValue(VecTy);
16323 Zero, Cast,
ArrayRef(Indices, NumElts),
"pslldq");
16324 return Builder.CreateBitCast(SV, Ops[0]->getType(),
"cast");
16326 case X86::BI__builtin_ia32_psrldqi128_byteshift:
16327 case X86::BI__builtin_ia32_psrldqi256_byteshift:
16328 case X86::BI__builtin_ia32_psrldqi512_byteshift: {
16329 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
16330 auto *ResultType = cast<llvm::FixedVectorType>(Ops[0]->getType());
16332 unsigned NumElts = ResultType->getNumElements() * 8;
16335 if (ShiftVal >= 16)
16336 return llvm::Constant::getNullValue(ResultType);
16340 for (
unsigned l = 0; l != NumElts; l += 16) {
16341 for (
unsigned i = 0; i != 16; ++i) {
16342 unsigned Idx = i + ShiftVal;
16343 if (Idx >= 16) Idx += NumElts - 16;
16344 Indices[l + i] = Idx + l;
16348 auto *VecTy = llvm::FixedVectorType::get(
Int8Ty, NumElts);
16350 Value *
Zero = llvm::Constant::getNullValue(VecTy);
16352 Cast, Zero,
ArrayRef(Indices, NumElts),
"psrldq");
16353 return Builder.CreateBitCast(SV, ResultType,
"cast");
16355 case X86::BI__builtin_ia32_kshiftliqi:
16356 case X86::BI__builtin_ia32_kshiftlihi:
16357 case X86::BI__builtin_ia32_kshiftlisi:
16358 case X86::BI__builtin_ia32_kshiftlidi: {
16359 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
16360 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
16362 if (ShiftVal >= NumElts)
16363 return llvm::Constant::getNullValue(Ops[0]->getType());
16368 for (
unsigned i = 0; i != NumElts; ++i)
16369 Indices[i] = NumElts + i - ShiftVal;
16371 Value *
Zero = llvm::Constant::getNullValue(
In->getType());
16373 Zero, In,
ArrayRef(Indices, NumElts),
"kshiftl");
16374 return Builder.CreateBitCast(SV, Ops[0]->getType());
16376 case X86::BI__builtin_ia32_kshiftriqi:
16377 case X86::BI__builtin_ia32_kshiftrihi:
16378 case X86::BI__builtin_ia32_kshiftrisi:
16379 case X86::BI__builtin_ia32_kshiftridi: {
16380 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
16381 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
16383 if (ShiftVal >= NumElts)
16384 return llvm::Constant::getNullValue(Ops[0]->getType());
16389 for (
unsigned i = 0; i != NumElts; ++i)
16390 Indices[i] = i + ShiftVal;
16392 Value *
Zero = llvm::Constant::getNullValue(
In->getType());
16394 In, Zero,
ArrayRef(Indices, NumElts),
"kshiftr");
16395 return Builder.CreateBitCast(SV, Ops[0]->getType());
16397 case X86::BI__builtin_ia32_movnti:
16398 case X86::BI__builtin_ia32_movnti64:
16399 case X86::BI__builtin_ia32_movntsd:
16400 case X86::BI__builtin_ia32_movntss: {
16401 llvm::MDNode *
Node = llvm::MDNode::get(
16404 Value *Ptr = Ops[0];
16405 Value *Src = Ops[1];
16408 if (BuiltinID == X86::BI__builtin_ia32_movntsd ||
16409 BuiltinID == X86::BI__builtin_ia32_movntss)
16410 Src =
Builder.CreateExtractElement(Src, (uint64_t)0,
"extract");
16414 SI->setMetadata(llvm::LLVMContext::MD_nontemporal,
Node);
16415 SI->setAlignment(llvm::Align(1));
16419 case X86::BI__builtin_ia32_vprotb:
16420 case X86::BI__builtin_ia32_vprotw:
16421 case X86::BI__builtin_ia32_vprotd:
16422 case X86::BI__builtin_ia32_vprotq:
16423 case X86::BI__builtin_ia32_vprotbi:
16424 case X86::BI__builtin_ia32_vprotwi:
16425 case X86::BI__builtin_ia32_vprotdi:
16426 case X86::BI__builtin_ia32_vprotqi:
16427 case X86::BI__builtin_ia32_prold128:
16428 case X86::BI__builtin_ia32_prold256:
16429 case X86::BI__builtin_ia32_prold512:
16430 case X86::BI__builtin_ia32_prolq128:
16431 case X86::BI__builtin_ia32_prolq256:
16432 case X86::BI__builtin_ia32_prolq512:
16433 case X86::BI__builtin_ia32_prolvd128:
16434 case X86::BI__builtin_ia32_prolvd256:
16435 case X86::BI__builtin_ia32_prolvd512:
16436 case X86::BI__builtin_ia32_prolvq128:
16437 case X86::BI__builtin_ia32_prolvq256:
16438 case X86::BI__builtin_ia32_prolvq512:
16440 case X86::BI__builtin_ia32_prord128:
16441 case X86::BI__builtin_ia32_prord256:
16442 case X86::BI__builtin_ia32_prord512:
16443 case X86::BI__builtin_ia32_prorq128:
16444 case X86::BI__builtin_ia32_prorq256:
16445 case X86::BI__builtin_ia32_prorq512:
16446 case X86::BI__builtin_ia32_prorvd128:
16447 case X86::BI__builtin_ia32_prorvd256:
16448 case X86::BI__builtin_ia32_prorvd512:
16449 case X86::BI__builtin_ia32_prorvq128:
16450 case X86::BI__builtin_ia32_prorvq256:
16451 case X86::BI__builtin_ia32_prorvq512:
16453 case X86::BI__builtin_ia32_selectb_128:
16454 case X86::BI__builtin_ia32_selectb_256:
16455 case X86::BI__builtin_ia32_selectb_512:
16456 case X86::BI__builtin_ia32_selectw_128:
16457 case X86::BI__builtin_ia32_selectw_256:
16458 case X86::BI__builtin_ia32_selectw_512:
16459 case X86::BI__builtin_ia32_selectd_128:
16460 case X86::BI__builtin_ia32_selectd_256:
16461 case X86::BI__builtin_ia32_selectd_512:
16462 case X86::BI__builtin_ia32_selectq_128:
16463 case X86::BI__builtin_ia32_selectq_256:
16464 case X86::BI__builtin_ia32_selectq_512:
16465 case X86::BI__builtin_ia32_selectph_128:
16466 case X86::BI__builtin_ia32_selectph_256:
16467 case X86::BI__builtin_ia32_selectph_512:
16468 case X86::BI__builtin_ia32_selectpbf_128:
16469 case X86::BI__builtin_ia32_selectpbf_256:
16470 case X86::BI__builtin_ia32_selectpbf_512:
16471 case X86::BI__builtin_ia32_selectps_128:
16472 case X86::BI__builtin_ia32_selectps_256:
16473 case X86::BI__builtin_ia32_selectps_512:
16474 case X86::BI__builtin_ia32_selectpd_128:
16475 case X86::BI__builtin_ia32_selectpd_256:
16476 case X86::BI__builtin_ia32_selectpd_512:
16478 case X86::BI__builtin_ia32_selectsh_128:
16479 case X86::BI__builtin_ia32_selectsbf_128:
16480 case X86::BI__builtin_ia32_selectss_128:
16481 case X86::BI__builtin_ia32_selectsd_128: {
16482 Value *A =
Builder.CreateExtractElement(Ops[1], (uint64_t)0);
16483 Value *B =
Builder.CreateExtractElement(Ops[2], (uint64_t)0);
16485 return Builder.CreateInsertElement(Ops[1], A, (uint64_t)0);
16487 case X86::BI__builtin_ia32_cmpb128_mask:
16488 case X86::BI__builtin_ia32_cmpb256_mask:
16489 case X86::BI__builtin_ia32_cmpb512_mask:
16490 case X86::BI__builtin_ia32_cmpw128_mask:
16491 case X86::BI__builtin_ia32_cmpw256_mask:
16492 case X86::BI__builtin_ia32_cmpw512_mask:
16493 case X86::BI__builtin_ia32_cmpd128_mask:
16494 case X86::BI__builtin_ia32_cmpd256_mask:
16495 case X86::BI__builtin_ia32_cmpd512_mask:
16496 case X86::BI__builtin_ia32_cmpq128_mask:
16497 case X86::BI__builtin_ia32_cmpq256_mask:
16498 case X86::BI__builtin_ia32_cmpq512_mask: {
16499 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
16502 case X86::BI__builtin_ia32_ucmpb128_mask:
16503 case X86::BI__builtin_ia32_ucmpb256_mask:
16504 case X86::BI__builtin_ia32_ucmpb512_mask:
16505 case X86::BI__builtin_ia32_ucmpw128_mask:
16506 case X86::BI__builtin_ia32_ucmpw256_mask:
16507 case X86::BI__builtin_ia32_ucmpw512_mask:
16508 case X86::BI__builtin_ia32_ucmpd128_mask:
16509 case X86::BI__builtin_ia32_ucmpd256_mask:
16510 case X86::BI__builtin_ia32_ucmpd512_mask:
16511 case X86::BI__builtin_ia32_ucmpq128_mask:
16512 case X86::BI__builtin_ia32_ucmpq256_mask:
16513 case X86::BI__builtin_ia32_ucmpq512_mask: {
16514 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
16517 case X86::BI__builtin_ia32_vpcomb:
16518 case X86::BI__builtin_ia32_vpcomw:
16519 case X86::BI__builtin_ia32_vpcomd:
16520 case X86::BI__builtin_ia32_vpcomq:
16522 case X86::BI__builtin_ia32_vpcomub:
16523 case X86::BI__builtin_ia32_vpcomuw:
16524 case X86::BI__builtin_ia32_vpcomud:
16525 case X86::BI__builtin_ia32_vpcomuq:
16528 case X86::BI__builtin_ia32_kortestcqi:
16529 case X86::BI__builtin_ia32_kortestchi:
16530 case X86::BI__builtin_ia32_kortestcsi:
16531 case X86::BI__builtin_ia32_kortestcdi: {
16533 Value *
C = llvm::Constant::getAllOnesValue(Ops[0]->getType());
16537 case X86::BI__builtin_ia32_kortestzqi:
16538 case X86::BI__builtin_ia32_kortestzhi:
16539 case X86::BI__builtin_ia32_kortestzsi:
16540 case X86::BI__builtin_ia32_kortestzdi: {
16542 Value *
C = llvm::Constant::getNullValue(Ops[0]->getType());
16547 case X86::BI__builtin_ia32_ktestcqi:
16548 case X86::BI__builtin_ia32_ktestzqi:
16549 case X86::BI__builtin_ia32_ktestchi:
16550 case X86::BI__builtin_ia32_ktestzhi:
16551 case X86::BI__builtin_ia32_ktestcsi:
16552 case X86::BI__builtin_ia32_ktestzsi:
16553 case X86::BI__builtin_ia32_ktestcdi:
16554 case X86::BI__builtin_ia32_ktestzdi: {
16556 switch (BuiltinID) {
16557 default: llvm_unreachable(
"Unsupported intrinsic!");
16558 case X86::BI__builtin_ia32_ktestcqi:
16559 IID = Intrinsic::x86_avx512_ktestc_b;
16561 case X86::BI__builtin_ia32_ktestzqi:
16562 IID = Intrinsic::x86_avx512_ktestz_b;
16564 case X86::BI__builtin_ia32_ktestchi:
16565 IID = Intrinsic::x86_avx512_ktestc_w;
16567 case X86::BI__builtin_ia32_ktestzhi:
16568 IID = Intrinsic::x86_avx512_ktestz_w;
16570 case X86::BI__builtin_ia32_ktestcsi:
16571 IID = Intrinsic::x86_avx512_ktestc_d;
16573 case X86::BI__builtin_ia32_ktestzsi:
16574 IID = Intrinsic::x86_avx512_ktestz_d;
16576 case X86::BI__builtin_ia32_ktestcdi:
16577 IID = Intrinsic::x86_avx512_ktestc_q;
16579 case X86::BI__builtin_ia32_ktestzdi:
16580 IID = Intrinsic::x86_avx512_ktestz_q;
16584 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
16588 return Builder.CreateCall(Intr, {LHS, RHS});
16591 case X86::BI__builtin_ia32_kaddqi:
16592 case X86::BI__builtin_ia32_kaddhi:
16593 case X86::BI__builtin_ia32_kaddsi:
16594 case X86::BI__builtin_ia32_kadddi: {
16596 switch (BuiltinID) {
16597 default: llvm_unreachable(
"Unsupported intrinsic!");
16598 case X86::BI__builtin_ia32_kaddqi:
16599 IID = Intrinsic::x86_avx512_kadd_b;
16601 case X86::BI__builtin_ia32_kaddhi:
16602 IID = Intrinsic::x86_avx512_kadd_w;
16604 case X86::BI__builtin_ia32_kaddsi:
16605 IID = Intrinsic::x86_avx512_kadd_d;
16607 case X86::BI__builtin_ia32_kadddi:
16608 IID = Intrinsic::x86_avx512_kadd_q;
16612 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
16617 return Builder.CreateBitCast(Res, Ops[0]->getType());
16619 case X86::BI__builtin_ia32_kandqi:
16620 case X86::BI__builtin_ia32_kandhi:
16621 case X86::BI__builtin_ia32_kandsi:
16622 case X86::BI__builtin_ia32_kanddi:
16624 case X86::BI__builtin_ia32_kandnqi:
16625 case X86::BI__builtin_ia32_kandnhi:
16626 case X86::BI__builtin_ia32_kandnsi:
16627 case X86::BI__builtin_ia32_kandndi:
16629 case X86::BI__builtin_ia32_korqi:
16630 case X86::BI__builtin_ia32_korhi:
16631 case X86::BI__builtin_ia32_korsi:
16632 case X86::BI__builtin_ia32_kordi:
16634 case X86::BI__builtin_ia32_kxnorqi:
16635 case X86::BI__builtin_ia32_kxnorhi:
16636 case X86::BI__builtin_ia32_kxnorsi:
16637 case X86::BI__builtin_ia32_kxnordi:
16639 case X86::BI__builtin_ia32_kxorqi:
16640 case X86::BI__builtin_ia32_kxorhi:
16641 case X86::BI__builtin_ia32_kxorsi:
16642 case X86::BI__builtin_ia32_kxordi:
16644 case X86::BI__builtin_ia32_knotqi:
16645 case X86::BI__builtin_ia32_knothi:
16646 case X86::BI__builtin_ia32_knotsi:
16647 case X86::BI__builtin_ia32_knotdi: {
16648 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
16651 Ops[0]->getType());
16653 case X86::BI__builtin_ia32_kmovb:
16654 case X86::BI__builtin_ia32_kmovw:
16655 case X86::BI__builtin_ia32_kmovd:
16656 case X86::BI__builtin_ia32_kmovq: {
16660 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
16662 return Builder.CreateBitCast(Res, Ops[0]->getType());
16665 case X86::BI__builtin_ia32_kunpckdi:
16666 case X86::BI__builtin_ia32_kunpcksi:
16667 case X86::BI__builtin_ia32_kunpckhi: {
16668 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
16672 for (
unsigned i = 0; i != NumElts; ++i)
16677 LHS =
Builder.CreateShuffleVector(LHS, LHS,
ArrayRef(Indices, NumElts / 2));
16678 RHS =
Builder.CreateShuffleVector(RHS, RHS,
ArrayRef(Indices, NumElts / 2));
16683 return Builder.CreateBitCast(Res, Ops[0]->getType());
16686 case X86::BI__builtin_ia32_vplzcntd_128:
16687 case X86::BI__builtin_ia32_vplzcntd_256:
16688 case X86::BI__builtin_ia32_vplzcntd_512:
16689 case X86::BI__builtin_ia32_vplzcntq_128:
16690 case X86::BI__builtin_ia32_vplzcntq_256:
16691 case X86::BI__builtin_ia32_vplzcntq_512: {
16695 case X86::BI__builtin_ia32_sqrtss:
16696 case X86::BI__builtin_ia32_sqrtsd: {
16697 Value *A =
Builder.CreateExtractElement(Ops[0], (uint64_t)0);
16699 if (
Builder.getIsFPConstrained()) {
16700 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
16703 A =
Builder.CreateConstrainedFPCall(F, {A});
16706 A =
Builder.CreateCall(F, {A});
16708 return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0);
16710 case X86::BI__builtin_ia32_sqrtsh_round_mask:
16711 case X86::BI__builtin_ia32_sqrtsd_round_mask:
16712 case X86::BI__builtin_ia32_sqrtss_round_mask: {
16713 unsigned CC = cast<llvm::ConstantInt>(Ops[4])->getZExtValue();
16719 switch (BuiltinID) {
16721 llvm_unreachable(
"Unsupported intrinsic!");
16722 case X86::BI__builtin_ia32_sqrtsh_round_mask:
16723 IID = Intrinsic::x86_avx512fp16_mask_sqrt_sh;
16725 case X86::BI__builtin_ia32_sqrtsd_round_mask:
16726 IID = Intrinsic::x86_avx512_mask_sqrt_sd;
16728 case X86::BI__builtin_ia32_sqrtss_round_mask:
16729 IID = Intrinsic::x86_avx512_mask_sqrt_ss;
16734 Value *A =
Builder.CreateExtractElement(Ops[1], (uint64_t)0);
16736 if (
Builder.getIsFPConstrained()) {
16737 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
16740 A =
Builder.CreateConstrainedFPCall(F, A);
16743 A =
Builder.CreateCall(F, A);
16745 Value *Src =
Builder.CreateExtractElement(Ops[2], (uint64_t)0);
16747 return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0);
16749 case X86::BI__builtin_ia32_sqrtpd256:
16750 case X86::BI__builtin_ia32_sqrtpd:
16751 case X86::BI__builtin_ia32_sqrtps256:
16752 case X86::BI__builtin_ia32_sqrtps:
16753 case X86::BI__builtin_ia32_sqrtph256:
16754 case X86::BI__builtin_ia32_sqrtph:
16755 case X86::BI__builtin_ia32_sqrtph512:
16756 case X86::BI__builtin_ia32_vsqrtbf16256:
16757 case X86::BI__builtin_ia32_vsqrtbf16:
16758 case X86::BI__builtin_ia32_vsqrtbf16512:
16759 case X86::BI__builtin_ia32_sqrtps512:
16760 case X86::BI__builtin_ia32_sqrtpd512: {
16761 if (Ops.size() == 2) {
16762 unsigned CC = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
16768 switch (BuiltinID) {
16770 llvm_unreachable(
"Unsupported intrinsic!");
16771 case X86::BI__builtin_ia32_sqrtph512:
16772 IID = Intrinsic::x86_avx512fp16_sqrt_ph_512;
16774 case X86::BI__builtin_ia32_sqrtps512:
16775 IID = Intrinsic::x86_avx512_sqrt_ps_512;
16777 case X86::BI__builtin_ia32_sqrtpd512:
16778 IID = Intrinsic::x86_avx512_sqrt_pd_512;
16784 if (
Builder.getIsFPConstrained()) {
16785 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
16787 Ops[0]->getType());
16788 return Builder.CreateConstrainedFPCall(F, Ops[0]);
16791 return Builder.CreateCall(F, Ops[0]);
16795 case X86::BI__builtin_ia32_pmuludq128:
16796 case X86::BI__builtin_ia32_pmuludq256:
16797 case X86::BI__builtin_ia32_pmuludq512:
16800 case X86::BI__builtin_ia32_pmuldq128:
16801 case X86::BI__builtin_ia32_pmuldq256:
16802 case X86::BI__builtin_ia32_pmuldq512:
16805 case X86::BI__builtin_ia32_pternlogd512_mask:
16806 case X86::BI__builtin_ia32_pternlogq512_mask:
16807 case X86::BI__builtin_ia32_pternlogd128_mask:
16808 case X86::BI__builtin_ia32_pternlogd256_mask:
16809 case X86::BI__builtin_ia32_pternlogq128_mask:
16810 case X86::BI__builtin_ia32_pternlogq256_mask:
16813 case X86::BI__builtin_ia32_pternlogd512_maskz:
16814 case X86::BI__builtin_ia32_pternlogq512_maskz:
16815 case X86::BI__builtin_ia32_pternlogd128_maskz:
16816 case X86::BI__builtin_ia32_pternlogd256_maskz:
16817 case X86::BI__builtin_ia32_pternlogq128_maskz:
16818 case X86::BI__builtin_ia32_pternlogq256_maskz:
16821 case X86::BI__builtin_ia32_vpshldd128:
16822 case X86::BI__builtin_ia32_vpshldd256:
16823 case X86::BI__builtin_ia32_vpshldd512:
16824 case X86::BI__builtin_ia32_vpshldq128:
16825 case X86::BI__builtin_ia32_vpshldq256:
16826 case X86::BI__builtin_ia32_vpshldq512:
16827 case X86::BI__builtin_ia32_vpshldw128:
16828 case X86::BI__builtin_ia32_vpshldw256:
16829 case X86::BI__builtin_ia32_vpshldw512:
16832 case X86::BI__builtin_ia32_vpshrdd128:
16833 case X86::BI__builtin_ia32_vpshrdd256:
16834 case X86::BI__builtin_ia32_vpshrdd512:
16835 case X86::BI__builtin_ia32_vpshrdq128:
16836 case X86::BI__builtin_ia32_vpshrdq256:
16837 case X86::BI__builtin_ia32_vpshrdq512:
16838 case X86::BI__builtin_ia32_vpshrdw128:
16839 case X86::BI__builtin_ia32_vpshrdw256:
16840 case X86::BI__builtin_ia32_vpshrdw512:
16844 case X86::BI__builtin_ia32_vpshldvd128:
16845 case X86::BI__builtin_ia32_vpshldvd256:
16846 case X86::BI__builtin_ia32_vpshldvd512:
16847 case X86::BI__builtin_ia32_vpshldvq128:
16848 case X86::BI__builtin_ia32_vpshldvq256:
16849 case X86::BI__builtin_ia32_vpshldvq512:
16850 case X86::BI__builtin_ia32_vpshldvw128:
16851 case X86::BI__builtin_ia32_vpshldvw256:
16852 case X86::BI__builtin_ia32_vpshldvw512:
16855 case X86::BI__builtin_ia32_vpshrdvd128:
16856 case X86::BI__builtin_ia32_vpshrdvd256:
16857 case X86::BI__builtin_ia32_vpshrdvd512:
16858 case X86::BI__builtin_ia32_vpshrdvq128:
16859 case X86::BI__builtin_ia32_vpshrdvq256:
16860 case X86::BI__builtin_ia32_vpshrdvq512:
16861 case X86::BI__builtin_ia32_vpshrdvw128:
16862 case X86::BI__builtin_ia32_vpshrdvw256:
16863 case X86::BI__builtin_ia32_vpshrdvw512:
16868 case X86::BI__builtin_ia32_reduce_fadd_pd512:
16869 case X86::BI__builtin_ia32_reduce_fadd_ps512:
16870 case X86::BI__builtin_ia32_reduce_fadd_ph512:
16871 case X86::BI__builtin_ia32_reduce_fadd_ph256:
16872 case X86::BI__builtin_ia32_reduce_fadd_ph128: {
16875 IRBuilder<>::FastMathFlagGuard FMFGuard(
Builder);
16876 Builder.getFastMathFlags().setAllowReassoc();
16877 return Builder.CreateCall(F, {Ops[0], Ops[1]});
16879 case X86::BI__builtin_ia32_reduce_fmul_pd512:
16880 case X86::BI__builtin_ia32_reduce_fmul_ps512:
16881 case X86::BI__builtin_ia32_reduce_fmul_ph512:
16882 case X86::BI__builtin_ia32_reduce_fmul_ph256:
16883 case X86::BI__builtin_ia32_reduce_fmul_ph128: {
16886 IRBuilder<>::FastMathFlagGuard FMFGuard(
Builder);
16887 Builder.getFastMathFlags().setAllowReassoc();
16888 return Builder.CreateCall(F, {Ops[0], Ops[1]});
16890 case X86::BI__builtin_ia32_reduce_fmax_pd512:
16891 case X86::BI__builtin_ia32_reduce_fmax_ps512:
16892 case X86::BI__builtin_ia32_reduce_fmax_ph512:
16893 case X86::BI__builtin_ia32_reduce_fmax_ph256:
16894 case X86::BI__builtin_ia32_reduce_fmax_ph128: {
16897 IRBuilder<>::FastMathFlagGuard FMFGuard(
Builder);
16898 Builder.getFastMathFlags().setNoNaNs();
16899 return Builder.CreateCall(F, {Ops[0]});
16901 case X86::BI__builtin_ia32_reduce_fmin_pd512:
16902 case X86::BI__builtin_ia32_reduce_fmin_ps512:
16903 case X86::BI__builtin_ia32_reduce_fmin_ph512:
16904 case X86::BI__builtin_ia32_reduce_fmin_ph256:
16905 case X86::BI__builtin_ia32_reduce_fmin_ph128: {
16908 IRBuilder<>::FastMathFlagGuard FMFGuard(
Builder);
16909 Builder.getFastMathFlags().setNoNaNs();
16910 return Builder.CreateCall(F, {Ops[0]});
16913 case X86::BI__builtin_ia32_rdrand16_step:
16914 case X86::BI__builtin_ia32_rdrand32_step:
16915 case X86::BI__builtin_ia32_rdrand64_step:
16916 case X86::BI__builtin_ia32_rdseed16_step:
16917 case X86::BI__builtin_ia32_rdseed32_step:
16918 case X86::BI__builtin_ia32_rdseed64_step: {
16920 switch (BuiltinID) {
16921 default: llvm_unreachable(
"Unsupported intrinsic!");
16922 case X86::BI__builtin_ia32_rdrand16_step:
16923 ID = Intrinsic::x86_rdrand_16;
16925 case X86::BI__builtin_ia32_rdrand32_step:
16926 ID = Intrinsic::x86_rdrand_32;
16928 case X86::BI__builtin_ia32_rdrand64_step:
16929 ID = Intrinsic::x86_rdrand_64;
16931 case X86::BI__builtin_ia32_rdseed16_step:
16932 ID = Intrinsic::x86_rdseed_16;
16934 case X86::BI__builtin_ia32_rdseed32_step:
16935 ID = Intrinsic::x86_rdseed_32;
16937 case X86::BI__builtin_ia32_rdseed64_step:
16938 ID = Intrinsic::x86_rdseed_64;
16947 case X86::BI__builtin_ia32_addcarryx_u32:
16948 case X86::BI__builtin_ia32_addcarryx_u64:
16949 case X86::BI__builtin_ia32_subborrow_u32:
16950 case X86::BI__builtin_ia32_subborrow_u64: {
16952 switch (BuiltinID) {
16953 default: llvm_unreachable(
"Unsupported intrinsic!");
16954 case X86::BI__builtin_ia32_addcarryx_u32:
16955 IID = Intrinsic::x86_addcarry_32;
16957 case X86::BI__builtin_ia32_addcarryx_u64:
16958 IID = Intrinsic::x86_addcarry_64;
16960 case X86::BI__builtin_ia32_subborrow_u32:
16961 IID = Intrinsic::x86_subborrow_32;
16963 case X86::BI__builtin_ia32_subborrow_u64:
16964 IID = Intrinsic::x86_subborrow_64;
16969 { Ops[0], Ops[1], Ops[2] });
16975 case X86::BI__builtin_ia32_fpclassps128_mask:
16976 case X86::BI__builtin_ia32_fpclassps256_mask:
16977 case X86::BI__builtin_ia32_fpclassps512_mask:
16978 case X86::BI__builtin_ia32_vfpclassbf16128_mask:
16979 case X86::BI__builtin_ia32_vfpclassbf16256_mask:
16980 case X86::BI__builtin_ia32_vfpclassbf16512_mask:
16981 case X86::BI__builtin_ia32_fpclassph128_mask:
16982 case X86::BI__builtin_ia32_fpclassph256_mask:
16983 case X86::BI__builtin_ia32_fpclassph512_mask:
16984 case X86::BI__builtin_ia32_fpclasspd128_mask:
16985 case X86::BI__builtin_ia32_fpclasspd256_mask:
16986 case X86::BI__builtin_ia32_fpclasspd512_mask: {
16988 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16989 Value *MaskIn = Ops[2];
16990 Ops.erase(&Ops[2]);
16993 switch (BuiltinID) {
16994 default: llvm_unreachable(
"Unsupported intrinsic!");
16995 case X86::BI__builtin_ia32_vfpclassbf16128_mask:
16996 ID = Intrinsic::x86_avx10_fpclass_bf16_128;
16998 case X86::BI__builtin_ia32_vfpclassbf16256_mask:
16999 ID = Intrinsic::x86_avx10_fpclass_bf16_256;
17001 case X86::BI__builtin_ia32_vfpclassbf16512_mask:
17002 ID = Intrinsic::x86_avx10_fpclass_bf16_512;
17004 case X86::BI__builtin_ia32_fpclassph128_mask:
17005 ID = Intrinsic::x86_avx512fp16_fpclass_ph_128;
17007 case X86::BI__builtin_ia32_fpclassph256_mask:
17008 ID = Intrinsic::x86_avx512fp16_fpclass_ph_256;
17010 case X86::BI__builtin_ia32_fpclassph512_mask:
17011 ID = Intrinsic::x86_avx512fp16_fpclass_ph_512;
17013 case X86::BI__builtin_ia32_fpclassps128_mask:
17014 ID = Intrinsic::x86_avx512_fpclass_ps_128;
17016 case X86::BI__builtin_ia32_fpclassps256_mask:
17017 ID = Intrinsic::x86_avx512_fpclass_ps_256;
17019 case X86::BI__builtin_ia32_fpclassps512_mask:
17020 ID = Intrinsic::x86_avx512_fpclass_ps_512;
17022 case X86::BI__builtin_ia32_fpclasspd128_mask:
17023 ID = Intrinsic::x86_avx512_fpclass_pd_128;
17025 case X86::BI__builtin_ia32_fpclasspd256_mask:
17026 ID = Intrinsic::x86_avx512_fpclass_pd_256;
17028 case X86::BI__builtin_ia32_fpclasspd512_mask:
17029 ID = Intrinsic::x86_avx512_fpclass_pd_512;
17037 case X86::BI__builtin_ia32_vp2intersect_q_512:
17038 case X86::BI__builtin_ia32_vp2intersect_q_256:
17039 case X86::BI__builtin_ia32_vp2intersect_q_128:
17040 case X86::BI__builtin_ia32_vp2intersect_d_512:
17041 case X86::BI__builtin_ia32_vp2intersect_d_256:
17042 case X86::BI__builtin_ia32_vp2intersect_d_128: {
17044 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
17047 switch (BuiltinID) {
17048 default: llvm_unreachable(
"Unsupported intrinsic!");
17049 case X86::BI__builtin_ia32_vp2intersect_q_512:
17050 ID = Intrinsic::x86_avx512_vp2intersect_q_512;
17052 case X86::BI__builtin_ia32_vp2intersect_q_256:
17053 ID = Intrinsic::x86_avx512_vp2intersect_q_256;
17055 case X86::BI__builtin_ia32_vp2intersect_q_128:
17056 ID = Intrinsic::x86_avx512_vp2intersect_q_128;
17058 case X86::BI__builtin_ia32_vp2intersect_d_512:
17059 ID = Intrinsic::x86_avx512_vp2intersect_d_512;
17061 case X86::BI__builtin_ia32_vp2intersect_d_256:
17062 ID = Intrinsic::x86_avx512_vp2intersect_d_256;
17064 case X86::BI__builtin_ia32_vp2intersect_d_128:
17065 ID = Intrinsic::x86_avx512_vp2intersect_d_128;
17079 case X86::BI__builtin_ia32_vpmultishiftqb128:
17080 case X86::BI__builtin_ia32_vpmultishiftqb256:
17081 case X86::BI__builtin_ia32_vpmultishiftqb512: {
17083 switch (BuiltinID) {
17084 default: llvm_unreachable(
"Unsupported intrinsic!");
17085 case X86::BI__builtin_ia32_vpmultishiftqb128:
17086 ID = Intrinsic::x86_avx512_pmultishift_qb_128;
17088 case X86::BI__builtin_ia32_vpmultishiftqb256:
17089 ID = Intrinsic::x86_avx512_pmultishift_qb_256;
17091 case X86::BI__builtin_ia32_vpmultishiftqb512:
17092 ID = Intrinsic::x86_avx512_pmultishift_qb_512;
17099 case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
17100 case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
17101 case X86::BI__builtin_ia32_vpshufbitqmb512_mask: {
17103 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
17104 Value *MaskIn = Ops[2];
17105 Ops.erase(&Ops[2]);
17108 switch (BuiltinID) {
17109 default: llvm_unreachable(
"Unsupported intrinsic!");
17110 case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
17111 ID = Intrinsic::x86_avx512_vpshufbitqmb_128;
17113 case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
17114 ID = Intrinsic::x86_avx512_vpshufbitqmb_256;
17116 case X86::BI__builtin_ia32_vpshufbitqmb512_mask:
17117 ID = Intrinsic::x86_avx512_vpshufbitqmb_512;
17126 case X86::BI__builtin_ia32_cmpeqps:
17127 case X86::BI__builtin_ia32_cmpeqpd:
17128 return getVectorFCmpIR(CmpInst::FCMP_OEQ,
false);
17129 case X86::BI__builtin_ia32_cmpltps:
17130 case X86::BI__builtin_ia32_cmpltpd:
17131 return getVectorFCmpIR(CmpInst::FCMP_OLT,
true);
17132 case X86::BI__builtin_ia32_cmpleps:
17133 case X86::BI__builtin_ia32_cmplepd:
17134 return getVectorFCmpIR(CmpInst::FCMP_OLE,
true);
17135 case X86::BI__builtin_ia32_cmpunordps:
17136 case X86::BI__builtin_ia32_cmpunordpd:
17137 return getVectorFCmpIR(CmpInst::FCMP_UNO,
false);
17138 case X86::BI__builtin_ia32_cmpneqps:
17139 case X86::BI__builtin_ia32_cmpneqpd:
17140 return getVectorFCmpIR(CmpInst::FCMP_UNE,
false);
17141 case X86::BI__builtin_ia32_cmpnltps:
17142 case X86::BI__builtin_ia32_cmpnltpd:
17143 return getVectorFCmpIR(CmpInst::FCMP_UGE,
true);
17144 case X86::BI__builtin_ia32_cmpnleps:
17145 case X86::BI__builtin_ia32_cmpnlepd:
17146 return getVectorFCmpIR(CmpInst::FCMP_UGT,
true);
17147 case X86::BI__builtin_ia32_cmpordps:
17148 case X86::BI__builtin_ia32_cmpordpd:
17149 return getVectorFCmpIR(CmpInst::FCMP_ORD,
false);
17150 case X86::BI__builtin_ia32_cmpph128_mask:
17151 case X86::BI__builtin_ia32_cmpph256_mask:
17152 case X86::BI__builtin_ia32_cmpph512_mask:
17153 case X86::BI__builtin_ia32_cmpps128_mask:
17154 case X86::BI__builtin_ia32_cmpps256_mask:
17155 case X86::BI__builtin_ia32_cmpps512_mask:
17156 case X86::BI__builtin_ia32_cmppd128_mask:
17157 case X86::BI__builtin_ia32_cmppd256_mask:
17158 case X86::BI__builtin_ia32_cmppd512_mask:
17159 case X86::BI__builtin_ia32_vcmppd256_round_mask:
17160 case X86::BI__builtin_ia32_vcmpps256_round_mask:
17161 case X86::BI__builtin_ia32_vcmpph256_round_mask:
17162 case X86::BI__builtin_ia32_vcmpbf16512_mask:
17163 case X86::BI__builtin_ia32_vcmpbf16256_mask:
17164 case X86::BI__builtin_ia32_vcmpbf16128_mask:
17167 case X86::BI__builtin_ia32_cmpps:
17168 case X86::BI__builtin_ia32_cmpps256:
17169 case X86::BI__builtin_ia32_cmppd:
17170 case X86::BI__builtin_ia32_cmppd256: {
17178 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x1f;
17183 FCmpInst::Predicate Pred;
17187 switch (CC & 0xf) {
17188 case 0x00: Pred = FCmpInst::FCMP_OEQ; IsSignaling =
false;
break;
17189 case 0x01: Pred = FCmpInst::FCMP_OLT; IsSignaling =
true;
break;
17190 case 0x02: Pred = FCmpInst::FCMP_OLE; IsSignaling =
true;
break;
17191 case 0x03: Pred = FCmpInst::FCMP_UNO; IsSignaling =
false;
break;
17192 case 0x04: Pred = FCmpInst::FCMP_UNE; IsSignaling =
false;
break;
17193 case 0x05: Pred = FCmpInst::FCMP_UGE; IsSignaling =
true;
break;
17194 case 0x06: Pred = FCmpInst::FCMP_UGT; IsSignaling =
true;
break;
17195 case 0x07: Pred = FCmpInst::FCMP_ORD; IsSignaling =
false;
break;
17196 case 0x08: Pred = FCmpInst::FCMP_UEQ; IsSignaling =
false;
break;
17197 case 0x09: Pred = FCmpInst::FCMP_ULT; IsSignaling =
true;
break;
17198 case 0x0a: Pred = FCmpInst::FCMP_ULE; IsSignaling =
true;
break;
17199 case 0x0b: Pred = FCmpInst::FCMP_FALSE; IsSignaling =
false;
break;
17200 case 0x0c: Pred = FCmpInst::FCMP_ONE; IsSignaling =
false;
break;
17201 case 0x0d: Pred = FCmpInst::FCMP_OGE; IsSignaling =
true;
break;
17202 case 0x0e: Pred = FCmpInst::FCMP_OGT; IsSignaling =
true;
break;
17203 case 0x0f: Pred = FCmpInst::FCMP_TRUE; IsSignaling =
false;
break;
17204 default: llvm_unreachable(
"Unhandled CC");
17209 IsSignaling = !IsSignaling;
17216 if (
Builder.getIsFPConstrained() &&
17217 (Pred == FCmpInst::FCMP_TRUE || Pred == FCmpInst::FCMP_FALSE ||
17221 switch (BuiltinID) {
17222 default: llvm_unreachable(
"Unexpected builtin");
17223 case X86::BI__builtin_ia32_cmpps:
17224 IID = Intrinsic::x86_sse_cmp_ps;
17226 case X86::BI__builtin_ia32_cmpps256:
17227 IID = Intrinsic::x86_avx_cmp_ps_256;
17229 case X86::BI__builtin_ia32_cmppd:
17230 IID = Intrinsic::x86_sse2_cmp_pd;
17232 case X86::BI__builtin_ia32_cmppd256:
17233 IID = Intrinsic::x86_avx_cmp_pd_256;
17235 case X86::BI__builtin_ia32_cmpph128_mask:
17236 IID = Intrinsic::x86_avx512fp16_mask_cmp_ph_128;
17238 case X86::BI__builtin_ia32_cmpph256_mask:
17239 IID = Intrinsic::x86_avx512fp16_mask_cmp_ph_256;
17241 case X86::BI__builtin_ia32_cmpph512_mask:
17242 IID = Intrinsic::x86_avx512fp16_mask_cmp_ph_512;
17244 case X86::BI__builtin_ia32_cmpps512_mask:
17245 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
17247 case X86::BI__builtin_ia32_cmppd512_mask:
17248 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
17250 case X86::BI__builtin_ia32_cmpps128_mask:
17251 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
17253 case X86::BI__builtin_ia32_cmpps256_mask:
17254 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
17256 case X86::BI__builtin_ia32_cmppd128_mask:
17257 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
17259 case X86::BI__builtin_ia32_cmppd256_mask:
17260 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
17267 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
17273 return Builder.CreateCall(Intr, Ops);
17284 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
17287 Cmp =
Builder.CreateFCmpS(Pred, Ops[0], Ops[1]);
17289 Cmp =
Builder.CreateFCmp(Pred, Ops[0], Ops[1]);
17293 return getVectorFCmpIR(Pred, IsSignaling);
17297 case X86::BI__builtin_ia32_cmpeqss:
17298 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 0);
17299 case X86::BI__builtin_ia32_cmpltss:
17300 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 1);
17301 case X86::BI__builtin_ia32_cmpless:
17302 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 2);
17303 case X86::BI__builtin_ia32_cmpunordss:
17304 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 3);
17305 case X86::BI__builtin_ia32_cmpneqss:
17306 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 4);
17307 case X86::BI__builtin_ia32_cmpnltss:
17308 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 5);
17309 case X86::BI__builtin_ia32_cmpnless:
17310 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 6);
17311 case X86::BI__builtin_ia32_cmpordss:
17312 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 7);
17313 case X86::BI__builtin_ia32_cmpeqsd:
17314 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 0);
17315 case X86::BI__builtin_ia32_cmpltsd:
17316 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 1);
17317 case X86::BI__builtin_ia32_cmplesd:
17318 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 2);
17319 case X86::BI__builtin_ia32_cmpunordsd:
17320 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 3);
17321 case X86::BI__builtin_ia32_cmpneqsd:
17322 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 4);
17323 case X86::BI__builtin_ia32_cmpnltsd:
17324 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 5);
17325 case X86::BI__builtin_ia32_cmpnlesd:
17326 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 6);
17327 case X86::BI__builtin_ia32_cmpordsd:
17328 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 7);
17331 case X86::BI__builtin_ia32_vcvtph2ps:
17332 case X86::BI__builtin_ia32_vcvtph2ps256:
17333 case X86::BI__builtin_ia32_vcvtph2ps_mask:
17334 case X86::BI__builtin_ia32_vcvtph2ps256_mask:
17335 case X86::BI__builtin_ia32_vcvtph2ps512_mask: {
17336 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
17341 case X86::BI__builtin_ia32_cvtneps2bf16_128_mask: {
17344 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements());
17345 Intrinsic::ID IID = Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128;
17348 case X86::BI__builtin_ia32_cvtsbf162ss_32:
17351 case X86::BI__builtin_ia32_cvtneps2bf16_256_mask:
17352 case X86::BI__builtin_ia32_cvtneps2bf16_512_mask: {
17354 switch (BuiltinID) {
17355 default: llvm_unreachable(
"Unsupported intrinsic!");
17356 case X86::BI__builtin_ia32_cvtneps2bf16_256_mask:
17357 IID = Intrinsic::x86_avx512bf16_cvtneps2bf16_256;
17359 case X86::BI__builtin_ia32_cvtneps2bf16_512_mask:
17360 IID = Intrinsic::x86_avx512bf16_cvtneps2bf16_512;
17367 case X86::BI__cpuid:
17368 case X86::BI__cpuidex: {
17370 Value *SubFuncId = BuiltinID == X86::BI__cpuidex
17374 llvm::StructType *CpuidRetTy =
17376 llvm::FunctionType *FTy =
17379 StringRef
Asm, Constraints;
17380 if (
getTarget().getTriple().getArch() == llvm::Triple::x86) {
17382 Constraints =
"={ax},={bx},={cx},={dx},{ax},{cx}";
17385 Asm =
"xchgq %rbx, ${1:q}\n"
17387 "xchgq %rbx, ${1:q}";
17388 Constraints =
"={ax},=r,={cx},={dx},0,2";
17391 llvm::InlineAsm *IA = llvm::InlineAsm::get(FTy,
Asm, Constraints,
17393 Value *IACall =
Builder.CreateCall(IA, {FuncId, SubFuncId});
17396 for (
unsigned i = 0; i < 4; i++) {
17397 Value *Extracted =
Builder.CreateExtractValue(IACall, i);
17407 case X86::BI__emul:
17408 case X86::BI__emulu: {
17410 bool isSigned = (BuiltinID == X86::BI__emul);
17413 return Builder.CreateMul(LHS, RHS,
"", !isSigned, isSigned);
17415 case X86::BI__mulh:
17416 case X86::BI__umulh:
17417 case X86::BI_mul128:
17418 case X86::BI_umul128: {
17420 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
17422 bool IsSigned = (BuiltinID == X86::BI__mulh || BuiltinID == X86::BI_mul128);
17423 Value *LHS =
Builder.CreateIntCast(Ops[0], Int128Ty, IsSigned);
17424 Value *RHS =
Builder.CreateIntCast(Ops[1], Int128Ty, IsSigned);
17426 Value *MulResult, *HigherBits;
17428 MulResult =
Builder.CreateNSWMul(LHS, RHS);
17429 HigherBits =
Builder.CreateAShr(MulResult, 64);
17431 MulResult =
Builder.CreateNUWMul(LHS, RHS);
17432 HigherBits =
Builder.CreateLShr(MulResult, 64);
17434 HigherBits =
Builder.CreateIntCast(HigherBits, ResType, IsSigned);
17436 if (BuiltinID == X86::BI__mulh || BuiltinID == X86::BI__umulh)
17441 return Builder.CreateIntCast(MulResult, ResType, IsSigned);
17444 case X86::BI__faststorefence: {
17445 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
17446 llvm::SyncScope::System);
17448 case X86::BI__shiftleft128:
17449 case X86::BI__shiftright128: {
17451 BuiltinID == X86::BI__shiftleft128 ? Intrinsic::fshl : Intrinsic::fshr,
17456 std::swap(Ops[0], Ops[1]);
17458 return Builder.CreateCall(F, Ops);
17460 case X86::BI_ReadWriteBarrier:
17461 case X86::BI_ReadBarrier:
17462 case X86::BI_WriteBarrier: {
17463 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
17464 llvm::SyncScope::SingleThread);
17467 case X86::BI_AddressOfReturnAddress: {
17470 return Builder.CreateCall(F);
17472 case X86::BI__stosb: {
17478 case X86::BI__builtin_ia32_t2rpntlvwz0_internal:
17479 case X86::BI__builtin_ia32_t2rpntlvwz0rs_internal:
17480 case X86::BI__builtin_ia32_t2rpntlvwz0t1_internal:
17481 case X86::BI__builtin_ia32_t2rpntlvwz0rst1_internal:
17482 case X86::BI__builtin_ia32_t2rpntlvwz1_internal:
17483 case X86::BI__builtin_ia32_t2rpntlvwz1rs_internal:
17484 case X86::BI__builtin_ia32_t2rpntlvwz1t1_internal:
17485 case X86::BI__builtin_ia32_t2rpntlvwz1rst1_internal: {
17487 switch (BuiltinID) {
17489 llvm_unreachable(
"Unsupported intrinsic!");
17490 case X86::BI__builtin_ia32_t2rpntlvwz0_internal:
17491 IID = Intrinsic::x86_t2rpntlvwz0_internal;
17493 case X86::BI__builtin_ia32_t2rpntlvwz0rs_internal:
17494 IID = Intrinsic::x86_t2rpntlvwz0rs_internal;
17496 case X86::BI__builtin_ia32_t2rpntlvwz0t1_internal:
17497 IID = Intrinsic::x86_t2rpntlvwz0t1_internal;
17499 case X86::BI__builtin_ia32_t2rpntlvwz0rst1_internal:
17500 IID = Intrinsic::x86_t2rpntlvwz0rst1_internal;
17502 case X86::BI__builtin_ia32_t2rpntlvwz1_internal:
17503 IID = Intrinsic::x86_t2rpntlvwz1_internal;
17505 case X86::BI__builtin_ia32_t2rpntlvwz1rs_internal:
17506 IID = Intrinsic::x86_t2rpntlvwz1rs_internal;
17508 case X86::BI__builtin_ia32_t2rpntlvwz1t1_internal:
17509 IID = Intrinsic::x86_t2rpntlvwz1t1_internal;
17511 case X86::BI__builtin_ia32_t2rpntlvwz1rst1_internal:
17512 IID = Intrinsic::x86_t2rpntlvwz1rst1_internal;
17518 {Ops[0], Ops[1], Ops[2], Ops[5], Ops[6]});
17521 assert(PtrTy &&
"arg3 must be of pointer type");
17528 Value *VecT0 =
Builder.CreateIntrinsic(Intrinsic::x86_cast_tile_to_vector,
17534 Value *VecT1 =
Builder.CreateIntrinsic(Intrinsic::x86_cast_tile_to_vector,
17548 case X86::BI__int2c: {
17550 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy,
false);
17551 llvm::InlineAsm *IA =
17552 llvm::InlineAsm::get(FTy,
"int $$0x2c",
"",
true);
17553 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
17555 llvm::Attribute::NoReturn);
17556 llvm::CallInst *CI =
Builder.CreateCall(IA);
17557 CI->setAttributes(NoReturnAttr);
17560 case X86::BI__readfsbyte:
17561 case X86::BI__readfsword:
17562 case X86::BI__readfsdword:
17563 case X86::BI__readfsqword: {
17569 Load->setVolatile(
true);
17572 case X86::BI__readgsbyte:
17573 case X86::BI__readgsword:
17574 case X86::BI__readgsdword:
17575 case X86::BI__readgsqword: {
17581 Load->setVolatile(
true);
17584 case X86::BI__builtin_ia32_encodekey128_u32: {
17585 Intrinsic::ID IID = Intrinsic::x86_encodekey128;
17589 for (
int i = 0; i < 3; ++i) {
17597 case X86::BI__builtin_ia32_encodekey256_u32: {
17598 Intrinsic::ID IID = Intrinsic::x86_encodekey256;
17603 for (
int i = 0; i < 4; ++i) {
17611 case X86::BI__builtin_ia32_aesenc128kl_u8:
17612 case X86::BI__builtin_ia32_aesdec128kl_u8:
17613 case X86::BI__builtin_ia32_aesenc256kl_u8:
17614 case X86::BI__builtin_ia32_aesdec256kl_u8: {
17616 StringRef BlockName;
17617 switch (BuiltinID) {
17619 llvm_unreachable(
"Unexpected builtin");
17620 case X86::BI__builtin_ia32_aesenc128kl_u8:
17621 IID = Intrinsic::x86_aesenc128kl;
17622 BlockName =
"aesenc128kl";
17624 case X86::BI__builtin_ia32_aesdec128kl_u8:
17625 IID = Intrinsic::x86_aesdec128kl;
17626 BlockName =
"aesdec128kl";
17628 case X86::BI__builtin_ia32_aesenc256kl_u8:
17629 IID = Intrinsic::x86_aesenc256kl;
17630 BlockName =
"aesenc256kl";
17632 case X86::BI__builtin_ia32_aesdec256kl_u8:
17633 IID = Intrinsic::x86_aesdec256kl;
17634 BlockName =
"aesdec256kl";
17640 BasicBlock *NoError =
17648 Builder.CreateCondBr(Succ, NoError, Error);
17650 Builder.SetInsertPoint(NoError);
17654 Builder.SetInsertPoint(Error);
17655 Constant *
Zero = llvm::Constant::getNullValue(Out->getType());
17662 case X86::BI__builtin_ia32_aesencwide128kl_u8:
17663 case X86::BI__builtin_ia32_aesdecwide128kl_u8:
17664 case X86::BI__builtin_ia32_aesencwide256kl_u8:
17665 case X86::BI__builtin_ia32_aesdecwide256kl_u8: {
17667 StringRef BlockName;
17668 switch (BuiltinID) {
17669 case X86::BI__builtin_ia32_aesencwide128kl_u8:
17670 IID = Intrinsic::x86_aesencwide128kl;
17671 BlockName =
"aesencwide128kl";
17673 case X86::BI__builtin_ia32_aesdecwide128kl_u8:
17674 IID = Intrinsic::x86_aesdecwide128kl;
17675 BlockName =
"aesdecwide128kl";
17677 case X86::BI__builtin_ia32_aesencwide256kl_u8:
17678 IID = Intrinsic::x86_aesencwide256kl;
17679 BlockName =
"aesencwide256kl";
17681 case X86::BI__builtin_ia32_aesdecwide256kl_u8:
17682 IID = Intrinsic::x86_aesdecwide256kl;
17683 BlockName =
"aesdecwide256kl";
17687 llvm::Type *Ty = FixedVectorType::get(
Builder.getInt64Ty(), 2);
17690 for (
int i = 0; i != 8; ++i) {
17691 Value *Ptr =
Builder.CreateConstGEP1_32(Ty, Ops[1], i);
17697 BasicBlock *NoError =
17704 Builder.CreateCondBr(Succ, NoError, Error);
17706 Builder.SetInsertPoint(NoError);
17707 for (
int i = 0; i != 8; ++i) {
17714 Builder.SetInsertPoint(Error);
17715 for (
int i = 0; i != 8; ++i) {
17717 Constant *
Zero = llvm::Constant::getNullValue(Out->getType());
17718 Value *Ptr =
Builder.CreateConstGEP1_32(Out->getType(), Ops[0], i);
17726 case X86::BI__builtin_ia32_vfcmaddcph512_mask:
17729 case X86::BI__builtin_ia32_vfmaddcph512_mask: {
17730 Intrinsic::ID IID = IsConjFMA
17731 ? Intrinsic::x86_avx512fp16_mask_vfcmadd_cph_512
17732 : Intrinsic::x86_avx512fp16_mask_vfmadd_cph_512;
17736 case X86::BI__builtin_ia32_vfcmaddcph256_round_mask:
17739 case X86::BI__builtin_ia32_vfmaddcph256_round_mask: {
17740 Intrinsic::ID IID = IsConjFMA ? Intrinsic::x86_avx10_mask_vfcmaddcph256
17741 : Intrinsic::x86_avx10_mask_vfmaddcph256;
17745 case X86::BI__builtin_ia32_vfcmaddcsh_round_mask:
17748 case X86::BI__builtin_ia32_vfmaddcsh_round_mask: {
17749 Intrinsic::ID IID = IsConjFMA ? Intrinsic::x86_avx512fp16_mask_vfcmadd_csh
17750 : Intrinsic::x86_avx512fp16_mask_vfmadd_csh;
17755 case X86::BI__builtin_ia32_vfcmaddcsh_round_mask3:
17758 case X86::BI__builtin_ia32_vfmaddcsh_round_mask3: {
17759 Intrinsic::ID IID = IsConjFMA ? Intrinsic::x86_avx512fp16_mask_vfcmadd_csh
17760 : Intrinsic::x86_avx512fp16_mask_vfmadd_csh;
17762 static constexpr int Mask[] = {0, 5, 6, 7};
17763 return Builder.CreateShuffleVector(
Call, Ops[2], Mask);
17765 case X86::BI__builtin_ia32_prefetchi:
17768 {Ops[0], llvm::ConstantInt::get(Int32Ty, 0), Ops[1],
17769 llvm::ConstantInt::get(Int32Ty, 0)});
17787 Intrinsic::ID
ID = Intrinsic::not_intrinsic;
17789#include "llvm/TargetParser/PPCTargetParser.def"
17790 auto GenAIXPPCBuiltinCpuExpr = [&](
unsigned SupportMethod,
unsigned FieldIdx,
17791 unsigned Mask, CmpInst::Predicate CompOp,
17792 unsigned OpValue) ->
Value * {
17793 if (SupportMethod == BUILTIN_PPC_FALSE)
17796 if (SupportMethod == BUILTIN_PPC_TRUE)
17799 assert(SupportMethod <= SYS_CALL &&
"Invalid value for SupportMethod.");
17801 llvm::Value *FieldValue =
nullptr;
17802 if (SupportMethod == USE_SYS_CONF) {
17803 llvm::Type *STy = llvm::StructType::get(PPC_SYSTEMCONFIG_TYPE);
17804 llvm::Constant *SysConf =
17808 llvm::Value *Idxs[] = {ConstantInt::get(
Int32Ty, 0),
17809 ConstantInt::get(
Int32Ty, FieldIdx)};
17814 }
else if (SupportMethod == SYS_CALL) {
17815 llvm::FunctionType *FTy =
17817 llvm::FunctionCallee
Func =
17823 assert(FieldValue &&
17824 "SupportMethod value is not defined in PPCTargetParser.def.");
17827 FieldValue =
Builder.CreateAnd(FieldValue, Mask);
17829 llvm::Type *ValueType = FieldValue->getType();
17830 bool IsValueType64Bit = ValueType->isIntegerTy(64);
17832 (IsValueType64Bit || ValueType->isIntegerTy(32)) &&
17833 "Only 32/64-bit integers are supported in GenAIXPPCBuiltinCpuExpr().");
17836 CompOp, FieldValue,
17837 ConstantInt::get(IsValueType64Bit ?
Int64Ty :
Int32Ty, OpValue));
17840 switch (BuiltinID) {
17841 default:
return nullptr;
17843 case Builtin::BI__builtin_cpu_is: {
17845 StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
17848 unsigned LinuxSupportMethod, LinuxIDValue, AIXSupportMethod, AIXIDValue;
17849 typedef std::tuple<unsigned, unsigned, unsigned, unsigned>
CPUInfo;
17851 std::tie(LinuxSupportMethod, LinuxIDValue, AIXSupportMethod, AIXIDValue) =
17852 static_cast<CPUInfo>(StringSwitch<CPUInfo>(CPUStr)
17853#define PPC_CPU(NAME, Linux_SUPPORT_METHOD, LinuxID, AIX_SUPPORT_METHOD, \
17855 .Case(NAME, {Linux_SUPPORT_METHOD, LinuxID, AIX_SUPPORT_METHOD, AIXID})
17856#include "llvm/TargetParser/PPCTargetParser.def"
17857 .Default({BUILTIN_PPC_UNSUPPORTED, 0,
17858 BUILTIN_PPC_UNSUPPORTED, 0}));
17860 if (Triple.isOSAIX()) {
17861 assert((AIXSupportMethod != BUILTIN_PPC_UNSUPPORTED) &&
17862 "Invalid CPU name. Missed by SemaChecking?");
17863 return GenAIXPPCBuiltinCpuExpr(AIXSupportMethod, AIX_SYSCON_IMPL_IDX, 0,
17864 ICmpInst::ICMP_EQ, AIXIDValue);
17867 assert(Triple.isOSLinux() &&
17868 "__builtin_cpu_is() is only supported for AIX and Linux.");
17870 assert((LinuxSupportMethod != BUILTIN_PPC_UNSUPPORTED) &&
17871 "Invalid CPU name. Missed by SemaChecking?");
17873 if (LinuxSupportMethod == BUILTIN_PPC_FALSE)
17876 Value *Op0 = llvm::ConstantInt::get(
Int32Ty, PPC_FAWORD_CPUID);
17878 Value *TheCall =
Builder.CreateCall(F, {Op0},
"cpu_is");
17879 return Builder.CreateICmpEQ(TheCall,
17880 llvm::ConstantInt::get(
Int32Ty, LinuxIDValue));
17882 case Builtin::BI__builtin_cpu_supports: {
17885 StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
17886 if (Triple.isOSAIX()) {
17887 unsigned SupportMethod, FieldIdx, Mask,
Value;
17888 CmpInst::Predicate CompOp;
17892 std::tie(SupportMethod, FieldIdx, Mask, CompOp,
Value) =
17893 static_cast<CPUSupportType
>(StringSwitch<CPUSupportType>(CPUStr)
17894#define PPC_AIX_FEATURE(NAME, DESC, SUPPORT_METHOD, INDEX, MASK, COMP_OP, \
17896 .Case(NAME, {SUPPORT_METHOD, INDEX, MASK, COMP_OP, VALUE})
17897#include "llvm/TargetParser/PPCTargetParser.def"
17898 .Default({BUILTIN_PPC_FALSE, 0, 0,
17899 CmpInst::Predicate(), 0}));
17900 return GenAIXPPCBuiltinCpuExpr(SupportMethod, FieldIdx, Mask, CompOp,
17904 assert(Triple.isOSLinux() &&
17905 "__builtin_cpu_supports() is only supported for AIX and Linux.");
17906 unsigned FeatureWord;
17908 std::tie(FeatureWord, BitMask) =
17909 StringSwitch<std::pair<unsigned, unsigned>>(CPUStr)
17910#define
PPC_LNX_FEATURE(Name, Description, EnumName, Bitmask, FA_WORD) \
17911 .Case(Name, {FA_WORD, Bitmask})
17912#include
"llvm/TargetParser/PPCTargetParser.def"
17916 Value *Op0 = llvm::ConstantInt::get(
Int32Ty, FeatureWord);
17918 Value *TheCall =
Builder.CreateCall(F, {Op0},
"cpu_supports");
17920 Builder.CreateAnd(TheCall, llvm::ConstantInt::get(
Int32Ty, BitMask));
17921 return Builder.CreateICmpNE(Mask, llvm::Constant::getNullValue(
Int32Ty));
17922#undef PPC_FAWORD_HWCAP
17923#undef PPC_FAWORD_HWCAP2
17924#undef PPC_FAWORD_CPUID
17929 case PPC::BI__builtin_ppc_get_timebase:
17933 case PPC::BI__builtin_altivec_lvx:
17934 case PPC::BI__builtin_altivec_lvxl:
17935 case PPC::BI__builtin_altivec_lvebx:
17936 case PPC::BI__builtin_altivec_lvehx:
17937 case PPC::BI__builtin_altivec_lvewx:
17938 case PPC::BI__builtin_altivec_lvsl:
17939 case PPC::BI__builtin_altivec_lvsr:
17940 case PPC::BI__builtin_vsx_lxvd2x:
17941 case PPC::BI__builtin_vsx_lxvw4x:
17942 case PPC::BI__builtin_vsx_lxvd2x_be:
17943 case PPC::BI__builtin_vsx_lxvw4x_be:
17944 case PPC::BI__builtin_vsx_lxvl:
17945 case PPC::BI__builtin_vsx_lxvll:
17950 if (!(BuiltinID == PPC::BI__builtin_vsx_lxvl ||
17951 BuiltinID == PPC::BI__builtin_vsx_lxvll)) {
17956 switch (BuiltinID) {
17957 default: llvm_unreachable(
"Unsupported ld/lvsl/lvsr intrinsic!");
17958 case PPC::BI__builtin_altivec_lvx:
17959 ID = Intrinsic::ppc_altivec_lvx;
17961 case PPC::BI__builtin_altivec_lvxl:
17962 ID = Intrinsic::ppc_altivec_lvxl;
17964 case PPC::BI__builtin_altivec_lvebx:
17965 ID = Intrinsic::ppc_altivec_lvebx;
17967 case PPC::BI__builtin_altivec_lvehx:
17968 ID = Intrinsic::ppc_altivec_lvehx;
17970 case PPC::BI__builtin_altivec_lvewx:
17971 ID = Intrinsic::ppc_altivec_lvewx;
17973 case PPC::BI__builtin_altivec_lvsl:
17974 ID = Intrinsic::ppc_altivec_lvsl;
17976 case PPC::BI__builtin_altivec_lvsr:
17977 ID = Intrinsic::ppc_altivec_lvsr;
17979 case PPC::BI__builtin_vsx_lxvd2x:
17980 ID = Intrinsic::ppc_vsx_lxvd2x;
17982 case PPC::BI__builtin_vsx_lxvw4x:
17983 ID = Intrinsic::ppc_vsx_lxvw4x;
17985 case PPC::BI__builtin_vsx_lxvd2x_be:
17986 ID = Intrinsic::ppc_vsx_lxvd2x_be;
17988 case PPC::BI__builtin_vsx_lxvw4x_be:
17989 ID = Intrinsic::ppc_vsx_lxvw4x_be;
17991 case PPC::BI__builtin_vsx_lxvl:
17992 ID = Intrinsic::ppc_vsx_lxvl;
17994 case PPC::BI__builtin_vsx_lxvll:
17995 ID = Intrinsic::ppc_vsx_lxvll;
17999 return Builder.CreateCall(F, Ops,
"");
18003 case PPC::BI__builtin_altivec_stvx:
18004 case PPC::BI__builtin_altivec_stvxl:
18005 case PPC::BI__builtin_altivec_stvebx:
18006 case PPC::BI__builtin_altivec_stvehx:
18007 case PPC::BI__builtin_altivec_stvewx:
18008 case PPC::BI__builtin_vsx_stxvd2x:
18009 case PPC::BI__builtin_vsx_stxvw4x:
18010 case PPC::BI__builtin_vsx_stxvd2x_be:
18011 case PPC::BI__builtin_vsx_stxvw4x_be:
18012 case PPC::BI__builtin_vsx_stxvl:
18013 case PPC::BI__builtin_vsx_stxvll:
18019 if (!(BuiltinID == PPC::BI__builtin_vsx_stxvl ||
18020 BuiltinID == PPC::BI__builtin_vsx_stxvll)) {
18025 switch (BuiltinID) {
18026 default: llvm_unreachable(
"Unsupported st intrinsic!");
18027 case PPC::BI__builtin_altivec_stvx:
18028 ID = Intrinsic::ppc_altivec_stvx;
18030 case PPC::BI__builtin_altivec_stvxl:
18031 ID = Intrinsic::ppc_altivec_stvxl;
18033 case PPC::BI__builtin_altivec_stvebx:
18034 ID = Intrinsic::ppc_altivec_stvebx;
18036 case PPC::BI__builtin_altivec_stvehx:
18037 ID = Intrinsic::ppc_altivec_stvehx;
18039 case PPC::BI__builtin_altivec_stvewx:
18040 ID = Intrinsic::ppc_altivec_stvewx;
18042 case PPC::BI__builtin_vsx_stxvd2x:
18043 ID = Intrinsic::ppc_vsx_stxvd2x;
18045 case PPC::BI__builtin_vsx_stxvw4x:
18046 ID = Intrinsic::ppc_vsx_stxvw4x;
18048 case PPC::BI__builtin_vsx_stxvd2x_be:
18049 ID = Intrinsic::ppc_vsx_stxvd2x_be;
18051 case PPC::BI__builtin_vsx_stxvw4x_be:
18052 ID = Intrinsic::ppc_vsx_stxvw4x_be;
18054 case PPC::BI__builtin_vsx_stxvl:
18055 ID = Intrinsic::ppc_vsx_stxvl;
18057 case PPC::BI__builtin_vsx_stxvll:
18058 ID = Intrinsic::ppc_vsx_stxvll;
18062 return Builder.CreateCall(F, Ops,
"");
18064 case PPC::BI__builtin_vsx_ldrmb: {
18070 int64_t NumBytes = cast<ConstantInt>(Op1)->getZExtValue();
18075 if (NumBytes == 16) {
18083 for (
int Idx = 0; Idx < 16; Idx++)
18084 RevMask.push_back(15 - Idx);
18085 return Builder.CreateShuffleVector(LD, LD, RevMask);
18089 llvm::Function *Lvs =
CGM.
getIntrinsic(IsLE ? Intrinsic::ppc_altivec_lvsr
18090 : Intrinsic::ppc_altivec_lvsl);
18091 llvm::Function *Vperm =
CGM.
getIntrinsic(Intrinsic::ppc_altivec_vperm);
18093 Int8Ty, Op0, ConstantInt::get(Op1->
getType(), NumBytes - 1));
18095 Value *HiLd =
Builder.CreateCall(Lvx, HiMem,
"ld.hi");
18098 Op0 = IsLE ? HiLd : LoLd;
18099 Op1 = IsLE ? LoLd : HiLd;
18100 Value *AllElts =
Builder.CreateCall(Vperm, {Op0, Op1, Mask1},
"shuffle1");
18101 Constant *
Zero = llvm::Constant::getNullValue(IsLE ? ResTy : AllElts->
getType());
18105 for (
int Idx = 0; Idx < 16; Idx++) {
18106 int Val = (NumBytes - Idx - 1 >= 0) ? (NumBytes - Idx - 1)
18107 : 16 - (NumBytes - Idx);
18108 Consts.push_back(Val);
18110 return Builder.CreateShuffleVector(
Builder.CreateBitCast(AllElts, ResTy),
18114 for (
int Idx = 0; Idx < 16; Idx++)
18115 Consts.push_back(
Builder.getInt8(NumBytes + Idx));
18116 Value *Mask2 = ConstantVector::get(Consts);
18117 return Builder.CreateBitCast(
18118 Builder.CreateCall(Vperm, {Zero, AllElts, Mask2},
"shuffle2"), ResTy);
18120 case PPC::BI__builtin_vsx_strmb: {
18124 int64_t NumBytes = cast<ConstantInt>(Op1)->getZExtValue();
18126 auto StoreSubVec = [&](
unsigned Width,
unsigned Offset,
unsigned EltNo) {
18130 Value *StVec = Op2;
18133 for (
int Idx = 0; Idx < 16; Idx++)
18134 RevMask.push_back(15 - Idx);
18135 StVec =
Builder.CreateShuffleVector(Op2, Op2, RevMask);
18141 unsigned NumElts = 0;
18144 llvm_unreachable(
"width for stores must be a power of 2");
18163 Op2, llvm::FixedVectorType::get(ConvTy, NumElts));
18166 Value *Elt =
Builder.CreateExtractElement(Vec, EltNo);
18167 if (IsLE && Width > 1) {
18169 Elt =
Builder.CreateCall(F, Elt);
18174 unsigned Stored = 0;
18175 unsigned RemainingBytes = NumBytes;
18177 if (NumBytes == 16)
18178 return StoreSubVec(16, 0, 0);
18179 if (NumBytes >= 8) {
18180 Result = StoreSubVec(8, NumBytes - 8, IsLE ? 0 : 1);
18181 RemainingBytes -= 8;
18184 if (RemainingBytes >= 4) {
18185 Result = StoreSubVec(4, NumBytes - Stored - 4,
18186 IsLE ? (Stored >> 2) : 3 - (Stored >> 2));
18187 RemainingBytes -= 4;
18190 if (RemainingBytes >= 2) {
18191 Result = StoreSubVec(2, NumBytes - Stored - 2,
18192 IsLE ? (Stored >> 1) : 7 - (Stored >> 1));
18193 RemainingBytes -= 2;
18196 if (RemainingBytes)
18198 StoreSubVec(1, NumBytes - Stored - 1, IsLE ? Stored : 15 - Stored);
18202 case PPC::BI__builtin_vsx_xvsqrtsp:
18203 case PPC::BI__builtin_vsx_xvsqrtdp: {
18206 if (
Builder.getIsFPConstrained()) {
18208 Intrinsic::experimental_constrained_sqrt, ResultType);
18209 return Builder.CreateConstrainedFPCall(F,
X);
18216 case PPC::BI__builtin_altivec_vclzb:
18217 case PPC::BI__builtin_altivec_vclzh:
18218 case PPC::BI__builtin_altivec_vclzw:
18219 case PPC::BI__builtin_altivec_vclzd: {
18222 Value *Undef = ConstantInt::get(
Builder.getInt1Ty(),
false);
18224 return Builder.CreateCall(F, {
X, Undef});
18226 case PPC::BI__builtin_altivec_vctzb:
18227 case PPC::BI__builtin_altivec_vctzh:
18228 case PPC::BI__builtin_altivec_vctzw:
18229 case PPC::BI__builtin_altivec_vctzd: {
18232 Value *Undef = ConstantInt::get(
Builder.getInt1Ty(),
false);
18234 return Builder.CreateCall(F, {
X, Undef});
18236 case PPC::BI__builtin_altivec_vinsd:
18237 case PPC::BI__builtin_altivec_vinsw:
18238 case PPC::BI__builtin_altivec_vinsd_elt:
18239 case PPC::BI__builtin_altivec_vinsw_elt: {
18245 bool IsUnaligned = (BuiltinID == PPC::BI__builtin_altivec_vinsw ||
18246 BuiltinID == PPC::BI__builtin_altivec_vinsd);
18248 bool Is32bit = (BuiltinID == PPC::BI__builtin_altivec_vinsw ||
18249 BuiltinID == PPC::BI__builtin_altivec_vinsw_elt);
18252 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
18254 "Third Arg to vinsw/vinsd intrinsic must be a constant integer!");
18258 int ValidMaxValue = 0;
18260 ValidMaxValue = (Is32bit) ? 12 : 8;
18262 ValidMaxValue = (Is32bit) ? 3 : 1;
18265 int64_t ConstArg = ArgCI->getSExtValue();
18268 std::string RangeErrMsg = IsUnaligned ?
"byte" :
"element";
18269 RangeErrMsg +=
" number " + llvm::to_string(ConstArg);
18270 RangeErrMsg +=
" is outside of the valid range [0, ";
18271 RangeErrMsg += llvm::to_string(ValidMaxValue) +
"]";
18274 if (ConstArg < 0 || ConstArg > ValidMaxValue)
18278 if (!IsUnaligned) {
18279 ConstArg *= Is32bit ? 4 : 8;
18282 ConstArg = (Is32bit ? 12 : 8) - ConstArg;
18285 ID = Is32bit ? Intrinsic::ppc_altivec_vinsw : Intrinsic::ppc_altivec_vinsd;
18286 Op2 = ConstantInt::getSigned(
Int32Ty, ConstArg);
18290 ?
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int32Ty, 4))
18292 llvm::FixedVectorType::get(
Int64Ty, 2));
18293 return Builder.CreateBitCast(
18296 case PPC::BI__builtin_altivec_vadduqm:
18297 case PPC::BI__builtin_altivec_vsubuqm: {
18300 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
18301 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(Int128Ty, 1));
18302 Op1 =
Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(Int128Ty, 1));
18303 if (BuiltinID == PPC::BI__builtin_altivec_vadduqm)
18304 return Builder.CreateAdd(Op0, Op1,
"vadduqm");
18306 return Builder.CreateSub(Op0, Op1,
"vsubuqm");
18308 case PPC::BI__builtin_altivec_vaddcuq_c:
18309 case PPC::BI__builtin_altivec_vsubcuq_c: {
18313 llvm::Type *V1I128Ty = llvm::FixedVectorType::get(
18315 Ops.push_back(
Builder.CreateBitCast(Op0, V1I128Ty));
18316 Ops.push_back(
Builder.CreateBitCast(Op1, V1I128Ty));
18317 ID = (BuiltinID == PPC::BI__builtin_altivec_vaddcuq_c)
18318 ? Intrinsic::ppc_altivec_vaddcuq
18319 : Intrinsic::ppc_altivec_vsubcuq;
18322 case PPC::BI__builtin_altivec_vaddeuqm_c:
18323 case PPC::BI__builtin_altivec_vaddecuq_c:
18324 case PPC::BI__builtin_altivec_vsubeuqm_c:
18325 case PPC::BI__builtin_altivec_vsubecuq_c: {
18330 llvm::Type *V1I128Ty = llvm::FixedVectorType::get(
18332 Ops.push_back(
Builder.CreateBitCast(Op0, V1I128Ty));
18333 Ops.push_back(
Builder.CreateBitCast(Op1, V1I128Ty));
18334 Ops.push_back(
Builder.CreateBitCast(Op2, V1I128Ty));
18335 switch (BuiltinID) {
18337 llvm_unreachable(
"Unsupported intrinsic!");
18338 case PPC::BI__builtin_altivec_vaddeuqm_c:
18339 ID = Intrinsic::ppc_altivec_vaddeuqm;
18341 case PPC::BI__builtin_altivec_vaddecuq_c:
18342 ID = Intrinsic::ppc_altivec_vaddecuq;
18344 case PPC::BI__builtin_altivec_vsubeuqm_c:
18345 ID = Intrinsic::ppc_altivec_vsubeuqm;
18347 case PPC::BI__builtin_altivec_vsubecuq_c:
18348 ID = Intrinsic::ppc_altivec_vsubecuq;
18353 case PPC::BI__builtin_ppc_rldimi:
18354 case PPC::BI__builtin_ppc_rlwimi: {
18361 if (BuiltinID == PPC::BI__builtin_ppc_rldimi &&
18371 ? Intrinsic::ppc_rldimi
18372 : Intrinsic::ppc_rlwimi),
18373 {Op0, Op1, Op2, Op3});
18375 case PPC::BI__builtin_ppc_rlwnm: {
18382 case PPC::BI__builtin_ppc_poppar4:
18383 case PPC::BI__builtin_ppc_poppar8: {
18385 llvm::Type *ArgType = Op0->
getType();
18391 if (
Result->getType() != ResultType)
18396 case PPC::BI__builtin_ppc_cmpb: {
18399 if (
getTarget().getTriple().isPPC64()) {
18402 return Builder.CreateCall(F, {Op0, Op1},
"cmpb");
18422 Constant *ShiftAmt = ConstantInt::get(
Int64Ty, 32);
18431 Value *ResHi =
Builder.CreateShl(ResHiShift, ShiftAmt);
18432 return Builder.CreateOr(ResLo, ResHi);
18435 case PPC::BI__builtin_vsx_xvcpsgnsp:
18436 case PPC::BI__builtin_vsx_xvcpsgndp: {
18440 ID = Intrinsic::copysign;
18442 return Builder.CreateCall(F, {
X, Y});
18445 case PPC::BI__builtin_vsx_xvrspip:
18446 case PPC::BI__builtin_vsx_xvrdpip:
18447 case PPC::BI__builtin_vsx_xvrdpim:
18448 case PPC::BI__builtin_vsx_xvrspim:
18449 case PPC::BI__builtin_vsx_xvrdpi:
18450 case PPC::BI__builtin_vsx_xvrspi:
18451 case PPC::BI__builtin_vsx_xvrdpic:
18452 case PPC::BI__builtin_vsx_xvrspic:
18453 case PPC::BI__builtin_vsx_xvrdpiz:
18454 case PPC::BI__builtin_vsx_xvrspiz: {
18457 if (BuiltinID == PPC::BI__builtin_vsx_xvrdpim ||
18458 BuiltinID == PPC::BI__builtin_vsx_xvrspim)
18460 ? Intrinsic::experimental_constrained_floor
18461 : Intrinsic::floor;
18462 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpi ||
18463 BuiltinID == PPC::BI__builtin_vsx_xvrspi)
18465 ? Intrinsic::experimental_constrained_round
18466 : Intrinsic::round;
18467 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpic ||
18468 BuiltinID == PPC::BI__builtin_vsx_xvrspic)
18470 ? Intrinsic::experimental_constrained_rint
18472 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpip ||
18473 BuiltinID == PPC::BI__builtin_vsx_xvrspip)
18475 ? Intrinsic::experimental_constrained_ceil
18477 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpiz ||
18478 BuiltinID == PPC::BI__builtin_vsx_xvrspiz)
18480 ? Intrinsic::experimental_constrained_trunc
18481 : Intrinsic::trunc;
18483 return Builder.getIsFPConstrained() ?
Builder.CreateConstrainedFPCall(F,
X)
18488 case PPC::BI__builtin_vsx_xvabsdp:
18489 case PPC::BI__builtin_vsx_xvabssp: {
18497 case PPC::BI__builtin_ppc_recipdivf:
18498 case PPC::BI__builtin_ppc_recipdivd:
18499 case PPC::BI__builtin_ppc_rsqrtf:
18500 case PPC::BI__builtin_ppc_rsqrtd: {
18501 FastMathFlags FMF =
Builder.getFastMathFlags();
18502 Builder.getFastMathFlags().setFast();
18506 if (BuiltinID == PPC::BI__builtin_ppc_recipdivf ||
18507 BuiltinID == PPC::BI__builtin_ppc_recipdivd) {
18510 Builder.getFastMathFlags() &= (FMF);
18513 auto *One = ConstantFP::get(ResultType, 1.0);
18516 Builder.getFastMathFlags() &= (FMF);
18519 case PPC::BI__builtin_ppc_alignx: {
18522 ConstantInt *AlignmentCI = cast<ConstantInt>(Op0);
18523 if (AlignmentCI->getValue().ugt(llvm::Value::MaximumAlignment))
18524 AlignmentCI = ConstantInt::get(AlignmentCI->getIntegerType(),
18525 llvm::Value::MaximumAlignment);
18529 AlignmentCI,
nullptr);
18532 case PPC::BI__builtin_ppc_rdlam: {
18536 llvm::Type *Ty = Op0->
getType();
18537 Value *ShiftAmt =
Builder.CreateIntCast(Op1, Ty,
false);
18539 Value *Rotate =
Builder.CreateCall(F, {Op0, Op0, ShiftAmt});
18540 return Builder.CreateAnd(Rotate, Op2);
18542 case PPC::BI__builtin_ppc_load2r: {
18549 case PPC::BI__builtin_ppc_fnmsub:
18550 case PPC::BI__builtin_ppc_fnmsubs:
18551 case PPC::BI__builtin_vsx_xvmaddadp:
18552 case PPC::BI__builtin_vsx_xvmaddasp:
18553 case PPC::BI__builtin_vsx_xvnmaddadp:
18554 case PPC::BI__builtin_vsx_xvnmaddasp:
18555 case PPC::BI__builtin_vsx_xvmsubadp:
18556 case PPC::BI__builtin_vsx_xvmsubasp:
18557 case PPC::BI__builtin_vsx_xvnmsubadp:
18558 case PPC::BI__builtin_vsx_xvnmsubasp: {
18564 if (
Builder.getIsFPConstrained())
18565 F =
CGM.
getIntrinsic(Intrinsic::experimental_constrained_fma, ResultType);
18568 switch (BuiltinID) {
18569 case PPC::BI__builtin_vsx_xvmaddadp:
18570 case PPC::BI__builtin_vsx_xvmaddasp:
18571 if (
Builder.getIsFPConstrained())
18572 return Builder.CreateConstrainedFPCall(F, {
X, Y, Z});
18574 return Builder.CreateCall(F, {
X, Y, Z});
18575 case PPC::BI__builtin_vsx_xvnmaddadp:
18576 case PPC::BI__builtin_vsx_xvnmaddasp:
18577 if (
Builder.getIsFPConstrained())
18579 Builder.CreateConstrainedFPCall(F, {X, Y, Z}),
"neg");
18581 return Builder.CreateFNeg(
Builder.CreateCall(F, {X, Y, Z}),
"neg");
18582 case PPC::BI__builtin_vsx_xvmsubadp:
18583 case PPC::BI__builtin_vsx_xvmsubasp:
18584 if (
Builder.getIsFPConstrained())
18585 return Builder.CreateConstrainedFPCall(
18586 F, {
X, Y,
Builder.CreateFNeg(Z,
"neg")});
18589 case PPC::BI__builtin_ppc_fnmsub:
18590 case PPC::BI__builtin_ppc_fnmsubs:
18591 case PPC::BI__builtin_vsx_xvnmsubadp:
18592 case PPC::BI__builtin_vsx_xvnmsubasp:
18593 if (
Builder.getIsFPConstrained())
18595 Builder.CreateConstrainedFPCall(
18596 F, {X, Y, Builder.CreateFNeg(Z,
"neg")}),
18602 llvm_unreachable(
"Unknown FMA operation");
18606 case PPC::BI__builtin_vsx_insertword: {
18614 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
18616 "Third arg to xxinsertw intrinsic must be constant integer");
18618 int64_t Index = std::clamp(ArgCI->getSExtValue(), (int64_t)0, MaxIndex);
18625 std::swap(Op0, Op1);
18629 Op1 =
Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(
Int64Ty, 2));
18633 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int64Ty, 2));
18637 Index = MaxIndex - Index;
18641 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int32Ty, 4));
18642 Op2 = ConstantInt::getSigned(
Int32Ty, Index);
18643 return Builder.CreateCall(F, {Op0, Op1, Op2});
18646 case PPC::BI__builtin_vsx_extractuword: {
18649 llvm::Function *F =
CGM.
getIntrinsic(Intrinsic::ppc_vsx_xxextractuw);
18652 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int64Ty, 2));
18656 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op1);
18658 "Second Arg to xxextractuw intrinsic must be a constant integer!");
18660 int64_t Index = std::clamp(ArgCI->getSExtValue(), (int64_t)0, MaxIndex);
18664 Index = MaxIndex - Index;
18665 Op1 = ConstantInt::getSigned(
Int32Ty, Index);
18670 Value *ShuffleCall =
18672 return ShuffleCall;
18674 Op1 = ConstantInt::getSigned(
Int32Ty, Index);
18675 return Builder.CreateCall(F, {Op0, Op1});
18679 case PPC::BI__builtin_vsx_xxpermdi: {
18683 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
18684 assert(ArgCI &&
"Third arg must be constant integer!");
18686 unsigned Index = ArgCI->getZExtValue();
18687 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int64Ty, 2));
18688 Op1 =
Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(
Int64Ty, 2));
18693 int ElemIdx0 = (Index & 2) >> 1;
18694 int ElemIdx1 = 2 + (Index & 1);
18696 int ShuffleElts[2] = {ElemIdx0, ElemIdx1};
18697 Value *ShuffleCall =
Builder.CreateShuffleVector(Op0, Op1, ShuffleElts);
18700 return Builder.CreateBitCast(ShuffleCall, RetTy);
18703 case PPC::BI__builtin_vsx_xxsldwi: {
18707 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
18708 assert(ArgCI &&
"Third argument must be a compile time constant");
18709 unsigned Index = ArgCI->getZExtValue() & 0x3;
18710 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int32Ty, 4));
18711 Op1 =
Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(
Int32Ty, 4));
18722 ElemIdx0 = (8 - Index) % 8;
18723 ElemIdx1 = (9 - Index) % 8;
18724 ElemIdx2 = (10 - Index) % 8;
18725 ElemIdx3 = (11 - Index) % 8;
18729 ElemIdx1 = Index + 1;
18730 ElemIdx2 = Index + 2;
18731 ElemIdx3 = Index + 3;
18734 int ShuffleElts[4] = {ElemIdx0, ElemIdx1, ElemIdx2, ElemIdx3};
18735 Value *ShuffleCall =
Builder.CreateShuffleVector(Op0, Op1, ShuffleElts);
18738 return Builder.CreateBitCast(ShuffleCall, RetTy);
18741 case PPC::BI__builtin_pack_vector_int128: {
18745 Value *PoisonValue =
18746 llvm::PoisonValue::get(llvm::FixedVectorType::get(Op0->
getType(), 2));
18748 PoisonValue, Op0, (uint64_t)(isLittleEndian ? 1 : 0));
18749 Res =
Builder.CreateInsertElement(Res, Op1,
18750 (uint64_t)(isLittleEndian ? 0 : 1));
18754 case PPC::BI__builtin_unpack_vector_int128: {
18757 ConstantInt *Index = cast<ConstantInt>(Op1);
18763 ConstantInt::get(Index->getIntegerType(), 1 - Index->getZExtValue());
18765 return Builder.CreateExtractElement(Unpacked, Index);
18768 case PPC::BI__builtin_ppc_sthcx: {
18772 return Builder.CreateCall(F, {Op0, Op1});
18781#define CUSTOM_BUILTIN(Name, Intr, Types, Accumulate, Feature) \
18782 case PPC::BI__builtin_##Name:
18783#include "clang/Basic/BuiltinsPPC.def"
18786 for (
unsigned i = 0, e =
E->getNumArgs(); i != e; i++)
18796 if (BuiltinID == PPC::BI__builtin_mma_disassemble_acc ||
18797 BuiltinID == PPC::BI__builtin_vsx_disassemble_pair ||
18798 BuiltinID == PPC::BI__builtin_mma_disassemble_pair) {
18799 unsigned NumVecs = 2;
18800 auto Intrinsic = Intrinsic::ppc_vsx_disassemble_pair;
18801 if (BuiltinID == PPC::BI__builtin_mma_disassemble_acc) {
18803 Intrinsic = Intrinsic::ppc_mma_disassemble_acc;
18809 llvm::Type *VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
18810 Value *Ptr = Ops[0];
18811 for (
unsigned i=0; i<NumVecs; i++) {
18813 llvm::ConstantInt* Index = llvm::ConstantInt::get(
IntTy, i);
18819 if (BuiltinID == PPC::BI__builtin_vsx_build_pair ||
18820 BuiltinID == PPC::BI__builtin_mma_build_acc) {
18828 std::reverse(Ops.begin() + 1, Ops.end());
18831 switch (BuiltinID) {
18832 #define CUSTOM_BUILTIN(Name, Intr, Types, Acc, Feature) \
18833 case PPC::BI__builtin_##Name: \
18834 ID = Intrinsic::ppc_##Intr; \
18835 Accumulate = Acc; \
18837 #include "clang/Basic/BuiltinsPPC.def"
18839 if (BuiltinID == PPC::BI__builtin_vsx_lxvp ||
18840 BuiltinID == PPC::BI__builtin_vsx_stxvp ||
18841 BuiltinID == PPC::BI__builtin_mma_lxvp ||
18842 BuiltinID == PPC::BI__builtin_mma_stxvp) {
18843 if (BuiltinID == PPC::BI__builtin_vsx_lxvp ||
18844 BuiltinID == PPC::BI__builtin_mma_lxvp) {
18851 return Builder.CreateCall(F, Ops,
"");
18857 CallOps.push_back(Acc);
18859 for (
unsigned i=1; i<Ops.size(); i++)
18860 CallOps.push_back(Ops[i]);
18866 case PPC::BI__builtin_ppc_compare_and_swap:
18867 case PPC::BI__builtin_ppc_compare_and_swaplp: {
18876 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Monotonic,
true);
18884 Value *LoadedVal = Pair.first.getScalarVal();
18888 case PPC::BI__builtin_ppc_fetch_and_add:
18889 case PPC::BI__builtin_ppc_fetch_and_addlp: {
18891 llvm::AtomicOrdering::Monotonic);
18893 case PPC::BI__builtin_ppc_fetch_and_and:
18894 case PPC::BI__builtin_ppc_fetch_and_andlp: {
18896 llvm::AtomicOrdering::Monotonic);
18899 case PPC::BI__builtin_ppc_fetch_and_or:
18900 case PPC::BI__builtin_ppc_fetch_and_orlp: {
18902 llvm::AtomicOrdering::Monotonic);
18904 case PPC::BI__builtin_ppc_fetch_and_swap:
18905 case PPC::BI__builtin_ppc_fetch_and_swaplp: {
18907 llvm::AtomicOrdering::Monotonic);
18909 case PPC::BI__builtin_ppc_ldarx:
18910 case PPC::BI__builtin_ppc_lwarx:
18911 case PPC::BI__builtin_ppc_lharx:
18912 case PPC::BI__builtin_ppc_lbarx:
18914 case PPC::BI__builtin_ppc_mfspr: {
18920 return Builder.CreateCall(F, {Op0});
18922 case PPC::BI__builtin_ppc_mtspr: {
18929 return Builder.CreateCall(F, {Op0, Op1});
18931 case PPC::BI__builtin_ppc_popcntb: {
18933 llvm::Type *ArgType = ArgValue->
getType();
18935 return Builder.CreateCall(F, {ArgValue},
"popcntb");
18937 case PPC::BI__builtin_ppc_mtfsf: {
18947 case PPC::BI__builtin_ppc_swdiv_nochk:
18948 case PPC::BI__builtin_ppc_swdivs_nochk: {
18951 FastMathFlags FMF =
Builder.getFastMathFlags();
18952 Builder.getFastMathFlags().setFast();
18953 Value *FDiv =
Builder.CreateFDiv(Op0, Op1,
"swdiv_nochk");
18954 Builder.getFastMathFlags() &= (FMF);
18957 case PPC::BI__builtin_ppc_fric:
18959 *
this,
E, Intrinsic::rint,
18960 Intrinsic::experimental_constrained_rint))
18962 case PPC::BI__builtin_ppc_frim:
18963 case PPC::BI__builtin_ppc_frims:
18965 *
this,
E, Intrinsic::floor,
18966 Intrinsic::experimental_constrained_floor))
18968 case PPC::BI__builtin_ppc_frin:
18969 case PPC::BI__builtin_ppc_frins:
18971 *
this,
E, Intrinsic::round,
18972 Intrinsic::experimental_constrained_round))
18974 case PPC::BI__builtin_ppc_frip:
18975 case PPC::BI__builtin_ppc_frips:
18977 *
this,
E, Intrinsic::ceil,
18978 Intrinsic::experimental_constrained_ceil))
18980 case PPC::BI__builtin_ppc_friz:
18981 case PPC::BI__builtin_ppc_frizs:
18983 *
this,
E, Intrinsic::trunc,
18984 Intrinsic::experimental_constrained_trunc))
18986 case PPC::BI__builtin_ppc_fsqrt:
18987 case PPC::BI__builtin_ppc_fsqrts:
18989 *
this,
E, Intrinsic::sqrt,
18990 Intrinsic::experimental_constrained_sqrt))
18992 case PPC::BI__builtin_ppc_test_data_class: {
18997 {Op0, Op1},
"test_data_class");
18999 case PPC::BI__builtin_ppc_maxfe: {
19005 {Op0, Op1, Op2, Op3});
19007 case PPC::BI__builtin_ppc_maxfl: {
19013 {Op0, Op1, Op2, Op3});
19015 case PPC::BI__builtin_ppc_maxfs: {
19021 {Op0, Op1, Op2, Op3});
19023 case PPC::BI__builtin_ppc_minfe: {
19029 {Op0, Op1, Op2, Op3});
19031 case PPC::BI__builtin_ppc_minfl: {
19037 {Op0, Op1, Op2, Op3});
19039 case PPC::BI__builtin_ppc_minfs: {
19045 {Op0, Op1, Op2, Op3});
19047 case PPC::BI__builtin_ppc_swdiv:
19048 case PPC::BI__builtin_ppc_swdivs: {
19051 return Builder.CreateFDiv(Op0, Op1,
"swdiv");
19053 case PPC::BI__builtin_ppc_set_fpscr_rn:
19055 {EmitScalarExpr(E->getArg(0))});
19056 case PPC::BI__builtin_ppc_mffs:
19069 Attribute::getWithDereferenceableBytes(
Call->getContext(), 64));
19070 Call->addRetAttr(Attribute::getWithAlignment(
Call->getContext(), Align(4)));
19074 auto *RetTy = cast<llvm::PointerType>(CGF.
ConvertType(BuiltinRetType));
19075 if (RetTy ==
Call->getType())
19084 Attribute::getWithDereferenceableBytes(
Call->getContext(), 256));
19085 Call->addRetAttr(Attribute::getWithAlignment(
Call->getContext(), Align(8)));
19100 llvm::LoadInst *LD;
19104 if (Cov == CodeObjectVersionKind::COV_None) {
19105 StringRef Name =
"__oclc_ABI_version";
19106 auto *ABIVersionC = CGF.
CGM.
getModule().getNamedGlobal(Name);
19108 ABIVersionC =
new llvm::GlobalVariable(
19110 llvm::GlobalValue::ExternalLinkage,
nullptr, Name,
nullptr,
19111 llvm::GlobalVariable::NotThreadLocal,
19122 llvm::ConstantInt::get(CGF.
Int32Ty, CodeObjectVersionKind::COV_5));
19126 CGF.
Int8Ty, EmitAMDGPUImplicitArgPtr(CGF), 12 + Index * 2);
19130 CGF.
Int8Ty, EmitAMDGPUDispatchPtr(CGF), 4 + Index * 2);
19132 auto Result = CGF.
Builder.CreateSelect(IsCOV5, ImplicitGEP, DispatchGEP);
19136 Value *GEP =
nullptr;
19137 if (Cov >= CodeObjectVersionKind::COV_5) {
19139 GEP = CGF.
Builder.CreateConstGEP1_32(
19140 CGF.
Int8Ty, EmitAMDGPUImplicitArgPtr(CGF), 12 + Index * 2);
19143 GEP = CGF.
Builder.CreateConstGEP1_32(
19144 CGF.
Int8Ty, EmitAMDGPUDispatchPtr(CGF), 4 + Index * 2);
19151 llvm::MDNode *RNode = MDHelper.createRange(
APInt(16, 1),
19153 LD->setMetadata(llvm::LLVMContext::MD_range, RNode);
19154 LD->setMetadata(llvm::LLVMContext::MD_noundef,
19156 LD->setMetadata(llvm::LLVMContext::MD_invariant_load,
19163 const unsigned XOffset = 12;
19164 auto *DP = EmitAMDGPUDispatchPtr(CGF);
19166 auto *Offset = llvm::ConstantInt::get(CGF.
Int32Ty, XOffset + Index * 4);
19174 LD->setMetadata(llvm::LLVMContext::MD_range,
19175 MDB.createRange(
APInt(32, 1), APInt::getZero(32)));
19176 LD->setMetadata(llvm::LLVMContext::MD_invariant_load,
19189 llvm::AtomicOrdering &AO,
19190 llvm::SyncScope::ID &SSID) {
19191 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
19194 assert(llvm::isValidAtomicOrderingCABI(ord));
19195 switch (
static_cast<llvm::AtomicOrderingCABI
>(ord)) {
19196 case llvm::AtomicOrderingCABI::acquire:
19197 case llvm::AtomicOrderingCABI::consume:
19198 AO = llvm::AtomicOrdering::Acquire;
19200 case llvm::AtomicOrderingCABI::release:
19201 AO = llvm::AtomicOrdering::Release;
19203 case llvm::AtomicOrderingCABI::acq_rel:
19204 AO = llvm::AtomicOrdering::AcquireRelease;
19206 case llvm::AtomicOrderingCABI::seq_cst:
19207 AO = llvm::AtomicOrdering::SequentiallyConsistent;
19209 case llvm::AtomicOrderingCABI::relaxed:
19210 AO = llvm::AtomicOrdering::Monotonic;
19216 if (llvm::getConstantStringInfo(
Scope, scp)) {
19222 int scope = cast<llvm::ConstantInt>(
Scope)->getZExtValue();
19225 SSID = llvm::SyncScope::System;
19237 SSID = llvm::SyncScope::SingleThread;
19240 SSID = llvm::SyncScope::System;
19248 llvm::Value *Arg =
nullptr;
19249 if ((ICEArguments & (1 << Idx)) == 0) {
19254 std::optional<llvm::APSInt>
Result =
19256 assert(
Result &&
"Expected argument to be a constant");
19265 return RT.getFDotIntrinsic();
19267 return RT.getSDotIntrinsic();
19269 return RT.getUDotIntrinsic();
19274 return RT.getFirstBitSHighIntrinsic();
19278 return RT.getFirstBitUHighIntrinsic();
19285 case llvm::Triple::spirv:
19286 return llvm::Intrinsic::spv_wave_reduce_sum;
19287 case llvm::Triple::dxil: {
19289 return llvm::Intrinsic::dx_wave_reduce_usum;
19290 return llvm::Intrinsic::dx_wave_reduce_sum;
19293 llvm_unreachable(
"Intrinsic WaveActiveSum"
19294 " not supported by target architecture");
19304 switch (BuiltinID) {
19305 case Builtin::BI__builtin_hlsl_resource_getpointer: {
19310 llvm::Type *RetTy = llvm::PointerType::getUnqual(
getLLVMContext());
19312 return Builder.CreateIntrinsic(
19316 case Builtin::BI__builtin_hlsl_all: {
19318 return Builder.CreateIntrinsic(
19323 case Builtin::BI__builtin_hlsl_any: {
19325 return Builder.CreateIntrinsic(
19330 case Builtin::BI__builtin_hlsl_asdouble:
19332 case Builtin::BI__builtin_hlsl_elementwise_clamp: {
19339 Ty = VecTy->getElementType();
19341 Intrinsic::ID Intr;
19350 return Builder.CreateIntrinsic(
19354 case Builtin::BI__builtin_hlsl_cross: {
19359 "cross operands must have a float representation");
19364 "input vectors must have 3 elements each");
19365 return Builder.CreateIntrinsic(
19369 case Builtin::BI__builtin_hlsl_dot: {
19372 llvm::Type *T0 = Op0->
getType();
19373 llvm::Type *T1 = Op1->
getType();
19376 if (!T0->isVectorTy() && !T1->isVectorTy()) {
19377 if (T0->isFloatingPointTy())
19378 return Builder.CreateFMul(Op0, Op1,
"hlsl.dot");
19380 if (T0->isIntegerTy())
19381 return Builder.CreateMul(Op0, Op1,
"hlsl.dot");
19384 "Scalar dot product is only supported on ints and floats.");
19389 assert(T0->isVectorTy() && T1->isVectorTy() &&
19390 "Dot product of vector and scalar is not supported.");
19393 [[maybe_unused]]
auto *VecTy1 =
19397 "Dot product of vectors need the same element types.");
19400 "Dot product requires vectors to be of the same size.");
19402 return Builder.CreateIntrinsic(
19403 T0->getScalarType(),
19407 case Builtin::BI__builtin_hlsl_dot4add_i8packed: {
19413 return Builder.CreateIntrinsic(
19415 "hlsl.dot4add.i8packed");
19417 case Builtin::BI__builtin_hlsl_dot4add_u8packed: {
19423 return Builder.CreateIntrinsic(
19425 "hlsl.dot4add.u8packed");
19427 case Builtin::BI__builtin_hlsl_elementwise_firstbithigh: {
19430 return Builder.CreateIntrinsic(
19435 case Builtin::BI__builtin_hlsl_elementwise_firstbitlow: {
19438 return Builder.CreateIntrinsic(
19441 nullptr,
"hlsl.firstbitlow");
19443 case Builtin::BI__builtin_hlsl_lerp: {
19448 llvm_unreachable(
"lerp operand must have a float representation");
19449 return Builder.CreateIntrinsic(
19453 case Builtin::BI__builtin_hlsl_normalize: {
19457 "normalize operand must have a float representation");
19459 return Builder.CreateIntrinsic(
19462 nullptr,
"hlsl.normalize");
19464 case Builtin::BI__builtin_hlsl_elementwise_degrees: {
19468 "degree operand must have a float representation");
19470 return Builder.CreateIntrinsic(
19474 case Builtin::BI__builtin_hlsl_elementwise_frac: {
19477 llvm_unreachable(
"frac operand must have a float representation");
19478 return Builder.CreateIntrinsic(
19482case Builtin::BI__builtin_hlsl_elementwise_isinf: {
19484 llvm::Type *Xty = Op0->
getType();
19485 llvm::Type *retType = llvm::Type::getInt1Ty(this->
getLLVMContext());
19486 if (Xty->isVectorTy()) {
19488 retType = llvm::VectorType::get(
19489 retType, ElementCount::getFixed(XVecTy->getNumElements()));
19492 llvm_unreachable(
"isinf operand must have a float representation");
19493 return Builder.CreateIntrinsic(retType, Intrinsic::dx_isinf,
19496 case Builtin::BI__builtin_hlsl_mad: {
19501 return Builder.CreateIntrinsic(
19502 M->
getType(), Intrinsic::fmuladd,
19507 return Builder.CreateIntrinsic(
19508 M->
getType(), Intrinsic::dx_imad,
19512 return Builder.CreateNSWAdd(Mul, B);
19516 return Builder.CreateIntrinsic(
19517 M->
getType(), Intrinsic::dx_umad,
19521 return Builder.CreateNUWAdd(Mul, B);
19523 case Builtin::BI__builtin_hlsl_elementwise_rcp: {
19526 llvm_unreachable(
"rcp operand must have a float representation");
19527 llvm::Type *Ty = Op0->
getType();
19528 llvm::Type *EltTy = Ty->getScalarType();
19529 Constant *One = Ty->isVectorTy()
19530 ? ConstantVector::getSplat(
19531 ElementCount::getFixed(
19532 cast<FixedVectorType>(Ty)->getNumElements()),
19533 ConstantFP::get(EltTy, 1.0))
19534 : ConstantFP::get(EltTy, 1.0);
19535 return Builder.CreateFDiv(One, Op0,
"hlsl.rcp");
19537 case Builtin::BI__builtin_hlsl_elementwise_rsqrt: {
19540 llvm_unreachable(
"rsqrt operand must have a float representation");
19541 return Builder.CreateIntrinsic(
19545 case Builtin::BI__builtin_hlsl_elementwise_saturate: {
19548 "saturate operand must have a float representation");
19549 return Builder.CreateIntrinsic(
19552 nullptr,
"hlsl.saturate");
19554 case Builtin::BI__builtin_hlsl_select: {
19568 Builder.CreateSelect(OpCond, OpTrue, OpFalse,
"hlsl.select");
19575 case Builtin::BI__builtin_hlsl_step: {
19580 "step operands must have a float representation");
19581 return Builder.CreateIntrinsic(
19585 case Builtin::BI__builtin_hlsl_wave_active_all_true: {
19587 assert(Op->
getType()->isIntegerTy(1) &&
19588 "Intrinsic WaveActiveAllTrue operand must be a bool");
19592 Intrinsic::getOrInsertDeclaration(&
CGM.
getModule(), ID), {Op});
19594 case Builtin::BI__builtin_hlsl_wave_active_any_true: {
19596 assert(Op->
getType()->isIntegerTy(1) &&
19597 "Intrinsic WaveActiveAnyTrue operand must be a bool");
19601 Intrinsic::getOrInsertDeclaration(&
CGM.
getModule(), ID), {Op});
19603 case Builtin::BI__builtin_hlsl_wave_active_count_bits: {
19607 Intrinsic::getOrInsertDeclaration(&
CGM.
getModule(), ID),
19610 case Builtin::BI__builtin_hlsl_wave_active_sum: {
19613 llvm::FunctionType *FT = llvm::FunctionType::get(
19625 ArrayRef{OpExpr},
"hlsl.wave.active.sum");
19627 case Builtin::BI__builtin_hlsl_wave_get_lane_index: {
19632 case llvm::Triple::dxil:
19635 case llvm::Triple::spirv:
19637 llvm::FunctionType::get(
IntTy, {},
false),
19638 "__hlsl_wave_get_lane_index", {},
false,
true));
19641 "Intrinsic WaveGetLaneIndex not supported by target architecture");
19644 case Builtin::BI__builtin_hlsl_wave_is_first_lane: {
19647 Intrinsic::getOrInsertDeclaration(&
CGM.
getModule(), ID));
19649 case Builtin::BI__builtin_hlsl_wave_read_lane_at: {
19654 llvm::FunctionType *FT = llvm::FunctionType::get(
19665 ArrayRef{OpExpr, OpIndex},
"hlsl.wave.readlane");
19667 case Builtin::BI__builtin_hlsl_elementwise_sign: {
19668 auto *Arg0 =
E->getArg(0);
19670 llvm::Type *Xty = Op0->
getType();
19671 llvm::Type *retType = llvm::Type::getInt32Ty(this->
getLLVMContext());
19672 if (Xty->isVectorTy()) {
19674 retType = llvm::VectorType::get(
19675 retType, ElementCount::getFixed(XVecTy->getNumElements()));
19679 "sign operand must have a float or int representation");
19682 Value *Cmp =
Builder.CreateICmpEQ(Op0, ConstantInt::get(Xty, 0));
19683 return Builder.CreateSelect(Cmp, ConstantInt::get(retType, 0),
19684 ConstantInt::get(retType, 1),
"hlsl.sign");
19687 return Builder.CreateIntrinsic(
19691 case Builtin::BI__builtin_hlsl_elementwise_radians: {
19694 "radians operand must have a float representation");
19695 return Builder.CreateIntrinsic(
19698 nullptr,
"hlsl.radians");
19700 case Builtin::BI__builtin_hlsl_buffer_update_counter: {
19704 return Builder.CreateIntrinsic(
19709 case Builtin::BI__builtin_hlsl_elementwise_splitdouble: {
19714 "asuint operands types mismatch");
19717 case Builtin::BI__builtin_hlsl_elementwise_clip:
19719 "clip operands types mismatch");
19721 case Builtin::BI__builtin_hlsl_group_memory_barrier_with_group_sync: {
19725 Intrinsic::getOrInsertDeclaration(&
CGM.
getModule(), ID));
19733 constexpr const char *
Tag =
"amdgpu-as";
19735 LLVMContext &Ctx = Inst->getContext();
19737 for (
unsigned K = 2; K <
E->getNumArgs(); ++K) {
19740 if (llvm::getConstantStringInfo(
V, AS)) {
19741 MMRAs.push_back({
Tag, AS});
19746 "expected an address space name as a string literal");
19750 MMRAs.erase(llvm::unique(MMRAs), MMRAs.end());
19751 Inst->setMetadata(LLVMContext::MD_mmra, MMRAMetadata::getMD(Ctx, MMRAs));
19756 llvm::AtomicOrdering AO = llvm::AtomicOrdering::SequentiallyConsistent;
19757 llvm::SyncScope::ID SSID;
19758 switch (BuiltinID) {
19759 case AMDGPU::BI__builtin_amdgcn_div_scale:
19760 case AMDGPU::BI__builtin_amdgcn_div_scalef: {
19773 llvm::Value *Tmp =
Builder.CreateCall(Callee, {
X, Y, Z});
19776 llvm::Value *Flag =
Builder.CreateExtractValue(Tmp, 1);
19780 llvm::Value *FlagExt =
Builder.CreateZExt(Flag, RealFlagType);
19784 case AMDGPU::BI__builtin_amdgcn_div_fmas:
19785 case AMDGPU::BI__builtin_amdgcn_div_fmasf: {
19793 llvm::Value *Src3ToBool =
Builder.CreateIsNotNull(Src3);
19794 return Builder.CreateCall(F, {Src0, Src1, Src2, Src3ToBool});
19797 case AMDGPU::BI__builtin_amdgcn_ds_swizzle:
19798 return emitBuiltinWithOneOverloadedType<2>(*
this,
E,
19799 Intrinsic::amdgcn_ds_swizzle);
19800 case AMDGPU::BI__builtin_amdgcn_mov_dpp8:
19801 case AMDGPU::BI__builtin_amdgcn_mov_dpp:
19802 case AMDGPU::BI__builtin_amdgcn_update_dpp: {
19806 unsigned ICEArguments = 0;
19811 unsigned Size = DataTy->getPrimitiveSizeInBits();
19812 llvm::Type *
IntTy =
19813 llvm::IntegerType::get(
Builder.getContext(), std::max(Size, 32u));
19816 ? Intrinsic::amdgcn_mov_dpp8
19817 : Intrinsic::amdgcn_update_dpp,
19819 assert(
E->getNumArgs() == 5 ||
E->getNumArgs() == 6 ||
19820 E->getNumArgs() == 2);
19821 bool InsertOld = BuiltinID == AMDGPU::BI__builtin_amdgcn_mov_dpp;
19823 Args.push_back(llvm::PoisonValue::get(
IntTy));
19824 for (
unsigned I = 0; I !=
E->getNumArgs(); ++I) {
19826 if (I < (BuiltinID == AMDGPU::BI__builtin_amdgcn_update_dpp ? 2u : 1u) &&
19828 if (!DataTy->isIntegerTy())
19830 V, llvm::IntegerType::get(
Builder.getContext(), Size));
19833 llvm::Type *ExpTy =
19834 F->getFunctionType()->getFunctionParamType(I + InsertOld);
19835 Args.push_back(
Builder.CreateTruncOrBitCast(
V, ExpTy));
19838 if (Size < 32 && !DataTy->isIntegerTy())
19840 V, llvm::IntegerType::get(
Builder.getContext(), Size));
19841 return Builder.CreateTruncOrBitCast(
V, DataTy);
19843 case AMDGPU::BI__builtin_amdgcn_permlane16:
19844 case AMDGPU::BI__builtin_amdgcn_permlanex16:
19845 return emitBuiltinWithOneOverloadedType<6>(
19847 BuiltinID == AMDGPU::BI__builtin_amdgcn_permlane16
19848 ? Intrinsic::amdgcn_permlane16
19849 : Intrinsic::amdgcn_permlanex16);
19850 case AMDGPU::BI__builtin_amdgcn_permlane64:
19851 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
19852 Intrinsic::amdgcn_permlane64);
19853 case AMDGPU::BI__builtin_amdgcn_readlane:
19854 return emitBuiltinWithOneOverloadedType<2>(*
this,
E,
19855 Intrinsic::amdgcn_readlane);
19856 case AMDGPU::BI__builtin_amdgcn_readfirstlane:
19857 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
19858 Intrinsic::amdgcn_readfirstlane);
19859 case AMDGPU::BI__builtin_amdgcn_div_fixup:
19860 case AMDGPU::BI__builtin_amdgcn_div_fixupf:
19861 case AMDGPU::BI__builtin_amdgcn_div_fixuph:
19862 return emitBuiltinWithOneOverloadedType<3>(*
this,
E,
19863 Intrinsic::amdgcn_div_fixup);
19864 case AMDGPU::BI__builtin_amdgcn_trig_preop:
19865 case AMDGPU::BI__builtin_amdgcn_trig_preopf:
19867 case AMDGPU::BI__builtin_amdgcn_rcp:
19868 case AMDGPU::BI__builtin_amdgcn_rcpf:
19869 case AMDGPU::BI__builtin_amdgcn_rcph:
19870 return emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::amdgcn_rcp);
19871 case AMDGPU::BI__builtin_amdgcn_sqrt:
19872 case AMDGPU::BI__builtin_amdgcn_sqrtf:
19873 case AMDGPU::BI__builtin_amdgcn_sqrth:
19874 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
19875 Intrinsic::amdgcn_sqrt);
19876 case AMDGPU::BI__builtin_amdgcn_rsq:
19877 case AMDGPU::BI__builtin_amdgcn_rsqf:
19878 case AMDGPU::BI__builtin_amdgcn_rsqh:
19879 return emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::amdgcn_rsq);
19880 case AMDGPU::BI__builtin_amdgcn_rsq_clamp:
19881 case AMDGPU::BI__builtin_amdgcn_rsq_clampf:
19882 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
19883 Intrinsic::amdgcn_rsq_clamp);
19884 case AMDGPU::BI__builtin_amdgcn_sinf:
19885 case AMDGPU::BI__builtin_amdgcn_sinh:
19886 return emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::amdgcn_sin);
19887 case AMDGPU::BI__builtin_amdgcn_cosf:
19888 case AMDGPU::BI__builtin_amdgcn_cosh:
19889 return emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::amdgcn_cos);
19890 case AMDGPU::BI__builtin_amdgcn_dispatch_ptr:
19891 return EmitAMDGPUDispatchPtr(*
this,
E);
19892 case AMDGPU::BI__builtin_amdgcn_logf:
19893 return emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::amdgcn_log);
19894 case AMDGPU::BI__builtin_amdgcn_exp2f:
19895 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
19896 Intrinsic::amdgcn_exp2);
19897 case AMDGPU::BI__builtin_amdgcn_log_clampf:
19898 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
19899 Intrinsic::amdgcn_log_clamp);
19900 case AMDGPU::BI__builtin_amdgcn_ldexp:
19901 case AMDGPU::BI__builtin_amdgcn_ldexpf: {
19904 llvm::Function *F =
19905 CGM.
getIntrinsic(Intrinsic::ldexp, {Src0->getType(), Src1->getType()});
19906 return Builder.CreateCall(F, {Src0, Src1});
19908 case AMDGPU::BI__builtin_amdgcn_ldexph: {
19913 llvm::Function *F =
19917 case AMDGPU::BI__builtin_amdgcn_frexp_mant:
19918 case AMDGPU::BI__builtin_amdgcn_frexp_mantf:
19919 case AMDGPU::BI__builtin_amdgcn_frexp_manth:
19920 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
19921 Intrinsic::amdgcn_frexp_mant);
19922 case AMDGPU::BI__builtin_amdgcn_frexp_exp:
19923 case AMDGPU::BI__builtin_amdgcn_frexp_expf: {
19927 return Builder.CreateCall(F, Src0);
19929 case AMDGPU::BI__builtin_amdgcn_frexp_exph: {
19933 return Builder.CreateCall(F, Src0);
19935 case AMDGPU::BI__builtin_amdgcn_fract:
19936 case AMDGPU::BI__builtin_amdgcn_fractf:
19937 case AMDGPU::BI__builtin_amdgcn_fracth:
19938 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
19939 Intrinsic::amdgcn_fract);
19940 case AMDGPU::BI__builtin_amdgcn_lerp:
19941 return emitBuiltinWithOneOverloadedType<3>(*
this,
E,
19942 Intrinsic::amdgcn_lerp);
19943 case AMDGPU::BI__builtin_amdgcn_ubfe:
19944 return emitBuiltinWithOneOverloadedType<3>(*
this,
E,
19945 Intrinsic::amdgcn_ubfe);
19946 case AMDGPU::BI__builtin_amdgcn_sbfe:
19947 return emitBuiltinWithOneOverloadedType<3>(*
this,
E,
19948 Intrinsic::amdgcn_sbfe);
19949 case AMDGPU::BI__builtin_amdgcn_ballot_w32:
19950 case AMDGPU::BI__builtin_amdgcn_ballot_w64: {
19954 return Builder.CreateCall(F, { Src });
19956 case AMDGPU::BI__builtin_amdgcn_uicmp:
19957 case AMDGPU::BI__builtin_amdgcn_uicmpl:
19958 case AMDGPU::BI__builtin_amdgcn_sicmp:
19959 case AMDGPU::BI__builtin_amdgcn_sicmpl: {
19966 {
Builder.getInt64Ty(), Src0->getType() });
19967 return Builder.CreateCall(F, { Src0, Src1, Src2 });
19969 case AMDGPU::BI__builtin_amdgcn_fcmp:
19970 case AMDGPU::BI__builtin_amdgcn_fcmpf: {
19977 {
Builder.getInt64Ty(), Src0->getType() });
19978 return Builder.CreateCall(F, { Src0, Src1, Src2 });
19980 case AMDGPU::BI__builtin_amdgcn_class:
19981 case AMDGPU::BI__builtin_amdgcn_classf:
19982 case AMDGPU::BI__builtin_amdgcn_classh:
19984 case AMDGPU::BI__builtin_amdgcn_fmed3f:
19985 case AMDGPU::BI__builtin_amdgcn_fmed3h:
19986 return emitBuiltinWithOneOverloadedType<3>(*
this,
E,
19987 Intrinsic::amdgcn_fmed3);
19988 case AMDGPU::BI__builtin_amdgcn_ds_append:
19989 case AMDGPU::BI__builtin_amdgcn_ds_consume: {
19990 Intrinsic::ID Intrin = BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_append ?
19991 Intrinsic::amdgcn_ds_append : Intrinsic::amdgcn_ds_consume;
19996 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_i32:
19997 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_v2i32:
19998 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4i16:
19999 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4f16:
20000 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4bf16:
20001 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16:
20002 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8f16:
20003 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8bf16:
20004 case AMDGPU::BI__builtin_amdgcn_ds_read_tr4_b64_v2i32:
20005 case AMDGPU::BI__builtin_amdgcn_ds_read_tr8_b64_v2i32:
20006 case AMDGPU::BI__builtin_amdgcn_ds_read_tr6_b96_v3i32:
20007 case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4f16:
20008 case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4bf16:
20009 case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4i16: {
20011 switch (BuiltinID) {
20012 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_i32:
20013 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_v2i32:
20014 IID = Intrinsic::amdgcn_global_load_tr_b64;
20016 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4i16:
20017 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4f16:
20018 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4bf16:
20019 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16:
20020 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8f16:
20021 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8bf16:
20022 IID = Intrinsic::amdgcn_global_load_tr_b128;
20024 case AMDGPU::BI__builtin_amdgcn_ds_read_tr4_b64_v2i32:
20025 IID = Intrinsic::amdgcn_ds_read_tr4_b64;
20027 case AMDGPU::BI__builtin_amdgcn_ds_read_tr8_b64_v2i32:
20028 IID = Intrinsic::amdgcn_ds_read_tr8_b64;
20030 case AMDGPU::BI__builtin_amdgcn_ds_read_tr6_b96_v3i32:
20031 IID = Intrinsic::amdgcn_ds_read_tr6_b96;
20033 case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4i16:
20034 case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4f16:
20035 case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4bf16:
20036 IID = Intrinsic::amdgcn_ds_read_tr16_b64;
20042 return Builder.CreateCall(F, {Addr});
20044 case AMDGPU::BI__builtin_amdgcn_get_fpenv: {
20047 return Builder.CreateCall(F);
20049 case AMDGPU::BI__builtin_amdgcn_set_fpenv: {
20055 case AMDGPU::BI__builtin_amdgcn_read_exec:
20057 case AMDGPU::BI__builtin_amdgcn_read_exec_lo:
20059 case AMDGPU::BI__builtin_amdgcn_read_exec_hi:
20061 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray:
20062 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_h:
20063 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_l:
20064 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_lh: {
20074 RayOrigin =
Builder.CreateShuffleVector(RayOrigin, RayOrigin,
20078 RayInverseDir =
Builder.CreateShuffleVector(RayInverseDir, RayInverseDir,
20082 {NodePtr->getType(), RayDir->getType()});
20083 return Builder.CreateCall(F, {NodePtr, RayExtent, RayOrigin, RayDir,
20084 RayInverseDir, TextureDescr});
20087 case AMDGPU::BI__builtin_amdgcn_ds_bvh_stack_rtn: {
20089 for (
int i = 0, e =
E->getNumArgs(); i != e; ++i)
20097 Value *I0 =
Builder.CreateInsertElement(PoisonValue::get(RetTy), Rtn,
20099 return Builder.CreateInsertElement(I0, A, 1);
20101 case AMDGPU::BI__builtin_amdgcn_mfma_scale_f32_16x16x128_f8f6f4:
20102 case AMDGPU::BI__builtin_amdgcn_mfma_scale_f32_32x32x64_f8f6f4: {
20103 llvm::FixedVectorType *VT = FixedVectorType::get(
Builder.getInt32Ty(), 8);
20105 BuiltinID == AMDGPU::BI__builtin_amdgcn_mfma_scale_f32_32x32x64_f8f6f4
20106 ? Intrinsic::amdgcn_mfma_scale_f32_32x32x64_f8f6f4
20107 : Intrinsic::amdgcn_mfma_scale_f32_16x16x128_f8f6f4,
20111 for (
unsigned I = 0, N =
E->getNumArgs(); I != N; ++I)
20113 return Builder.CreateCall(F, Args);
20115 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32:
20116 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w32:
20117 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64:
20118 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w64:
20119 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32:
20120 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w32:
20121 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64:
20122 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w64:
20123 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32:
20124 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64:
20125 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32:
20126 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64:
20127 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32:
20128 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64:
20129 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32:
20130 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64:
20131 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32_gfx12:
20132 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64_gfx12:
20133 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32_gfx12:
20134 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64_gfx12:
20135 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32_gfx12:
20136 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64_gfx12:
20137 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12:
20138 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64_gfx12:
20139 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32_gfx12:
20140 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64_gfx12:
20141 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12:
20142 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64_gfx12:
20143 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w32_gfx12:
20144 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w64_gfx12:
20145 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w32_gfx12:
20146 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w64_gfx12:
20147 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w32_gfx12:
20148 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w64_gfx12:
20149 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w32_gfx12:
20150 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w64_gfx12:
20151 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w32_gfx12:
20152 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w64_gfx12:
20153 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w32:
20154 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w64:
20155 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w32:
20156 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w64:
20157 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w32:
20158 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w64:
20159 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w32:
20160 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w64:
20161 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w32:
20162 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w64:
20163 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w32:
20164 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w64:
20165 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w32:
20166 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w64:
20167 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w32:
20168 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w64:
20169 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w32:
20170 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w64:
20171 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w32:
20172 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w64:
20173 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w32:
20174 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w64: {
20187 bool AppendFalseForOpselArg =
false;
20188 unsigned BuiltinWMMAOp;
20190 switch (BuiltinID) {
20191 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32:
20192 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64:
20193 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12:
20194 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64_gfx12:
20195 ArgsForMatchingMatrixTypes = {2, 0};
20196 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_f16;
20198 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32:
20199 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64:
20200 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32_gfx12:
20201 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64_gfx12:
20202 ArgsForMatchingMatrixTypes = {2, 0};
20203 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_bf16;
20205 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32_gfx12:
20206 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64_gfx12:
20207 AppendFalseForOpselArg =
true;
20209 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32:
20210 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64:
20211 ArgsForMatchingMatrixTypes = {2, 0};
20212 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f16_16x16x16_f16;
20214 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32_gfx12:
20215 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64_gfx12:
20216 AppendFalseForOpselArg =
true;
20218 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32:
20219 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64:
20220 ArgsForMatchingMatrixTypes = {2, 0};
20221 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_bf16_16x16x16_bf16;
20223 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w32:
20224 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w64:
20225 ArgsForMatchingMatrixTypes = {2, 0};
20226 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f16_16x16x16_f16_tied;
20228 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w32:
20229 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w64:
20230 ArgsForMatchingMatrixTypes = {2, 0};
20231 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_bf16_16x16x16_bf16_tied;
20233 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32:
20234 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64:
20235 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12:
20236 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64_gfx12:
20237 ArgsForMatchingMatrixTypes = {4, 1};
20238 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x16_iu8;
20240 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32:
20241 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64:
20242 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32_gfx12:
20243 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64_gfx12:
20244 ArgsForMatchingMatrixTypes = {4, 1};
20245 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x16_iu4;
20247 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w32_gfx12:
20248 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w64_gfx12:
20249 ArgsForMatchingMatrixTypes = {2, 0};
20250 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_fp8_fp8;
20252 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w32_gfx12:
20253 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w64_gfx12:
20254 ArgsForMatchingMatrixTypes = {2, 0};
20255 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_fp8_bf8;
20257 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w32_gfx12:
20258 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w64_gfx12:
20259 ArgsForMatchingMatrixTypes = {2, 0};
20260 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_bf8_fp8;
20262 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w32_gfx12:
20263 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w64_gfx12:
20264 ArgsForMatchingMatrixTypes = {2, 0};
20265 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_bf8_bf8;
20267 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w32_gfx12:
20268 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w64_gfx12:
20269 ArgsForMatchingMatrixTypes = {4, 1};
20270 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x32_iu4;
20272 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w32:
20273 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w64:
20274 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
20275 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_f16;
20277 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w32:
20278 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w64:
20279 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
20280 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_bf16;
20282 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w32:
20283 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w64:
20284 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
20285 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f16_16x16x32_f16;
20287 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w32:
20288 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w64:
20289 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
20290 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_bf16_16x16x32_bf16;
20292 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w32:
20293 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w64:
20294 ArgsForMatchingMatrixTypes = {4, 1, 3, 5};
20295 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_i32_16x16x32_iu8;
20297 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w32:
20298 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w64:
20299 ArgsForMatchingMatrixTypes = {4, 1, 3, 5};
20300 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_i32_16x16x32_iu4;
20302 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w32:
20303 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w64:
20304 ArgsForMatchingMatrixTypes = {4, 1, 3, 5};
20305 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_i32_16x16x64_iu4;
20307 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w32:
20308 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w64:
20309 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
20310 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_fp8;
20312 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w32:
20313 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w64:
20314 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
20315 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_bf8;
20317 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w32:
20318 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w64:
20319 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
20320 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_fp8;
20322 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w32:
20323 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w64:
20324 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
20325 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_bf8;
20330 for (
int i = 0, e =
E->getNumArgs(); i != e; ++i)
20332 if (AppendFalseForOpselArg)
20333 Args.push_back(
Builder.getFalse());
20336 for (
auto ArgIdx : ArgsForMatchingMatrixTypes)
20337 ArgTypes.push_back(Args[ArgIdx]->getType());
20340 return Builder.CreateCall(F, Args);
20344 case AMDGPU::BI__builtin_amdgcn_workitem_id_x:
20346 case AMDGPU::BI__builtin_amdgcn_workitem_id_y:
20348 case AMDGPU::BI__builtin_amdgcn_workitem_id_z:
20352 case AMDGPU::BI__builtin_amdgcn_workgroup_size_x:
20353 return EmitAMDGPUWorkGroupSize(*
this, 0);
20354 case AMDGPU::BI__builtin_amdgcn_workgroup_size_y:
20355 return EmitAMDGPUWorkGroupSize(*
this, 1);
20356 case AMDGPU::BI__builtin_amdgcn_workgroup_size_z:
20357 return EmitAMDGPUWorkGroupSize(*
this, 2);
20360 case AMDGPU::BI__builtin_amdgcn_grid_size_x:
20361 return EmitAMDGPUGridSize(*
this, 0);
20362 case AMDGPU::BI__builtin_amdgcn_grid_size_y:
20363 return EmitAMDGPUGridSize(*
this, 1);
20364 case AMDGPU::BI__builtin_amdgcn_grid_size_z:
20365 return EmitAMDGPUGridSize(*
this, 2);
20368 case AMDGPU::BI__builtin_r600_recipsqrt_ieee:
20369 case AMDGPU::BI__builtin_r600_recipsqrt_ieeef:
20370 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
20371 Intrinsic::r600_recipsqrt_ieee);
20372 case AMDGPU::BI__builtin_r600_read_tidig_x:
20374 case AMDGPU::BI__builtin_r600_read_tidig_y:
20376 case AMDGPU::BI__builtin_r600_read_tidig_z:
20378 case AMDGPU::BI__builtin_amdgcn_alignbit: {
20383 return Builder.CreateCall(F, { Src0, Src1, Src2 });
20385 case AMDGPU::BI__builtin_amdgcn_fence: {
20388 FenceInst *Fence =
Builder.CreateFence(AO, SSID);
20389 if (
E->getNumArgs() > 2)
20393 case AMDGPU::BI__builtin_amdgcn_atomic_inc32:
20394 case AMDGPU::BI__builtin_amdgcn_atomic_inc64:
20395 case AMDGPU::BI__builtin_amdgcn_atomic_dec32:
20396 case AMDGPU::BI__builtin_amdgcn_atomic_dec64:
20397 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f64:
20398 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f32:
20399 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2f16:
20400 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2bf16:
20401 case AMDGPU::BI__builtin_amdgcn_ds_faddf:
20402 case AMDGPU::BI__builtin_amdgcn_ds_fminf:
20403 case AMDGPU::BI__builtin_amdgcn_ds_fmaxf:
20404 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f32:
20405 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f64:
20406 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2f16:
20407 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2f16:
20408 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f32:
20409 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f64:
20410 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2bf16:
20411 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2bf16:
20412 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmin_f64:
20413 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmax_f64:
20414 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmin_f64:
20415 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmax_f64: {
20416 llvm::AtomicRMWInst::BinOp BinOp;
20417 switch (BuiltinID) {
20418 case AMDGPU::BI__builtin_amdgcn_atomic_inc32:
20419 case AMDGPU::BI__builtin_amdgcn_atomic_inc64:
20420 BinOp = llvm::AtomicRMWInst::UIncWrap;
20422 case AMDGPU::BI__builtin_amdgcn_atomic_dec32:
20423 case AMDGPU::BI__builtin_amdgcn_atomic_dec64:
20424 BinOp = llvm::AtomicRMWInst::UDecWrap;
20426 case AMDGPU::BI__builtin_amdgcn_ds_faddf:
20427 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f64:
20428 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f32:
20429 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2f16:
20430 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2bf16:
20431 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f32:
20432 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f64:
20433 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2f16:
20434 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2f16:
20435 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f32:
20436 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f64:
20437 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2bf16:
20438 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2bf16:
20439 BinOp = llvm::AtomicRMWInst::FAdd;
20441 case AMDGPU::BI__builtin_amdgcn_ds_fminf:
20442 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmin_f64:
20443 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmin_f64:
20444 BinOp = llvm::AtomicRMWInst::FMin;
20446 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmax_f64:
20447 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmax_f64:
20448 case AMDGPU::BI__builtin_amdgcn_ds_fmaxf:
20449 BinOp = llvm::AtomicRMWInst::FMax;
20455 llvm::Type *OrigTy = Val->
getType();
20460 if (BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_faddf ||
20461 BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_fminf ||
20462 BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_fmaxf) {
20472 if (
E->getNumArgs() >= 4) {
20484 AO = AtomicOrdering::Monotonic;
20487 if (BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2bf16 ||
20488 BuiltinID == AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2bf16 ||
20489 BuiltinID == AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2bf16) {
20490 llvm::Type *V2BF16Ty = FixedVectorType::get(
20491 llvm::Type::getBFloatTy(
Builder.getContext()), 2);
20492 Val =
Builder.CreateBitCast(Val, V2BF16Ty);
20496 llvm::AtomicRMWInst *RMW =
20499 RMW->setVolatile(
true);
20501 unsigned AddrSpace = Ptr.
getType()->getAddressSpace();
20502 if (AddrSpace != llvm::AMDGPUAS::LOCAL_ADDRESS) {
20506 RMW->setMetadata(
"amdgpu.no.fine.grained.memory", EmptyMD);
20510 if (BinOp == llvm::AtomicRMWInst::FAdd && Val->
getType()->isFloatTy())
20511 RMW->setMetadata(
"amdgpu.ignore.denormal.mode", EmptyMD);
20514 return Builder.CreateBitCast(RMW, OrigTy);
20516 case AMDGPU::BI__builtin_amdgcn_s_sendmsg_rtn:
20517 case AMDGPU::BI__builtin_amdgcn_s_sendmsg_rtnl: {
20523 return Builder.CreateCall(F, {Arg});
20525 case AMDGPU::BI__builtin_amdgcn_permlane16_swap:
20526 case AMDGPU::BI__builtin_amdgcn_permlane32_swap: {
20534 CGM.
getIntrinsic(BuiltinID == AMDGPU::BI__builtin_amdgcn_permlane16_swap
20535 ? Intrinsic::amdgcn_permlane16_swap
20536 : Intrinsic::amdgcn_permlane32_swap);
20537 llvm::CallInst *
Call =
20538 Builder.CreateCall(F, {VDstOld, VSrcOld, FI, BoundCtrl});
20540 llvm::Value *Elt0 =
Builder.CreateExtractValue(
Call, 0);
20541 llvm::Value *Elt1 =
Builder.CreateExtractValue(
Call, 1);
20545 llvm::Value *Insert0 =
Builder.CreateInsertElement(
20546 llvm::PoisonValue::get(ResultType), Elt0, UINT64_C(0));
20547 llvm::Value *AsVector =
20548 Builder.CreateInsertElement(Insert0, Elt1, UINT64_C(1));
20551 case AMDGPU::BI__builtin_amdgcn_bitop3_b32:
20552 case AMDGPU::BI__builtin_amdgcn_bitop3_b16:
20553 return emitBuiltinWithOneOverloadedType<4>(*
this,
E,
20554 Intrinsic::amdgcn_bitop3);
20555 case AMDGPU::BI__builtin_amdgcn_make_buffer_rsrc:
20556 return emitBuiltinWithOneOverloadedType<4>(
20557 *
this,
E, Intrinsic::amdgcn_make_buffer_rsrc);
20558 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b8:
20559 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b16:
20560 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b32:
20561 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b64:
20562 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b96:
20563 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b128:
20564 return emitBuiltinWithOneOverloadedType<5>(
20565 *
this,
E, Intrinsic::amdgcn_raw_ptr_buffer_store);
20566 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b8:
20567 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b16:
20568 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b32:
20569 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b64:
20570 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b96:
20571 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b128: {
20572 llvm::Type *RetTy =
nullptr;
20573 switch (BuiltinID) {
20574 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b8:
20577 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b16:
20580 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b32:
20583 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b64:
20584 RetTy = llvm::FixedVectorType::get(
Int32Ty, 2);
20586 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b96:
20587 RetTy = llvm::FixedVectorType::get(
Int32Ty, 3);
20589 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b128:
20590 RetTy = llvm::FixedVectorType::get(
Int32Ty, 4);
20599 case AMDGPU::BI__builtin_amdgcn_s_prefetch_data:
20600 return emitBuiltinWithOneOverloadedType<2>(
20601 *
this,
E, Intrinsic::amdgcn_s_prefetch_data);
20609 switch (BuiltinID) {
20610 case SPIRV::BI__builtin_spirv_distance: {
20615 "Distance operands must have a float representation");
20618 "Distance operands must be a vector");
20619 return Builder.CreateIntrinsic(
20620 X->getType()->getScalarType(), Intrinsic::spv_distance,
20623 case SPIRV::BI__builtin_spirv_length: {
20626 "length operand must have a float representation");
20628 "length operand must be a vector");
20629 return Builder.CreateIntrinsic(
20630 X->getType()->getScalarType(), Intrinsic::spv_length,
20641 unsigned IntrinsicID,
20643 unsigned NumArgs =
E->getNumArgs() - 1;
20645 for (
unsigned I = 0; I < NumArgs; ++I)
20657 switch (BuiltinID) {
20658 case SystemZ::BI__builtin_tbegin: {
20660 Value *Control = llvm::ConstantInt::get(
Int32Ty, 0xff0c);
20662 return Builder.CreateCall(F, {TDB, Control});
20664 case SystemZ::BI__builtin_tbegin_nofloat: {
20666 Value *Control = llvm::ConstantInt::get(
Int32Ty, 0xff0c);
20668 return Builder.CreateCall(F, {TDB, Control});
20670 case SystemZ::BI__builtin_tbeginc: {
20672 Value *Control = llvm::ConstantInt::get(
Int32Ty, 0xff08);
20674 return Builder.CreateCall(F, {TDB, Control});
20676 case SystemZ::BI__builtin_tabort: {
20681 case SystemZ::BI__builtin_non_tx_store: {
20693 case SystemZ::BI__builtin_s390_vclzb:
20694 case SystemZ::BI__builtin_s390_vclzh:
20695 case SystemZ::BI__builtin_s390_vclzf:
20696 case SystemZ::BI__builtin_s390_vclzg:
20697 case SystemZ::BI__builtin_s390_vclzq: {
20700 Value *Undef = ConstantInt::get(
Builder.getInt1Ty(),
false);
20702 return Builder.CreateCall(F, {
X, Undef});
20705 case SystemZ::BI__builtin_s390_vctzb:
20706 case SystemZ::BI__builtin_s390_vctzh:
20707 case SystemZ::BI__builtin_s390_vctzf:
20708 case SystemZ::BI__builtin_s390_vctzg:
20709 case SystemZ::BI__builtin_s390_vctzq: {
20712 Value *Undef = ConstantInt::get(
Builder.getInt1Ty(),
false);
20714 return Builder.CreateCall(F, {
X, Undef});
20717 case SystemZ::BI__builtin_s390_verllb:
20718 case SystemZ::BI__builtin_s390_verllh:
20719 case SystemZ::BI__builtin_s390_verllf:
20720 case SystemZ::BI__builtin_s390_verllg: {
20725 unsigned NumElts = cast<llvm::FixedVectorType>(ResultType)->getNumElements();
20726 Amt =
Builder.CreateIntCast(Amt, ResultType->getScalarType(),
false);
20727 Amt =
Builder.CreateVectorSplat(NumElts, Amt);
20729 return Builder.CreateCall(F, { Src, Src, Amt });
20732 case SystemZ::BI__builtin_s390_verllvb:
20733 case SystemZ::BI__builtin_s390_verllvh:
20734 case SystemZ::BI__builtin_s390_verllvf:
20735 case SystemZ::BI__builtin_s390_verllvg: {
20740 return Builder.CreateCall(F, { Src, Src, Amt });
20743 case SystemZ::BI__builtin_s390_vfsqsb:
20744 case SystemZ::BI__builtin_s390_vfsqdb: {
20747 if (
Builder.getIsFPConstrained()) {
20749 return Builder.CreateConstrainedFPCall(F, {
X });
20755 case SystemZ::BI__builtin_s390_vfmasb:
20756 case SystemZ::BI__builtin_s390_vfmadb: {
20761 if (
Builder.getIsFPConstrained()) {
20763 return Builder.CreateConstrainedFPCall(F, {
X, Y, Z});
20766 return Builder.CreateCall(F, {
X, Y, Z});
20769 case SystemZ::BI__builtin_s390_vfmssb:
20770 case SystemZ::BI__builtin_s390_vfmsdb: {
20775 if (
Builder.getIsFPConstrained()) {
20777 return Builder.CreateConstrainedFPCall(F, {
X, Y,
Builder.CreateFNeg(Z,
"neg")});
20783 case SystemZ::BI__builtin_s390_vfnmasb:
20784 case SystemZ::BI__builtin_s390_vfnmadb: {
20789 if (
Builder.getIsFPConstrained()) {
20791 return Builder.CreateFNeg(
Builder.CreateConstrainedFPCall(F, {X, Y, Z}),
"neg");
20794 return Builder.CreateFNeg(
Builder.CreateCall(F, {X, Y, Z}),
"neg");
20797 case SystemZ::BI__builtin_s390_vfnmssb:
20798 case SystemZ::BI__builtin_s390_vfnmsdb: {
20803 if (
Builder.getIsFPConstrained()) {
20806 return Builder.CreateFNeg(
Builder.CreateConstrainedFPCall(F, {X, Y, NegZ}));
20813 case SystemZ::BI__builtin_s390_vflpsb:
20814 case SystemZ::BI__builtin_s390_vflpdb: {
20820 case SystemZ::BI__builtin_s390_vflnsb:
20821 case SystemZ::BI__builtin_s390_vflndb: {
20827 case SystemZ::BI__builtin_s390_vfisb:
20828 case SystemZ::BI__builtin_s390_vfidb: {
20836 Intrinsic::ID
ID = Intrinsic::not_intrinsic;
20838 switch (M4.getZExtValue()) {
20841 switch (M5.getZExtValue()) {
20843 case 0:
ID = Intrinsic::rint;
20844 CI = Intrinsic::experimental_constrained_rint;
break;
20848 switch (M5.getZExtValue()) {
20850 case 0:
ID = Intrinsic::nearbyint;
20851 CI = Intrinsic::experimental_constrained_nearbyint;
break;
20852 case 1:
ID = Intrinsic::round;
20853 CI = Intrinsic::experimental_constrained_round;
break;
20854 case 5:
ID = Intrinsic::trunc;
20855 CI = Intrinsic::experimental_constrained_trunc;
break;
20856 case 6:
ID = Intrinsic::ceil;
20857 CI = Intrinsic::experimental_constrained_ceil;
break;
20858 case 7:
ID = Intrinsic::floor;
20859 CI = Intrinsic::experimental_constrained_floor;
break;
20863 if (ID != Intrinsic::not_intrinsic) {
20864 if (
Builder.getIsFPConstrained()) {
20866 return Builder.CreateConstrainedFPCall(F,
X);
20872 switch (BuiltinID) {
20873 case SystemZ::BI__builtin_s390_vfisb:
ID = Intrinsic::s390_vfisb;
break;
20874 case SystemZ::BI__builtin_s390_vfidb:
ID = Intrinsic::s390_vfidb;
break;
20875 default: llvm_unreachable(
"Unknown BuiltinID");
20880 return Builder.CreateCall(F, {
X, M4Value, M5Value});
20882 case SystemZ::BI__builtin_s390_vfmaxsb:
20883 case SystemZ::BI__builtin_s390_vfmaxdb: {
20891 Intrinsic::ID
ID = Intrinsic::not_intrinsic;
20893 switch (M4.getZExtValue()) {
20895 case 4:
ID = Intrinsic::maxnum;
20896 CI = Intrinsic::experimental_constrained_maxnum;
break;
20898 if (ID != Intrinsic::not_intrinsic) {
20899 if (
Builder.getIsFPConstrained()) {
20901 return Builder.CreateConstrainedFPCall(F, {
X, Y});
20904 return Builder.CreateCall(F, {
X, Y});
20907 switch (BuiltinID) {
20908 case SystemZ::BI__builtin_s390_vfmaxsb:
ID = Intrinsic::s390_vfmaxsb;
break;
20909 case SystemZ::BI__builtin_s390_vfmaxdb:
ID = Intrinsic::s390_vfmaxdb;
break;
20910 default: llvm_unreachable(
"Unknown BuiltinID");
20914 return Builder.CreateCall(F, {
X, Y, M4Value});
20916 case SystemZ::BI__builtin_s390_vfminsb:
20917 case SystemZ::BI__builtin_s390_vfmindb: {
20925 Intrinsic::ID
ID = Intrinsic::not_intrinsic;
20927 switch (M4.getZExtValue()) {
20929 case 4:
ID = Intrinsic::minnum;
20930 CI = Intrinsic::experimental_constrained_minnum;
break;
20932 if (ID != Intrinsic::not_intrinsic) {
20933 if (
Builder.getIsFPConstrained()) {
20935 return Builder.CreateConstrainedFPCall(F, {
X, Y});
20938 return Builder.CreateCall(F, {
X, Y});
20941 switch (BuiltinID) {
20942 case SystemZ::BI__builtin_s390_vfminsb:
ID = Intrinsic::s390_vfminsb;
break;
20943 case SystemZ::BI__builtin_s390_vfmindb:
ID = Intrinsic::s390_vfmindb;
break;
20944 default: llvm_unreachable(
"Unknown BuiltinID");
20948 return Builder.CreateCall(F, {
X, Y, M4Value});
20951 case SystemZ::BI__builtin_s390_vlbrh:
20952 case SystemZ::BI__builtin_s390_vlbrf:
20953 case SystemZ::BI__builtin_s390_vlbrg:
20954 case SystemZ::BI__builtin_s390_vlbrq: {
20963#define INTRINSIC_WITH_CC(NAME) \
20964 case SystemZ::BI__builtin_##NAME: \
20965 return EmitSystemZIntrinsicWithCC(*this, Intrinsic::NAME, E)
21047#undef INTRINSIC_WITH_CC
21056struct NVPTXMmaLdstInfo {
21057 unsigned NumResults;
21063#define MMA_INTR(geom_op_type, layout) \
21064 Intrinsic::nvvm_wmma_##geom_op_type##_##layout##_stride
21065#define MMA_LDST(n, geom_op_type) \
21066 { n, MMA_INTR(geom_op_type, col), MMA_INTR(geom_op_type, row) }
21068static NVPTXMmaLdstInfo getNVPTXMmaLdstInfo(
unsigned BuiltinID) {
21069 switch (BuiltinID) {
21071 case NVPTX::BI__hmma_m16n16k16_ld_a:
21072 return MMA_LDST(8, m16n16k16_load_a_f16);
21073 case NVPTX::BI__hmma_m16n16k16_ld_b:
21074 return MMA_LDST(8, m16n16k16_load_b_f16);
21075 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
21076 return MMA_LDST(4, m16n16k16_load_c_f16);
21077 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
21078 return MMA_LDST(8, m16n16k16_load_c_f32);
21079 case NVPTX::BI__hmma_m32n8k16_ld_a:
21080 return MMA_LDST(8, m32n8k16_load_a_f16);
21081 case NVPTX::BI__hmma_m32n8k16_ld_b:
21082 return MMA_LDST(8, m32n8k16_load_b_f16);
21083 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
21084 return MMA_LDST(4, m32n8k16_load_c_f16);
21085 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
21086 return MMA_LDST(8, m32n8k16_load_c_f32);
21087 case NVPTX::BI__hmma_m8n32k16_ld_a:
21088 return MMA_LDST(8, m8n32k16_load_a_f16);
21089 case NVPTX::BI__hmma_m8n32k16_ld_b:
21090 return MMA_LDST(8, m8n32k16_load_b_f16);
21091 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
21092 return MMA_LDST(4, m8n32k16_load_c_f16);
21093 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
21094 return MMA_LDST(8, m8n32k16_load_c_f32);
21097 case NVPTX::BI__imma_m16n16k16_ld_a_s8:
21098 return MMA_LDST(2, m16n16k16_load_a_s8);
21099 case NVPTX::BI__imma_m16n16k16_ld_a_u8:
21100 return MMA_LDST(2, m16n16k16_load_a_u8);
21101 case NVPTX::BI__imma_m16n16k16_ld_b_s8:
21102 return MMA_LDST(2, m16n16k16_load_b_s8);
21103 case NVPTX::BI__imma_m16n16k16_ld_b_u8:
21104 return MMA_LDST(2, m16n16k16_load_b_u8);
21105 case NVPTX::BI__imma_m16n16k16_ld_c:
21106 return MMA_LDST(8, m16n16k16_load_c_s32);
21107 case NVPTX::BI__imma_m32n8k16_ld_a_s8:
21108 return MMA_LDST(4, m32n8k16_load_a_s8);
21109 case NVPTX::BI__imma_m32n8k16_ld_a_u8:
21110 return MMA_LDST(4, m32n8k16_load_a_u8);
21111 case NVPTX::BI__imma_m32n8k16_ld_b_s8:
21112 return MMA_LDST(1, m32n8k16_load_b_s8);
21113 case NVPTX::BI__imma_m32n8k16_ld_b_u8:
21114 return MMA_LDST(1, m32n8k16_load_b_u8);
21115 case NVPTX::BI__imma_m32n8k16_ld_c:
21116 return MMA_LDST(8, m32n8k16_load_c_s32);
21117 case NVPTX::BI__imma_m8n32k16_ld_a_s8:
21118 return MMA_LDST(1, m8n32k16_load_a_s8);
21119 case NVPTX::BI__imma_m8n32k16_ld_a_u8:
21120 return MMA_LDST(1, m8n32k16_load_a_u8);
21121 case NVPTX::BI__imma_m8n32k16_ld_b_s8:
21122 return MMA_LDST(4, m8n32k16_load_b_s8);
21123 case NVPTX::BI__imma_m8n32k16_ld_b_u8:
21124 return MMA_LDST(4, m8n32k16_load_b_u8);
21125 case NVPTX::BI__imma_m8n32k16_ld_c:
21126 return MMA_LDST(8, m8n32k16_load_c_s32);
21130 case NVPTX::BI__imma_m8n8k32_ld_a_s4:
21131 return {1, 0,
MMA_INTR(m8n8k32_load_a_s4, row)};
21132 case NVPTX::BI__imma_m8n8k32_ld_a_u4:
21133 return {1, 0,
MMA_INTR(m8n8k32_load_a_u4, row)};
21134 case NVPTX::BI__imma_m8n8k32_ld_b_s4:
21135 return {1,
MMA_INTR(m8n8k32_load_b_s4, col), 0};
21136 case NVPTX::BI__imma_m8n8k32_ld_b_u4:
21137 return {1,
MMA_INTR(m8n8k32_load_b_u4, col), 0};
21138 case NVPTX::BI__imma_m8n8k32_ld_c:
21139 return MMA_LDST(2, m8n8k32_load_c_s32);
21140 case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
21141 return {1, 0,
MMA_INTR(m8n8k128_load_a_b1, row)};
21142 case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
21143 return {1,
MMA_INTR(m8n8k128_load_b_b1, col), 0};
21144 case NVPTX::BI__bmma_m8n8k128_ld_c:
21145 return MMA_LDST(2, m8n8k128_load_c_s32);
21148 case NVPTX::BI__dmma_m8n8k4_ld_a:
21149 return MMA_LDST(1, m8n8k4_load_a_f64);
21150 case NVPTX::BI__dmma_m8n8k4_ld_b:
21151 return MMA_LDST(1, m8n8k4_load_b_f64);
21152 case NVPTX::BI__dmma_m8n8k4_ld_c:
21153 return MMA_LDST(2, m8n8k4_load_c_f64);
21156 case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
21157 return MMA_LDST(4, m16n16k16_load_a_bf16);
21158 case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
21159 return MMA_LDST(4, m16n16k16_load_b_bf16);
21160 case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
21161 return MMA_LDST(2, m8n32k16_load_a_bf16);
21162 case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
21163 return MMA_LDST(8, m8n32k16_load_b_bf16);
21164 case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
21165 return MMA_LDST(8, m32n8k16_load_a_bf16);
21166 case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
21167 return MMA_LDST(2, m32n8k16_load_b_bf16);
21168 case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
21169 return MMA_LDST(4, m16n16k8_load_a_tf32);
21170 case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
21171 return MMA_LDST(4, m16n16k8_load_b_tf32);
21172 case NVPTX::BI__mma_tf32_m16n16k8_ld_c:
21173 return MMA_LDST(8, m16n16k8_load_c_f32);
21179 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
21180 return MMA_LDST(4, m16n16k16_store_d_f16);
21181 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
21182 return MMA_LDST(8, m16n16k16_store_d_f32);
21183 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
21184 return MMA_LDST(4, m32n8k16_store_d_f16);
21185 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
21186 return MMA_LDST(8, m32n8k16_store_d_f32);
21187 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
21188 return MMA_LDST(4, m8n32k16_store_d_f16);
21189 case NVPTX::BI__hmma_m8n32k16_st_c_f32:
21190 return MMA_LDST(8, m8n32k16_store_d_f32);
21195 case NVPTX::BI__imma_m16n16k16_st_c_i32:
21196 return MMA_LDST(8, m16n16k16_store_d_s32);
21197 case NVPTX::BI__imma_m32n8k16_st_c_i32:
21198 return MMA_LDST(8, m32n8k16_store_d_s32);
21199 case NVPTX::BI__imma_m8n32k16_st_c_i32:
21200 return MMA_LDST(8, m8n32k16_store_d_s32);
21201 case NVPTX::BI__imma_m8n8k32_st_c_i32:
21202 return MMA_LDST(2, m8n8k32_store_d_s32);
21203 case NVPTX::BI__bmma_m8n8k128_st_c_i32:
21204 return MMA_LDST(2, m8n8k128_store_d_s32);
21207 case NVPTX::BI__dmma_m8n8k4_st_c_f64:
21208 return MMA_LDST(2, m8n8k4_store_d_f64);
21211 case NVPTX::BI__mma_m16n16k8_st_c_f32:
21212 return MMA_LDST(8, m16n16k8_store_d_f32);
21215 llvm_unreachable(
"Unknown MMA builtin");
21222struct NVPTXMmaInfo {
21231 std::array<unsigned, 8> Variants;
21233 unsigned getMMAIntrinsic(
int Layout,
bool Satf) {
21234 unsigned Index = Layout + 4 * Satf;
21235 if (Index >= Variants.size())
21237 return Variants[Index];
21243static NVPTXMmaInfo getNVPTXMmaInfo(
unsigned BuiltinID) {
21245#define MMA_VARIANTS(geom, type) \
21246 Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type, \
21247 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type, \
21248 Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type, \
21249 Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type
21250#define MMA_SATF_VARIANTS(geom, type) \
21251 MMA_VARIANTS(geom, type), \
21252 Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type##_satfinite, \
21253 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type##_satfinite, \
21254 Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type##_satfinite, \
21255 Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type##_satfinite
21257#define MMA_VARIANTS_I4(geom, type) \
21259 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type, \
21263 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type##_satfinite, \
21267#define MMA_VARIANTS_B1_XOR(geom, type) \
21269 Intrinsic::nvvm_wmma_##geom##_mma_xor_popc_row_col_##type, \
21276#define MMA_VARIANTS_B1_AND(geom, type) \
21278 Intrinsic::nvvm_wmma_##geom##_mma_and_popc_row_col_##type, \
21286 switch (BuiltinID) {
21290 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
21292 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
21294 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
21296 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
21298 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
21300 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
21302 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
21304 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
21306 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
21308 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
21310 case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
21312 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
21316 case NVPTX::BI__imma_m16n16k16_mma_s8:
21318 case NVPTX::BI__imma_m16n16k16_mma_u8:
21320 case NVPTX::BI__imma_m32n8k16_mma_s8:
21322 case NVPTX::BI__imma_m32n8k16_mma_u8:
21324 case NVPTX::BI__imma_m8n32k16_mma_s8:
21326 case NVPTX::BI__imma_m8n32k16_mma_u8:
21330 case NVPTX::BI__imma_m8n8k32_mma_s4:
21332 case NVPTX::BI__imma_m8n8k32_mma_u4:
21334 case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1:
21336 case NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1:
21340 case NVPTX::BI__dmma_m8n8k4_mma_f64:
21344 case NVPTX::BI__mma_bf16_m16n16k16_mma_f32:
21345 return {4, 4, 8, 8, {{
MMA_VARIANTS(m16n16k16, bf16)}}};
21346 case NVPTX::BI__mma_bf16_m8n32k16_mma_f32:
21347 return {2, 8, 8, 8, {{
MMA_VARIANTS(m8n32k16, bf16)}}};
21348 case NVPTX::BI__mma_bf16_m32n8k16_mma_f32:
21349 return {8, 2, 8, 8, {{
MMA_VARIANTS(m32n8k16, bf16)}}};
21350 case NVPTX::BI__mma_tf32_m16n16k8_mma_f32:
21351 return {4, 4, 8, 8, {{
MMA_VARIANTS(m16n16k8, tf32)}}};
21353 llvm_unreachable(
"Unexpected builtin ID.");
21356#undef MMA_SATF_VARIANTS
21357#undef MMA_VARIANTS_I4
21358#undef MMA_VARIANTS_B1_AND
21359#undef MMA_VARIANTS_B1_XOR
21368 return CGF.
Builder.CreateCall(
21370 {Ptr, ConstantInt::get(CGF.Builder.getInt32Ty(), Align.getQuantity())});
21382 MDNode *MD = MDNode::get(CGF.
Builder.getContext(), {});
21383 LD->setMetadata(LLVMContext::MD_invariant_load, MD);
21391 llvm::Type *ElemTy =
21393 return CGF.
Builder.CreateCall(
21395 {Ptr, CGF.EmitScalarExpr(E->getArg(1))});
21398static Value *MakeCpAsync(
unsigned IntrinsicID,
unsigned IntrinsicIDS,
21401 return E->getNumArgs() == 3
21403 {CGF.EmitScalarExpr(E->getArg(0)),
21404 CGF.EmitScalarExpr(E->getArg(1)),
21405 CGF.EmitScalarExpr(E->getArg(2))})
21407 {CGF.EmitScalarExpr(E->getArg(0)),
21408 CGF.EmitScalarExpr(E->getArg(1))});
21411static Value *MakeHalfType(
unsigned IntrinsicID,
unsigned BuiltinID,
21414 if (!(
C.getLangOpts().NativeHalfType ||
21415 !
C.getTargetInfo().useFP16ConversionIntrinsics())) {
21417 " requires native half type support.");
21421 if (BuiltinID == NVPTX::BI__nvvm_ldg_h || BuiltinID == NVPTX::BI__nvvm_ldg_h2)
21422 return MakeLdg(CGF,
E);
21424 if (IntrinsicID == Intrinsic::nvvm_ldu_global_f)
21425 return MakeLdu(IntrinsicID, CGF,
E);
21429 auto *FTy = F->getFunctionType();
21430 unsigned ICEArguments = 0;
21432 C.GetBuiltinType(BuiltinID, Error, &ICEArguments);
21434 for (
unsigned i = 0, e =
E->getNumArgs(); i != e; ++i) {
21435 assert((ICEArguments & (1 << i)) == 0);
21437 auto *PTy = FTy->getParamType(i);
21438 if (PTy != ArgValue->
getType())
21439 ArgValue = CGF.
Builder.CreateBitCast(ArgValue, PTy);
21440 Args.push_back(ArgValue);
21443 return CGF.
Builder.CreateCall(F, Args);
21449 switch (BuiltinID) {
21450 case NVPTX::BI__nvvm_atom_add_gen_i:
21451 case NVPTX::BI__nvvm_atom_add_gen_l:
21452 case NVPTX::BI__nvvm_atom_add_gen_ll:
21455 case NVPTX::BI__nvvm_atom_sub_gen_i:
21456 case NVPTX::BI__nvvm_atom_sub_gen_l:
21457 case NVPTX::BI__nvvm_atom_sub_gen_ll:
21460 case NVPTX::BI__nvvm_atom_and_gen_i:
21461 case NVPTX::BI__nvvm_atom_and_gen_l:
21462 case NVPTX::BI__nvvm_atom_and_gen_ll:
21465 case NVPTX::BI__nvvm_atom_or_gen_i:
21466 case NVPTX::BI__nvvm_atom_or_gen_l:
21467 case NVPTX::BI__nvvm_atom_or_gen_ll:
21470 case NVPTX::BI__nvvm_atom_xor_gen_i:
21471 case NVPTX::BI__nvvm_atom_xor_gen_l:
21472 case NVPTX::BI__nvvm_atom_xor_gen_ll:
21475 case NVPTX::BI__nvvm_atom_xchg_gen_i:
21476 case NVPTX::BI__nvvm_atom_xchg_gen_l:
21477 case NVPTX::BI__nvvm_atom_xchg_gen_ll:
21480 case NVPTX::BI__nvvm_atom_max_gen_i:
21481 case NVPTX::BI__nvvm_atom_max_gen_l:
21482 case NVPTX::BI__nvvm_atom_max_gen_ll:
21485 case NVPTX::BI__nvvm_atom_max_gen_ui:
21486 case NVPTX::BI__nvvm_atom_max_gen_ul:
21487 case NVPTX::BI__nvvm_atom_max_gen_ull:
21490 case NVPTX::BI__nvvm_atom_min_gen_i:
21491 case NVPTX::BI__nvvm_atom_min_gen_l:
21492 case NVPTX::BI__nvvm_atom_min_gen_ll:
21495 case NVPTX::BI__nvvm_atom_min_gen_ui:
21496 case NVPTX::BI__nvvm_atom_min_gen_ul:
21497 case NVPTX::BI__nvvm_atom_min_gen_ull:
21500 case NVPTX::BI__nvvm_atom_cas_gen_us:
21501 case NVPTX::BI__nvvm_atom_cas_gen_i:
21502 case NVPTX::BI__nvvm_atom_cas_gen_l:
21503 case NVPTX::BI__nvvm_atom_cas_gen_ll:
21508 case NVPTX::BI__nvvm_atom_add_gen_f:
21509 case NVPTX::BI__nvvm_atom_add_gen_d: {
21514 AtomicOrdering::SequentiallyConsistent);
21517 case NVPTX::BI__nvvm_atom_inc_gen_ui: {
21522 return Builder.CreateCall(FnALI32, {Ptr, Val});
21525 case NVPTX::BI__nvvm_atom_dec_gen_ui: {
21530 return Builder.CreateCall(FnALD32, {Ptr, Val});
21533 case NVPTX::BI__nvvm_ldg_c:
21534 case NVPTX::BI__nvvm_ldg_sc:
21535 case NVPTX::BI__nvvm_ldg_c2:
21536 case NVPTX::BI__nvvm_ldg_sc2:
21537 case NVPTX::BI__nvvm_ldg_c4:
21538 case NVPTX::BI__nvvm_ldg_sc4:
21539 case NVPTX::BI__nvvm_ldg_s:
21540 case NVPTX::BI__nvvm_ldg_s2:
21541 case NVPTX::BI__nvvm_ldg_s4:
21542 case NVPTX::BI__nvvm_ldg_i:
21543 case NVPTX::BI__nvvm_ldg_i2:
21544 case NVPTX::BI__nvvm_ldg_i4:
21545 case NVPTX::BI__nvvm_ldg_l:
21546 case NVPTX::BI__nvvm_ldg_l2:
21547 case NVPTX::BI__nvvm_ldg_ll:
21548 case NVPTX::BI__nvvm_ldg_ll2:
21549 case NVPTX::BI__nvvm_ldg_uc:
21550 case NVPTX::BI__nvvm_ldg_uc2:
21551 case NVPTX::BI__nvvm_ldg_uc4:
21552 case NVPTX::BI__nvvm_ldg_us:
21553 case NVPTX::BI__nvvm_ldg_us2:
21554 case NVPTX::BI__nvvm_ldg_us4:
21555 case NVPTX::BI__nvvm_ldg_ui:
21556 case NVPTX::BI__nvvm_ldg_ui2:
21557 case NVPTX::BI__nvvm_ldg_ui4:
21558 case NVPTX::BI__nvvm_ldg_ul:
21559 case NVPTX::BI__nvvm_ldg_ul2:
21560 case NVPTX::BI__nvvm_ldg_ull:
21561 case NVPTX::BI__nvvm_ldg_ull2:
21562 case NVPTX::BI__nvvm_ldg_f:
21563 case NVPTX::BI__nvvm_ldg_f2:
21564 case NVPTX::BI__nvvm_ldg_f4:
21565 case NVPTX::BI__nvvm_ldg_d:
21566 case NVPTX::BI__nvvm_ldg_d2:
21570 return MakeLdg(*
this,
E);
21572 case NVPTX::BI__nvvm_ldu_c:
21573 case NVPTX::BI__nvvm_ldu_sc:
21574 case NVPTX::BI__nvvm_ldu_c2:
21575 case NVPTX::BI__nvvm_ldu_sc2:
21576 case NVPTX::BI__nvvm_ldu_c4:
21577 case NVPTX::BI__nvvm_ldu_sc4:
21578 case NVPTX::BI__nvvm_ldu_s:
21579 case NVPTX::BI__nvvm_ldu_s2:
21580 case NVPTX::BI__nvvm_ldu_s4:
21581 case NVPTX::BI__nvvm_ldu_i:
21582 case NVPTX::BI__nvvm_ldu_i2:
21583 case NVPTX::BI__nvvm_ldu_i4:
21584 case NVPTX::BI__nvvm_ldu_l:
21585 case NVPTX::BI__nvvm_ldu_l2:
21586 case NVPTX::BI__nvvm_ldu_ll:
21587 case NVPTX::BI__nvvm_ldu_ll2:
21588 case NVPTX::BI__nvvm_ldu_uc:
21589 case NVPTX::BI__nvvm_ldu_uc2:
21590 case NVPTX::BI__nvvm_ldu_uc4:
21591 case NVPTX::BI__nvvm_ldu_us:
21592 case NVPTX::BI__nvvm_ldu_us2:
21593 case NVPTX::BI__nvvm_ldu_us4:
21594 case NVPTX::BI__nvvm_ldu_ui:
21595 case NVPTX::BI__nvvm_ldu_ui2:
21596 case NVPTX::BI__nvvm_ldu_ui4:
21597 case NVPTX::BI__nvvm_ldu_ul:
21598 case NVPTX::BI__nvvm_ldu_ul2:
21599 case NVPTX::BI__nvvm_ldu_ull:
21600 case NVPTX::BI__nvvm_ldu_ull2:
21601 return MakeLdu(Intrinsic::nvvm_ldu_global_i, *
this,
E);
21602 case NVPTX::BI__nvvm_ldu_f:
21603 case NVPTX::BI__nvvm_ldu_f2:
21604 case NVPTX::BI__nvvm_ldu_f4:
21605 case NVPTX::BI__nvvm_ldu_d:
21606 case NVPTX::BI__nvvm_ldu_d2:
21607 return MakeLdu(Intrinsic::nvvm_ldu_global_f, *
this,
E);
21609 case NVPTX::BI__nvvm_atom_cta_add_gen_i:
21610 case NVPTX::BI__nvvm_atom_cta_add_gen_l:
21611 case NVPTX::BI__nvvm_atom_cta_add_gen_ll:
21612 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_cta, *
this,
E);
21613 case NVPTX::BI__nvvm_atom_sys_add_gen_i:
21614 case NVPTX::BI__nvvm_atom_sys_add_gen_l:
21615 case NVPTX::BI__nvvm_atom_sys_add_gen_ll:
21616 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_sys, *
this,
E);
21617 case NVPTX::BI__nvvm_atom_cta_add_gen_f:
21618 case NVPTX::BI__nvvm_atom_cta_add_gen_d:
21619 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_cta, *
this,
E);
21620 case NVPTX::BI__nvvm_atom_sys_add_gen_f:
21621 case NVPTX::BI__nvvm_atom_sys_add_gen_d:
21622 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_sys, *
this,
E);
21623 case NVPTX::BI__nvvm_atom_cta_xchg_gen_i:
21624 case NVPTX::BI__nvvm_atom_cta_xchg_gen_l:
21625 case NVPTX::BI__nvvm_atom_cta_xchg_gen_ll:
21626 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_cta, *
this,
E);
21627 case NVPTX::BI__nvvm_atom_sys_xchg_gen_i:
21628 case NVPTX::BI__nvvm_atom_sys_xchg_gen_l:
21629 case NVPTX::BI__nvvm_atom_sys_xchg_gen_ll:
21630 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_sys, *
this,
E);
21631 case NVPTX::BI__nvvm_atom_cta_max_gen_i:
21632 case NVPTX::BI__nvvm_atom_cta_max_gen_ui:
21633 case NVPTX::BI__nvvm_atom_cta_max_gen_l:
21634 case NVPTX::BI__nvvm_atom_cta_max_gen_ul:
21635 case NVPTX::BI__nvvm_atom_cta_max_gen_ll:
21636 case NVPTX::BI__nvvm_atom_cta_max_gen_ull:
21637 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_cta, *
this,
E);
21638 case NVPTX::BI__nvvm_atom_sys_max_gen_i:
21639 case NVPTX::BI__nvvm_atom_sys_max_gen_ui:
21640 case NVPTX::BI__nvvm_atom_sys_max_gen_l:
21641 case NVPTX::BI__nvvm_atom_sys_max_gen_ul:
21642 case NVPTX::BI__nvvm_atom_sys_max_gen_ll:
21643 case NVPTX::BI__nvvm_atom_sys_max_gen_ull:
21644 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_sys, *
this,
E);
21645 case NVPTX::BI__nvvm_atom_cta_min_gen_i:
21646 case NVPTX::BI__nvvm_atom_cta_min_gen_ui:
21647 case NVPTX::BI__nvvm_atom_cta_min_gen_l:
21648 case NVPTX::BI__nvvm_atom_cta_min_gen_ul:
21649 case NVPTX::BI__nvvm_atom_cta_min_gen_ll:
21650 case NVPTX::BI__nvvm_atom_cta_min_gen_ull:
21651 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_cta, *
this,
E);
21652 case NVPTX::BI__nvvm_atom_sys_min_gen_i:
21653 case NVPTX::BI__nvvm_atom_sys_min_gen_ui:
21654 case NVPTX::BI__nvvm_atom_sys_min_gen_l:
21655 case NVPTX::BI__nvvm_atom_sys_min_gen_ul:
21656 case NVPTX::BI__nvvm_atom_sys_min_gen_ll:
21657 case NVPTX::BI__nvvm_atom_sys_min_gen_ull:
21658 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_sys, *
this,
E);
21659 case NVPTX::BI__nvvm_atom_cta_inc_gen_ui:
21660 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_cta, *
this,
E);
21661 case NVPTX::BI__nvvm_atom_cta_dec_gen_ui:
21662 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_cta, *
this,
E);
21663 case NVPTX::BI__nvvm_atom_sys_inc_gen_ui:
21664 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_sys, *
this,
E);
21665 case NVPTX::BI__nvvm_atom_sys_dec_gen_ui:
21666 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_sys, *
this,
E);
21667 case NVPTX::BI__nvvm_atom_cta_and_gen_i:
21668 case NVPTX::BI__nvvm_atom_cta_and_gen_l:
21669 case NVPTX::BI__nvvm_atom_cta_and_gen_ll:
21670 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_cta, *
this,
E);
21671 case NVPTX::BI__nvvm_atom_sys_and_gen_i:
21672 case NVPTX::BI__nvvm_atom_sys_and_gen_l:
21673 case NVPTX::BI__nvvm_atom_sys_and_gen_ll:
21674 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_sys, *
this,
E);
21675 case NVPTX::BI__nvvm_atom_cta_or_gen_i:
21676 case NVPTX::BI__nvvm_atom_cta_or_gen_l:
21677 case NVPTX::BI__nvvm_atom_cta_or_gen_ll:
21678 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_cta, *
this,
E);
21679 case NVPTX::BI__nvvm_atom_sys_or_gen_i:
21680 case NVPTX::BI__nvvm_atom_sys_or_gen_l:
21681 case NVPTX::BI__nvvm_atom_sys_or_gen_ll:
21682 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_sys, *
this,
E);
21683 case NVPTX::BI__nvvm_atom_cta_xor_gen_i:
21684 case NVPTX::BI__nvvm_atom_cta_xor_gen_l:
21685 case NVPTX::BI__nvvm_atom_cta_xor_gen_ll:
21686 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_cta, *
this,
E);
21687 case NVPTX::BI__nvvm_atom_sys_xor_gen_i:
21688 case NVPTX::BI__nvvm_atom_sys_xor_gen_l:
21689 case NVPTX::BI__nvvm_atom_sys_xor_gen_ll:
21690 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_sys, *
this,
E);
21691 case NVPTX::BI__nvvm_atom_cta_cas_gen_us:
21692 case NVPTX::BI__nvvm_atom_cta_cas_gen_i:
21693 case NVPTX::BI__nvvm_atom_cta_cas_gen_l:
21694 case NVPTX::BI__nvvm_atom_cta_cas_gen_ll: {
21696 llvm::Type *ElemTy =
21700 Intrinsic::nvvm_atomic_cas_gen_i_cta, {ElemTy, Ptr->getType()}),
21701 {Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))});
21703 case NVPTX::BI__nvvm_atom_sys_cas_gen_us:
21704 case NVPTX::BI__nvvm_atom_sys_cas_gen_i:
21705 case NVPTX::BI__nvvm_atom_sys_cas_gen_l:
21706 case NVPTX::BI__nvvm_atom_sys_cas_gen_ll: {
21708 llvm::Type *ElemTy =
21712 Intrinsic::nvvm_atomic_cas_gen_i_sys, {ElemTy, Ptr->getType()}),
21713 {Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))});
21715 case NVPTX::BI__nvvm_match_all_sync_i32p:
21716 case NVPTX::BI__nvvm_match_all_sync_i64p: {
21722 ? Intrinsic::nvvm_match_all_sync_i32p
21723 : Intrinsic::nvvm_match_all_sync_i64p),
21728 return Builder.CreateExtractValue(ResultPair, 0);
21732 case NVPTX::BI__hmma_m16n16k16_ld_a:
21733 case NVPTX::BI__hmma_m16n16k16_ld_b:
21734 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
21735 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
21736 case NVPTX::BI__hmma_m32n8k16_ld_a:
21737 case NVPTX::BI__hmma_m32n8k16_ld_b:
21738 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
21739 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
21740 case NVPTX::BI__hmma_m8n32k16_ld_a:
21741 case NVPTX::BI__hmma_m8n32k16_ld_b:
21742 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
21743 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
21745 case NVPTX::BI__imma_m16n16k16_ld_a_s8:
21746 case NVPTX::BI__imma_m16n16k16_ld_a_u8:
21747 case NVPTX::BI__imma_m16n16k16_ld_b_s8:
21748 case NVPTX::BI__imma_m16n16k16_ld_b_u8:
21749 case NVPTX::BI__imma_m16n16k16_ld_c:
21750 case NVPTX::BI__imma_m32n8k16_ld_a_s8:
21751 case NVPTX::BI__imma_m32n8k16_ld_a_u8:
21752 case NVPTX::BI__imma_m32n8k16_ld_b_s8:
21753 case NVPTX::BI__imma_m32n8k16_ld_b_u8:
21754 case NVPTX::BI__imma_m32n8k16_ld_c:
21755 case NVPTX::BI__imma_m8n32k16_ld_a_s8:
21756 case NVPTX::BI__imma_m8n32k16_ld_a_u8:
21757 case NVPTX::BI__imma_m8n32k16_ld_b_s8:
21758 case NVPTX::BI__imma_m8n32k16_ld_b_u8:
21759 case NVPTX::BI__imma_m8n32k16_ld_c:
21761 case NVPTX::BI__imma_m8n8k32_ld_a_s4:
21762 case NVPTX::BI__imma_m8n8k32_ld_a_u4:
21763 case NVPTX::BI__imma_m8n8k32_ld_b_s4:
21764 case NVPTX::BI__imma_m8n8k32_ld_b_u4:
21765 case NVPTX::BI__imma_m8n8k32_ld_c:
21766 case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
21767 case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
21768 case NVPTX::BI__bmma_m8n8k128_ld_c:
21770 case NVPTX::BI__dmma_m8n8k4_ld_a:
21771 case NVPTX::BI__dmma_m8n8k4_ld_b:
21772 case NVPTX::BI__dmma_m8n8k4_ld_c:
21774 case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
21775 case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
21776 case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
21777 case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
21778 case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
21779 case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
21780 case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
21781 case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
21782 case NVPTX::BI__mma_tf32_m16n16k8_ld_c: {
21786 std::optional<llvm::APSInt> isColMajorArg =
21788 if (!isColMajorArg)
21790 bool isColMajor = isColMajorArg->getSExtValue();
21791 NVPTXMmaLdstInfo II = getNVPTXMmaLdstInfo(BuiltinID);
21792 unsigned IID = isColMajor ? II.IID_col : II.IID_row;
21800 assert(II.NumResults);
21801 if (II.NumResults == 1) {
21805 for (
unsigned i = 0; i < II.NumResults; ++i) {
21810 llvm::ConstantInt::get(
IntTy, i)),
21817 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
21818 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
21819 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
21820 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
21821 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
21822 case NVPTX::BI__hmma_m8n32k16_st_c_f32:
21823 case NVPTX::BI__imma_m16n16k16_st_c_i32:
21824 case NVPTX::BI__imma_m32n8k16_st_c_i32:
21825 case NVPTX::BI__imma_m8n32k16_st_c_i32:
21826 case NVPTX::BI__imma_m8n8k32_st_c_i32:
21827 case NVPTX::BI__bmma_m8n8k128_st_c_i32:
21828 case NVPTX::BI__dmma_m8n8k4_st_c_f64:
21829 case NVPTX::BI__mma_m16n16k8_st_c_f32: {
21833 std::optional<llvm::APSInt> isColMajorArg =
21835 if (!isColMajorArg)
21837 bool isColMajor = isColMajorArg->getSExtValue();
21838 NVPTXMmaLdstInfo II = getNVPTXMmaLdstInfo(BuiltinID);
21839 unsigned IID = isColMajor ? II.IID_col : II.IID_row;
21844 llvm::Type *ParamType = Intrinsic->getFunctionType()->getParamType(1);
21846 for (
unsigned i = 0; i < II.NumResults; ++i) {
21850 llvm::ConstantInt::get(
IntTy, i)),
21852 Values.push_back(
Builder.CreateBitCast(
V, ParamType));
21854 Values.push_back(Ldm);
21861 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
21862 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
21863 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
21864 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
21865 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
21866 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
21867 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
21868 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
21869 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
21870 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
21871 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
21872 case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
21873 case NVPTX::BI__imma_m16n16k16_mma_s8:
21874 case NVPTX::BI__imma_m16n16k16_mma_u8:
21875 case NVPTX::BI__imma_m32n8k16_mma_s8:
21876 case NVPTX::BI__imma_m32n8k16_mma_u8:
21877 case NVPTX::BI__imma_m8n32k16_mma_s8:
21878 case NVPTX::BI__imma_m8n32k16_mma_u8:
21879 case NVPTX::BI__imma_m8n8k32_mma_s4:
21880 case NVPTX::BI__imma_m8n8k32_mma_u4:
21881 case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1:
21882 case NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1:
21883 case NVPTX::BI__dmma_m8n8k4_mma_f64:
21884 case NVPTX::BI__mma_bf16_m16n16k16_mma_f32:
21885 case NVPTX::BI__mma_bf16_m8n32k16_mma_f32:
21886 case NVPTX::BI__mma_bf16_m32n8k16_mma_f32:
21887 case NVPTX::BI__mma_tf32_m16n16k8_mma_f32: {
21892 std::optional<llvm::APSInt> LayoutArg =
21896 int Layout = LayoutArg->getSExtValue();
21897 if (Layout < 0 || Layout > 3)
21899 llvm::APSInt SatfArg;
21900 if (BuiltinID == NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1 ||
21901 BuiltinID == NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1)
21903 else if (std::optional<llvm::APSInt> OptSatfArg =
21905 SatfArg = *OptSatfArg;
21908 bool Satf = SatfArg.getSExtValue();
21909 NVPTXMmaInfo MI = getNVPTXMmaInfo(BuiltinID);
21910 unsigned IID = MI.getMMAIntrinsic(Layout, Satf);
21916 llvm::Type *AType = Intrinsic->getFunctionType()->getParamType(0);
21918 for (
unsigned i = 0; i < MI.NumEltsA; ++i) {
21922 llvm::ConstantInt::get(
IntTy, i)),
21924 Values.push_back(
Builder.CreateBitCast(
V, AType));
21927 llvm::Type *BType = Intrinsic->getFunctionType()->getParamType(MI.NumEltsA);
21928 for (
unsigned i = 0; i < MI.NumEltsB; ++i) {
21932 llvm::ConstantInt::get(
IntTy, i)),
21934 Values.push_back(
Builder.CreateBitCast(
V, BType));
21937 llvm::Type *CType =
21938 Intrinsic->getFunctionType()->getParamType(MI.NumEltsA + MI.NumEltsB);
21939 for (
unsigned i = 0; i < MI.NumEltsC; ++i) {
21943 llvm::ConstantInt::get(
IntTy, i)),
21945 Values.push_back(
Builder.CreateBitCast(
V, CType));
21949 for (
unsigned i = 0; i < MI.NumEltsD; ++i)
21953 llvm::ConstantInt::get(
IntTy, i)),
21958 case NVPTX::BI__nvvm_ex2_approx_f16:
21959 return MakeHalfType(Intrinsic::nvvm_ex2_approx_f16, BuiltinID,
E, *
this);
21960 case NVPTX::BI__nvvm_ex2_approx_f16x2:
21961 return MakeHalfType(Intrinsic::nvvm_ex2_approx_f16x2, BuiltinID,
E, *
this);
21962 case NVPTX::BI__nvvm_ff2f16x2_rn:
21963 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rn, BuiltinID,
E, *
this);
21964 case NVPTX::BI__nvvm_ff2f16x2_rn_relu:
21965 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rn_relu, BuiltinID,
E, *
this);
21966 case NVPTX::BI__nvvm_ff2f16x2_rz:
21967 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rz, BuiltinID,
E, *
this);
21968 case NVPTX::BI__nvvm_ff2f16x2_rz_relu:
21969 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rz_relu, BuiltinID,
E, *
this);
21970 case NVPTX::BI__nvvm_fma_rn_f16:
21971 return MakeHalfType(Intrinsic::nvvm_fma_rn_f16, BuiltinID,
E, *
this);
21972 case NVPTX::BI__nvvm_fma_rn_f16x2:
21973 return MakeHalfType(Intrinsic::nvvm_fma_rn_f16x2, BuiltinID,
E, *
this);
21974 case NVPTX::BI__nvvm_fma_rn_ftz_f16:
21975 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_f16, BuiltinID,
E, *
this);
21976 case NVPTX::BI__nvvm_fma_rn_ftz_f16x2:
21977 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_f16x2, BuiltinID,
E, *
this);
21978 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16:
21979 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_relu_f16, BuiltinID,
E,
21981 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16x2:
21982 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_relu_f16x2, BuiltinID,
E,
21984 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16:
21985 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_sat_f16, BuiltinID,
E,
21987 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16x2:
21988 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_sat_f16x2, BuiltinID,
E,
21990 case NVPTX::BI__nvvm_fma_rn_relu_f16:
21991 return MakeHalfType(Intrinsic::nvvm_fma_rn_relu_f16, BuiltinID,
E, *
this);
21992 case NVPTX::BI__nvvm_fma_rn_relu_f16x2:
21993 return MakeHalfType(Intrinsic::nvvm_fma_rn_relu_f16x2, BuiltinID,
E, *
this);
21994 case NVPTX::BI__nvvm_fma_rn_sat_f16:
21995 return MakeHalfType(Intrinsic::nvvm_fma_rn_sat_f16, BuiltinID,
E, *
this);
21996 case NVPTX::BI__nvvm_fma_rn_sat_f16x2:
21997 return MakeHalfType(Intrinsic::nvvm_fma_rn_sat_f16x2, BuiltinID,
E, *
this);
21998 case NVPTX::BI__nvvm_fmax_f16:
21999 return MakeHalfType(Intrinsic::nvvm_fmax_f16, BuiltinID,
E, *
this);
22000 case NVPTX::BI__nvvm_fmax_f16x2:
22001 return MakeHalfType(Intrinsic::nvvm_fmax_f16x2, BuiltinID,
E, *
this);
22002 case NVPTX::BI__nvvm_fmax_ftz_f16:
22003 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_f16, BuiltinID,
E, *
this);
22004 case NVPTX::BI__nvvm_fmax_ftz_f16x2:
22005 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_f16x2, BuiltinID,
E, *
this);
22006 case NVPTX::BI__nvvm_fmax_ftz_nan_f16:
22007 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_f16, BuiltinID,
E, *
this);
22008 case NVPTX::BI__nvvm_fmax_ftz_nan_f16x2:
22009 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_f16x2, BuiltinID,
E,
22011 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16:
22012 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f16, BuiltinID,
22014 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16x2:
22015 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f16x2,
22016 BuiltinID,
E, *
this);
22017 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16:
22018 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_xorsign_abs_f16, BuiltinID,
E,
22020 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16x2:
22021 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_xorsign_abs_f16x2, BuiltinID,
22023 case NVPTX::BI__nvvm_fmax_nan_f16:
22024 return MakeHalfType(Intrinsic::nvvm_fmax_nan_f16, BuiltinID,
E, *
this);
22025 case NVPTX::BI__nvvm_fmax_nan_f16x2:
22026 return MakeHalfType(Intrinsic::nvvm_fmax_nan_f16x2, BuiltinID,
E, *
this);
22027 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16:
22028 return MakeHalfType(Intrinsic::nvvm_fmax_nan_xorsign_abs_f16, BuiltinID,
E,
22030 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16x2:
22031 return MakeHalfType(Intrinsic::nvvm_fmax_nan_xorsign_abs_f16x2, BuiltinID,
22033 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16:
22034 return MakeHalfType(Intrinsic::nvvm_fmax_xorsign_abs_f16, BuiltinID,
E,
22036 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16x2:
22037 return MakeHalfType(Intrinsic::nvvm_fmax_xorsign_abs_f16x2, BuiltinID,
E,
22039 case NVPTX::BI__nvvm_fmin_f16:
22040 return MakeHalfType(Intrinsic::nvvm_fmin_f16, BuiltinID,
E, *
this);
22041 case NVPTX::BI__nvvm_fmin_f16x2:
22042 return MakeHalfType(Intrinsic::nvvm_fmin_f16x2, BuiltinID,
E, *
this);
22043 case NVPTX::BI__nvvm_fmin_ftz_f16:
22044 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_f16, BuiltinID,
E, *
this);
22045 case NVPTX::BI__nvvm_fmin_ftz_f16x2:
22046 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_f16x2, BuiltinID,
E, *
this);
22047 case NVPTX::BI__nvvm_fmin_ftz_nan_f16:
22048 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_f16, BuiltinID,
E, *
this);
22049 case NVPTX::BI__nvvm_fmin_ftz_nan_f16x2:
22050 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_f16x2, BuiltinID,
E,
22052 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16:
22053 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f16, BuiltinID,
22055 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16x2:
22056 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f16x2,
22057 BuiltinID,
E, *
this);
22058 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16:
22059 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_xorsign_abs_f16, BuiltinID,
E,
22061 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16x2:
22062 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_xorsign_abs_f16x2, BuiltinID,
22064 case NVPTX::BI__nvvm_fmin_nan_f16:
22065 return MakeHalfType(Intrinsic::nvvm_fmin_nan_f16, BuiltinID,
E, *
this);
22066 case NVPTX::BI__nvvm_fmin_nan_f16x2:
22067 return MakeHalfType(Intrinsic::nvvm_fmin_nan_f16x2, BuiltinID,
E, *
this);
22068 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16:
22069 return MakeHalfType(Intrinsic::nvvm_fmin_nan_xorsign_abs_f16, BuiltinID,
E,
22071 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16x2:
22072 return MakeHalfType(Intrinsic::nvvm_fmin_nan_xorsign_abs_f16x2, BuiltinID,
22074 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16:
22075 return MakeHalfType(Intrinsic::nvvm_fmin_xorsign_abs_f16, BuiltinID,
E,
22077 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16x2:
22078 return MakeHalfType(Intrinsic::nvvm_fmin_xorsign_abs_f16x2, BuiltinID,
E,
22080 case NVPTX::BI__nvvm_ldg_h:
22081 case NVPTX::BI__nvvm_ldg_h2:
22082 return MakeHalfType(Intrinsic::not_intrinsic, BuiltinID,
E, *
this);
22083 case NVPTX::BI__nvvm_ldu_h:
22084 case NVPTX::BI__nvvm_ldu_h2:
22085 return MakeHalfType(Intrinsic::nvvm_ldu_global_f, BuiltinID,
E, *
this);
22086 case NVPTX::BI__nvvm_cp_async_ca_shared_global_4:
22087 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_4,
22088 Intrinsic::nvvm_cp_async_ca_shared_global_4_s, *
this,
E,
22090 case NVPTX::BI__nvvm_cp_async_ca_shared_global_8:
22091 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_8,
22092 Intrinsic::nvvm_cp_async_ca_shared_global_8_s, *
this,
E,
22094 case NVPTX::BI__nvvm_cp_async_ca_shared_global_16:
22095 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_16,
22096 Intrinsic::nvvm_cp_async_ca_shared_global_16_s, *
this,
E,
22098 case NVPTX::BI__nvvm_cp_async_cg_shared_global_16:
22099 return MakeCpAsync(Intrinsic::nvvm_cp_async_cg_shared_global_16,
22100 Intrinsic::nvvm_cp_async_cg_shared_global_16_s, *
this,
E,
22102 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_x:
22105 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_y:
22108 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_z:
22111 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_w:
22114 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_x:
22117 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_y:
22120 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_z:
22123 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_w:
22126 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_x:
22129 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_y:
22132 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_z:
22135 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_w:
22138 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_x:
22141 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_y:
22144 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_z:
22147 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_w:
22150 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctarank:
22153 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctarank:
22156 case NVPTX::BI__nvvm_is_explicit_cluster:
22159 case NVPTX::BI__nvvm_isspacep_shared_cluster:
22163 case NVPTX::BI__nvvm_mapa:
22166 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
22167 case NVPTX::BI__nvvm_mapa_shared_cluster:
22170 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
22171 case NVPTX::BI__nvvm_getctarank:
22175 case NVPTX::BI__nvvm_getctarank_shared_cluster:
22179 case NVPTX::BI__nvvm_barrier_cluster_arrive:
22182 case NVPTX::BI__nvvm_barrier_cluster_arrive_relaxed:
22185 case NVPTX::BI__nvvm_barrier_cluster_wait:
22188 case NVPTX::BI__nvvm_fence_sc_cluster:
22197struct BuiltinAlignArgs {
22198 llvm::Value *Src =
nullptr;
22199 llvm::Type *SrcType =
nullptr;
22200 llvm::Value *Alignment =
nullptr;
22201 llvm::Value *Mask =
nullptr;
22202 llvm::IntegerType *IntType =
nullptr;
22210 SrcType = Src->getType();
22211 if (SrcType->isPointerTy()) {
22212 IntType = IntegerType::get(
22216 assert(SrcType->isIntegerTy());
22217 IntType = cast<llvm::IntegerType>(SrcType);
22220 Alignment = CGF.
Builder.CreateZExtOrTrunc(Alignment, IntType,
"alignment");
22221 auto *One = llvm::ConstantInt::get(IntType, 1);
22222 Mask = CGF.
Builder.CreateSub(Alignment, One,
"mask");
22229 BuiltinAlignArgs Args(
E, *
this);
22230 llvm::Value *SrcAddress = Args.Src;
22231 if (Args.SrcType->isPointerTy())
22233 Builder.CreateBitOrPointerCast(Args.Src, Args.IntType,
"src_addr");
22235 Builder.CreateAnd(SrcAddress, Args.Mask,
"set_bits"),
22236 llvm::Constant::getNullValue(Args.IntType),
"is_aligned"));
22243 BuiltinAlignArgs Args(
E, *
this);
22244 llvm::Value *SrcForMask = Args.Src;
22250 if (Args.Src->getType()->isPointerTy()) {
22260 SrcForMask =
Builder.CreateAdd(SrcForMask, Args.Mask,
"over_boundary");
22264 llvm::Value *InvertedMask =
Builder.CreateNot(Args.Mask,
"inverted_mask");
22265 llvm::Value *
Result =
nullptr;
22266 if (Args.Src->getType()->isPointerTy()) {
22268 Intrinsic::ptrmask, {Args.SrcType, Args.IntType},
22269 {SrcForMask, InvertedMask},
nullptr,
"aligned_result");
22271 Result =
Builder.CreateAnd(SrcForMask, InvertedMask,
"aligned_result");
22273 assert(
Result->getType() == Args.SrcType);
22279 switch (BuiltinID) {
22280 case WebAssembly::BI__builtin_wasm_memory_size: {
22285 return Builder.CreateCall(Callee, I);
22287 case WebAssembly::BI__builtin_wasm_memory_grow: {
22293 return Builder.CreateCall(Callee, Args);
22295 case WebAssembly::BI__builtin_wasm_tls_size: {
22298 return Builder.CreateCall(Callee);
22300 case WebAssembly::BI__builtin_wasm_tls_align: {
22303 return Builder.CreateCall(Callee);
22305 case WebAssembly::BI__builtin_wasm_tls_base: {
22307 return Builder.CreateCall(Callee);
22309 case WebAssembly::BI__builtin_wasm_throw: {
22313 return Builder.CreateCall(Callee, {
Tag, Obj});
22315 case WebAssembly::BI__builtin_wasm_rethrow: {
22317 return Builder.CreateCall(Callee);
22319 case WebAssembly::BI__builtin_wasm_memory_atomic_wait32: {
22326 case WebAssembly::BI__builtin_wasm_memory_atomic_wait64: {
22333 case WebAssembly::BI__builtin_wasm_memory_atomic_notify: {
22337 return Builder.CreateCall(Callee, {Addr, Count});
22339 case WebAssembly::BI__builtin_wasm_trunc_s_i32_f32:
22340 case WebAssembly::BI__builtin_wasm_trunc_s_i32_f64:
22341 case WebAssembly::BI__builtin_wasm_trunc_s_i64_f32:
22342 case WebAssembly::BI__builtin_wasm_trunc_s_i64_f64: {
22347 return Builder.CreateCall(Callee, {Src});
22349 case WebAssembly::BI__builtin_wasm_trunc_u_i32_f32:
22350 case WebAssembly::BI__builtin_wasm_trunc_u_i32_f64:
22351 case WebAssembly::BI__builtin_wasm_trunc_u_i64_f32:
22352 case WebAssembly::BI__builtin_wasm_trunc_u_i64_f64: {
22357 return Builder.CreateCall(Callee, {Src});
22359 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32_f32:
22360 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32_f64:
22361 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i64_f32:
22362 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i64_f64:
22363 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i16x8_f16x8:
22364 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32x4_f32x4: {
22369 return Builder.CreateCall(Callee, {Src});
22371 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32_f32:
22372 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32_f64:
22373 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i64_f32:
22374 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i64_f64:
22375 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i16x8_f16x8:
22376 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32x4_f32x4: {
22381 return Builder.CreateCall(Callee, {Src});
22383 case WebAssembly::BI__builtin_wasm_min_f32:
22384 case WebAssembly::BI__builtin_wasm_min_f64:
22385 case WebAssembly::BI__builtin_wasm_min_f16x8:
22386 case WebAssembly::BI__builtin_wasm_min_f32x4:
22387 case WebAssembly::BI__builtin_wasm_min_f64x2: {
22392 return Builder.CreateCall(Callee, {LHS, RHS});
22394 case WebAssembly::BI__builtin_wasm_max_f32:
22395 case WebAssembly::BI__builtin_wasm_max_f64:
22396 case WebAssembly::BI__builtin_wasm_max_f16x8:
22397 case WebAssembly::BI__builtin_wasm_max_f32x4:
22398 case WebAssembly::BI__builtin_wasm_max_f64x2: {
22403 return Builder.CreateCall(Callee, {LHS, RHS});
22405 case WebAssembly::BI__builtin_wasm_pmin_f16x8:
22406 case WebAssembly::BI__builtin_wasm_pmin_f32x4:
22407 case WebAssembly::BI__builtin_wasm_pmin_f64x2: {
22412 return Builder.CreateCall(Callee, {LHS, RHS});
22414 case WebAssembly::BI__builtin_wasm_pmax_f16x8:
22415 case WebAssembly::BI__builtin_wasm_pmax_f32x4:
22416 case WebAssembly::BI__builtin_wasm_pmax_f64x2: {
22421 return Builder.CreateCall(Callee, {LHS, RHS});
22423 case WebAssembly::BI__builtin_wasm_ceil_f16x8:
22424 case WebAssembly::BI__builtin_wasm_floor_f16x8:
22425 case WebAssembly::BI__builtin_wasm_trunc_f16x8:
22426 case WebAssembly::BI__builtin_wasm_nearest_f16x8:
22427 case WebAssembly::BI__builtin_wasm_ceil_f32x4:
22428 case WebAssembly::BI__builtin_wasm_floor_f32x4:
22429 case WebAssembly::BI__builtin_wasm_trunc_f32x4:
22430 case WebAssembly::BI__builtin_wasm_nearest_f32x4:
22431 case WebAssembly::BI__builtin_wasm_ceil_f64x2:
22432 case WebAssembly::BI__builtin_wasm_floor_f64x2:
22433 case WebAssembly::BI__builtin_wasm_trunc_f64x2:
22434 case WebAssembly::BI__builtin_wasm_nearest_f64x2: {
22436 switch (BuiltinID) {
22437 case WebAssembly::BI__builtin_wasm_ceil_f16x8:
22438 case WebAssembly::BI__builtin_wasm_ceil_f32x4:
22439 case WebAssembly::BI__builtin_wasm_ceil_f64x2:
22440 IntNo = Intrinsic::ceil;
22442 case WebAssembly::BI__builtin_wasm_floor_f16x8:
22443 case WebAssembly::BI__builtin_wasm_floor_f32x4:
22444 case WebAssembly::BI__builtin_wasm_floor_f64x2:
22445 IntNo = Intrinsic::floor;
22447 case WebAssembly::BI__builtin_wasm_trunc_f16x8:
22448 case WebAssembly::BI__builtin_wasm_trunc_f32x4:
22449 case WebAssembly::BI__builtin_wasm_trunc_f64x2:
22450 IntNo = Intrinsic::trunc;
22452 case WebAssembly::BI__builtin_wasm_nearest_f16x8:
22453 case WebAssembly::BI__builtin_wasm_nearest_f32x4:
22454 case WebAssembly::BI__builtin_wasm_nearest_f64x2:
22455 IntNo = Intrinsic::nearbyint;
22458 llvm_unreachable(
"unexpected builtin ID");
22464 case WebAssembly::BI__builtin_wasm_ref_null_extern: {
22466 return Builder.CreateCall(Callee);
22468 case WebAssembly::BI__builtin_wasm_ref_null_func: {
22470 return Builder.CreateCall(Callee);
22472 case WebAssembly::BI__builtin_wasm_swizzle_i8x16: {
22476 return Builder.CreateCall(Callee, {Src, Indices});
22478 case WebAssembly::BI__builtin_wasm_abs_i8x16:
22479 case WebAssembly::BI__builtin_wasm_abs_i16x8:
22480 case WebAssembly::BI__builtin_wasm_abs_i32x4:
22481 case WebAssembly::BI__builtin_wasm_abs_i64x2: {
22484 Constant *
Zero = llvm::Constant::getNullValue(Vec->
getType());
22485 Value *ICmp =
Builder.CreateICmpSLT(Vec, Zero,
"abscond");
22486 return Builder.CreateSelect(ICmp, Neg, Vec,
"abs");
22488 case WebAssembly::BI__builtin_wasm_avgr_u_i8x16:
22489 case WebAssembly::BI__builtin_wasm_avgr_u_i16x8: {
22494 return Builder.CreateCall(Callee, {LHS, RHS});
22496 case WebAssembly::BI__builtin_wasm_q15mulr_sat_s_i16x8: {
22500 return Builder.CreateCall(Callee, {LHS, RHS});
22502 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_s_i16x8:
22503 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_u_i16x8:
22504 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_s_i32x4:
22505 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_u_i32x4: {
22508 switch (BuiltinID) {
22509 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_s_i16x8:
22510 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_s_i32x4:
22511 IntNo = Intrinsic::wasm_extadd_pairwise_signed;
22513 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_u_i16x8:
22514 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_u_i32x4:
22515 IntNo = Intrinsic::wasm_extadd_pairwise_unsigned;
22518 llvm_unreachable(
"unexpected builtin ID");
22522 return Builder.CreateCall(Callee, Vec);
22524 case WebAssembly::BI__builtin_wasm_bitselect: {
22530 return Builder.CreateCall(Callee, {V1, V2,
C});
22532 case WebAssembly::BI__builtin_wasm_dot_s_i32x4_i16x8: {
22536 return Builder.CreateCall(Callee, {LHS, RHS});
22538 case WebAssembly::BI__builtin_wasm_any_true_v128:
22539 case WebAssembly::BI__builtin_wasm_all_true_i8x16:
22540 case WebAssembly::BI__builtin_wasm_all_true_i16x8:
22541 case WebAssembly::BI__builtin_wasm_all_true_i32x4:
22542 case WebAssembly::BI__builtin_wasm_all_true_i64x2: {
22544 switch (BuiltinID) {
22545 case WebAssembly::BI__builtin_wasm_any_true_v128:
22546 IntNo = Intrinsic::wasm_anytrue;
22548 case WebAssembly::BI__builtin_wasm_all_true_i8x16:
22549 case WebAssembly::BI__builtin_wasm_all_true_i16x8:
22550 case WebAssembly::BI__builtin_wasm_all_true_i32x4:
22551 case WebAssembly::BI__builtin_wasm_all_true_i64x2:
22552 IntNo = Intrinsic::wasm_alltrue;
22555 llvm_unreachable(
"unexpected builtin ID");
22559 return Builder.CreateCall(Callee, {Vec});
22561 case WebAssembly::BI__builtin_wasm_bitmask_i8x16:
22562 case WebAssembly::BI__builtin_wasm_bitmask_i16x8:
22563 case WebAssembly::BI__builtin_wasm_bitmask_i32x4:
22564 case WebAssembly::BI__builtin_wasm_bitmask_i64x2: {
22568 return Builder.CreateCall(Callee, {Vec});
22570 case WebAssembly::BI__builtin_wasm_abs_f16x8:
22571 case WebAssembly::BI__builtin_wasm_abs_f32x4:
22572 case WebAssembly::BI__builtin_wasm_abs_f64x2: {
22575 return Builder.CreateCall(Callee, {Vec});
22577 case WebAssembly::BI__builtin_wasm_sqrt_f16x8:
22578 case WebAssembly::BI__builtin_wasm_sqrt_f32x4:
22579 case WebAssembly::BI__builtin_wasm_sqrt_f64x2: {
22582 return Builder.CreateCall(Callee, {Vec});
22584 case WebAssembly::BI__builtin_wasm_narrow_s_i8x16_i16x8:
22585 case WebAssembly::BI__builtin_wasm_narrow_u_i8x16_i16x8:
22586 case WebAssembly::BI__builtin_wasm_narrow_s_i16x8_i32x4:
22587 case WebAssembly::BI__builtin_wasm_narrow_u_i16x8_i32x4: {
22591 switch (BuiltinID) {
22592 case WebAssembly::BI__builtin_wasm_narrow_s_i8x16_i16x8:
22593 case WebAssembly::BI__builtin_wasm_narrow_s_i16x8_i32x4:
22594 IntNo = Intrinsic::wasm_narrow_signed;
22596 case WebAssembly::BI__builtin_wasm_narrow_u_i8x16_i16x8:
22597 case WebAssembly::BI__builtin_wasm_narrow_u_i16x8_i32x4:
22598 IntNo = Intrinsic::wasm_narrow_unsigned;
22601 llvm_unreachable(
"unexpected builtin ID");
22605 return Builder.CreateCall(Callee, {Low, High});
22607 case WebAssembly::BI__builtin_wasm_trunc_sat_s_zero_f64x2_i32x4:
22608 case WebAssembly::BI__builtin_wasm_trunc_sat_u_zero_f64x2_i32x4: {
22611 switch (BuiltinID) {
22612 case WebAssembly::BI__builtin_wasm_trunc_sat_s_zero_f64x2_i32x4:
22613 IntNo = Intrinsic::fptosi_sat;
22615 case WebAssembly::BI__builtin_wasm_trunc_sat_u_zero_f64x2_i32x4:
22616 IntNo = Intrinsic::fptoui_sat;
22619 llvm_unreachable(
"unexpected builtin ID");
22621 llvm::Type *SrcT = Vec->
getType();
22622 llvm::Type *TruncT = SrcT->getWithNewType(
Builder.getInt32Ty());
22625 Value *Splat = Constant::getNullValue(TruncT);
22628 case WebAssembly::BI__builtin_wasm_shuffle_i8x16: {
22633 while (OpIdx < 18) {
22634 std::optional<llvm::APSInt> LaneConst =
22636 assert(LaneConst &&
"Constant arg isn't actually constant?");
22637 Ops[OpIdx++] = llvm::ConstantInt::get(
getLLVMContext(), *LaneConst);
22640 return Builder.CreateCall(Callee, Ops);
22642 case WebAssembly::BI__builtin_wasm_relaxed_madd_f16x8:
22643 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f16x8:
22644 case WebAssembly::BI__builtin_wasm_relaxed_madd_f32x4:
22645 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f32x4:
22646 case WebAssembly::BI__builtin_wasm_relaxed_madd_f64x2:
22647 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f64x2: {
22652 switch (BuiltinID) {
22653 case WebAssembly::BI__builtin_wasm_relaxed_madd_f16x8:
22654 case WebAssembly::BI__builtin_wasm_relaxed_madd_f32x4:
22655 case WebAssembly::BI__builtin_wasm_relaxed_madd_f64x2:
22656 IntNo = Intrinsic::wasm_relaxed_madd;
22658 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f16x8:
22659 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f32x4:
22660 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f64x2:
22661 IntNo = Intrinsic::wasm_relaxed_nmadd;
22664 llvm_unreachable(
"unexpected builtin ID");
22667 return Builder.CreateCall(Callee, {A, B,
C});
22669 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i8x16:
22670 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i16x8:
22671 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i32x4:
22672 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i64x2: {
22678 return Builder.CreateCall(Callee, {A, B,
C});
22680 case WebAssembly::BI__builtin_wasm_relaxed_swizzle_i8x16: {
22684 return Builder.CreateCall(Callee, {Src, Indices});
22686 case WebAssembly::BI__builtin_wasm_relaxed_min_f32x4:
22687 case WebAssembly::BI__builtin_wasm_relaxed_max_f32x4:
22688 case WebAssembly::BI__builtin_wasm_relaxed_min_f64x2:
22689 case WebAssembly::BI__builtin_wasm_relaxed_max_f64x2: {
22693 switch (BuiltinID) {
22694 case WebAssembly::BI__builtin_wasm_relaxed_min_f32x4:
22695 case WebAssembly::BI__builtin_wasm_relaxed_min_f64x2:
22696 IntNo = Intrinsic::wasm_relaxed_min;
22698 case WebAssembly::BI__builtin_wasm_relaxed_max_f32x4:
22699 case WebAssembly::BI__builtin_wasm_relaxed_max_f64x2:
22700 IntNo = Intrinsic::wasm_relaxed_max;
22703 llvm_unreachable(
"unexpected builtin ID");
22706 return Builder.CreateCall(Callee, {LHS, RHS});
22708 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_i32x4_f32x4:
22709 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_i32x4_f32x4:
22710 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_zero_i32x4_f64x2:
22711 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_zero_i32x4_f64x2: {
22714 switch (BuiltinID) {
22715 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_i32x4_f32x4:
22716 IntNo = Intrinsic::wasm_relaxed_trunc_signed;
22718 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_i32x4_f32x4:
22719 IntNo = Intrinsic::wasm_relaxed_trunc_unsigned;
22721 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_zero_i32x4_f64x2:
22722 IntNo = Intrinsic::wasm_relaxed_trunc_signed_zero;
22724 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_zero_i32x4_f64x2:
22725 IntNo = Intrinsic::wasm_relaxed_trunc_unsigned_zero;
22728 llvm_unreachable(
"unexpected builtin ID");
22731 return Builder.CreateCall(Callee, {Vec});
22733 case WebAssembly::BI__builtin_wasm_relaxed_q15mulr_s_i16x8: {
22737 return Builder.CreateCall(Callee, {LHS, RHS});
22739 case WebAssembly::BI__builtin_wasm_relaxed_dot_i8x16_i7x16_s_i16x8: {
22744 return Builder.CreateCall(Callee, {LHS, RHS});
22746 case WebAssembly::BI__builtin_wasm_relaxed_dot_i8x16_i7x16_add_s_i32x4: {
22751 CGM.
getIntrinsic(Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed);
22752 return Builder.CreateCall(Callee, {LHS, RHS, Acc});
22754 case WebAssembly::BI__builtin_wasm_relaxed_dot_bf16x8_add_f32_f32x4: {
22760 return Builder.CreateCall(Callee, {LHS, RHS, Acc});
22762 case WebAssembly::BI__builtin_wasm_loadf16_f32: {
22765 return Builder.CreateCall(Callee, {Addr});
22767 case WebAssembly::BI__builtin_wasm_storef16_f32: {
22771 return Builder.CreateCall(Callee, {Val, Addr});
22773 case WebAssembly::BI__builtin_wasm_splat_f16x8: {
22776 return Builder.CreateCall(Callee, {Val});
22778 case WebAssembly::BI__builtin_wasm_extract_lane_f16x8: {
22784 case WebAssembly::BI__builtin_wasm_replace_lane_f16x8: {
22791 case WebAssembly::BI__builtin_wasm_table_get: {
22802 "Unexpected reference type for __builtin_wasm_table_get");
22803 return Builder.CreateCall(Callee, {Table, Index});
22805 case WebAssembly::BI__builtin_wasm_table_set: {
22817 "Unexpected reference type for __builtin_wasm_table_set");
22818 return Builder.CreateCall(Callee, {Table, Index, Val});
22820 case WebAssembly::BI__builtin_wasm_table_size: {
22826 case WebAssembly::BI__builtin_wasm_table_grow: {
22839 "Unexpected reference type for __builtin_wasm_table_grow");
22841 return Builder.CreateCall(Callee, {Table, Val, NElems});
22843 case WebAssembly::BI__builtin_wasm_table_fill: {
22857 "Unexpected reference type for __builtin_wasm_table_fill");
22859 return Builder.CreateCall(Callee, {Table, Index, Val, NElems});
22861 case WebAssembly::BI__builtin_wasm_table_copy: {
22871 return Builder.CreateCall(Callee, {TableX, TableY, SrcIdx, DstIdx, NElems});
22878static std::pair<Intrinsic::ID, unsigned>
22881 unsigned BuiltinID;
22882 Intrinsic::ID IntrinsicID;
22885 static Info Infos[] = {
22886#define CUSTOM_BUILTIN_MAPPING(x,s) \
22887 { Hexagon::BI__builtin_HEXAGON_##x, Intrinsic::hexagon_##x, s },
22919#include "clang/Basic/BuiltinsHexagonMapCustomDep.def"
22920#undef CUSTOM_BUILTIN_MAPPING
22923 auto CmpInfo = [] (Info A, Info B) {
return A.BuiltinID < B.BuiltinID; };
22924 static const bool SortOnce = (llvm::sort(Infos, CmpInfo),
true);
22927 const Info *F = llvm::lower_bound(Infos, Info{BuiltinID, 0, 0}, CmpInfo);
22928 if (F == std::end(Infos) || F->BuiltinID != BuiltinID)
22929 return {Intrinsic::not_intrinsic, 0};
22931 return {F->IntrinsicID, F->VecLen};
22940 auto MakeCircOp = [
this,
E](
unsigned IntID,
bool IsLoad) {
22954 for (
unsigned i = 1, e =
E->getNumArgs(); i != e; ++i)
22960 llvm::Value *NewBase = IsLoad ?
Builder.CreateExtractValue(
Result, 1)
22964 llvm::Value *RetVal =
22974 auto MakeBrevLd = [
this,
E](
unsigned IntID, llvm::Type *DestTy) {
22991 CGM.
getIntrinsic(IntID), {BaseAddress, EmitScalarExpr(E->getArg(2))});
22994 llvm::Value *DestVal =
Builder.CreateExtractValue(
Result, 0);
22999 DestVal =
Builder.CreateTrunc(DestVal, DestTy);
23006 auto V2Q = [
this, VecLen] (llvm::Value *Vec) {
23007 Intrinsic::ID
ID = VecLen == 128 ? Intrinsic::hexagon_V6_vandvrt_128B
23008 : Intrinsic::hexagon_V6_vandvrt;
23010 {Vec,
Builder.getInt32(-1)});
23012 auto Q2V = [
this, VecLen] (llvm::Value *Pred) {
23013 Intrinsic::ID
ID = VecLen == 128 ? Intrinsic::hexagon_V6_vandqrt_128B
23014 : Intrinsic::hexagon_V6_vandqrt;
23016 {Pred,
Builder.getInt32(-1)});
23019 switch (BuiltinID) {
23023 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarry:
23024 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarry_128B:
23025 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarry:
23026 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarry_128B: {
23033 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1)), PredIn});
23035 llvm::Value *PredOut =
Builder.CreateExtractValue(
Result, 1);
23043 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarryo:
23044 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarryo_128B:
23045 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarryo:
23046 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarryo_128B: {
23052 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
23054 llvm::Value *PredOut =
Builder.CreateExtractValue(
Result, 1);
23060 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstoreq:
23061 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorenq:
23062 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentq:
23063 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentnq:
23064 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstoreq_128B:
23065 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorenq_128B:
23066 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentq_128B:
23067 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentnq_128B: {
23069 const Expr *PredOp =
E->getArg(0);
23071 if (
auto *Cast = dyn_cast<ImplicitCastExpr>(PredOp)) {
23072 if (
Cast->getCastKind() == CK_BitCast)
23073 PredOp =
Cast->getSubExpr();
23076 for (
int i = 1, e =
E->getNumArgs(); i != e; ++i)
23081 case Hexagon::BI__builtin_HEXAGON_L2_loadrub_pci:
23082 case Hexagon::BI__builtin_HEXAGON_L2_loadrb_pci:
23083 case Hexagon::BI__builtin_HEXAGON_L2_loadruh_pci:
23084 case Hexagon::BI__builtin_HEXAGON_L2_loadrh_pci:
23085 case Hexagon::BI__builtin_HEXAGON_L2_loadri_pci:
23086 case Hexagon::BI__builtin_HEXAGON_L2_loadrd_pci:
23087 case Hexagon::BI__builtin_HEXAGON_L2_loadrub_pcr:
23088 case Hexagon::BI__builtin_HEXAGON_L2_loadrb_pcr:
23089 case Hexagon::BI__builtin_HEXAGON_L2_loadruh_pcr:
23090 case Hexagon::BI__builtin_HEXAGON_L2_loadrh_pcr:
23091 case Hexagon::BI__builtin_HEXAGON_L2_loadri_pcr:
23092 case Hexagon::BI__builtin_HEXAGON_L2_loadrd_pcr:
23093 return MakeCircOp(ID,
true);
23094 case Hexagon::BI__builtin_HEXAGON_S2_storerb_pci:
23095 case Hexagon::BI__builtin_HEXAGON_S2_storerh_pci:
23096 case Hexagon::BI__builtin_HEXAGON_S2_storerf_pci:
23097 case Hexagon::BI__builtin_HEXAGON_S2_storeri_pci:
23098 case Hexagon::BI__builtin_HEXAGON_S2_storerd_pci:
23099 case Hexagon::BI__builtin_HEXAGON_S2_storerb_pcr:
23100 case Hexagon::BI__builtin_HEXAGON_S2_storerh_pcr:
23101 case Hexagon::BI__builtin_HEXAGON_S2_storerf_pcr:
23102 case Hexagon::BI__builtin_HEXAGON_S2_storeri_pcr:
23103 case Hexagon::BI__builtin_HEXAGON_S2_storerd_pcr:
23104 return MakeCircOp(ID,
false);
23105 case Hexagon::BI__builtin_brev_ldub:
23106 return MakeBrevLd(Intrinsic::hexagon_L2_loadrub_pbr,
Int8Ty);
23107 case Hexagon::BI__builtin_brev_ldb:
23108 return MakeBrevLd(Intrinsic::hexagon_L2_loadrb_pbr,
Int8Ty);
23109 case Hexagon::BI__builtin_brev_lduh:
23110 return MakeBrevLd(Intrinsic::hexagon_L2_loadruh_pbr,
Int16Ty);
23111 case Hexagon::BI__builtin_brev_ldh:
23112 return MakeBrevLd(Intrinsic::hexagon_L2_loadrh_pbr,
Int16Ty);
23113 case Hexagon::BI__builtin_brev_ldw:
23114 return MakeBrevLd(Intrinsic::hexagon_L2_loadri_pbr,
Int32Ty);
23115 case Hexagon::BI__builtin_brev_ldd:
23116 return MakeBrevLd(Intrinsic::hexagon_L2_loadrd_pbr,
Int64Ty);
23124 StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
23132 llvm::Constant *RISCVCPUModel =
23134 cast<llvm::GlobalValue>(RISCVCPUModel)->setDSOLocal(
true);
23136 auto loadRISCVCPUID = [&](
unsigned Index) {
23139 Ptr, llvm::MaybeAlign());
23143 const llvm::RISCV::CPUModel Model = llvm::RISCV::getCPUModel(CPUStr);
23146 Value *VendorID = loadRISCVCPUID(0);
23148 Builder.CreateICmpEQ(VendorID,
Builder.getInt32(Model.MVendorID));
23151 Value *ArchID = loadRISCVCPUID(1);
23156 Value *ImpID = loadRISCVCPUID(2);
23167 if (BuiltinID == Builtin::BI__builtin_cpu_supports)
23169 if (BuiltinID == Builtin::BI__builtin_cpu_init)
23171 if (BuiltinID == Builtin::BI__builtin_cpu_is)
23178 unsigned ICEArguments = 0;
23186 if (BuiltinID == RISCVVector::BI__builtin_rvv_vget_v ||
23187 BuiltinID == RISCVVector::BI__builtin_rvv_vset_v)
23188 ICEArguments = 1 << 1;
23193 if (BuiltinID == RISCV::BI__builtin_riscv_ntl_load)
23194 ICEArguments |= (1 << 1);
23195 if (BuiltinID == RISCV::BI__builtin_riscv_ntl_store)
23196 ICEArguments |= (1 << 2);
23198 for (
unsigned i = 0, e =
E->getNumArgs(); i != e; i++) {
23203 Ops.push_back(AggValue);
23209 Intrinsic::ID
ID = Intrinsic::not_intrinsic;
23212 constexpr unsigned RVV_VTA = 0x1;
23213 constexpr unsigned RVV_VMA = 0x2;
23214 int PolicyAttrs = 0;
23215 bool IsMasked =
false;
23217 unsigned SegInstSEW = 8;
23221 switch (BuiltinID) {
23222 default: llvm_unreachable(
"unexpected builtin ID");
23223 case RISCV::BI__builtin_riscv_orc_b_32:
23224 case RISCV::BI__builtin_riscv_orc_b_64:
23225 case RISCV::BI__builtin_riscv_clmul_32:
23226 case RISCV::BI__builtin_riscv_clmul_64:
23227 case RISCV::BI__builtin_riscv_clmulh_32:
23228 case RISCV::BI__builtin_riscv_clmulh_64:
23229 case RISCV::BI__builtin_riscv_clmulr_32:
23230 case RISCV::BI__builtin_riscv_clmulr_64:
23231 case RISCV::BI__builtin_riscv_xperm4_32:
23232 case RISCV::BI__builtin_riscv_xperm4_64:
23233 case RISCV::BI__builtin_riscv_xperm8_32:
23234 case RISCV::BI__builtin_riscv_xperm8_64:
23235 case RISCV::BI__builtin_riscv_brev8_32:
23236 case RISCV::BI__builtin_riscv_brev8_64:
23237 case RISCV::BI__builtin_riscv_zip_32:
23238 case RISCV::BI__builtin_riscv_unzip_32: {
23239 switch (BuiltinID) {
23240 default: llvm_unreachable(
"unexpected builtin ID");
23242 case RISCV::BI__builtin_riscv_orc_b_32:
23243 case RISCV::BI__builtin_riscv_orc_b_64:
23244 ID = Intrinsic::riscv_orc_b;
23248 case RISCV::BI__builtin_riscv_clmul_32:
23249 case RISCV::BI__builtin_riscv_clmul_64:
23250 ID = Intrinsic::riscv_clmul;
23252 case RISCV::BI__builtin_riscv_clmulh_32:
23253 case RISCV::BI__builtin_riscv_clmulh_64:
23254 ID = Intrinsic::riscv_clmulh;
23256 case RISCV::BI__builtin_riscv_clmulr_32:
23257 case RISCV::BI__builtin_riscv_clmulr_64:
23258 ID = Intrinsic::riscv_clmulr;
23262 case RISCV::BI__builtin_riscv_xperm8_32:
23263 case RISCV::BI__builtin_riscv_xperm8_64:
23264 ID = Intrinsic::riscv_xperm8;
23266 case RISCV::BI__builtin_riscv_xperm4_32:
23267 case RISCV::BI__builtin_riscv_xperm4_64:
23268 ID = Intrinsic::riscv_xperm4;
23272 case RISCV::BI__builtin_riscv_brev8_32:
23273 case RISCV::BI__builtin_riscv_brev8_64:
23274 ID = Intrinsic::riscv_brev8;
23276 case RISCV::BI__builtin_riscv_zip_32:
23277 ID = Intrinsic::riscv_zip;
23279 case RISCV::BI__builtin_riscv_unzip_32:
23280 ID = Intrinsic::riscv_unzip;
23284 IntrinsicTypes = {ResultType};
23291 case RISCV::BI__builtin_riscv_sha256sig0:
23292 ID = Intrinsic::riscv_sha256sig0;
23294 case RISCV::BI__builtin_riscv_sha256sig1:
23295 ID = Intrinsic::riscv_sha256sig1;
23297 case RISCV::BI__builtin_riscv_sha256sum0:
23298 ID = Intrinsic::riscv_sha256sum0;
23300 case RISCV::BI__builtin_riscv_sha256sum1:
23301 ID = Intrinsic::riscv_sha256sum1;
23305 case RISCV::BI__builtin_riscv_sm4ks:
23306 ID = Intrinsic::riscv_sm4ks;
23308 case RISCV::BI__builtin_riscv_sm4ed:
23309 ID = Intrinsic::riscv_sm4ed;
23313 case RISCV::BI__builtin_riscv_sm3p0:
23314 ID = Intrinsic::riscv_sm3p0;
23316 case RISCV::BI__builtin_riscv_sm3p1:
23317 ID = Intrinsic::riscv_sm3p1;
23320 case RISCV::BI__builtin_riscv_clz_32:
23321 case RISCV::BI__builtin_riscv_clz_64: {
23324 if (
Result->getType() != ResultType)
23329 case RISCV::BI__builtin_riscv_ctz_32:
23330 case RISCV::BI__builtin_riscv_ctz_64: {
23333 if (
Result->getType() != ResultType)
23340 case RISCV::BI__builtin_riscv_ntl_load: {
23342 unsigned DomainVal = 5;
23343 if (Ops.size() == 2)
23344 DomainVal = cast<ConstantInt>(Ops[1])->getZExtValue();
23346 llvm::MDNode *RISCVDomainNode = llvm::MDNode::get(
23348 llvm::ConstantAsMetadata::get(
Builder.getInt32(DomainVal)));
23349 llvm::MDNode *NontemporalNode = llvm::MDNode::get(
23353 if(ResTy->isScalableTy()) {
23354 const ScalableVectorType *SVTy = cast<ScalableVectorType>(ResTy);
23355 llvm::Type *ScalarTy = ResTy->getScalarType();
23356 Width = ScalarTy->getPrimitiveSizeInBits() *
23357 SVTy->getElementCount().getKnownMinValue();
23359 Width = ResTy->getPrimitiveSizeInBits();
23363 Load->setMetadata(llvm::LLVMContext::MD_nontemporal, NontemporalNode);
23364 Load->setMetadata(
CGM.
getModule().getMDKindID(
"riscv-nontemporal-domain"),
23369 case RISCV::BI__builtin_riscv_ntl_store: {
23370 unsigned DomainVal = 5;
23371 if (Ops.size() == 3)
23372 DomainVal = cast<ConstantInt>(Ops[2])->getZExtValue();
23374 llvm::MDNode *RISCVDomainNode = llvm::MDNode::get(
23376 llvm::ConstantAsMetadata::get(
Builder.getInt32(DomainVal)));
23377 llvm::MDNode *NontemporalNode = llvm::MDNode::get(
23381 Store->setMetadata(llvm::LLVMContext::MD_nontemporal, NontemporalNode);
23388 case RISCV::BI__builtin_riscv_cv_alu_addN:
23389 ID = Intrinsic::riscv_cv_alu_addN;
23391 case RISCV::BI__builtin_riscv_cv_alu_addRN:
23392 ID = Intrinsic::riscv_cv_alu_addRN;
23394 case RISCV::BI__builtin_riscv_cv_alu_adduN:
23395 ID = Intrinsic::riscv_cv_alu_adduN;
23397 case RISCV::BI__builtin_riscv_cv_alu_adduRN:
23398 ID = Intrinsic::riscv_cv_alu_adduRN;
23400 case RISCV::BI__builtin_riscv_cv_alu_clip:
23401 ID = Intrinsic::riscv_cv_alu_clip;
23403 case RISCV::BI__builtin_riscv_cv_alu_clipu:
23404 ID = Intrinsic::riscv_cv_alu_clipu;
23406 case RISCV::BI__builtin_riscv_cv_alu_extbs:
23409 case RISCV::BI__builtin_riscv_cv_alu_extbz:
23412 case RISCV::BI__builtin_riscv_cv_alu_exths:
23415 case RISCV::BI__builtin_riscv_cv_alu_exthz:
23418 case RISCV::BI__builtin_riscv_cv_alu_slet:
23421 case RISCV::BI__builtin_riscv_cv_alu_sletu:
23424 case RISCV::BI__builtin_riscv_cv_alu_subN:
23425 ID = Intrinsic::riscv_cv_alu_subN;
23427 case RISCV::BI__builtin_riscv_cv_alu_subRN:
23428 ID = Intrinsic::riscv_cv_alu_subRN;
23430 case RISCV::BI__builtin_riscv_cv_alu_subuN:
23431 ID = Intrinsic::riscv_cv_alu_subuN;
23433 case RISCV::BI__builtin_riscv_cv_alu_subuRN:
23434 ID = Intrinsic::riscv_cv_alu_subuRN;
23438#include "clang/Basic/riscv_vector_builtin_cg.inc"
23441#include "clang/Basic/riscv_sifive_vector_builtin_cg.inc"
23444 assert(ID != Intrinsic::not_intrinsic);
23447 return Builder.CreateCall(F, Ops,
"");
Defines the clang::ASTContext interface.
#define PPC_LNX_FEATURE(NAME, DESC, ENUMNAME, ENUMVAL, HWCAPN)
static constexpr SparcCPUInfo CPUInfo[]
#define X86_CPU_SUBTYPE(ENUM, STR)
#define X86_CPU_SUBTYPE_ALIAS(ENUM, ALIAS)
#define X86_VENDOR(ENUM, STRING)
#define X86_CPU_TYPE_ALIAS(ENUM, ALIAS)
#define X86_CPU_TYPE(ENUM, STR)
static constexpr Builtin::Info BuiltinInfo[]
static void Accumulate(SMap &SM, CFGBlock *B)
static Value * EmitSpecialRegisterBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Type *RegisterType, llvm::Type *ValueType, SpecialRegisterAccessKind AccessKind, StringRef SysReg="")
static llvm::Value * ARMMVEVectorReinterpret(CGBuilderTy &Builder, CodeGenFunction *CGF, llvm::Value *V, llvm::Type *DestType)
static Value * MakeBinaryAtomicValue(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E, AtomicOrdering Ordering=AtomicOrdering::SequentiallyConsistent)
Utility to insert an atomic instruction based on Intrinsic::ID and the expression node.
static char bitActionToX86BTCode(BitTest::ActionKind A)
#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier)
static Intrinsic::ID getWaveActiveSumIntrinsic(llvm::Triple::ArchType Arch, CGHLSLRuntime &RT, QualType QT)
static Value * EmitAtomicCmpXchg128ForMSIntrin(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering SuccessOrdering)
#define INTRINSIC_X86_XSAVE_ID(NAME)
static CanQualType getOSLogArgType(ASTContext &C, int Size)
Get the argument type for arguments to os_log_helper.
static Value * EmitOverflowCheckedAbs(CodeGenFunction &CGF, const CallExpr *E, bool SanitizeOverflow)
static llvm::VectorType * GetFloatNeonType(CodeGenFunction *CGF, NeonTypeFlags IntTypeFlags)
static Value * tryUseTestFPKind(CodeGenFunction &CGF, unsigned BuiltinID, Value *V)
static llvm::Value * MVEImmediateShr(CGBuilderTy &Builder, llvm::Value *V, uint32_t Shift, bool Unsigned)
static bool areBOSTypesCompatible(int From, int To)
Checks if using the result of __builtin_object_size(p, From) in place of __builtin_object_size(p,...
static llvm::Value * SignOrZeroExtend(CGBuilderTy &Builder, llvm::Value *V, llvm::Type *T, bool Unsigned)
static void InsertExplicitZeroOperand(CGBuilderTy &Builder, llvm::Type *Ty, SmallVectorImpl< Value * > &Ops)
static const ARMVectorIntrinsicInfo AArch64SMEIntrinsicMap[]
static Value * EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E, SmallVectorImpl< Value * > &Ops, llvm::Triple::ArchType Arch)
#define MMA_VARIANTS_B1_AND(geom, type)
static void swapCommutativeSMEOperands(unsigned BuiltinID, SmallVectorImpl< Value * > &Ops)
static bool AArch64SISDIntrinsicsProvenSorted
static Value * EmitX86CompressExpand(CodeGenFunction &CGF, ArrayRef< Value * > Ops, bool IsCompress)
static const ARMVectorIntrinsicInfo AArch64SVEIntrinsicMap[]
static bool HasExtraNeonArgument(unsigned BuiltinID)
Return true if BuiltinID is an overloaded Neon intrinsic with an extra argument that specifies the ve...
static bool TypeRequiresBuiltinLaunderImp(const ASTContext &Ctx, QualType Ty, llvm::SmallPtrSetImpl< const Decl * > &Seen)
static Value * EmitAtomicIncrementValue(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering Ordering=AtomicOrdering::SequentiallyConsistent)
static std::pair< Intrinsic::ID, unsigned > getIntrinsicForHexagonNonClangBuiltin(unsigned BuiltinID)
static Value * emitRangedBuiltin(CodeGenFunction &CGF, unsigned IntrinsicID, int low, int high)
#define MMA_INTR(geom_op_type, layout)
static Value * EmitX86MaskedCompare(CodeGenFunction &CGF, unsigned CC, bool Signed, ArrayRef< Value * > Ops)
static Value * emitUnaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
static Intrinsic::ID getDotProductIntrinsic(CGHLSLRuntime &RT, QualType QT)
#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier)
static bool AArch64SVEIntrinsicsProvenSorted
static RValue EmitMSVCRTSetJmp(CodeGenFunction &CGF, MSVCSetJmpKind SJKind, const CallExpr *E)
MSVC handles setjmp a bit differently on different platforms.
static const ARMVectorIntrinsicInfo * findARMVectorIntrinsicInMap(ArrayRef< ARMVectorIntrinsicInfo > IntrinsicMap, unsigned BuiltinID, bool &MapProvenSorted)
static Value * EmitScalarFMAExpr(CodeGenFunction &CGF, const CallExpr *E, MutableArrayRef< Value * > Ops, Value *Upper, bool ZeroMask=false, unsigned PTIdx=0, bool NegAcc=false)
static Value * loadRISCVFeatureBits(unsigned Index, CGBuilderTy &Builder, CodeGenModule &CGM)
#define MUTATE_LDBL(func)
static Value * EmitX86ExpandLoad(CodeGenFunction &CGF, ArrayRef< Value * > Ops)
static unsigned CountCountedByAttrs(const RecordDecl *RD)
static Value * emitMaybeConstrainedFPToIntRoundBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
static Value * EmitX86MaskedLoad(CodeGenFunction &CGF, ArrayRef< Value * > Ops, Align Alignment)
static bool TypeRequiresBuiltinLaunder(CodeGenModule &CGM, QualType Ty)
Determine if the specified type requires laundering by checking if it is a dynamic class type or cont...
static void InsertExplicitUndefOperand(CGBuilderTy &Builder, llvm::Type *Ty, SmallVectorImpl< Value * > &Ops)
static Value * EmitISOVolatileLoad(CodeGenFunction &CGF, const CallExpr *E)
static struct WidthAndSignedness EncompassingIntegerType(ArrayRef< struct WidthAndSignedness > Types)
static Integer GetIntegerConstantValue(const Expr *E, ASTContext &Context)
#define MMA_VARIANTS(geom, type)
static bool AArch64SMEIntrinsicsProvenSorted
static llvm::Value * VectorZip(CGBuilderTy &Builder, llvm::Value *V0, llvm::Value *V1)
static Value * EmitTargetArchBuiltinExpr(CodeGenFunction *CGF, unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
constexpr unsigned SVEBitsPerBlock
static std::optional< CodeGenFunction::MSVCIntrin > translateX86ToMsvcIntrin(unsigned BuiltinID)
static const std::pair< unsigned, unsigned > NEONEquivalentIntrinsicMap[]
#define NEONMAP0(NameBase)
static Value * EmitX86MaskedStore(CodeGenFunction &CGF, ArrayRef< Value * > Ops, Align Alignment)
static Value * handleHlslSplitdouble(const CallExpr *E, CodeGenFunction *CGF)
static Value * emitBinaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E, Instruction::BinaryOps Op, bool Invert=false)
Utility to insert an atomic instruction based Intrinsic::ID and the expression node,...
static bool HasNoIndirectArgumentsOrResults(CGFunctionInfo const &FnInfo)
Checks no arguments or results are passed indirectly in the ABI (i.e.
static Value * EmitX86Muldq(CodeGenFunction &CGF, bool IsSigned, ArrayRef< Value * > Ops)
Value * readX18AsPtr(CodeGenFunction &CGF)
Helper for the read/write/add/inc X18 builtins: read the X18 register and return it as an i8 pointer.
static llvm::Value * ARMMVEVectorElementReverse(CGBuilderTy &Builder, llvm::Value *V, unsigned ReverseWidth)
#define MMA_SATF_VARIANTS(geom, type)
static std::optional< CodeGenFunction::MSVCIntrin > translateAarch64ToMsvcIntrin(unsigned BuiltinID)
static std::optional< CodeGenFunction::MSVCIntrin > translateArmToMsvcIntrin(unsigned BuiltinID)
static llvm::Value * EmitBitTestIntrinsic(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E)
Emit a _bittest* intrinsic.
static Value * emitBuiltinWithOneOverloadedType(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, llvm::StringRef Name="")
static const ARMVectorIntrinsicInfo ARMSIMDIntrinsicMap[]
static Value * EmitSignBit(CodeGenFunction &CGF, Value *V)
Emit the computation of the sign bit for a floating point value.
static Value * EmitFAbs(CodeGenFunction &CGF, Value *V)
EmitFAbs - Emit a call to @llvm.fabs().
#define CUSTOM_BUILTIN_MAPPING(x, s)
static Value * EmitX86CvtF16ToFloatExpr(CodeGenFunction &CGF, ArrayRef< Value * > Ops, llvm::Type *DstTy)
static bool isSpecialUnsignedMultiplySignedResult(unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info, WidthAndSignedness ResultInfo)
static llvm::Value * getDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType)
static llvm::Value * emitPPCLoadReserveIntrinsic(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E)
static llvm::Value * VectorUnzip(CGBuilderTy &Builder, llvm::Value *V, bool Odd)
static Value * EmitX86SExtMask(CodeGenFunction &CGF, Value *Op, llvm::Type *DstTy)
static Value * emitTernaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
static WidthAndSignedness getIntegerWidthAndSignedness(const clang::ASTContext &context, const clang::QualType Type)
static Value * EmitX86FunnelShift(CodeGenFunction &CGF, Value *Op0, Value *Op1, Value *Amt, bool IsRight)
static RValue EmitCheckedMixedSignMultiply(CodeGenFunction &CGF, const clang::Expr *Op1, WidthAndSignedness Op1Info, const clang::Expr *Op2, WidthAndSignedness Op2Info, const clang::Expr *ResultArg, QualType ResultQTy, WidthAndSignedness ResultInfo)
Emit a checked mixed-sign multiply.
static llvm::ScalableVectorType * getSVEVectorForElementType(llvm::Type *EltTy)
static unsigned mutateLongDoubleBuiltin(unsigned BuiltinID)
#define INTRINSIC_WITH_CC(NAME)
static llvm::FixedVectorType * GetNeonType(CodeGenFunction *CGF, NeonTypeFlags TypeFlags, bool HasLegalHalfType=true, bool V1Ty=false, bool AllowBFloatArgsAndRet=true)
static RValue EmitBinaryAtomic(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E)
static llvm::Value * ARMMVEConstantSplat(CGBuilderTy &Builder, llvm::Type *VT)
static Value * EmitX86MaskLogic(CodeGenFunction &CGF, Instruction::BinaryOps Opc, ArrayRef< Value * > Ops, bool InvertLHS=false)
static Value * EmitFromInt(CodeGenFunction &CGF, llvm::Value *V, QualType T, llvm::Type *ResultType)
static Value * EmitAMDGCNBallotForExec(CodeGenFunction &CGF, const CallExpr *E, llvm::Type *RegisterType, llvm::Type *ValueType, bool isExecHi)
static void emitSincosBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Intrinsic::ID IntrinsicID)
static void initializeAlloca(CodeGenFunction &CGF, AllocaInst *AI, Value *Size, Align AlignmentInBytes)
static Value * EmitX86Select(CodeGenFunction &CGF, Value *Mask, Value *Op0, Value *Op1)
SpecialRegisterAccessKind
static Value * EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering SuccessOrdering=AtomicOrdering::SequentiallyConsistent)
This function should be invoked to emit atomic cmpxchg for Microsoft's _InterlockedCompareExchange* i...
static Address CheckAtomicAlignment(CodeGenFunction &CGF, const CallExpr *E)
static Value * EmitX86FMAExpr(CodeGenFunction &CGF, const CallExpr *E, ArrayRef< Value * > Ops, unsigned BuiltinID, bool IsAddSub)
static Value * getMaskVecValue(CodeGenFunction &CGF, Value *Mask, unsigned NumElts)
static bool isSpecialMixedSignMultiply(unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info, WidthAndSignedness ResultInfo)
Determine if a binop is a checked mixed-sign multiply we can specialize.
static Value * MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E, bool ReturnBool)
Utility to insert an atomic cmpxchg instruction.
static Value * emitBinaryExpMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Intrinsic::ID IntrinsicID, llvm::Intrinsic::ID ConstrainedIntrinsicID)
static Value * EmitToInt(CodeGenFunction &CGF, llvm::Value *V, QualType T, llvm::IntegerType *IntType)
Emit the conversions required to turn the given value into an integer of the given size.
static llvm::Value * ARMMVEVectorSplat(CGBuilderTy &Builder, llvm::Value *V)
static Value * EmitX86MaskedCompareResult(CodeGenFunction &CGF, Value *Cmp, unsigned NumElts, Value *MaskIn)
static Value * EmitX86CompressStore(CodeGenFunction &CGF, ArrayRef< Value * > Ops)
static bool NEONSIMDIntrinsicsProvenSorted
static const ARMVectorIntrinsicInfo AArch64SIMDIntrinsicMap[]
static Value * EmitNontemporalStore(CodeGenFunction &CGF, const CallExpr *E)
static Intrinsic::ID getFirstBitHighIntrinsic(CGHLSLRuntime &RT, QualType QT)
static llvm::Value * EmitOverflowIntrinsic(CodeGenFunction &CGF, const llvm::Intrinsic::ID IntrinsicID, llvm::Value *X, llvm::Value *Y, llvm::Value *&Carry)
Emit a call to llvm.
static Value * EmitCommonNeonSISDBuiltinExpr(CodeGenFunction &CGF, const ARMVectorIntrinsicInfo &SISDInfo, SmallVectorImpl< Value * > &Ops, const CallExpr *E)
static Value * emitFPIntBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
#define MMA_LDST(n, geom_op_type)
static Value * EmitX86vpcom(CodeGenFunction &CGF, ArrayRef< Value * > Ops, bool IsSigned)
static Value * emitFrexpBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Intrinsic::ID IntrinsicID)
@ _InterlockedExchangeAdd_rel
@ _InterlockedIncrement_acq
@ _InterlockedExchange_nf
@ _InterlockedIncrement_nf
@ _InterlockedExchange_acq
@ _InterlockedCompareExchange128_rel
@ _InterlockedCompareExchange128_acq
@ _InterlockedCompareExchange_acq
@ _InterlockedExchangeAdd_nf
@ _InterlockedCompareExchange_nf
@ _InterlockedDecrement_rel
@ _InterlockedExchangeSub
@ _InterlockedExchangeAdd_acq
@ _InterlockedIncrement_rel
@ _InterlockedCompareExchange128_nf
@ _InterlockedCompareExchange128
@ _InterlockedExchange_rel
@ _InterlockedCompareExchange
@ _InterlockedDecrement_nf
@ _InterlockedExchangeAdd
@ _InterlockedDecrement_acq
@ _InterlockedCompareExchange_rel
static Value * EmitX86ConvertToMask(CodeGenFunction &CGF, Value *In)
static Value * EmitISOVolatileStore(CodeGenFunction &CGF, const CallExpr *E)
static Value * EmitSystemZIntrinsicWithCC(CodeGenFunction &CGF, unsigned IntrinsicID, const CallExpr *E)
Handle a SystemZ function in which the final argument is a pointer to an int that receives the post-i...
static RValue EmitHipStdParUnsupportedBuiltin(CodeGenFunction *CGF, const FunctionDecl *FD)
static llvm::Value * EmitX86BitTestIntrinsic(CodeGenFunction &CGF, BitTest BT, const CallExpr *E, Value *BitBase, Value *BitPos)
static RValue EmitCheckedUnsignedMultiplySignedResult(CodeGenFunction &CGF, const clang::Expr *Op1, WidthAndSignedness Op1Info, const clang::Expr *Op2, WidthAndSignedness Op2Info, const clang::Expr *ResultArg, QualType ResultQTy, WidthAndSignedness ResultInfo)
static Value * emitCallMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID, llvm::Type *Ty, ArrayRef< Value * > Args)
static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *FD, const CallExpr *E, llvm::Constant *calleeValue)
static Value * handleAsDoubleBuiltin(CodeGenFunction &CGF, const CallExpr *E)
static Value * EmitNontemporalLoad(CodeGenFunction &CGF, const CallExpr *E)
static Value * EmitX86Ternlog(CodeGenFunction &CGF, bool ZeroMask, ArrayRef< Value * > Ops)
static llvm::AtomicOrdering getBitTestAtomicOrdering(BitTest::InterlockingKind I)
#define MMA_VARIANTS_B1_XOR(geom, type)
#define MMA_VARIANTS_I4(geom, type)
static Value * EmitX86ConvertIntToFp(CodeGenFunction &CGF, const CallExpr *E, ArrayRef< Value * > Ops, bool IsSigned)
static Value * packTBLDVectorList(CodeGenFunction &CGF, ArrayRef< Value * > Ops, Value *ExtOp, Value *IndexOp, llvm::Type *ResTy, unsigned IntID, const char *Name)
static Value * EmitAbs(CodeGenFunction &CGF, Value *ArgValue, bool HasNSW)
static Value * EmitX86ScalarSelect(CodeGenFunction &CGF, Value *Mask, Value *Op0, Value *Op1)
static Value * EmitAtomicDecrementValue(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering Ordering=AtomicOrdering::SequentiallyConsistent)
static Value * handleHlslClip(const CallExpr *E, CodeGenFunction *CGF)
static bool AArch64SIMDIntrinsicsProvenSorted
static const ARMVectorIntrinsicInfo AArch64SISDIntrinsicMap[]
CodeGenFunction::ComplexPairTy ComplexPairTy
#define ALIAS(NAME, TOK, FLAGS)
llvm::MachO::Record Record
static std::string getName(const CallEvent &Call)
static std::string toString(const clang::SanitizerSet &Sanitizers)
Produce a string containing comma-separated names of sanitizers in Sanitizers set.
HLSLResourceBindingAttr::RegisterType RegisterType
static QualType getPointeeType(const MemRegion *R)
Enumerates target-specific builtins in their own namespaces within namespace clang.
Defines the clang::TargetOptions class.
C Language Family Type Representation.
__DEVICE__ float modf(float __x, float *__iptr)
__DEVICE__ double nan(const char *)
__device__ __2f16 float __ockl_bool s
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
CharUnits getTypeAlignInChars(QualType T) const
Return the ABI-specified alignment of a (complete) type T, in characters.
unsigned getIntWidth(QualType T) const
const ASTRecordLayout & getASTRecordLayout(const RecordDecl *D) const
Get or compute information about the layout of the specified record (struct/union/class) D,...
Builtin::Context & BuiltinInfo
QualType getConstantArrayType(QualType EltTy, const llvm::APInt &ArySize, const Expr *SizeExpr, ArraySizeModifier ASM, unsigned IndexTypeQuals) const
Return the unique reference to the type for a constant array of the specified element type.
QualType getBaseElementType(const ArrayType *VAT) const
Return the innermost element type of an array type.
QualType getObjCIdType() const
Represents the Objective-CC id type.
bool hasSameUnqualifiedType(QualType T1, QualType T2) const
Determine whether the given types are equivalent after cvr-qualifiers have been removed.
const ArrayType * getAsArrayType(QualType T) const
Type Query functions.
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
QualType GetBuiltinType(unsigned ID, GetBuiltinTypeError &Error, unsigned *IntegerConstantArgs=nullptr) const
Return the type for the specified builtin.
const TargetInfo & getTargetInfo() const
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
unsigned getTargetAddressSpace(LangAS AS) const
@ GE_Missing_type
Missing a type.
ASTRecordLayout - This class contains layout information for one RecordDecl, which is a struct/union/...
uint64_t getFieldOffset(unsigned FieldNo) const
getFieldOffset - Get the offset of the given field index, in bits.
Represents an array type, per C99 6.7.5.2 - Array Declarators.
QualType getElementType() const
static std::unique_ptr< AtomicScopeModel > create(AtomicScopeModelKind K)
Create an atomic scope model by AtomicScopeModelKind.
bool isLibFunction(unsigned ID) const
Return true if this is a builtin for a libc/libm function, with a "__builtin_" prefix (e....
llvm::StringRef getName(unsigned ID) const
Return the identifier name for the specified builtin, e.g.
bool isConstWithoutErrnoAndExceptions(unsigned ID) const
Return true if this function has no side effects and doesn't read memory, except for possibly errno o...
bool isConstWithoutExceptions(unsigned ID) const
bool isConst(unsigned ID) const
Return true if this function has no side effects and doesn't read memory.
CallExpr - Represents a function call (C99 6.5.2.2, C++ [expr.call]).
CharUnits - This is an opaque type for sizes expressed in character units.
llvm::Align getAsAlign() const
getAsAlign - Returns Quantity as a valid llvm::Align, Beware llvm::Align assumes power of two 8-bit b...
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
static CharUnits One()
One - Construct a CharUnits quantity of one.
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
XRayInstrSet XRayInstrumentationBundle
Set of XRay instrumentation kinds to emit.
ABIArgInfo - Helper class to encapsulate information about how a specific C type should be passed to ...
Like RawAddress, an abstract representation of an aligned address, but the pointer contained in this ...
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
Return the pointer contained in this class after authenticating it and adding offset to it if necessa...
CharUnits getAlignment() const
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
Address withAlignment(CharUnits NewAlignment) const
Return address with different alignment, but same pointer and element type.
llvm::PointerType * getType() const
Return the type of the pointer value.
Address getAddress() const
A scoped helper to set the current debug location to the specified location or preferred location of ...
static ApplyDebugLocation CreateArtificial(CodeGenFunction &CGF)
Apply TemporaryLocation if it is valid.
static ApplyDebugLocation CreateEmpty(CodeGenFunction &CGF)
Set the IRBuilder to not attach debug locations.
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
llvm::StoreInst * CreateAlignedStore(llvm::Value *Val, llvm::Value *Addr, CharUnits Align, bool IsVolatile=false)
Address CreateGEP(CodeGenFunction &CGF, Address Addr, llvm::Value *Index, const llvm::Twine &Name="")
llvm::CallInst * CreateMemMove(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
llvm::CallInst * CreateMemCpyInline(Address Dest, Address Src, uint64_t Size)
llvm::AtomicRMWInst * CreateAtomicRMW(llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val, llvm::AtomicOrdering Ordering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
llvm::CallInst * CreateMemSetInline(Address Dest, llvm::Value *Value, uint64_t Size)
llvm::StoreInst * CreateDefaultAlignedStore(llvm::Value *Val, llvm::Value *Addr, bool IsVolatile=false)
llvm::CallInst * CreateMemSet(Address Dest, llvm::Value *Value, llvm::Value *Size, bool IsVolatile=false)
Address CreateStructGEP(Address Addr, unsigned Index, const llvm::Twine &Name="")
llvm::AtomicCmpXchgInst * CreateAtomicCmpXchg(Address Addr, llvm::Value *Cmp, llvm::Value *New, llvm::AtomicOrdering SuccessOrdering, llvm::AtomicOrdering FailureOrdering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
Address CreateConstByteGEP(Address Addr, CharUnits Offset, const llvm::Twine &Name="")
Address CreateLaunderInvariantGroup(Address Addr)
llvm::CallInst * CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
llvm::LoadInst * CreateAlignedLoad(llvm::Type *Ty, llvm::Value *Addr, CharUnits Align, const llvm::Twine &Name="")
Address CreateAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
Address CreateConstInBoundsGEP(Address Addr, uint64_t Index, const llvm::Twine &Name="")
Given addr = T* ... produce name = getelementptr inbounds addr, i64 index where i64 is actually the t...
Address CreateInBoundsGEP(Address Addr, ArrayRef< llvm::Value * > IdxList, llvm::Type *ElementType, CharUnits Align, const Twine &Name="")
virtual std::string getDeviceSideName(const NamedDecl *ND)=0
Returns function or variable name on device side even if the current compilation is for host.
virtual llvm::GlobalVariable * getThrowInfo(QualType T)
All available information about a concrete callee.
static CGCallee forDirect(llvm::Constant *functionPtr, const CGCalleeInfo &abstractInfo=CGCalleeInfo())
llvm::DIType * getOrCreateStandaloneType(QualType Ty, SourceLocation Loc)
Emit standalone debug info for a type.
llvm::DILocation * CreateTrapFailureMessageFor(llvm::DebugLoc TrapLocation, StringRef Category, StringRef FailureMsg)
Create a debug location from TrapLocation that adds an artificial inline frame where the frame name i...
CGFunctionInfo - Class to encapsulate the information about a function definition.
ABIArgInfo & getReturnInfo()
MutableArrayRef< ArgInfo > arguments()
virtual void EmitGCMemmoveCollectable(CodeGen::CodeGenFunction &CGF, Address DestPtr, Address SrcPtr, llvm::Value *Size)=0
EnqueuedBlockInfo emitOpenCLEnqueuedBlock(CodeGenFunction &CGF, const Expr *E)
CallArgList - Type for representing both the value and type of arguments in a call.
void add(RValue rvalue, QualType type)
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
llvm::Value * EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E, llvm::Triple::ArchType Arch)
llvm::Value * EmitFromMemory(llvm::Value *Value, QualType Ty)
EmitFromMemory - Change a scalar value from its memory representation to its value representation.
llvm::Value * EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID, const CallExpr *E)
void FinishFunction(SourceLocation EndLoc=SourceLocation())
FinishFunction - Complete IR generation of the current function.
llvm::Value * EmitLifetimeStart(llvm::TypeSize Size, llvm::Value *Addr)
std::pair< RValue, llvm::Value * > EmitAtomicCompareExchange(LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, llvm::AtomicOrdering Success=llvm::AtomicOrdering::SequentiallyConsistent, llvm::AtomicOrdering Failure=llvm::AtomicOrdering::SequentiallyConsistent, bool IsWeak=false, AggValueSlot Slot=AggValueSlot::ignored())
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
llvm::Value * EmitSVEPredicateCast(llvm::Value *Pred, llvm::ScalableVectorType *VTy)
llvm::CallInst * EmitTrapCall(llvm::Intrinsic::ID IntrID)
Emit a call to trap or debugtrap and attach function attribute "trap-func-name" if specified.
SanitizerSet SanOpts
Sanitizers enabled for this function.
RValue EmitBuiltinIsAligned(const CallExpr *E)
Emit IR for __builtin_is_aligned.
LValue EmitAggExprToLValue(const Expr *E)
EmitAggExprToLValue - Emit the computation of the specified expression of aggregate type into a tempo...
void EmitNonNullArgCheck(RValue RV, QualType ArgType, SourceLocation ArgLoc, AbstractCallee AC, unsigned ParmNum)
Create a check for a function parameter that may potentially be declared as non-null.
llvm::Value * EmitHLSLBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
llvm::Value * EmitHexagonBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
void GetAArch64SVEProcessedOperands(unsigned BuiltinID, const CallExpr *E, SmallVectorImpl< llvm::Value * > &Ops, SVETypeFlags TypeFlags)
llvm::Value * EmitAMDGPUBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
void EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr)
void pushLifetimeExtendedDestroy(CleanupKind kind, Address addr, QualType type, Destroyer *destroyer, bool useEHCleanupForArray)
void EmitTrapCheck(llvm::Value *Checked, SanitizerHandler CheckHandlerID, bool NoMerge=false)
Create a basic block that will call the trap intrinsic, and emit a conditional branch to it,...
llvm::Value * EmitCheckedArgForBuiltin(const Expr *E, BuiltinCheckKind Kind)
Emits an argument for a call to a builtin.
llvm::Value * EmitSVEGatherLoad(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
LValue EmitHLSLOutArgExpr(const HLSLOutArgExpr *E, CallArgList &Args, QualType Ty)
CleanupKind getARCCleanupKind()
Retrieves the default cleanup kind for an ARC cleanup.
llvm::Value * EmitRISCVCpuSupports(const CallExpr *E)
llvm::Value * EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitAArch64SVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
RValue emitBuiltinOSLogFormat(const CallExpr &E)
Emit IR for __builtin_os_log_format.
LValue EmitLValue(const Expr *E, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitLValue - Emit code to compute a designator that specifies the location of the expression.
llvm::Value * EmitVAStartEnd(llvm::Value *ArgValue, bool IsStart)
Emits a call to an LLVM variable-argument intrinsic, either llvm.va_start or llvm....
llvm::Value * EmitSVEMaskedStore(const CallExpr *, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
llvm::Value * EmitSVEReinterpret(llvm::Value *Val, llvm::Type *Ty)
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
llvm::Value * EmitSEHExceptionInfo()
RValue EmitBuiltinAlignTo(const CallExpr *E, bool AlignUp)
Emit IR for __builtin_align_up/__builtin_align_down.
const LangOptions & getLangOpts() const
llvm::Value * EmitLoadOfCountedByField(const Expr *Base, const FieldDecl *FAMDecl, const FieldDecl *CountDecl)
Build an expression accessing the "counted_by" field.
void ProcessOrderScopeAMDGCN(llvm::Value *Order, llvm::Value *Scope, llvm::AtomicOrdering &AO, llvm::SyncScope::ID &SSID)
llvm::Constant * EmitCheckTypeDescriptor(QualType T)
Emit a description of a type in a format suitable for passing to a runtime sanitizer handler.
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
void EmitUnreachable(SourceLocation Loc)
Emit a reached-unreachable diagnostic if Loc is valid and runtime checking is enabled.
llvm::Value * EmitSVETupleCreate(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType, ArrayRef< llvm::Value * > Ops)
Address makeNaturalAddressForPointer(llvm::Value *Ptr, QualType T, CharUnits Alignment=CharUnits::Zero(), bool ForPointeeType=false, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
Construct an address with the natural alignment of T.
llvm::Value * EmitCheckedArgForAssume(const Expr *E)
Emits an argument for a call to a __builtin_assume.
ComplexPairTy EmitComplexExpr(const Expr *E, bool IgnoreReal=false, bool IgnoreImag=false)
EmitComplexExpr - Emit the computation of the specified expression of complex type,...
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
TypeCheckKind
Situations in which we might emit a check for the suitability of a pointer or glvalue.
@ TCK_Store
Checking the destination of a store. Must be suitably sized and aligned.
@ TCK_Load
Checking the operand of a load. Must be suitably sized and aligned.
llvm::Value * EmitSMELdrStr(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::Value * EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx, const llvm::ElementCount &Count)
llvm::Type * ConvertTypeForMem(QualType T)
llvm::Value * EmitSVEMaskedLoad(const CallExpr *, llvm::Type *ReturnTy, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID, bool IsZExtReturn)
bool AlwaysEmitXRayCustomEvents() const
AlwaysEmitXRayCustomEvents - Return true if we must unconditionally emit XRay custom event handling c...
llvm::Value * EmitFP8NeonCall(unsigned IID, ArrayRef< llvm::Type * > Tys, SmallVectorImpl< llvm::Value * > &O, const CallExpr *E, const char *name)
llvm::Value * EmitSVEDupX(llvm::Value *Scalar)
RawAddress CreateMemTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
@ Default
! No language constraints on evaluation order.
const TargetInfo & getTarget() const
llvm::Value * vectorWrapScalar16(llvm::Value *Op)
llvm::Function * LookupNeonLLVMIntrinsic(unsigned IntrinsicID, unsigned Modifier, llvm::Type *ArgTy, const CallExpr *E)
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
llvm::Value * EmitSEHExceptionCode()
Address EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitPointerWithAlignment - Given an expression with a pointer type, emit the value and compute our be...
llvm::Value * EmitTargetBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
EmitTargetBuiltinExpr - Emit the given builtin call.
RValue EmitAnyExprToTemp(const Expr *E)
EmitAnyExprToTemp - Similarly to EmitAnyExpr(), however, the result will always be accessible even if...
RValue EmitCoroutineIntrinsic(const CallExpr *E, unsigned int IID)
llvm::Value * EmitAArch64SMEBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
RValue EmitAMDGPUDevicePrintfCallExpr(const CallExpr *E)
Address EmitArrayToPointerDecay(const Expr *Array, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
RValue EmitBuiltinNewDeleteCall(const FunctionProtoType *Type, const CallExpr *TheCallExpr, bool IsDelete)
llvm::Value * EmitRISCVCpuInit()
void EmitCheck(ArrayRef< std::pair< llvm::Value *, SanitizerKind::SanitizerOrdinal > > Checked, SanitizerHandler Check, ArrayRef< llvm::Constant * > StaticArgs, ArrayRef< llvm::Value * > DynamicArgs)
Create a basic block that will either trap or call a handler function in the UBSan runtime with the p...
RValue EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
RValue EmitAnyExpr(const Expr *E, AggValueSlot aggSlot=AggValueSlot::ignored(), bool ignoreResult=false)
EmitAnyExpr - Emit code to compute the specified expression which can have any type.
llvm::Value * EmitSVEPrefetchLoad(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
llvm::Value * EmitAArch64CompareBuiltinExpr(llvm::Value *Op, llvm::Type *Ty, const llvm::CmpInst::Predicate Fp, const llvm::CmpInst::Predicate Ip, const llvm::Twine &Name="")
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
AggValueSlot CreateAggTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateAggTemp - Create a temporary memory object for the given aggregate type.
llvm::ScalableVectorType * getSVEType(const SVETypeFlags &TypeFlags)
RValue emitRotate(const CallExpr *E, bool IsRotateRight)
llvm::Constant * EmitCheckSourceLocation(SourceLocation Loc)
Emit a description of a source location in a format suitable for passing to a runtime sanitizer handl...
void ErrorUnsupported(const Stmt *S, const char *Type)
ErrorUnsupported - Print out an error that codegen doesn't support the specified stmt yet.
CGDebugInfo * getDebugInfo()
const FieldDecl * FindFlexibleArrayMemberFieldAndOffset(ASTContext &Ctx, const RecordDecl *RD, const FieldDecl *FAMDecl, uint64_t &Offset)
llvm::Value * EmitRISCVCpuIs(const CallExpr *E)
Address EmitVAListRef(const Expr *E)
llvm::Value * EmitNeonShiftVector(llvm::Value *V, llvm::Type *Ty, bool negateForRightShift)
llvm::Value * EmitSVEMovl(const SVETypeFlags &TypeFlags, llvm::ArrayRef< llvm::Value * > Ops, unsigned BuiltinID)
void emitAlignmentAssumption(llvm::Value *PtrValue, QualType Ty, SourceLocation Loc, SourceLocation AssumptionLoc, llvm::Value *Alignment, llvm::Value *OffsetValue=nullptr)
const TargetCodeGenInfo & getTargetHooks() const
llvm::Value * EmitARMBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
llvm::Type * getEltType(const SVETypeFlags &TypeFlags)
void EmitAggExpr(const Expr *E, AggValueSlot AS)
EmitAggExpr - Emit the computation of the specified expression of aggregate type.
bool ShouldXRayInstrumentFunction() const
ShouldXRayInstrument - Return true if the current function should be instrumented with XRay nop sleds...
llvm::Value * EmitSVEPMull(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
llvm::Value * EmitToMemory(llvm::Value *Value, QualType Ty)
EmitToMemory - Change a scalar value from its value representation to its in-memory representation.
bool IsInPreservedAIRegion
True if CodeGen currently emits code inside presereved access index region.
llvm::Value * EmitARCRetain(QualType type, llvm::Value *value)
bool AlwaysEmitXRayTypedEvents() const
AlwaysEmitXRayTypedEvents - Return true if clang must unconditionally emit XRay typed event handling ...
void SetSqrtFPAccuracy(llvm::Value *Val)
Set the minimum required accuracy of the given sqrt operation based on CodeGenOpts.
RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee, ReturnValueSlot ReturnValue, const CallArgList &Args, llvm::CallBase **CallOrInvoke, bool IsMustTail, SourceLocation Loc, bool IsVirtualFunctionPointerThunk=false)
EmitCall - Generate a call of the given function, expecting the given result type,...
llvm::Value * EmitSVEScatterStore(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::CallInst * EmitNounwindRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
ASTContext & getContext() const
llvm::Function * generateBuiltinOSLogHelperFunction(const analyze_os_log::OSLogBufferLayout &Layout, CharUnits BufferAlignment)
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
CGCallee EmitCallee(const Expr *E)
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
llvm::Value * EmitScalarOrConstFoldImmArg(unsigned ICEArguments, unsigned Idx, const CallExpr *E)
void checkTargetFeatures(const CallExpr *E, const FunctionDecl *TargetDecl)
llvm::Value * BuildVector(ArrayRef< llvm::Value * > Ops)
llvm::Value * EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitARMCDEBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
llvm::Value * GetCountedByFieldExprGEP(const Expr *Base, const FieldDecl *FAMDecl, const FieldDecl *CountDecl)
llvm::CallInst * EmitRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
llvm::Value * EmitFP8NeonFDOTCall(unsigned IID, bool ExtendLaneArg, llvm::Type *RetTy, SmallVectorImpl< llvm::Value * > &Ops, const CallExpr *E, const char *name)
llvm::Value * EmitSVEPredicateTupleCast(llvm::Value *PredTuple, llvm::StructType *Ty)
llvm::Type * ConvertType(QualType T)
llvm::Value * EmitFP8NeonFMLACall(unsigned IID, bool ExtendLaneArg, llvm::Type *RetTy, SmallVectorImpl< llvm::Value * > &Ops, const CallExpr *E, const char *name)
void EmitWritebacks(const CallArgList &Args)
EmitWriteback - Emit callbacks for function.
llvm::CallBase * EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee, ArrayRef< llvm::Value * > args, const Twine &name="")
llvm::Value * EmitSystemZBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitSMEReadWrite(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, LValue LV, QualType Type, SanitizerSet SkippedChecks=SanitizerSet(), llvm::Value *ArraySize=nullptr)
llvm::Value * EmitSMELd1St1(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::Value * EmitPPCBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitFP8NeonCvtCall(unsigned IID, llvm::Type *Ty0, llvm::Type *Ty1, bool Extract, SmallVectorImpl< llvm::Value * > &Ops, const CallExpr *E, const char *name)
static Destroyer destroyARCStrongPrecise
void EmitARCIntrinsicUse(ArrayRef< llvm::Value * > values)
RValue EmitNVPTXDevicePrintfCallExpr(const CallExpr *E)
llvm::Value * EvaluateExprAsBool(const Expr *E)
EvaluateExprAsBool - Perform the usual unary conversions on the specified expression and compare the ...
llvm::Value * EmitSVEStructLoad(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Address EmitMSVAListRef(const Expr *E)
Emit a "reference" to a __builtin_ms_va_list; this is always the value of the expression,...
llvm::Value * EmitCheckedInBoundsGEP(llvm::Type *ElemTy, llvm::Value *Ptr, ArrayRef< llvm::Value * > IdxList, bool SignedIndices, bool IsSubtraction, SourceLocation Loc, const Twine &Name="")
Same as IRBuilder::CreateInBoundsGEP, but additionally emits a check to detect undefined behavior whe...
llvm::Value * EmitNeonRShiftImm(llvm::Value *Vec, llvm::Value *Amt, llvm::Type *Ty, bool usgn, const char *name)
SmallVector< llvm::Type *, 2 > getSVEOverloadTypes(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType, ArrayRef< llvm::Value * > Ops)
static bool hasAggregateEvaluationKind(QualType T)
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
llvm::Value * EmitARMMVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
llvm::Value * EmitSVEStructStore(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
llvm::Value * EmitSEHAbnormalTermination()
llvm::Value * EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E)
Address ReturnValue
ReturnValue - The temporary alloca to hold the return value.
llvm::Value * EmitSVEAllTruePred(const SVETypeFlags &TypeFlags)
RValue GetUndefRValue(QualType Ty)
GetUndefRValue - Get an appropriate 'undef' rvalue for the given type.
llvm::Type * SVEBuiltinMemEltTy(const SVETypeFlags &TypeFlags)
SVEBuiltinMemEltTy - Returns the memory element type for this memory access builtin.
llvm::LLVMContext & getLLVMContext()
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
void AddAMDGPUFenceAddressSpaceMMRA(llvm::Instruction *Inst, const CallExpr *E)
llvm::Value * EmitSMEZero(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::Value * EmitRISCVBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
llvm::Value * EmitCommonNeonBuiltinExpr(unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic, const char *NameHint, unsigned Modifier, const CallExpr *E, SmallVectorImpl< llvm::Value * > &Ops, Address PtrOp0, Address PtrOp1, llvm::Triple::ArchType Arch)
llvm::Value * EmitNeonCall(llvm::Function *F, SmallVectorImpl< llvm::Value * > &O, const char *name, unsigned shift=0, bool rightshift=false)
llvm::Value * EmitAnnotationCall(llvm::Function *AnnotationFn, llvm::Value *AnnotatedVal, StringRef AnnotationStr, SourceLocation Location, const AnnotateAttr *Attr)
Emit an annotation call (intrinsic).
llvm::ScalableVectorType * getSVEPredType(const SVETypeFlags &TypeFlags)
llvm::Value * EmitSVEGatherPrefetch(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
llvm::Value * EmitSPIRVBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitBPFBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitSVETupleSetOrGet(const SVETypeFlags &TypeFlags, ArrayRef< llvm::Value * > Ops)
This class organizes the cross-function state that is used while generating LLVM code.
CGHLSLRuntime & getHLSLRuntime()
Return a reference to the configured HLSL runtime.
llvm::Module & getModule() const
llvm::FunctionCallee CreateRuntimeFunction(llvm::FunctionType *Ty, StringRef Name, llvm::AttributeList ExtraAttrs=llvm::AttributeList(), bool Local=false, bool AssumeConvergent=false)
Create or return a runtime function declaration with the specified type and name.
llvm::Constant * getRawFunctionPointer(GlobalDecl GD, llvm::Type *Ty=nullptr)
Return a function pointer for a reference to the given function.
llvm::Constant * getBuiltinLibFunction(const FunctionDecl *FD, unsigned BuiltinID)
Given a builtin id for a function like "__builtin_fabsf", return a Function* for "fabsf".
DiagnosticsEngine & getDiags() const
void ErrorUnsupported(const Stmt *S, const char *Type)
Print out an error that codegen doesn't support the specified stmt yet.
const LangOptions & getLangOpts() const
CGCUDARuntime & getCUDARuntime()
Return a reference to the configured CUDA runtime.
CodeGenTypes & getTypes()
CGOpenCLRuntime & getOpenCLRuntime()
Return a reference to the configured OpenCL runtime.
const TargetInfo & getTarget() const
const llvm::DataLayout & getDataLayout() const
void Error(SourceLocation loc, StringRef error)
Emit a general error that something can't be done.
CGCXXABI & getCXXABI() const
llvm::Constant * GetFunctionStart(const ValueDecl *Decl)
const llvm::Triple & getTriple() const
void DecorateInstructionWithTBAA(llvm::Instruction *Inst, TBAAAccessInfo TBAAInfo)
DecorateInstructionWithTBAA - Decorate the instruction with a TBAA tag.
llvm::Constant * CreateRuntimeVariable(llvm::Type *Ty, StringRef Name)
Create a new runtime global variable with the specified type and name.
TBAAAccessInfo getTBAAAccessInfo(QualType AccessType)
getTBAAAccessInfo - Get TBAA information that describes an access to an object of the given type.
ASTContext & getContext() const
const CodeGenOptions & getCodeGenOpts() const
StringRef getMangledName(GlobalDecl GD)
CharUnits getNaturalPointeeTypeAlignment(QualType T, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
llvm::LLVMContext & getLLVMContext()
llvm::Function * getIntrinsic(unsigned IID, ArrayRef< llvm::Type * > Tys={})
CGObjCRuntime & getObjCRuntime()
Return a reference to the configured Objective-C runtime.
void SetLLVMFunctionAttributes(GlobalDecl GD, const CGFunctionInfo &Info, llvm::Function *F, bool IsThunk)
Set the LLVM function attributes (sext, zext, etc).
void SetLLVMFunctionAttributesForDefinition(const Decl *D, llvm::Function *F)
Set the LLVM function attributes which only apply to a function definition.
ConstantAddress GetAddrOfConstantCString(const std::string &Str, const char *GlobalName=nullptr)
Returns a pointer to a character array containing the literal and a terminating '\0' character.
llvm::Type * ConvertType(QualType T)
ConvertType - Convert type T into a llvm::Type.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
const CGFunctionInfo & arrangeBuiltinFunctionCall(QualType resultType, const CallArgList &args)
llvm::Constant * emitAbstract(const Expr *E, QualType T)
Emit the result of the given expression as an abstract constant, asserting that it succeeded.
Information for lazily generating a cleanup.
FunctionArgList - Type for representing both the decl and type of parameters to a function.
LValue - This represents an lvalue references.
llvm::Value * getRawBitFieldPointer(CodeGenFunction &CGF) const
llvm::Value * getPointer(CodeGenFunction &CGF) const
Address getAddress() const
void setNontemporal(bool Value)
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
llvm::Value * getAggregatePointer(QualType PointeeType, CodeGenFunction &CGF) const
static RValue getIgnored()
static RValue get(llvm::Value *V)
static RValue getAggregate(Address addr, bool isVolatile=false)
Convert an Address to an RValue.
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
An abstract representation of an aligned address.
llvm::Value * getPointer() const
static RawAddress invalid()
ReturnValueSlot - Contains the address where the return value of a function can be stored,...
virtual llvm::Value * encodeReturnAddress(CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const
Performs the code-generation required to convert the address of an instruction into a return address ...
virtual llvm::Value * decodeReturnAddress(CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const
Performs the code-generation required to convert a return address as stored by the system into the ac...
const T & getABIInfo() const
virtual int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const
Determines the DWARF register number for the stack pointer, for exception-handling purposes.
virtual llvm::Value * testFPKind(llvm::Value *V, unsigned BuiltinID, CGBuilderTy &Builder, CodeGenModule &CGM) const
Performs a target specific test of a floating point value for things like IsNaN, Infinity,...
Complex values, per C99 6.2.5p11.
Represents a concrete matrix type with constant number of rows and columns.
Represents a sugar type with __counted_by or __sized_by annotations, including their _or_null variant...
DynamicCountPointerKind getKind() const
RecordDecl * getOuterLexicalRecordContext()
Retrieve the outermost lexically enclosing record context.
bool isImplicit() const
isImplicit - Indicates whether the declaration was implicitly generated by the implementation.
FunctionDecl * getAsFunction() LLVM_READONLY
Returns the function itself, or the templated function if this is a function template.
DeclContext * getDeclContext()
static bool isFlexibleArrayMemberLike(ASTContext &Context, const Decl *D, QualType Ty, LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel, bool IgnoreTemplateOrMacroSubstitution)
Whether it resembles a flexible array member.
Concrete class used by the front-end to report problems and issues.
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
This represents one expression.
bool EvaluateAsInt(EvalResult &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer,...
Expr * IgnoreParenCasts() LLVM_READONLY
Skip past any parentheses and casts which might surround this expression until reaching a fixed point...
llvm::APSInt EvaluateKnownConstInt(const ASTContext &Ctx, SmallVectorImpl< PartialDiagnosticAt > *Diag=nullptr) const
EvaluateKnownConstInt - Call EvaluateAsRValue and return the folded integer.
Expr * IgnoreParenImpCasts() LLVM_READONLY
Skip past any parentheses and implicit casts which might surround this expression until reaching a fi...
bool EvaluateAsFloat(llvm::APFloat &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsFloat - Return true if this is a constant which we can fold and convert to a floating point...
Expr * IgnoreParens() LLVM_READONLY
Skip past any parentheses which might surround this expression until reaching a fixed point.
@ NPC_ValueDependentIsNotNull
Specifies that a value-dependent expression should be considered to never be a null pointer constant.
ExprObjectKind getObjectKind() const
getObjectKind - The object kind that this expression produces.
bool EvaluateAsRValue(EvalResult &Result, const ASTContext &Ctx, bool InConstantContext=false) const
EvaluateAsRValue - Return true if this is a constant which we can fold to an rvalue using any crazy t...
bool HasSideEffects(const ASTContext &Ctx, bool IncludePossibleEffects=true) const
HasSideEffects - This routine returns true for all those expressions which have any effect other than...
std::optional< std::string > tryEvaluateString(ASTContext &Ctx) const
If the current Expr can be evaluated to a pointer to a null-terminated constant string,...
Expr * IgnoreImpCasts() LLVM_READONLY
Skip past any implicit casts which might surround this expression until reaching a fixed point.
NullPointerConstantKind isNullPointerConstant(ASTContext &Ctx, NullPointerConstantValueDependence NPC) const
isNullPointerConstant - C99 6.3.2.3p3 - Test if this reduces down to a Null pointer constant.
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
std::optional< llvm::APSInt > getIntegerConstantExpr(const ASTContext &Ctx, SourceLocation *Loc=nullptr) const
isIntegerConstantExpr - Return the value if this expression is a valid integer constant expression.
bool tryEvaluateObjectSize(uint64_t &Result, ASTContext &Ctx, unsigned Type) const
If the current Expr is a pointer, this will try to statically determine the number of bytes available...
const ValueDecl * getAsBuiltinConstantDeclRef(const ASTContext &Context) const
If this expression is an unambiguous reference to a single declaration, in the style of __builtin_fun...
Represents difference between two FPOptions values.
Represents a member of a struct/union/class.
const FieldDecl * findCountedByField() const
Find the FieldDecl specified in a FAM's "counted_by" attribute.
Represents a function declaration or definition.
const ParmVarDecl * getParamDecl(unsigned i) const
unsigned getBuiltinID(bool ConsiderWrapperFunctions=false) const
Returns a value indicating whether this function corresponds to a builtin function.
Represents a prototype with parameter type info, e.g.
@ SME_PStateSMEnabledMask
@ SME_PStateSMCompatibleMask
GlobalDecl - represents a global declaration.
const Decl * getDecl() const
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
StrictFlexArraysLevelKind
MemberExpr - [C99 6.5.2.3] Structure and Union Members.
StringRef getName() const
Get the name of identifier for this declaration as a StringRef.
std::string getNameAsString() const
Get a human-readable name for the declaration, even if it is one of the special kinds of names (C++ c...
Flags to identify the types for overloaded Neon builtins.
EltType getEltType() const
PointerType - C99 6.7.5.1 - Pointer Declarators.
QualType getPointeeType() const
A (possibly-)qualified type.
bool isVolatileQualified() const
Determine whether this type is volatile-qualified.
bool isWebAssemblyFuncrefType() const
Returns true if it is a WebAssembly Funcref Type.
LangAS getAddressSpace() const
Return the address space of this type.
bool isWebAssemblyExternrefType() const
Returns true if it is a WebAssembly Externref Type.
The collection of all-type qualifiers we support.
Represents a struct/union/class.
field_range fields() const
Flags to identify the types for overloaded SVE builtins.
bool isZExtReturn() const
bool isReverseUSDOT() const
bool isOverloadNone() const
MemEltType getMemEltType() const
bool isGatherLoad() const
bool isOverloadCvt() const
EltType getEltType() const
bool isOverloadDefault() const
bool isOverloadWhileRW() const
bool isReverseMergeAnyAccOp() const
bool isReductionQV() const
bool isInsertOp1SVALL() const
bool isAppendSVALL() const
bool isReverseMergeAnyBinOp() const
bool isStructStore() const
bool isTupleCreate() const
bool isGatherPrefetch() const
bool hasSplatOperand() const
MergeType getMergeType() const
bool isByteIndexed() const
bool isStructLoad() const
bool isOverloadWhileOrMultiVecCvt() const
unsigned getSplatOperand() const
bool isScatterStore() const
bool isReverseCompare() const
Scope - A scope is a transient data structure that is used while parsing the program.
Encodes a location in the source.
SourceLocation getBeginLoc() const LLVM_READONLY
bool areArgsDestroyedLeftToRightInCallee() const
Are arguments to a call destroyed left to right in the callee? This is a fundamental language change,...
Exposes information about the current target.
TargetOptions & getTargetOpts() const
Retrieve the target options.
virtual bool hasLegalHalfType() const
Determine whether _Float16 is supported on this target.
const llvm::Triple & getTriple() const
Returns the target triple of the primary target.
bool isLittleEndian() const
unsigned getMaxOpenCLWorkGroupSize() const
TargetCXXABI getCXXABI() const
Get the C++ ABI currently in use.
virtual bool checkArithmeticFenceSupported() const
Controls if __arithmetic_fence is supported in the targeted backend.
unsigned getSuitableAlign() const
Return the alignment that is the largest alignment ever used for any scalar/SIMD data type on the tar...
virtual std::string_view getClobbers() const =0
Returns a string of target-specific clobbers, in LLVM format.
llvm::CodeObjectVersionKind CodeObjectVersion
Code object version for AMDGPU.
The base class of the type hierarchy.
CXXRecordDecl * getAsCXXRecordDecl() const
Retrieves the CXXRecordDecl that this type refers to, either because the type is a RecordType or beca...
bool isBlockPointerType() const
bool isSignedIntegerType() const
Return true if this is an integer type that is signed, according to C99 6.2.5p4 [char,...
bool isComplexType() const
isComplexType() does not include complex integers (a GCC extension).
bool hasIntegerRepresentation() const
Determine whether this type has an integer representation of some sort, e.g., it is an integer type o...
bool isCountAttributedType() const
bool isPointerType() const
bool isIntegerType() const
isIntegerType() does not include complex integers (a GCC extension).
const T * castAs() const
Member-template castAs<specific type>.
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
bool isIntegralOrEnumerationType() const
Determine whether this type is an integral or enumeration type.
bool hasUnsignedIntegerRepresentation() const
Determine whether this type has an unsigned integer representation of some sort, e....
bool hasSignedIntegerRepresentation() const
Determine whether this type has an signed integer representation of some sort, e.g....
bool isObjCObjectPointerType() const
bool hasFloatingRepresentation() const
Determine whether this type has a floating-point representation of some sort, e.g....
bool isVectorType() const
bool isFloatingType() const
bool isUnsignedIntegerType() const
Return true if this is an integer type that is unsigned, according to C99 6.2.5p6 [which returns true...
const T * getAs() const
Member-template getAs<specific type>'.
bool isRecordType() const
bool isSizelessVectorType() const
Returns true for all scalable vector types.
QualType getSizelessVectorEltType(const ASTContext &Ctx) const
Returns the representative type for the element of a sizeless vector builtin type.
RecordDecl * getAsRecordDecl() const
Retrieves the RecordDecl this type refers to.
Represent the declaration of a variable (in which case it is an lvalue) a function (in which case it ...
Represents a GCC generic vector type.
unsigned getNumElements() const
QualType getElementType() const
SmallVector< OSLogBufferItem, 4 > Items
unsigned char getNumArgsByte() const
unsigned char getSummaryByte() const
Defines the clang::TargetInfo interface.
@ Type
The l-value was considered opaque, so the alignment was determined from a type.
llvm::Constant * initializationPatternFor(CodeGenModule &, llvm::Type *)
TypeEvaluationKind
The kind of evaluation to perform on values of a particular type.
@ EHCleanup
Denotes a cleanup that should run when a scope is exited using exceptional control flow (a throw stat...
constexpr XRayInstrMask Typed
constexpr XRayInstrMask Custom
bool computeOSLogBufferLayout(clang::ASTContext &Ctx, const clang::CallExpr *E, OSLogBufferLayout &layout)
const void * Store
Store - This opaque type encapsulates an immutable mapping from locations to values.
bool Dup(InterpState &S, CodePtr OpPC)
bool Zero(InterpState &S, CodePtr OpPC)
bool Mul(InterpState &S, CodePtr OpPC)
bool Neg(InterpState &S, CodePtr OpPC)
bool Load(InterpState &S, CodePtr OpPC)
bool Cast(InterpState &S, CodePtr OpPC)
bool Ret(InterpState &S, CodePtr &PC)
The JSON file list parser is used to communicate input to InstallAPI.
@ OK_BitField
A bitfield object is a bitfield on a C or C++ record.
@ Vector
'vector' clause, allowed on 'loop', Combined, and 'routine' directives.
@ DType
'dtype' clause, an alias for 'device_type', stored separately for diagnostic purposes.
bool operator<(DeclarationName LHS, DeclarationName RHS)
Ordering on two declaration names.
@ Asm
Assembly: we accept this only so that we can preprocess it.
@ Result
The result type of a method or function.
LangAS
Defines the address space values used by the address space qualifier of QualType.
const FunctionProtoType * T
SyncScope
Defines synch scope values used internally by clang.
llvm::StringRef getAsString(SyncScope S)
@ Success
Template argument deduction was successful.
@ Other
Other implicit parameter.
Diagnostic wrappers for TextAPI types for error reporting.
llvm::PointerType * VoidPtrTy
llvm::IntegerType * Int64Ty
llvm::PointerType * ConstGlobalsPtrTy
void* in the address space for constant globals
CharUnits getIntAlign() const
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
llvm::Type * HalfTy
half, bfloat, float, double
llvm::IntegerType * SizeTy
llvm::IntegerType * Int32Ty
llvm::IntegerType * IntPtrTy
llvm::IntegerType * IntTy
int
llvm::IntegerType * Int16Ty
llvm::PointerType * Int8PtrTy
llvm::PointerType * UnqualPtrTy
llvm::PointerType * AllocaInt8PtrTy
LangAS getASTAllocaAddressSpace() const
EvalResult is a struct with detailed info about an evaluated expression.
APValue Val
Val - This is the value the expression can be folded to.
void clear(SanitizerMask K=SanitizerKind::All)
Disable the sanitizers specified in K.
void set(SanitizerMask K, bool Value)
Enable or disable a certain (single) sanitizer.
bool has(SanitizerMask K) const
Check if a certain (single) sanitizer is enabled.
bool has(XRayInstrMask K) const
#define scalbln(__x, __y)
#define copysign(__x, __y)
#define remquo(__x, __y, __z)
#define nextafter(__x, __y)
#define nexttoward(__x, __y)
#define remainder(__x, __y)
#define fma(__x, __y, __z)