38#include "llvm/ADT/APFloat.h"
39#include "llvm/ADT/APInt.h"
40#include "llvm/ADT/FloatingPointMode.h"
41#include "llvm/ADT/SmallPtrSet.h"
42#include "llvm/ADT/StringExtras.h"
43#include "llvm/Analysis/ValueTracking.h"
44#include "llvm/IR/DataLayout.h"
45#include "llvm/IR/InlineAsm.h"
46#include "llvm/IR/Intrinsics.h"
47#include "llvm/IR/IntrinsicsAArch64.h"
48#include "llvm/IR/IntrinsicsAMDGPU.h"
49#include "llvm/IR/IntrinsicsARM.h"
50#include "llvm/IR/IntrinsicsBPF.h"
51#include "llvm/IR/IntrinsicsDirectX.h"
52#include "llvm/IR/IntrinsicsHexagon.h"
53#include "llvm/IR/IntrinsicsNVPTX.h"
54#include "llvm/IR/IntrinsicsPowerPC.h"
55#include "llvm/IR/IntrinsicsR600.h"
56#include "llvm/IR/IntrinsicsRISCV.h"
57#include "llvm/IR/IntrinsicsS390.h"
58#include "llvm/IR/IntrinsicsWebAssembly.h"
59#include "llvm/IR/IntrinsicsX86.h"
60#include "llvm/IR/MDBuilder.h"
61#include "llvm/IR/MatrixBuilder.h"
62#include "llvm/IR/MemoryModelRelaxationAnnotations.h"
63#include "llvm/Support/AMDGPUAddrSpace.h"
64#include "llvm/Support/ConvertUTF.h"
65#include "llvm/Support/MathExtras.h"
66#include "llvm/Support/ScopedPrinter.h"
67#include "llvm/TargetParser/AArch64TargetParser.h"
68#include "llvm/TargetParser/RISCVISAInfo.h"
69#include "llvm/TargetParser/RISCVTargetParser.h"
70#include "llvm/TargetParser/X86TargetParser.h"
75using namespace CodeGen;
79 Align AlignmentInBytes) {
81 switch (CGF.
getLangOpts().getTrivialAutoVarInit()) {
82 case LangOptions::TrivialAutoVarInitKind::Uninitialized:
85 case LangOptions::TrivialAutoVarInitKind::Zero:
86 Byte = CGF.
Builder.getInt8(0x00);
88 case LangOptions::TrivialAutoVarInitKind::Pattern: {
90 Byte = llvm::dyn_cast<llvm::ConstantInt>(
98 I->addAnnotationMetadata(
"auto-init");
104 Constant *FZeroConst = ConstantFP::getZero(CGF->
FloatTy);
109 FZeroConst = ConstantVector::getSplat(
110 ElementCount::getFixed(VecTy->getNumElements()), FZeroConst);
111 auto *FCompInst = CGF->
Builder.CreateFCmpOLT(Op0, FZeroConst);
112 CMP = CGF->
Builder.CreateIntrinsic(
114 {FCompInst},
nullptr);
116 CMP = CGF->
Builder.CreateFCmpOLT(Op0, FZeroConst);
119 LastInstr = CGF->
Builder.CreateIntrinsic(
120 CGF->
VoidTy, llvm::Intrinsic::dx_discard, {CMP},
nullptr);
125 CGF->
Builder.CreateCondBr(CMP, LT0, End);
127 CGF->
Builder.SetInsertPoint(LT0);
129 CGF->
Builder.CreateIntrinsic(CGF->
VoidTy, llvm::Intrinsic::spv_discard, {},
132 LastInstr = CGF->
Builder.CreateBr(End);
134 CGF->
Builder.SetInsertPoint(End);
136 llvm_unreachable(
"Backend Codegen not supported.");
144 const auto *OutArg1 = dyn_cast<HLSLOutArgExpr>(
E->getArg(1));
145 const auto *OutArg2 = dyn_cast<HLSLOutArgExpr>(
E->getArg(2));
156 Value *LowBits =
nullptr;
157 Value *HighBits =
nullptr;
161 llvm::Type *RetElementTy = CGF->
Int32Ty;
163 RetElementTy = llvm::VectorType::get(
164 CGF->
Int32Ty, ElementCount::getFixed(Op0VecTy->getNumElements()));
165 auto *RetTy = llvm::StructType::get(RetElementTy, RetElementTy);
167 CallInst *CI = CGF->
Builder.CreateIntrinsic(
168 RetTy, Intrinsic::dx_splitdouble, {Op0},
nullptr,
"hlsl.splitdouble");
170 LowBits = CGF->
Builder.CreateExtractValue(CI, 0);
171 HighBits = CGF->
Builder.CreateExtractValue(CI, 1);
176 if (!Op0->
getType()->isVectorTy()) {
177 FixedVectorType *DestTy = FixedVectorType::get(CGF->
Int32Ty, 2);
178 Value *Bitcast = CGF->
Builder.CreateBitCast(Op0, DestTy);
180 LowBits = CGF->
Builder.CreateExtractElement(Bitcast, (uint64_t)0);
181 HighBits = CGF->
Builder.CreateExtractElement(Bitcast, 1);
184 if (
const auto *VecTy =
186 NumElements = VecTy->getNumElements();
188 FixedVectorType *Uint32VecTy =
189 FixedVectorType::get(CGF->
Int32Ty, NumElements * 2);
190 Value *Uint32Vec = CGF->
Builder.CreateBitCast(Op0, Uint32VecTy);
191 if (NumElements == 1) {
192 LowBits = CGF->
Builder.CreateExtractElement(Uint32Vec, (uint64_t)0);
193 HighBits = CGF->
Builder.CreateExtractElement(Uint32Vec, 1);
196 for (
int I = 0,
E = NumElements; I !=
E; ++I) {
197 EvenMask.push_back(I * 2);
198 OddMask.push_back(I * 2 + 1);
200 LowBits = CGF->
Builder.CreateShuffleVector(Uint32Vec, EvenMask);
201 HighBits = CGF->
Builder.CreateShuffleVector(Uint32Vec, OddMask);
215 "asdouble operands types mismatch");
219 llvm::Type *ResultType = CGF.
DoubleTy;
222 N = VTy->getNumElements();
223 ResultType = llvm::FixedVectorType::get(CGF.
DoubleTy, N);
227 return CGF.
Builder.CreateIntrinsic(
228 ResultType, Intrinsic::dx_asdouble,
232 OpLowBits = CGF.
Builder.CreateVectorSplat(1, OpLowBits);
233 OpHighBits = CGF.
Builder.CreateVectorSplat(1, OpHighBits);
237 for (
int i = 0; i < N; i++) {
239 Mask.push_back(i + N);
242 Value *BitVec = CGF.
Builder.CreateShuffleVector(OpLowBits, OpHighBits, Mask);
244 return CGF.
Builder.CreateBitCast(BitVec, ResultType);
251 llvm::Metadata *Ops[] = {llvm::MDString::get(Context,
"x18")};
252 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
253 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
256 llvm::Value *X18 = CGF.
Builder.CreateCall(F, Metadata);
263 unsigned BuiltinID) {
272 static SmallDenseMap<unsigned, StringRef, 64> F128Builtins{
273 {Builtin::BI__builtin___fprintf_chk,
"__fprintf_chkieee128"},
274 {Builtin::BI__builtin___printf_chk,
"__printf_chkieee128"},
275 {Builtin::BI__builtin___snprintf_chk,
"__snprintf_chkieee128"},
276 {Builtin::BI__builtin___sprintf_chk,
"__sprintf_chkieee128"},
277 {Builtin::BI__builtin___vfprintf_chk,
"__vfprintf_chkieee128"},
278 {Builtin::BI__builtin___vprintf_chk,
"__vprintf_chkieee128"},
279 {Builtin::BI__builtin___vsnprintf_chk,
"__vsnprintf_chkieee128"},
280 {Builtin::BI__builtin___vsprintf_chk,
"__vsprintf_chkieee128"},
281 {Builtin::BI__builtin_fprintf,
"__fprintfieee128"},
282 {Builtin::BI__builtin_printf,
"__printfieee128"},
283 {Builtin::BI__builtin_snprintf,
"__snprintfieee128"},
284 {Builtin::BI__builtin_sprintf,
"__sprintfieee128"},
285 {Builtin::BI__builtin_vfprintf,
"__vfprintfieee128"},
286 {Builtin::BI__builtin_vprintf,
"__vprintfieee128"},
287 {Builtin::BI__builtin_vsnprintf,
"__vsnprintfieee128"},
288 {Builtin::BI__builtin_vsprintf,
"__vsprintfieee128"},
289 {Builtin::BI__builtin_fscanf,
"__fscanfieee128"},
290 {Builtin::BI__builtin_scanf,
"__scanfieee128"},
291 {Builtin::BI__builtin_sscanf,
"__sscanfieee128"},
292 {Builtin::BI__builtin_vfscanf,
"__vfscanfieee128"},
293 {Builtin::BI__builtin_vscanf,
"__vscanfieee128"},
294 {Builtin::BI__builtin_vsscanf,
"__vsscanfieee128"},
295 {Builtin::BI__builtin_nexttowardf128,
"__nexttowardieee128"},
301 static SmallDenseMap<unsigned, StringRef, 4> AIXLongDouble64Builtins{
302 {Builtin::BI__builtin_frexpl,
"frexp"},
303 {Builtin::BI__builtin_ldexpl,
"ldexp"},
304 {Builtin::BI__builtin_modfl,
"modf"},
310 if (FD->
hasAttr<AsmLabelAttr>())
316 &
getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad() &&
317 F128Builtins.contains(BuiltinID))
318 Name = F128Builtins[BuiltinID];
321 &llvm::APFloat::IEEEdouble() &&
322 AIXLongDouble64Builtins.contains(BuiltinID))
323 Name = AIXLongDouble64Builtins[BuiltinID];
328 llvm::FunctionType *Ty =
331 return GetOrCreateLLVMFunction(Name, Ty,
D,
false);
337 QualType T, llvm::IntegerType *IntType) {
340 if (
V->getType()->isPointerTy())
341 return CGF.
Builder.CreatePtrToInt(
V, IntType);
343 assert(
V->getType() == IntType);
351 if (ResultType->isPointerTy())
352 return CGF.
Builder.CreateIntToPtr(
V, ResultType);
354 assert(
V->getType() == ResultType);
365 if (Align % Bytes != 0) {
378 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
388 llvm::IntegerType *IntType = llvm::IntegerType::get(
392 llvm::Type *ValueType = Val->getType();
420 llvm::AtomicRMWInst::BinOp Kind,
429 llvm::AtomicRMWInst::BinOp Kind,
431 Instruction::BinaryOps Op,
432 bool Invert =
false) {
441 llvm::IntegerType *IntType = llvm::IntegerType::get(
445 llvm::Type *ValueType = Val->getType();
449 Kind, DestAddr, Val, llvm::AtomicOrdering::SequentiallyConsistent);
454 llvm::ConstantInt::getAllOnesValue(IntType));
478 llvm::IntegerType *IntType = llvm::IntegerType::get(
482 llvm::Type *ValueType = Cmp->getType();
487 DestAddr, Cmp, New, llvm::AtomicOrdering::SequentiallyConsistent,
488 llvm::AtomicOrdering::SequentiallyConsistent);
491 return CGF.
Builder.CreateZExt(CGF.
Builder.CreateExtractValue(Pair, 1),
514 AtomicOrdering SuccessOrdering = AtomicOrdering::SequentiallyConsistent) {
526 auto *RTy = Exchange->getType();
530 if (RTy->isPointerTy()) {
536 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release ?
537 AtomicOrdering::Monotonic :
545 DestAddr, Comparand, Exchange, SuccessOrdering, FailureOrdering);
546 CmpXchg->setVolatile(
true);
549 if (RTy->isPointerTy()) {
570 AtomicOrdering SuccessOrdering) {
571 assert(
E->getNumArgs() == 4);
577 assert(DestPtr->getType()->isPointerTy());
578 assert(!ExchangeHigh->getType()->isPointerTy());
579 assert(!ExchangeLow->getType()->isPointerTy());
582 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release
583 ? AtomicOrdering::Monotonic
588 llvm::Type *Int128Ty = llvm::IntegerType::get(CGF.
getLLVMContext(), 128);
589 Address DestAddr(DestPtr, Int128Ty,
594 ExchangeHigh = CGF.
Builder.CreateZExt(ExchangeHigh, Int128Ty);
595 ExchangeLow = CGF.
Builder.CreateZExt(ExchangeLow, Int128Ty);
597 CGF.
Builder.CreateShl(ExchangeHigh, llvm::ConstantInt::get(Int128Ty, 64));
598 llvm::Value *Exchange = CGF.
Builder.CreateOr(ExchangeHigh, ExchangeLow);
604 SuccessOrdering, FailureOrdering);
610 CXI->setVolatile(
true);
622 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
628 AtomicRMWInst::Add, DestAddr, ConstantInt::get(IntTy, 1), Ordering);
629 return CGF.
Builder.CreateAdd(
Result, ConstantInt::get(IntTy, 1));
634 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
640 AtomicRMWInst::Sub, DestAddr, ConstantInt::get(IntTy, 1), Ordering);
641 return CGF.
Builder.CreateSub(
Result, ConstantInt::get(IntTy, 1));
652 Load->setVolatile(
true);
662 llvm::StoreInst *Store =
664 Store->setVolatile(
true);
673 unsigned ConstrainedIntrinsicID) {
676 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
677 if (CGF.
Builder.getIsFPConstrained()) {
679 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0 });
682 return CGF.
Builder.CreateCall(F, Src0);
690 unsigned ConstrainedIntrinsicID) {
694 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
695 if (CGF.
Builder.getIsFPConstrained()) {
697 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0, Src1 });
700 return CGF.
Builder.CreateCall(F, { Src0, Src1 });
707 llvm::Intrinsic::ID ConstrainedIntrinsicID) {
711 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
712 if (CGF.
Builder.getIsFPConstrained()) {
714 {Src0->getType(), Src1->getType()});
715 return CGF.
Builder.CreateConstrainedFPCall(F, {Src0, Src1});
720 return CGF.
Builder.CreateCall(F, {Src0, Src1});
727 unsigned ConstrainedIntrinsicID) {
732 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
733 if (CGF.
Builder.getIsFPConstrained()) {
735 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0, Src1, Src2 });
738 return CGF.
Builder.CreateCall(F, { Src0, Src1, Src2 });
745 unsigned IntrinsicID,
746 unsigned ConstrainedIntrinsicID,
750 if (CGF.
Builder.getIsFPConstrained())
755 if (CGF.
Builder.getIsFPConstrained())
756 return CGF.
Builder.CreateConstrainedFPCall(F, Args);
758 return CGF.
Builder.CreateCall(F, Args);
767 unsigned IntrinsicID,
768 llvm::StringRef Name =
"") {
769 static_assert(N,
"expect non-empty argument");
771 for (
unsigned I = 0; I < N; ++I)
774 return CGF.
Builder.CreateCall(F, Args, Name);
780 unsigned IntrinsicID) {
785 return CGF.
Builder.CreateCall(F, {Src0, Src1});
791 unsigned IntrinsicID,
792 unsigned ConstrainedIntrinsicID) {
796 if (CGF.
Builder.getIsFPConstrained()) {
797 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
799 {ResultType, Src0->getType()});
800 return CGF.
Builder.CreateConstrainedFPCall(F, {Src0});
804 return CGF.
Builder.CreateCall(F, Src0);
809 llvm::Intrinsic::ID IntrinsicID) {
817 llvm::Value *
Call = CGF.
Builder.CreateCall(F, Src0);
819 llvm::Value *Exp = CGF.
Builder.CreateExtractValue(
Call, 1);
827 llvm::Intrinsic::ID IntrinsicID) {
832 llvm::Function *F = CGF.
CGM.
getIntrinsic(IntrinsicID, {Val->getType()});
833 llvm::Value *
Call = CGF.
Builder.CreateCall(F, Val);
835 llvm::Value *SinResult = CGF.
Builder.CreateExtractValue(
Call, 0);
836 llvm::Value *CosResult = CGF.
Builder.CreateExtractValue(
Call, 1);
842 llvm::StoreInst *StoreSin =
844 llvm::StoreInst *StoreCos =
851 MDNode *
Domain = MDHelper.createAnonymousAliasScopeDomain();
852 MDNode *AliasScope = MDHelper.createAnonymousAliasScope(
Domain);
853 MDNode *AliasScopeList = MDNode::get(
Call->getContext(), AliasScope);
854 StoreSin->setMetadata(LLVMContext::MD_alias_scope, AliasScopeList);
855 StoreCos->setMetadata(LLVMContext::MD_noalias, AliasScopeList);
862 Call->setDoesNotAccessMemory();
871 llvm::Type *Ty =
V->getType();
872 int Width = Ty->getPrimitiveSizeInBits();
873 llvm::Type *IntTy = llvm::IntegerType::get(
C, Width);
875 if (Ty->isPPC_FP128Ty()) {
885 Value *ShiftCst = llvm::ConstantInt::get(IntTy, Width);
890 IntTy = llvm::IntegerType::get(
C, Width);
893 Value *Zero = llvm::Constant::getNullValue(IntTy);
894 return CGF.
Builder.CreateICmpSLT(
V, Zero);
903 auto IsIndirect = [&](
ABIArgInfo const &info) {
904 return info.isIndirect() || info.isIndirectAliased() || info.isInAlloca();
909 return IsIndirect(ArgInfo.info);
914 const CallExpr *
E, llvm::Constant *calleeValue) {
915 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
917 llvm::CallBase *callOrInvoke =
nullptr;
921 nullptr, &callOrInvoke, &FnInfo);
926 bool ConstWithoutErrnoAndExceptions =
930 if (ConstWithoutErrnoAndExceptions && CGF.
CGM.
getLangOpts().MathErrno &&
931 !CGF.
Builder.getIsFPConstrained() &&
Call.isScalar() &&
952 const llvm::Intrinsic::ID IntrinsicID,
953 llvm::Value *
X, llvm::Value *Y,
954 llvm::Value *&Carry) {
956 assert(
X->getType() == Y->getType() &&
957 "Arguments must be the same type. (Did you forget to make sure both "
958 "arguments have the same integer width?)");
961 llvm::Value *Tmp = CGF.
Builder.CreateCall(Callee, {
X, Y});
962 Carry = CGF.
Builder.CreateExtractValue(Tmp, 1);
963 return CGF.
Builder.CreateExtractValue(Tmp, 0);
970 llvm::ConstantRange CR(APInt(32, low), APInt(32, high));
971 Call->addRangeRetAttr(CR);
972 Call->addRetAttr(llvm::Attribute::AttrKind::NoUndef);
977 struct WidthAndSignedness {
983static WidthAndSignedness
995static struct WidthAndSignedness
997 assert(Types.size() > 0 &&
"Empty list of types.");
1001 for (
const auto &
Type : Types) {
1010 for (
const auto &
Type : Types) {
1012 if (Width < MinWidth) {
1021 Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend;
1032 return From == To || (From == 0 && To == 1) || (From == 3 && To == 2);
1037 return ConstantInt::get(ResType, (
Type & 2) ? 0 : -1,
true);
1041CodeGenFunction::evaluateOrEmitBuiltinObjectSize(
const Expr *
E,
unsigned Type,
1042 llvm::IntegerType *ResType,
1043 llvm::Value *EmittedE,
1047 return emitBuiltinObjectSize(
E,
Type, ResType, EmittedE, IsDynamic);
1048 return ConstantInt::get(ResType, ObjectSize,
true);
1062 if ((!FAMDecl || FD == FAMDecl) &&
1064 Ctx, FD, FD->getType(), StrictFlexArraysLevel,
1092 if (FD->getType()->isCountAttributedType())
1104CodeGenFunction::emitFlexibleArrayMemberSize(
const Expr *
E,
unsigned Type,
1105 llvm::IntegerType *ResType) {
1134 const Expr *Idx =
nullptr;
1136 if (
const auto *UO = dyn_cast<UnaryOperator>(
Base);
1137 UO && UO->getOpcode() == UO_AddrOf) {
1139 if (
const auto *ASE = dyn_cast<ArraySubscriptExpr>(SubExpr)) {
1140 Base = ASE->getBase()->IgnoreParenImpCasts();
1143 if (
const auto *IL = dyn_cast<IntegerLiteral>(Idx)) {
1144 int64_t Val = IL->getValue().getSExtValue();
1161 if (
const auto *ME = dyn_cast<MemberExpr>(
Base)) {
1163 const ValueDecl *VD = ME->getMemberDecl();
1165 FAMDecl = dyn_cast<FieldDecl>(VD);
1168 }
else if (
const auto *DRE = dyn_cast<DeclRefExpr>(
Base)) {
1170 QualType Ty = DRE->getDecl()->getType();
1223 if (isa<DeclRefExpr>(
Base))
1247 CountedByInst =
Builder.CreateIntCast(CountedByInst, ResType, IsSigned);
1250 Value *IdxInst =
nullptr;
1258 IdxInst =
Builder.CreateIntCast(IdxInst, ResType, IdxSigned);
1263 Builder.CreateSub(CountedByInst, IdxInst,
"", !IsSigned, IsSigned);
1269 llvm::Constant *ElemSize =
1270 llvm::ConstantInt::get(ResType,
Size.getQuantity(), IsSigned);
1272 Builder.CreateMul(CountedByInst, ElemSize,
"", !IsSigned, IsSigned);
1273 Res =
Builder.CreateIntCast(Res, ResType, IsSigned);
1282 return Builder.CreateSelect(Cmp, Res, ConstantInt::get(ResType, 0, IsSigned));
1295CodeGenFunction::emitBuiltinObjectSize(
const Expr *
E,
unsigned Type,
1296 llvm::IntegerType *ResType,
1297 llvm::Value *EmittedE,
bool IsDynamic) {
1301 auto *Param = dyn_cast<ParmVarDecl>(
D->getDecl());
1302 auto *PS =
D->getDecl()->
getAttr<PassObjectSizeAttr>();
1303 if (Param !=
nullptr && PS !=
nullptr &&
1305 auto Iter = SizeArguments.find(Param);
1306 assert(
Iter != SizeArguments.end());
1309 auto DIter = LocalDeclMap.find(
D);
1310 assert(DIter != LocalDeclMap.end());
1320 if (
Value *
V = emitFlexibleArrayMemberSize(
E,
Type, ResType))
1331 assert(Ptr->
getType()->isPointerTy() &&
1332 "Non-pointer passed to __builtin_object_size?");
1348 enum ActionKind : uint8_t { TestOnly, Complement, Reset,
Set };
1349 enum InterlockingKind : uint8_t {
1358 InterlockingKind Interlocking;
1361 static BitTest decodeBitTestBuiltin(
unsigned BuiltinID);
1366BitTest BitTest::decodeBitTestBuiltin(
unsigned BuiltinID) {
1367 switch (BuiltinID) {
1369 case Builtin::BI_bittest:
1370 return {TestOnly, Unlocked,
false};
1371 case Builtin::BI_bittestandcomplement:
1372 return {Complement, Unlocked,
false};
1373 case Builtin::BI_bittestandreset:
1374 return {Reset, Unlocked,
false};
1375 case Builtin::BI_bittestandset:
1376 return {
Set, Unlocked,
false};
1377 case Builtin::BI_interlockedbittestandreset:
1378 return {Reset, Sequential,
false};
1379 case Builtin::BI_interlockedbittestandset:
1380 return {
Set, Sequential,
false};
1383 case Builtin::BI_bittest64:
1384 return {TestOnly, Unlocked,
true};
1385 case Builtin::BI_bittestandcomplement64:
1386 return {Complement, Unlocked,
true};
1387 case Builtin::BI_bittestandreset64:
1388 return {Reset, Unlocked,
true};
1389 case Builtin::BI_bittestandset64:
1390 return {
Set, Unlocked,
true};
1391 case Builtin::BI_interlockedbittestandreset64:
1392 return {Reset, Sequential,
true};
1393 case Builtin::BI_interlockedbittestandset64:
1394 return {
Set, Sequential,
true};
1397 case Builtin::BI_interlockedbittestandset_acq:
1398 return {
Set, Acquire,
false};
1399 case Builtin::BI_interlockedbittestandset_rel:
1400 return {
Set, Release,
false};
1401 case Builtin::BI_interlockedbittestandset_nf:
1402 return {
Set, NoFence,
false};
1403 case Builtin::BI_interlockedbittestandreset_acq:
1404 return {Reset, Acquire,
false};
1405 case Builtin::BI_interlockedbittestandreset_rel:
1406 return {Reset, Release,
false};
1407 case Builtin::BI_interlockedbittestandreset_nf:
1408 return {Reset, NoFence,
false};
1410 llvm_unreachable(
"expected only bittest intrinsics");
1415 case BitTest::TestOnly:
return '\0';
1416 case BitTest::Complement:
return 'c';
1417 case BitTest::Reset:
return 'r';
1418 case BitTest::Set:
return 's';
1420 llvm_unreachable(
"invalid action");
1428 char SizeSuffix = BT.Is64Bit ?
'q' :
'l';
1432 raw_svector_ostream AsmOS(
Asm);
1433 if (BT.Interlocking != BitTest::Unlocked)
1438 AsmOS << SizeSuffix <<
" $2, ($1)";
1441 std::string Constraints =
"={@ccc},r,r,~{cc},~{memory}";
1443 if (!MachineClobbers.empty()) {
1445 Constraints += MachineClobbers;
1447 llvm::IntegerType *IntType = llvm::IntegerType::get(
1450 llvm::FunctionType *FTy =
1451 llvm::FunctionType::get(CGF.
Int8Ty, {CGF.UnqualPtrTy, IntType},
false);
1453 llvm::InlineAsm *IA =
1454 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
1455 return CGF.
Builder.CreateCall(IA, {BitBase, BitPos});
1458static llvm::AtomicOrdering
1461 case BitTest::Unlocked:
return llvm::AtomicOrdering::NotAtomic;
1462 case BitTest::Sequential:
return llvm::AtomicOrdering::SequentiallyConsistent;
1463 case BitTest::Acquire:
return llvm::AtomicOrdering::Acquire;
1464 case BitTest::Release:
return llvm::AtomicOrdering::Release;
1465 case BitTest::NoFence:
return llvm::AtomicOrdering::Monotonic;
1467 llvm_unreachable(
"invalid interlocking");
1480 BitTest BT = BitTest::decodeBitTestBuiltin(BuiltinID);
1492 BitPos, llvm::ConstantInt::get(BitPos->
getType(), 3),
"bittest.byteidx");
1494 "bittest.byteaddr"),
1498 llvm::ConstantInt::get(CGF.
Int8Ty, 0x7));
1501 Value *Mask =
nullptr;
1502 if (BT.Action != BitTest::TestOnly) {
1503 Mask = CGF.
Builder.CreateShl(llvm::ConstantInt::get(CGF.
Int8Ty, 1), PosLow,
1510 Value *OldByte =
nullptr;
1511 if (Ordering != llvm::AtomicOrdering::NotAtomic) {
1514 llvm::AtomicRMWInst::BinOp RMWOp = llvm::AtomicRMWInst::Or;
1515 if (BT.Action == BitTest::Reset) {
1516 Mask = CGF.
Builder.CreateNot(Mask);
1517 RMWOp = llvm::AtomicRMWInst::And;
1523 Value *NewByte =
nullptr;
1524 switch (BT.Action) {
1525 case BitTest::TestOnly:
1528 case BitTest::Complement:
1529 NewByte = CGF.
Builder.CreateXor(OldByte, Mask);
1531 case BitTest::Reset:
1532 NewByte = CGF.
Builder.CreateAnd(OldByte, CGF.
Builder.CreateNot(Mask));
1535 NewByte = CGF.
Builder.CreateOr(OldByte, Mask);
1544 Value *ShiftedByte = CGF.
Builder.CreateLShr(OldByte, PosLow,
"bittest.shr");
1546 ShiftedByte, llvm::ConstantInt::get(CGF.
Int8Ty, 1),
"bittest.res");
1555 raw_svector_ostream AsmOS(
Asm);
1556 llvm::IntegerType *RetType = CGF.
Int32Ty;
1558 switch (BuiltinID) {
1559 case clang::PPC::BI__builtin_ppc_ldarx:
1563 case clang::PPC::BI__builtin_ppc_lwarx:
1567 case clang::PPC::BI__builtin_ppc_lharx:
1571 case clang::PPC::BI__builtin_ppc_lbarx:
1576 llvm_unreachable(
"Expected only PowerPC load reserve intrinsics");
1579 AsmOS <<
"$0, ${1:y}";
1581 std::string Constraints =
"=r,*Z,~{memory}";
1583 if (!MachineClobbers.empty()) {
1585 Constraints += MachineClobbers;
1589 llvm::FunctionType *FTy = llvm::FunctionType::get(RetType, {PtrType},
false);
1591 llvm::InlineAsm *IA =
1592 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
1593 llvm::CallInst *CI = CGF.
Builder.CreateCall(IA, {Addr});
1595 0, Attribute::get(CGF.
getLLVMContext(), Attribute::ElementType, RetType));
1600enum class MSVCSetJmpKind {
1612 llvm::Value *Arg1 =
nullptr;
1613 llvm::Type *Arg1Ty =
nullptr;
1615 bool IsVarArg =
false;
1616 if (SJKind == MSVCSetJmpKind::_setjmp3) {
1619 Arg1 = llvm::ConstantInt::get(CGF.
IntTy, 0);
1622 Name = SJKind == MSVCSetJmpKind::_setjmp ?
"_setjmp" :
"_setjmpex";
1625 Arg1 = CGF.
Builder.CreateCall(
1628 Arg1 = CGF.
Builder.CreateCall(
1630 llvm::ConstantInt::get(CGF.
Int32Ty, 0));
1634 llvm::Type *ArgTypes[2] = {CGF.
Int8PtrTy, Arg1Ty};
1635 llvm::AttributeList ReturnsTwiceAttr = llvm::AttributeList::get(
1637 llvm::Attribute::ReturnsTwice);
1639 llvm::FunctionType::get(CGF.
IntTy, ArgTypes, IsVarArg), Name,
1640 ReturnsTwiceAttr,
true);
1642 llvm::Value *Buf = CGF.
Builder.CreateBitOrPointerCast(
1644 llvm::Value *Args[] = {Buf, Arg1};
1646 CB->setAttributes(ReturnsTwiceAttr);
1695static std::optional<CodeGenFunction::MSVCIntrin>
1698 switch (BuiltinID) {
1700 return std::nullopt;
1701 case clang::ARM::BI_BitScanForward:
1702 case clang::ARM::BI_BitScanForward64:
1703 return MSVCIntrin::_BitScanForward;
1704 case clang::ARM::BI_BitScanReverse:
1705 case clang::ARM::BI_BitScanReverse64:
1706 return MSVCIntrin::_BitScanReverse;
1707 case clang::ARM::BI_InterlockedAnd64:
1708 return MSVCIntrin::_InterlockedAnd;
1709 case clang::ARM::BI_InterlockedExchange64:
1710 return MSVCIntrin::_InterlockedExchange;
1711 case clang::ARM::BI_InterlockedExchangeAdd64:
1712 return MSVCIntrin::_InterlockedExchangeAdd;
1713 case clang::ARM::BI_InterlockedExchangeSub64:
1714 return MSVCIntrin::_InterlockedExchangeSub;
1715 case clang::ARM::BI_InterlockedOr64:
1716 return MSVCIntrin::_InterlockedOr;
1717 case clang::ARM::BI_InterlockedXor64:
1718 return MSVCIntrin::_InterlockedXor;
1719 case clang::ARM::BI_InterlockedDecrement64:
1720 return MSVCIntrin::_InterlockedDecrement;
1721 case clang::ARM::BI_InterlockedIncrement64:
1722 return MSVCIntrin::_InterlockedIncrement;
1723 case clang::ARM::BI_InterlockedExchangeAdd8_acq:
1724 case clang::ARM::BI_InterlockedExchangeAdd16_acq:
1725 case clang::ARM::BI_InterlockedExchangeAdd_acq:
1726 case clang::ARM::BI_InterlockedExchangeAdd64_acq:
1727 return MSVCIntrin::_InterlockedExchangeAdd_acq;
1728 case clang::ARM::BI_InterlockedExchangeAdd8_rel:
1729 case clang::ARM::BI_InterlockedExchangeAdd16_rel:
1730 case clang::ARM::BI_InterlockedExchangeAdd_rel:
1731 case clang::ARM::BI_InterlockedExchangeAdd64_rel:
1732 return MSVCIntrin::_InterlockedExchangeAdd_rel;
1733 case clang::ARM::BI_InterlockedExchangeAdd8_nf:
1734 case clang::ARM::BI_InterlockedExchangeAdd16_nf:
1735 case clang::ARM::BI_InterlockedExchangeAdd_nf:
1736 case clang::ARM::BI_InterlockedExchangeAdd64_nf:
1737 return MSVCIntrin::_InterlockedExchangeAdd_nf;
1738 case clang::ARM::BI_InterlockedExchange8_acq:
1739 case clang::ARM::BI_InterlockedExchange16_acq:
1740 case clang::ARM::BI_InterlockedExchange_acq:
1741 case clang::ARM::BI_InterlockedExchange64_acq:
1742 case clang::ARM::BI_InterlockedExchangePointer_acq:
1743 return MSVCIntrin::_InterlockedExchange_acq;
1744 case clang::ARM::BI_InterlockedExchange8_rel:
1745 case clang::ARM::BI_InterlockedExchange16_rel:
1746 case clang::ARM::BI_InterlockedExchange_rel:
1747 case clang::ARM::BI_InterlockedExchange64_rel:
1748 case clang::ARM::BI_InterlockedExchangePointer_rel:
1749 return MSVCIntrin::_InterlockedExchange_rel;
1750 case clang::ARM::BI_InterlockedExchange8_nf:
1751 case clang::ARM::BI_InterlockedExchange16_nf:
1752 case clang::ARM::BI_InterlockedExchange_nf:
1753 case clang::ARM::BI_InterlockedExchange64_nf:
1754 case clang::ARM::BI_InterlockedExchangePointer_nf:
1755 return MSVCIntrin::_InterlockedExchange_nf;
1756 case clang::ARM::BI_InterlockedCompareExchange8_acq:
1757 case clang::ARM::BI_InterlockedCompareExchange16_acq:
1758 case clang::ARM::BI_InterlockedCompareExchange_acq:
1759 case clang::ARM::BI_InterlockedCompareExchange64_acq:
1760 case clang::ARM::BI_InterlockedCompareExchangePointer_acq:
1761 return MSVCIntrin::_InterlockedCompareExchange_acq;
1762 case clang::ARM::BI_InterlockedCompareExchange8_rel:
1763 case clang::ARM::BI_InterlockedCompareExchange16_rel:
1764 case clang::ARM::BI_InterlockedCompareExchange_rel:
1765 case clang::ARM::BI_InterlockedCompareExchange64_rel:
1766 case clang::ARM::BI_InterlockedCompareExchangePointer_rel:
1767 return MSVCIntrin::_InterlockedCompareExchange_rel;
1768 case clang::ARM::BI_InterlockedCompareExchange8_nf:
1769 case clang::ARM::BI_InterlockedCompareExchange16_nf:
1770 case clang::ARM::BI_InterlockedCompareExchange_nf:
1771 case clang::ARM::BI_InterlockedCompareExchange64_nf:
1772 return MSVCIntrin::_InterlockedCompareExchange_nf;
1773 case clang::ARM::BI_InterlockedOr8_acq:
1774 case clang::ARM::BI_InterlockedOr16_acq:
1775 case clang::ARM::BI_InterlockedOr_acq:
1776 case clang::ARM::BI_InterlockedOr64_acq:
1777 return MSVCIntrin::_InterlockedOr_acq;
1778 case clang::ARM::BI_InterlockedOr8_rel:
1779 case clang::ARM::BI_InterlockedOr16_rel:
1780 case clang::ARM::BI_InterlockedOr_rel:
1781 case clang::ARM::BI_InterlockedOr64_rel:
1782 return MSVCIntrin::_InterlockedOr_rel;
1783 case clang::ARM::BI_InterlockedOr8_nf:
1784 case clang::ARM::BI_InterlockedOr16_nf:
1785 case clang::ARM::BI_InterlockedOr_nf:
1786 case clang::ARM::BI_InterlockedOr64_nf:
1787 return MSVCIntrin::_InterlockedOr_nf;
1788 case clang::ARM::BI_InterlockedXor8_acq:
1789 case clang::ARM::BI_InterlockedXor16_acq:
1790 case clang::ARM::BI_InterlockedXor_acq:
1791 case clang::ARM::BI_InterlockedXor64_acq:
1792 return MSVCIntrin::_InterlockedXor_acq;
1793 case clang::ARM::BI_InterlockedXor8_rel:
1794 case clang::ARM::BI_InterlockedXor16_rel:
1795 case clang::ARM::BI_InterlockedXor_rel:
1796 case clang::ARM::BI_InterlockedXor64_rel:
1797 return MSVCIntrin::_InterlockedXor_rel;
1798 case clang::ARM::BI_InterlockedXor8_nf:
1799 case clang::ARM::BI_InterlockedXor16_nf:
1800 case clang::ARM::BI_InterlockedXor_nf:
1801 case clang::ARM::BI_InterlockedXor64_nf:
1802 return MSVCIntrin::_InterlockedXor_nf;
1803 case clang::ARM::BI_InterlockedAnd8_acq:
1804 case clang::ARM::BI_InterlockedAnd16_acq:
1805 case clang::ARM::BI_InterlockedAnd_acq:
1806 case clang::ARM::BI_InterlockedAnd64_acq:
1807 return MSVCIntrin::_InterlockedAnd_acq;
1808 case clang::ARM::BI_InterlockedAnd8_rel:
1809 case clang::ARM::BI_InterlockedAnd16_rel:
1810 case clang::ARM::BI_InterlockedAnd_rel:
1811 case clang::ARM::BI_InterlockedAnd64_rel:
1812 return MSVCIntrin::_InterlockedAnd_rel;
1813 case clang::ARM::BI_InterlockedAnd8_nf:
1814 case clang::ARM::BI_InterlockedAnd16_nf:
1815 case clang::ARM::BI_InterlockedAnd_nf:
1816 case clang::ARM::BI_InterlockedAnd64_nf:
1817 return MSVCIntrin::_InterlockedAnd_nf;
1818 case clang::ARM::BI_InterlockedIncrement16_acq:
1819 case clang::ARM::BI_InterlockedIncrement_acq:
1820 case clang::ARM::BI_InterlockedIncrement64_acq:
1821 return MSVCIntrin::_InterlockedIncrement_acq;
1822 case clang::ARM::BI_InterlockedIncrement16_rel:
1823 case clang::ARM::BI_InterlockedIncrement_rel:
1824 case clang::ARM::BI_InterlockedIncrement64_rel:
1825 return MSVCIntrin::_InterlockedIncrement_rel;
1826 case clang::ARM::BI_InterlockedIncrement16_nf:
1827 case clang::ARM::BI_InterlockedIncrement_nf:
1828 case clang::ARM::BI_InterlockedIncrement64_nf:
1829 return MSVCIntrin::_InterlockedIncrement_nf;
1830 case clang::ARM::BI_InterlockedDecrement16_acq:
1831 case clang::ARM::BI_InterlockedDecrement_acq:
1832 case clang::ARM::BI_InterlockedDecrement64_acq:
1833 return MSVCIntrin::_InterlockedDecrement_acq;
1834 case clang::ARM::BI_InterlockedDecrement16_rel:
1835 case clang::ARM::BI_InterlockedDecrement_rel:
1836 case clang::ARM::BI_InterlockedDecrement64_rel:
1837 return MSVCIntrin::_InterlockedDecrement_rel;
1838 case clang::ARM::BI_InterlockedDecrement16_nf:
1839 case clang::ARM::BI_InterlockedDecrement_nf:
1840 case clang::ARM::BI_InterlockedDecrement64_nf:
1841 return MSVCIntrin::_InterlockedDecrement_nf;
1843 llvm_unreachable(
"must return from switch");
1846static std::optional<CodeGenFunction::MSVCIntrin>
1849 switch (BuiltinID) {
1851 return std::nullopt;
1852 case clang::AArch64::BI_BitScanForward:
1853 case clang::AArch64::BI_BitScanForward64:
1854 return MSVCIntrin::_BitScanForward;
1855 case clang::AArch64::BI_BitScanReverse:
1856 case clang::AArch64::BI_BitScanReverse64:
1857 return MSVCIntrin::_BitScanReverse;
1858 case clang::AArch64::BI_InterlockedAnd64:
1859 return MSVCIntrin::_InterlockedAnd;
1860 case clang::AArch64::BI_InterlockedExchange64:
1861 return MSVCIntrin::_InterlockedExchange;
1862 case clang::AArch64::BI_InterlockedExchangeAdd64:
1863 return MSVCIntrin::_InterlockedExchangeAdd;
1864 case clang::AArch64::BI_InterlockedExchangeSub64:
1865 return MSVCIntrin::_InterlockedExchangeSub;
1866 case clang::AArch64::BI_InterlockedOr64:
1867 return MSVCIntrin::_InterlockedOr;
1868 case clang::AArch64::BI_InterlockedXor64:
1869 return MSVCIntrin::_InterlockedXor;
1870 case clang::AArch64::BI_InterlockedDecrement64:
1871 return MSVCIntrin::_InterlockedDecrement;
1872 case clang::AArch64::BI_InterlockedIncrement64:
1873 return MSVCIntrin::_InterlockedIncrement;
1874 case clang::AArch64::BI_InterlockedExchangeAdd8_acq:
1875 case clang::AArch64::BI_InterlockedExchangeAdd16_acq:
1876 case clang::AArch64::BI_InterlockedExchangeAdd_acq:
1877 case clang::AArch64::BI_InterlockedExchangeAdd64_acq:
1878 return MSVCIntrin::_InterlockedExchangeAdd_acq;
1879 case clang::AArch64::BI_InterlockedExchangeAdd8_rel:
1880 case clang::AArch64::BI_InterlockedExchangeAdd16_rel:
1881 case clang::AArch64::BI_InterlockedExchangeAdd_rel:
1882 case clang::AArch64::BI_InterlockedExchangeAdd64_rel:
1883 return MSVCIntrin::_InterlockedExchangeAdd_rel;
1884 case clang::AArch64::BI_InterlockedExchangeAdd8_nf:
1885 case clang::AArch64::BI_InterlockedExchangeAdd16_nf:
1886 case clang::AArch64::BI_InterlockedExchangeAdd_nf:
1887 case clang::AArch64::BI_InterlockedExchangeAdd64_nf:
1888 return MSVCIntrin::_InterlockedExchangeAdd_nf;
1889 case clang::AArch64::BI_InterlockedExchange8_acq:
1890 case clang::AArch64::BI_InterlockedExchange16_acq:
1891 case clang::AArch64::BI_InterlockedExchange_acq:
1892 case clang::AArch64::BI_InterlockedExchange64_acq:
1893 case clang::AArch64::BI_InterlockedExchangePointer_acq:
1894 return MSVCIntrin::_InterlockedExchange_acq;
1895 case clang::AArch64::BI_InterlockedExchange8_rel:
1896 case clang::AArch64::BI_InterlockedExchange16_rel:
1897 case clang::AArch64::BI_InterlockedExchange_rel:
1898 case clang::AArch64::BI_InterlockedExchange64_rel:
1899 case clang::AArch64::BI_InterlockedExchangePointer_rel:
1900 return MSVCIntrin::_InterlockedExchange_rel;
1901 case clang::AArch64::BI_InterlockedExchange8_nf:
1902 case clang::AArch64::BI_InterlockedExchange16_nf:
1903 case clang::AArch64::BI_InterlockedExchange_nf:
1904 case clang::AArch64::BI_InterlockedExchange64_nf:
1905 case clang::AArch64::BI_InterlockedExchangePointer_nf:
1906 return MSVCIntrin::_InterlockedExchange_nf;
1907 case clang::AArch64::BI_InterlockedCompareExchange8_acq:
1908 case clang::AArch64::BI_InterlockedCompareExchange16_acq:
1909 case clang::AArch64::BI_InterlockedCompareExchange_acq:
1910 case clang::AArch64::BI_InterlockedCompareExchange64_acq:
1911 case clang::AArch64::BI_InterlockedCompareExchangePointer_acq:
1912 return MSVCIntrin::_InterlockedCompareExchange_acq;
1913 case clang::AArch64::BI_InterlockedCompareExchange8_rel:
1914 case clang::AArch64::BI_InterlockedCompareExchange16_rel:
1915 case clang::AArch64::BI_InterlockedCompareExchange_rel:
1916 case clang::AArch64::BI_InterlockedCompareExchange64_rel:
1917 case clang::AArch64::BI_InterlockedCompareExchangePointer_rel:
1918 return MSVCIntrin::_InterlockedCompareExchange_rel;
1919 case clang::AArch64::BI_InterlockedCompareExchange8_nf:
1920 case clang::AArch64::BI_InterlockedCompareExchange16_nf:
1921 case clang::AArch64::BI_InterlockedCompareExchange_nf:
1922 case clang::AArch64::BI_InterlockedCompareExchange64_nf:
1923 return MSVCIntrin::_InterlockedCompareExchange_nf;
1924 case clang::AArch64::BI_InterlockedCompareExchange128:
1925 return MSVCIntrin::_InterlockedCompareExchange128;
1926 case clang::AArch64::BI_InterlockedCompareExchange128_acq:
1927 return MSVCIntrin::_InterlockedCompareExchange128_acq;
1928 case clang::AArch64::BI_InterlockedCompareExchange128_nf:
1929 return MSVCIntrin::_InterlockedCompareExchange128_nf;
1930 case clang::AArch64::BI_InterlockedCompareExchange128_rel:
1931 return MSVCIntrin::_InterlockedCompareExchange128_rel;
1932 case clang::AArch64::BI_InterlockedOr8_acq:
1933 case clang::AArch64::BI_InterlockedOr16_acq:
1934 case clang::AArch64::BI_InterlockedOr_acq:
1935 case clang::AArch64::BI_InterlockedOr64_acq:
1936 return MSVCIntrin::_InterlockedOr_acq;
1937 case clang::AArch64::BI_InterlockedOr8_rel:
1938 case clang::AArch64::BI_InterlockedOr16_rel:
1939 case clang::AArch64::BI_InterlockedOr_rel:
1940 case clang::AArch64::BI_InterlockedOr64_rel:
1941 return MSVCIntrin::_InterlockedOr_rel;
1942 case clang::AArch64::BI_InterlockedOr8_nf:
1943 case clang::AArch64::BI_InterlockedOr16_nf:
1944 case clang::AArch64::BI_InterlockedOr_nf:
1945 case clang::AArch64::BI_InterlockedOr64_nf:
1946 return MSVCIntrin::_InterlockedOr_nf;
1947 case clang::AArch64::BI_InterlockedXor8_acq:
1948 case clang::AArch64::BI_InterlockedXor16_acq:
1949 case clang::AArch64::BI_InterlockedXor_acq:
1950 case clang::AArch64::BI_InterlockedXor64_acq:
1951 return MSVCIntrin::_InterlockedXor_acq;
1952 case clang::AArch64::BI_InterlockedXor8_rel:
1953 case clang::AArch64::BI_InterlockedXor16_rel:
1954 case clang::AArch64::BI_InterlockedXor_rel:
1955 case clang::AArch64::BI_InterlockedXor64_rel:
1956 return MSVCIntrin::_InterlockedXor_rel;
1957 case clang::AArch64::BI_InterlockedXor8_nf:
1958 case clang::AArch64::BI_InterlockedXor16_nf:
1959 case clang::AArch64::BI_InterlockedXor_nf:
1960 case clang::AArch64::BI_InterlockedXor64_nf:
1961 return MSVCIntrin::_InterlockedXor_nf;
1962 case clang::AArch64::BI_InterlockedAnd8_acq:
1963 case clang::AArch64::BI_InterlockedAnd16_acq:
1964 case clang::AArch64::BI_InterlockedAnd_acq:
1965 case clang::AArch64::BI_InterlockedAnd64_acq:
1966 return MSVCIntrin::_InterlockedAnd_acq;
1967 case clang::AArch64::BI_InterlockedAnd8_rel:
1968 case clang::AArch64::BI_InterlockedAnd16_rel:
1969 case clang::AArch64::BI_InterlockedAnd_rel:
1970 case clang::AArch64::BI_InterlockedAnd64_rel:
1971 return MSVCIntrin::_InterlockedAnd_rel;
1972 case clang::AArch64::BI_InterlockedAnd8_nf:
1973 case clang::AArch64::BI_InterlockedAnd16_nf:
1974 case clang::AArch64::BI_InterlockedAnd_nf:
1975 case clang::AArch64::BI_InterlockedAnd64_nf:
1976 return MSVCIntrin::_InterlockedAnd_nf;
1977 case clang::AArch64::BI_InterlockedIncrement16_acq:
1978 case clang::AArch64::BI_InterlockedIncrement_acq:
1979 case clang::AArch64::BI_InterlockedIncrement64_acq:
1980 return MSVCIntrin::_InterlockedIncrement_acq;
1981 case clang::AArch64::BI_InterlockedIncrement16_rel:
1982 case clang::AArch64::BI_InterlockedIncrement_rel:
1983 case clang::AArch64::BI_InterlockedIncrement64_rel:
1984 return MSVCIntrin::_InterlockedIncrement_rel;
1985 case clang::AArch64::BI_InterlockedIncrement16_nf:
1986 case clang::AArch64::BI_InterlockedIncrement_nf:
1987 case clang::AArch64::BI_InterlockedIncrement64_nf:
1988 return MSVCIntrin::_InterlockedIncrement_nf;
1989 case clang::AArch64::BI_InterlockedDecrement16_acq:
1990 case clang::AArch64::BI_InterlockedDecrement_acq:
1991 case clang::AArch64::BI_InterlockedDecrement64_acq:
1992 return MSVCIntrin::_InterlockedDecrement_acq;
1993 case clang::AArch64::BI_InterlockedDecrement16_rel:
1994 case clang::AArch64::BI_InterlockedDecrement_rel:
1995 case clang::AArch64::BI_InterlockedDecrement64_rel:
1996 return MSVCIntrin::_InterlockedDecrement_rel;
1997 case clang::AArch64::BI_InterlockedDecrement16_nf:
1998 case clang::AArch64::BI_InterlockedDecrement_nf:
1999 case clang::AArch64::BI_InterlockedDecrement64_nf:
2000 return MSVCIntrin::_InterlockedDecrement_nf;
2002 llvm_unreachable(
"must return from switch");
2005static std::optional<CodeGenFunction::MSVCIntrin>
2008 switch (BuiltinID) {
2010 return std::nullopt;
2011 case clang::X86::BI_BitScanForward:
2012 case clang::X86::BI_BitScanForward64:
2013 return MSVCIntrin::_BitScanForward;
2014 case clang::X86::BI_BitScanReverse:
2015 case clang::X86::BI_BitScanReverse64:
2016 return MSVCIntrin::_BitScanReverse;
2017 case clang::X86::BI_InterlockedAnd64:
2018 return MSVCIntrin::_InterlockedAnd;
2019 case clang::X86::BI_InterlockedCompareExchange128:
2020 return MSVCIntrin::_InterlockedCompareExchange128;
2021 case clang::X86::BI_InterlockedExchange64:
2022 return MSVCIntrin::_InterlockedExchange;
2023 case clang::X86::BI_InterlockedExchangeAdd64:
2024 return MSVCIntrin::_InterlockedExchangeAdd;
2025 case clang::X86::BI_InterlockedExchangeSub64:
2026 return MSVCIntrin::_InterlockedExchangeSub;
2027 case clang::X86::BI_InterlockedOr64:
2028 return MSVCIntrin::_InterlockedOr;
2029 case clang::X86::BI_InterlockedXor64:
2030 return MSVCIntrin::_InterlockedXor;
2031 case clang::X86::BI_InterlockedDecrement64:
2032 return MSVCIntrin::_InterlockedDecrement;
2033 case clang::X86::BI_InterlockedIncrement64:
2034 return MSVCIntrin::_InterlockedIncrement;
2036 llvm_unreachable(
"must return from switch");
2042 switch (BuiltinID) {
2043 case MSVCIntrin::_BitScanForward:
2044 case MSVCIntrin::_BitScanReverse: {
2048 llvm::Type *ArgType = ArgValue->
getType();
2049 llvm::Type *IndexType = IndexAddress.getElementType();
2052 Value *ArgZero = llvm::Constant::getNullValue(ArgType);
2053 Value *ResZero = llvm::Constant::getNullValue(ResultType);
2054 Value *ResOne = llvm::ConstantInt::get(ResultType, 1);
2059 PHINode *
Result =
Builder.CreatePHI(ResultType, 2,
"bitscan_result");
2062 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, ArgZero);
2064 Builder.CreateCondBr(IsZero, End, NotZero);
2067 Builder.SetInsertPoint(NotZero);
2069 if (BuiltinID == MSVCIntrin::_BitScanForward) {
2072 ZeroCount =
Builder.CreateIntCast(ZeroCount, IndexType,
false);
2075 unsigned ArgWidth = cast<llvm::IntegerType>(ArgType)->getBitWidth();
2076 Value *ArgTypeLastIndex = llvm::ConstantInt::get(IndexType, ArgWidth - 1);
2080 ZeroCount =
Builder.CreateIntCast(ZeroCount, IndexType,
false);
2081 Value *Index =
Builder.CreateNSWSub(ArgTypeLastIndex, ZeroCount);
2085 Result->addIncoming(ResOne, NotZero);
2090 case MSVCIntrin::_InterlockedAnd:
2092 case MSVCIntrin::_InterlockedExchange:
2094 case MSVCIntrin::_InterlockedExchangeAdd:
2096 case MSVCIntrin::_InterlockedExchangeSub:
2098 case MSVCIntrin::_InterlockedOr:
2100 case MSVCIntrin::_InterlockedXor:
2102 case MSVCIntrin::_InterlockedExchangeAdd_acq:
2104 AtomicOrdering::Acquire);
2105 case MSVCIntrin::_InterlockedExchangeAdd_rel:
2107 AtomicOrdering::Release);
2108 case MSVCIntrin::_InterlockedExchangeAdd_nf:
2110 AtomicOrdering::Monotonic);
2111 case MSVCIntrin::_InterlockedExchange_acq:
2113 AtomicOrdering::Acquire);
2114 case MSVCIntrin::_InterlockedExchange_rel:
2116 AtomicOrdering::Release);
2117 case MSVCIntrin::_InterlockedExchange_nf:
2119 AtomicOrdering::Monotonic);
2120 case MSVCIntrin::_InterlockedCompareExchange:
2122 case MSVCIntrin::_InterlockedCompareExchange_acq:
2124 case MSVCIntrin::_InterlockedCompareExchange_rel:
2126 case MSVCIntrin::_InterlockedCompareExchange_nf:
2128 case MSVCIntrin::_InterlockedCompareExchange128:
2130 *
this,
E, AtomicOrdering::SequentiallyConsistent);
2131 case MSVCIntrin::_InterlockedCompareExchange128_acq:
2133 case MSVCIntrin::_InterlockedCompareExchange128_rel:
2135 case MSVCIntrin::_InterlockedCompareExchange128_nf:
2137 case MSVCIntrin::_InterlockedOr_acq:
2139 AtomicOrdering::Acquire);
2140 case MSVCIntrin::_InterlockedOr_rel:
2142 AtomicOrdering::Release);
2143 case MSVCIntrin::_InterlockedOr_nf:
2145 AtomicOrdering::Monotonic);
2146 case MSVCIntrin::_InterlockedXor_acq:
2148 AtomicOrdering::Acquire);
2149 case MSVCIntrin::_InterlockedXor_rel:
2151 AtomicOrdering::Release);
2152 case MSVCIntrin::_InterlockedXor_nf:
2154 AtomicOrdering::Monotonic);
2155 case MSVCIntrin::_InterlockedAnd_acq:
2157 AtomicOrdering::Acquire);
2158 case MSVCIntrin::_InterlockedAnd_rel:
2160 AtomicOrdering::Release);
2161 case MSVCIntrin::_InterlockedAnd_nf:
2163 AtomicOrdering::Monotonic);
2164 case MSVCIntrin::_InterlockedIncrement_acq:
2166 case MSVCIntrin::_InterlockedIncrement_rel:
2168 case MSVCIntrin::_InterlockedIncrement_nf:
2170 case MSVCIntrin::_InterlockedDecrement_acq:
2172 case MSVCIntrin::_InterlockedDecrement_rel:
2174 case MSVCIntrin::_InterlockedDecrement_nf:
2177 case MSVCIntrin::_InterlockedDecrement:
2179 case MSVCIntrin::_InterlockedIncrement:
2182 case MSVCIntrin::__fastfail: {
2187 StringRef
Asm, Constraints;
2192 case llvm::Triple::x86:
2193 case llvm::Triple::x86_64:
2195 Constraints =
"{cx}";
2197 case llvm::Triple::thumb:
2199 Constraints =
"{r0}";
2201 case llvm::Triple::aarch64:
2202 Asm =
"brk #0xF003";
2203 Constraints =
"{w0}";
2205 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy, {
Int32Ty},
false);
2206 llvm::InlineAsm *IA =
2207 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
2208 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
2210 llvm::Attribute::NoReturn);
2212 CI->setAttributes(NoReturnAttr);
2216 llvm_unreachable(
"Incorrect MSVC intrinsic!");
2222 CallObjCArcUse(llvm::Value *
object) : object(object) {}
2223 llvm::Value *object;
2232 BuiltinCheckKind Kind) {
2234 "Unsupported builtin check kind");
2240 SanitizerScope SanScope(
this);
2242 ArgValue, llvm::Constant::getNullValue(ArgValue->
getType()));
2243 EmitCheck(std::make_pair(Cond, SanitizerKind::Builtin),
2244 SanitizerHandler::InvalidBuiltin,
2246 llvm::ConstantInt::get(
Builder.getInt8Ty(), Kind)},
2256 SanitizerScope SanScope(
this);
2258 std::make_pair(ArgValue, SanitizerKind::Builtin),
2259 SanitizerHandler::InvalidBuiltin,
2267 return CGF.
Builder.CreateBinaryIntrinsic(
2268 Intrinsic::abs, ArgValue,
2269 ConstantInt::get(CGF.
Builder.getInt1Ty(), HasNSW));
2273 bool SanitizeOverflow) {
2277 if (
const auto *VCI = dyn_cast<llvm::ConstantInt>(ArgValue)) {
2278 if (!VCI->isMinSignedValue())
2279 return EmitAbs(CGF, ArgValue,
true);
2282 CodeGenFunction::SanitizerScope SanScope(&CGF);
2284 Constant *Zero = Constant::getNullValue(ArgValue->
getType());
2285 Value *ResultAndOverflow = CGF.
Builder.CreateBinaryIntrinsic(
2286 Intrinsic::ssub_with_overflow, Zero, ArgValue);
2289 CGF.
Builder.CreateExtractValue(ResultAndOverflow, 1));
2292 if (SanitizeOverflow) {
2293 CGF.
EmitCheck({{NotOverflow, SanitizerKind::SignedIntegerOverflow}},
2294 SanitizerHandler::NegateOverflow,
2299 CGF.
EmitTrapCheck(NotOverflow, SanitizerHandler::SubOverflow);
2301 Value *CmpResult = CGF.
Builder.CreateICmpSLT(ArgValue, Zero,
"abscond");
2302 return CGF.
Builder.CreateSelect(CmpResult,
Result, ArgValue,
"abs");
2307 QualType UnsignedTy =
C.getIntTypeForBitwidth(Size * 8,
false);
2308 return C.getCanonicalType(UnsignedTy);
2318 raw_svector_ostream OS(Name);
2319 OS <<
"__os_log_helper";
2323 for (
const auto &Item : Layout.
Items)
2324 OS <<
"_" <<
int(Item.getSizeByte()) <<
"_"
2325 <<
int(Item.getDescriptorByte());
2328 if (llvm::Function *F =
CGM.
getModule().getFunction(Name))
2338 for (
unsigned int I = 0,
E = Layout.
Items.size(); I <
E; ++I) {
2339 char Size = Layout.
Items[I].getSizeByte();
2346 &Ctx.
Idents.
get(std::string(
"arg") + llvm::to_string(I)), ArgTy,
2348 ArgTys.emplace_back(ArgTy);
2359 llvm::Function *
Fn = llvm::Function::Create(
2360 FuncTy, llvm::GlobalValue::LinkOnceODRLinkage, Name, &
CGM.
getModule());
2361 Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);
2364 Fn->setDoesNotThrow();
2368 Fn->addFnAttr(llvm::Attribute::NoInline);
2386 for (
const auto &Item : Layout.
Items) {
2388 Builder.getInt8(Item.getDescriptorByte()),
2391 Builder.getInt8(Item.getSizeByte()),
2395 if (!
Size.getQuantity())
2412 assert(
E.getNumArgs() >= 2 &&
2413 "__builtin_os_log_format takes at least 2 arguments");
2424 for (
const auto &Item : Layout.
Items) {
2425 int Size = Item.getSizeByte();
2429 llvm::Value *ArgVal;
2433 for (
unsigned I = 0,
E = Item.getMaskType().size(); I <
E; ++I)
2434 Val |= ((
uint64_t)Item.getMaskType()[I]) << I * 8;
2435 ArgVal = llvm::Constant::getIntegerValue(
Int64Ty, llvm::APInt(64, Val));
2436 }
else if (
const Expr *TheExpr = Item.getExpr()) {
2442 auto LifetimeExtendObject = [&](
const Expr *
E) {
2450 if (isa<CallExpr>(
E) || isa<ObjCMessageExpr>(
E))
2455 if (TheExpr->getType()->isObjCRetainableType() &&
2456 getLangOpts().ObjCAutoRefCount && LifetimeExtendObject(TheExpr)) {
2458 "Only scalar can be a ObjC retainable type");
2459 if (!isa<Constant>(ArgVal)) {
2473 pushCleanupAfterFullExpr<CallObjCArcUse>(Cleanup, ArgVal);
2477 ArgVal =
Builder.getInt32(Item.getConstValue().getQuantity());
2480 unsigned ArgValSize =
2484 ArgVal =
Builder.CreateBitOrPointerCast(ArgVal,
IntTy);
2500 unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info,
2501 WidthAndSignedness ResultInfo) {
2502 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2503 Op1Info.Width == Op2Info.Width && Op2Info.Width == ResultInfo.Width &&
2504 !Op1Info.Signed && !Op2Info.Signed && ResultInfo.Signed;
2509 const clang::Expr *Op2, WidthAndSignedness Op2Info,
2511 WidthAndSignedness ResultInfo) {
2513 Builtin::BI__builtin_mul_overflow, Op1Info, Op2Info, ResultInfo) &&
2514 "Cannot specialize this multiply");
2519 llvm::Value *HasOverflow;
2521 CGF, llvm::Intrinsic::umul_with_overflow, V1, V2, HasOverflow);
2526 auto IntMax = llvm::APInt::getSignedMaxValue(ResultInfo.Width);
2527 llvm::Value *IntMaxValue = llvm::ConstantInt::get(
Result->getType(), IntMax);
2529 llvm::Value *IntMaxOverflow = CGF.
Builder.CreateICmpUGT(
Result, IntMaxValue);
2530 HasOverflow = CGF.
Builder.CreateOr(HasOverflow, IntMaxOverflow);
2542 WidthAndSignedness Op1Info,
2543 WidthAndSignedness Op2Info,
2544 WidthAndSignedness ResultInfo) {
2545 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2546 std::max(Op1Info.Width, Op2Info.Width) >= ResultInfo.Width &&
2547 Op1Info.Signed != Op2Info.Signed;
2554 WidthAndSignedness Op1Info,
const clang::Expr *Op2,
2555 WidthAndSignedness Op2Info,
2557 WidthAndSignedness ResultInfo) {
2559 Op2Info, ResultInfo) &&
2560 "Not a mixed-sign multipliction we can specialize");
2563 const clang::Expr *SignedOp = Op1Info.Signed ? Op1 : Op2;
2564 const clang::Expr *UnsignedOp = Op1Info.Signed ? Op2 : Op1;
2567 unsigned SignedOpWidth = Op1Info.Signed ? Op1Info.Width : Op2Info.Width;
2568 unsigned UnsignedOpWidth = Op1Info.Signed ? Op2Info.Width : Op1Info.Width;
2571 if (SignedOpWidth < UnsignedOpWidth)
2573 if (UnsignedOpWidth < SignedOpWidth)
2576 llvm::Type *OpTy =
Signed->getType();
2577 llvm::Value *Zero = llvm::Constant::getNullValue(OpTy);
2580 unsigned OpWidth = std::max(Op1Info.Width, Op2Info.Width);
2583 llvm::Value *IsNegative = CGF.
Builder.CreateICmpSLT(
Signed, Zero);
2584 llvm::Value *AbsOfNegative = CGF.
Builder.CreateSub(Zero,
Signed);
2585 llvm::Value *AbsSigned =
2586 CGF.
Builder.CreateSelect(IsNegative, AbsOfNegative,
Signed);
2589 llvm::Value *UnsignedOverflow;
2590 llvm::Value *UnsignedResult =
2594 llvm::Value *Overflow, *
Result;
2595 if (ResultInfo.Signed) {
2599 llvm::APInt::getSignedMaxValue(ResultInfo.Width).zext(OpWidth);
2600 llvm::Value *MaxResult =
2601 CGF.
Builder.CreateAdd(llvm::ConstantInt::get(OpTy, IntMax),
2602 CGF.
Builder.CreateZExt(IsNegative, OpTy));
2603 llvm::Value *SignedOverflow =
2604 CGF.
Builder.CreateICmpUGT(UnsignedResult, MaxResult);
2605 Overflow = CGF.
Builder.CreateOr(UnsignedOverflow, SignedOverflow);
2608 llvm::Value *NegativeResult = CGF.
Builder.CreateNeg(UnsignedResult);
2609 llvm::Value *SignedResult =
2610 CGF.
Builder.CreateSelect(IsNegative, NegativeResult, UnsignedResult);
2614 llvm::Value *Underflow = CGF.
Builder.CreateAnd(
2615 IsNegative, CGF.
Builder.CreateIsNotNull(UnsignedResult));
2616 Overflow = CGF.
Builder.CreateOr(UnsignedOverflow, Underflow);
2617 if (ResultInfo.Width < OpWidth) {
2619 llvm::APInt::getMaxValue(ResultInfo.Width).zext(OpWidth);
2620 llvm::Value *TruncOverflow = CGF.
Builder.CreateICmpUGT(
2621 UnsignedResult, llvm::ConstantInt::get(OpTy, IntMax));
2622 Overflow = CGF.
Builder.CreateOr(Overflow, TruncOverflow);
2627 IsNegative, CGF.
Builder.CreateNeg(UnsignedResult), UnsignedResult);
2631 assert(Overflow &&
Result &&
"Missing overflow or result");
2642 llvm::SmallPtrSetImpl<const Decl *> &Seen) {
2651 if (!Seen.insert(
Record).second)
2654 assert(
Record->hasDefinition() &&
2655 "Incomplete types should already be diagnosed");
2657 if (
Record->isDynamicClass())
2682 llvm::Type *Ty = Src->getType();
2683 ShiftAmt =
Builder.CreateIntCast(ShiftAmt, Ty,
false);
2686 unsigned IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2693 switch (BuiltinID) {
2694#define MUTATE_LDBL(func) \
2695 case Builtin::BI__builtin_##func##l: \
2696 return Builtin::BI__builtin_##func##f128;
2765 if (CGF.
Builder.getIsFPConstrained() &&
2766 CGF.
Builder.getDefaultConstrainedExcept() != fp::ebIgnore) {
2778 auto UBF = CGF->
CGM.
getModule().getOrInsertFunction(Name, FnTy);
2781 for (
auto &&FormalTy : FnTy->params())
2782 Args.push_back(llvm::PoisonValue::get(FormalTy));
2791 "Should not codegen for consteval builtins");
2798 !
Result.hasSideEffects()) {
2802 if (
Result.Val.isFloat())
2811 if (
getTarget().getTriple().isPPC64() &&
2812 &
getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad())
2819 const unsigned BuiltinIDIfNoAsmLabel =
2820 FD->
hasAttr<AsmLabelAttr>() ? 0 : BuiltinID;
2822 std::optional<bool> ErrnoOverriden;
2826 if (
E->hasStoredFPFeatures()) {
2828 if (OP.hasMathErrnoOverride())
2829 ErrnoOverriden = OP.getMathErrnoOverride();
2838 bool ErrnoOverridenToFalseWithOpt =
2839 ErrnoOverriden.has_value() && !ErrnoOverriden.value() && !OptNone &&
2857 switch (BuiltinID) {
2858 case Builtin::BI__builtin_fma:
2859 case Builtin::BI__builtin_fmaf:
2860 case Builtin::BI__builtin_fmal:
2861 case Builtin::BI__builtin_fmaf16:
2862 case Builtin::BIfma:
2863 case Builtin::BIfmaf:
2864 case Builtin::BIfmal: {
2866 if (Trip.isGNUEnvironment() || Trip.isOSMSVCRT())
2874 bool ConstWithoutErrnoAndExceptions =
2876 bool ConstWithoutExceptions =
2894 bool ConstWithoutErrnoOrExceptions =
2895 ConstWithoutErrnoAndExceptions || ConstWithoutExceptions;
2896 bool GenerateIntrinsics =
2897 (ConstAlways && !OptNone) ||
2899 !(ErrnoOverriden.has_value() && ErrnoOverriden.value()) && !OptNone);
2900 if (!GenerateIntrinsics) {
2901 GenerateIntrinsics =
2902 ConstWithoutErrnoOrExceptions && !ConstWithoutErrnoAndExceptions;
2903 if (!GenerateIntrinsics)
2904 GenerateIntrinsics =
2905 ConstWithoutErrnoOrExceptions &&
2907 !(ErrnoOverriden.has_value() && ErrnoOverriden.value()) && !OptNone);
2908 if (!GenerateIntrinsics)
2909 GenerateIntrinsics =
2910 ConstWithoutErrnoOrExceptions && ErrnoOverridenToFalseWithOpt;
2912 if (GenerateIntrinsics) {
2913 switch (BuiltinIDIfNoAsmLabel) {
2914 case Builtin::BIacos:
2915 case Builtin::BIacosf:
2916 case Builtin::BIacosl:
2917 case Builtin::BI__builtin_acos:
2918 case Builtin::BI__builtin_acosf:
2919 case Builtin::BI__builtin_acosf16:
2920 case Builtin::BI__builtin_acosl:
2921 case Builtin::BI__builtin_acosf128:
2923 *
this,
E, Intrinsic::acos, Intrinsic::experimental_constrained_acos));
2925 case Builtin::BIasin:
2926 case Builtin::BIasinf:
2927 case Builtin::BIasinl:
2928 case Builtin::BI__builtin_asin:
2929 case Builtin::BI__builtin_asinf:
2930 case Builtin::BI__builtin_asinf16:
2931 case Builtin::BI__builtin_asinl:
2932 case Builtin::BI__builtin_asinf128:
2934 *
this,
E, Intrinsic::asin, Intrinsic::experimental_constrained_asin));
2936 case Builtin::BIatan:
2937 case Builtin::BIatanf:
2938 case Builtin::BIatanl:
2939 case Builtin::BI__builtin_atan:
2940 case Builtin::BI__builtin_atanf:
2941 case Builtin::BI__builtin_atanf16:
2942 case Builtin::BI__builtin_atanl:
2943 case Builtin::BI__builtin_atanf128:
2945 *
this,
E, Intrinsic::atan, Intrinsic::experimental_constrained_atan));
2947 case Builtin::BIatan2:
2948 case Builtin::BIatan2f:
2949 case Builtin::BIatan2l:
2950 case Builtin::BI__builtin_atan2:
2951 case Builtin::BI__builtin_atan2f:
2952 case Builtin::BI__builtin_atan2f16:
2953 case Builtin::BI__builtin_atan2l:
2954 case Builtin::BI__builtin_atan2f128:
2956 *
this,
E, Intrinsic::atan2,
2957 Intrinsic::experimental_constrained_atan2));
2959 case Builtin::BIceil:
2960 case Builtin::BIceilf:
2961 case Builtin::BIceill:
2962 case Builtin::BI__builtin_ceil:
2963 case Builtin::BI__builtin_ceilf:
2964 case Builtin::BI__builtin_ceilf16:
2965 case Builtin::BI__builtin_ceill:
2966 case Builtin::BI__builtin_ceilf128:
2969 Intrinsic::experimental_constrained_ceil));
2971 case Builtin::BIcopysign:
2972 case Builtin::BIcopysignf:
2973 case Builtin::BIcopysignl:
2974 case Builtin::BI__builtin_copysign:
2975 case Builtin::BI__builtin_copysignf:
2976 case Builtin::BI__builtin_copysignf16:
2977 case Builtin::BI__builtin_copysignl:
2978 case Builtin::BI__builtin_copysignf128:
2980 emitBuiltinWithOneOverloadedType<2>(*
this,
E, Intrinsic::copysign));
2982 case Builtin::BIcos:
2983 case Builtin::BIcosf:
2984 case Builtin::BIcosl:
2985 case Builtin::BI__builtin_cos:
2986 case Builtin::BI__builtin_cosf:
2987 case Builtin::BI__builtin_cosf16:
2988 case Builtin::BI__builtin_cosl:
2989 case Builtin::BI__builtin_cosf128:
2992 Intrinsic::experimental_constrained_cos));
2994 case Builtin::BIcosh:
2995 case Builtin::BIcoshf:
2996 case Builtin::BIcoshl:
2997 case Builtin::BI__builtin_cosh:
2998 case Builtin::BI__builtin_coshf:
2999 case Builtin::BI__builtin_coshf16:
3000 case Builtin::BI__builtin_coshl:
3001 case Builtin::BI__builtin_coshf128:
3003 *
this,
E, Intrinsic::cosh, Intrinsic::experimental_constrained_cosh));
3005 case Builtin::BIexp:
3006 case Builtin::BIexpf:
3007 case Builtin::BIexpl:
3008 case Builtin::BI__builtin_exp:
3009 case Builtin::BI__builtin_expf:
3010 case Builtin::BI__builtin_expf16:
3011 case Builtin::BI__builtin_expl:
3012 case Builtin::BI__builtin_expf128:
3015 Intrinsic::experimental_constrained_exp));
3017 case Builtin::BIexp2:
3018 case Builtin::BIexp2f:
3019 case Builtin::BIexp2l:
3020 case Builtin::BI__builtin_exp2:
3021 case Builtin::BI__builtin_exp2f:
3022 case Builtin::BI__builtin_exp2f16:
3023 case Builtin::BI__builtin_exp2l:
3024 case Builtin::BI__builtin_exp2f128:
3027 Intrinsic::experimental_constrained_exp2));
3028 case Builtin::BI__builtin_exp10:
3029 case Builtin::BI__builtin_exp10f:
3030 case Builtin::BI__builtin_exp10f16:
3031 case Builtin::BI__builtin_exp10l:
3032 case Builtin::BI__builtin_exp10f128: {
3034 if (
Builder.getIsFPConstrained())
3037 emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::exp10));
3039 case Builtin::BIfabs:
3040 case Builtin::BIfabsf:
3041 case Builtin::BIfabsl:
3042 case Builtin::BI__builtin_fabs:
3043 case Builtin::BI__builtin_fabsf:
3044 case Builtin::BI__builtin_fabsf16:
3045 case Builtin::BI__builtin_fabsl:
3046 case Builtin::BI__builtin_fabsf128:
3048 emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::fabs));
3050 case Builtin::BIfloor:
3051 case Builtin::BIfloorf:
3052 case Builtin::BIfloorl:
3053 case Builtin::BI__builtin_floor:
3054 case Builtin::BI__builtin_floorf:
3055 case Builtin::BI__builtin_floorf16:
3056 case Builtin::BI__builtin_floorl:
3057 case Builtin::BI__builtin_floorf128:
3060 Intrinsic::experimental_constrained_floor));
3062 case Builtin::BIfma:
3063 case Builtin::BIfmaf:
3064 case Builtin::BIfmal:
3065 case Builtin::BI__builtin_fma:
3066 case Builtin::BI__builtin_fmaf:
3067 case Builtin::BI__builtin_fmaf16:
3068 case Builtin::BI__builtin_fmal:
3069 case Builtin::BI__builtin_fmaf128:
3072 Intrinsic::experimental_constrained_fma));
3074 case Builtin::BIfmax:
3075 case Builtin::BIfmaxf:
3076 case Builtin::BIfmaxl:
3077 case Builtin::BI__builtin_fmax:
3078 case Builtin::BI__builtin_fmaxf:
3079 case Builtin::BI__builtin_fmaxf16:
3080 case Builtin::BI__builtin_fmaxl:
3081 case Builtin::BI__builtin_fmaxf128:
3084 Intrinsic::experimental_constrained_maxnum));
3086 case Builtin::BIfmin:
3087 case Builtin::BIfminf:
3088 case Builtin::BIfminl:
3089 case Builtin::BI__builtin_fmin:
3090 case Builtin::BI__builtin_fminf:
3091 case Builtin::BI__builtin_fminf16:
3092 case Builtin::BI__builtin_fminl:
3093 case Builtin::BI__builtin_fminf128:
3096 Intrinsic::experimental_constrained_minnum));
3098 case Builtin::BIfmaximum_num:
3099 case Builtin::BIfmaximum_numf:
3100 case Builtin::BIfmaximum_numl:
3101 case Builtin::BI__builtin_fmaximum_num:
3102 case Builtin::BI__builtin_fmaximum_numf:
3103 case Builtin::BI__builtin_fmaximum_numf16:
3104 case Builtin::BI__builtin_fmaximum_numl:
3105 case Builtin::BI__builtin_fmaximum_numf128:
3107 emitBuiltinWithOneOverloadedType<2>(*
this,
E, Intrinsic::maximumnum));
3109 case Builtin::BIfminimum_num:
3110 case Builtin::BIfminimum_numf:
3111 case Builtin::BIfminimum_numl:
3112 case Builtin::BI__builtin_fminimum_num:
3113 case Builtin::BI__builtin_fminimum_numf:
3114 case Builtin::BI__builtin_fminimum_numf16:
3115 case Builtin::BI__builtin_fminimum_numl:
3116 case Builtin::BI__builtin_fminimum_numf128:
3118 emitBuiltinWithOneOverloadedType<2>(*
this,
E, Intrinsic::minimumnum));
3122 case Builtin::BIfmod:
3123 case Builtin::BIfmodf:
3124 case Builtin::BIfmodl:
3125 case Builtin::BI__builtin_fmod:
3126 case Builtin::BI__builtin_fmodf:
3127 case Builtin::BI__builtin_fmodf16:
3128 case Builtin::BI__builtin_fmodl:
3129 case Builtin::BI__builtin_fmodf128:
3130 case Builtin::BI__builtin_elementwise_fmod: {
3131 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
3137 case Builtin::BIlog:
3138 case Builtin::BIlogf:
3139 case Builtin::BIlogl:
3140 case Builtin::BI__builtin_log:
3141 case Builtin::BI__builtin_logf:
3142 case Builtin::BI__builtin_logf16:
3143 case Builtin::BI__builtin_logl:
3144 case Builtin::BI__builtin_logf128:
3147 Intrinsic::experimental_constrained_log));
3149 case Builtin::BIlog10:
3150 case Builtin::BIlog10f:
3151 case Builtin::BIlog10l:
3152 case Builtin::BI__builtin_log10:
3153 case Builtin::BI__builtin_log10f:
3154 case Builtin::BI__builtin_log10f16:
3155 case Builtin::BI__builtin_log10l:
3156 case Builtin::BI__builtin_log10f128:
3159 Intrinsic::experimental_constrained_log10));
3161 case Builtin::BIlog2:
3162 case Builtin::BIlog2f:
3163 case Builtin::BIlog2l:
3164 case Builtin::BI__builtin_log2:
3165 case Builtin::BI__builtin_log2f:
3166 case Builtin::BI__builtin_log2f16:
3167 case Builtin::BI__builtin_log2l:
3168 case Builtin::BI__builtin_log2f128:
3171 Intrinsic::experimental_constrained_log2));
3173 case Builtin::BInearbyint:
3174 case Builtin::BInearbyintf:
3175 case Builtin::BInearbyintl:
3176 case Builtin::BI__builtin_nearbyint:
3177 case Builtin::BI__builtin_nearbyintf:
3178 case Builtin::BI__builtin_nearbyintl:
3179 case Builtin::BI__builtin_nearbyintf128:
3181 Intrinsic::nearbyint,
3182 Intrinsic::experimental_constrained_nearbyint));
3184 case Builtin::BIpow:
3185 case Builtin::BIpowf:
3186 case Builtin::BIpowl:
3187 case Builtin::BI__builtin_pow:
3188 case Builtin::BI__builtin_powf:
3189 case Builtin::BI__builtin_powf16:
3190 case Builtin::BI__builtin_powl:
3191 case Builtin::BI__builtin_powf128:
3194 Intrinsic::experimental_constrained_pow));
3196 case Builtin::BIrint:
3197 case Builtin::BIrintf:
3198 case Builtin::BIrintl:
3199 case Builtin::BI__builtin_rint:
3200 case Builtin::BI__builtin_rintf:
3201 case Builtin::BI__builtin_rintf16:
3202 case Builtin::BI__builtin_rintl:
3203 case Builtin::BI__builtin_rintf128:
3206 Intrinsic::experimental_constrained_rint));
3208 case Builtin::BIround:
3209 case Builtin::BIroundf:
3210 case Builtin::BIroundl:
3211 case Builtin::BI__builtin_round:
3212 case Builtin::BI__builtin_roundf:
3213 case Builtin::BI__builtin_roundf16:
3214 case Builtin::BI__builtin_roundl:
3215 case Builtin::BI__builtin_roundf128:
3218 Intrinsic::experimental_constrained_round));
3220 case Builtin::BIroundeven:
3221 case Builtin::BIroundevenf:
3222 case Builtin::BIroundevenl:
3223 case Builtin::BI__builtin_roundeven:
3224 case Builtin::BI__builtin_roundevenf:
3225 case Builtin::BI__builtin_roundevenf16:
3226 case Builtin::BI__builtin_roundevenl:
3227 case Builtin::BI__builtin_roundevenf128:
3229 Intrinsic::roundeven,
3230 Intrinsic::experimental_constrained_roundeven));
3232 case Builtin::BIsin:
3233 case Builtin::BIsinf:
3234 case Builtin::BIsinl:
3235 case Builtin::BI__builtin_sin:
3236 case Builtin::BI__builtin_sinf:
3237 case Builtin::BI__builtin_sinf16:
3238 case Builtin::BI__builtin_sinl:
3239 case Builtin::BI__builtin_sinf128:
3242 Intrinsic::experimental_constrained_sin));
3244 case Builtin::BIsinh:
3245 case Builtin::BIsinhf:
3246 case Builtin::BIsinhl:
3247 case Builtin::BI__builtin_sinh:
3248 case Builtin::BI__builtin_sinhf:
3249 case Builtin::BI__builtin_sinhf16:
3250 case Builtin::BI__builtin_sinhl:
3251 case Builtin::BI__builtin_sinhf128:
3253 *
this,
E, Intrinsic::sinh, Intrinsic::experimental_constrained_sinh));
3255 case Builtin::BI__builtin_sincos:
3256 case Builtin::BI__builtin_sincosf:
3257 case Builtin::BI__builtin_sincosf16:
3258 case Builtin::BI__builtin_sincosl:
3259 case Builtin::BI__builtin_sincosf128:
3263 case Builtin::BIsqrt:
3264 case Builtin::BIsqrtf:
3265 case Builtin::BIsqrtl:
3266 case Builtin::BI__builtin_sqrt:
3267 case Builtin::BI__builtin_sqrtf:
3268 case Builtin::BI__builtin_sqrtf16:
3269 case Builtin::BI__builtin_sqrtl:
3270 case Builtin::BI__builtin_sqrtf128:
3271 case Builtin::BI__builtin_elementwise_sqrt: {
3273 *
this,
E, Intrinsic::sqrt, Intrinsic::experimental_constrained_sqrt);
3278 case Builtin::BItan:
3279 case Builtin::BItanf:
3280 case Builtin::BItanl:
3281 case Builtin::BI__builtin_tan:
3282 case Builtin::BI__builtin_tanf:
3283 case Builtin::BI__builtin_tanf16:
3284 case Builtin::BI__builtin_tanl:
3285 case Builtin::BI__builtin_tanf128:
3287 *
this,
E, Intrinsic::tan, Intrinsic::experimental_constrained_tan));
3289 case Builtin::BItanh:
3290 case Builtin::BItanhf:
3291 case Builtin::BItanhl:
3292 case Builtin::BI__builtin_tanh:
3293 case Builtin::BI__builtin_tanhf:
3294 case Builtin::BI__builtin_tanhf16:
3295 case Builtin::BI__builtin_tanhl:
3296 case Builtin::BI__builtin_tanhf128:
3298 *
this,
E, Intrinsic::tanh, Intrinsic::experimental_constrained_tanh));
3300 case Builtin::BItrunc:
3301 case Builtin::BItruncf:
3302 case Builtin::BItruncl:
3303 case Builtin::BI__builtin_trunc:
3304 case Builtin::BI__builtin_truncf:
3305 case Builtin::BI__builtin_truncf16:
3306 case Builtin::BI__builtin_truncl:
3307 case Builtin::BI__builtin_truncf128:
3310 Intrinsic::experimental_constrained_trunc));
3312 case Builtin::BIlround:
3313 case Builtin::BIlroundf:
3314 case Builtin::BIlroundl:
3315 case Builtin::BI__builtin_lround:
3316 case Builtin::BI__builtin_lroundf:
3317 case Builtin::BI__builtin_lroundl:
3318 case Builtin::BI__builtin_lroundf128:
3320 *
this,
E, Intrinsic::lround,
3321 Intrinsic::experimental_constrained_lround));
3323 case Builtin::BIllround:
3324 case Builtin::BIllroundf:
3325 case Builtin::BIllroundl:
3326 case Builtin::BI__builtin_llround:
3327 case Builtin::BI__builtin_llroundf:
3328 case Builtin::BI__builtin_llroundl:
3329 case Builtin::BI__builtin_llroundf128:
3331 *
this,
E, Intrinsic::llround,
3332 Intrinsic::experimental_constrained_llround));
3334 case Builtin::BIlrint:
3335 case Builtin::BIlrintf:
3336 case Builtin::BIlrintl:
3337 case Builtin::BI__builtin_lrint:
3338 case Builtin::BI__builtin_lrintf:
3339 case Builtin::BI__builtin_lrintl:
3340 case Builtin::BI__builtin_lrintf128:
3342 *
this,
E, Intrinsic::lrint,
3343 Intrinsic::experimental_constrained_lrint));
3345 case Builtin::BIllrint:
3346 case Builtin::BIllrintf:
3347 case Builtin::BIllrintl:
3348 case Builtin::BI__builtin_llrint:
3349 case Builtin::BI__builtin_llrintf:
3350 case Builtin::BI__builtin_llrintl:
3351 case Builtin::BI__builtin_llrintf128:
3353 *
this,
E, Intrinsic::llrint,
3354 Intrinsic::experimental_constrained_llrint));
3355 case Builtin::BI__builtin_ldexp:
3356 case Builtin::BI__builtin_ldexpf:
3357 case Builtin::BI__builtin_ldexpl:
3358 case Builtin::BI__builtin_ldexpf16:
3359 case Builtin::BI__builtin_ldexpf128: {
3361 *
this,
E, Intrinsic::ldexp,
3362 Intrinsic::experimental_constrained_ldexp));
3372 Value *Val = A.emitRawPointer(*
this);
3378 SkippedChecks.
set(SanitizerKind::All);
3379 SkippedChecks.
clear(SanitizerKind::Alignment);
3382 if (
auto *CE = dyn_cast<ImplicitCastExpr>(Arg))
3383 if (CE->getCastKind() == CK_BitCast)
3384 Arg = CE->getSubExpr();
3390 switch (BuiltinIDIfNoAsmLabel) {
3392 case Builtin::BI__builtin___CFStringMakeConstantString:
3393 case Builtin::BI__builtin___NSStringMakeConstantString:
3395 case Builtin::BI__builtin_stdarg_start:
3396 case Builtin::BI__builtin_va_start:
3397 case Builtin::BI__va_start:
3398 case Builtin::BI__builtin_va_end:
3402 BuiltinID != Builtin::BI__builtin_va_end);
3404 case Builtin::BI__builtin_va_copy: {
3411 case Builtin::BIabs:
3412 case Builtin::BIlabs:
3413 case Builtin::BIllabs:
3414 case Builtin::BI__builtin_abs:
3415 case Builtin::BI__builtin_labs:
3416 case Builtin::BI__builtin_llabs: {
3417 bool SanitizeOverflow =
SanOpts.
has(SanitizerKind::SignedIntegerOverflow);
3420 switch (
getLangOpts().getSignedOverflowBehavior()) {
3425 if (!SanitizeOverflow) {
3437 case Builtin::BI__builtin_complex: {
3442 case Builtin::BI__builtin_conj:
3443 case Builtin::BI__builtin_conjf:
3444 case Builtin::BI__builtin_conjl:
3445 case Builtin::BIconj:
3446 case Builtin::BIconjf:
3447 case Builtin::BIconjl: {
3449 Value *Real = ComplexVal.first;
3450 Value *Imag = ComplexVal.second;
3451 Imag =
Builder.CreateFNeg(Imag,
"neg");
3454 case Builtin::BI__builtin_creal:
3455 case Builtin::BI__builtin_crealf:
3456 case Builtin::BI__builtin_creall:
3457 case Builtin::BIcreal:
3458 case Builtin::BIcrealf:
3459 case Builtin::BIcreall: {
3464 case Builtin::BI__builtin_preserve_access_index: {
3485 case Builtin::BI__builtin_cimag:
3486 case Builtin::BI__builtin_cimagf:
3487 case Builtin::BI__builtin_cimagl:
3488 case Builtin::BIcimag:
3489 case Builtin::BIcimagf:
3490 case Builtin::BIcimagl: {
3495 case Builtin::BI__builtin_clrsb:
3496 case Builtin::BI__builtin_clrsbl:
3497 case Builtin::BI__builtin_clrsbll: {
3501 llvm::Type *ArgType = ArgValue->
getType();
3505 Value *
Zero = llvm::Constant::getNullValue(ArgType);
3506 Value *IsNeg =
Builder.CreateICmpSLT(ArgValue, Zero,
"isneg");
3508 Value *Tmp =
Builder.CreateSelect(IsNeg, Inverse, ArgValue);
3515 case Builtin::BI__builtin_ctzs:
3516 case Builtin::BI__builtin_ctz:
3517 case Builtin::BI__builtin_ctzl:
3518 case Builtin::BI__builtin_ctzll:
3519 case Builtin::BI__builtin_ctzg: {
3520 bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_ctzg &&
3521 E->getNumArgs() > 1;
3527 llvm::Type *ArgType = ArgValue->
getType();
3534 if (
Result->getType() != ResultType)
3540 Value *
Zero = Constant::getNullValue(ArgType);
3541 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, Zero,
"iszero");
3543 Value *ResultOrFallback =
3544 Builder.CreateSelect(IsZero, FallbackValue,
Result,
"ctzg");
3547 case Builtin::BI__builtin_clzs:
3548 case Builtin::BI__builtin_clz:
3549 case Builtin::BI__builtin_clzl:
3550 case Builtin::BI__builtin_clzll:
3551 case Builtin::BI__builtin_clzg: {
3552 bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_clzg &&
3553 E->getNumArgs() > 1;
3559 llvm::Type *ArgType = ArgValue->
getType();
3566 if (
Result->getType() != ResultType)
3572 Value *
Zero = Constant::getNullValue(ArgType);
3573 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, Zero,
"iszero");
3575 Value *ResultOrFallback =
3576 Builder.CreateSelect(IsZero, FallbackValue,
Result,
"clzg");
3579 case Builtin::BI__builtin_ffs:
3580 case Builtin::BI__builtin_ffsl:
3581 case Builtin::BI__builtin_ffsll: {
3585 llvm::Type *ArgType = ArgValue->
getType();
3590 Builder.CreateAdd(
Builder.CreateCall(F, {ArgValue, Builder.getTrue()}),
3591 llvm::ConstantInt::get(ArgType, 1));
3592 Value *
Zero = llvm::Constant::getNullValue(ArgType);
3593 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, Zero,
"iszero");
3595 if (
Result->getType() != ResultType)
3600 case Builtin::BI__builtin_parity:
3601 case Builtin::BI__builtin_parityl:
3602 case Builtin::BI__builtin_parityll: {
3606 llvm::Type *ArgType = ArgValue->
getType();
3612 if (
Result->getType() != ResultType)
3617 case Builtin::BI__lzcnt16:
3618 case Builtin::BI__lzcnt:
3619 case Builtin::BI__lzcnt64: {
3622 llvm::Type *ArgType = ArgValue->
getType();
3627 if (
Result->getType() != ResultType)
3632 case Builtin::BI__popcnt16:
3633 case Builtin::BI__popcnt:
3634 case Builtin::BI__popcnt64:
3635 case Builtin::BI__builtin_popcount:
3636 case Builtin::BI__builtin_popcountl:
3637 case Builtin::BI__builtin_popcountll:
3638 case Builtin::BI__builtin_popcountg: {
3641 llvm::Type *ArgType = ArgValue->
getType();
3646 if (
Result->getType() != ResultType)
3651 case Builtin::BI__builtin_unpredictable: {
3657 case Builtin::BI__builtin_expect: {
3659 llvm::Type *ArgType = ArgValue->
getType();
3670 Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue},
"expval");
3673 case Builtin::BI__builtin_expect_with_probability: {
3675 llvm::Type *ArgType = ArgValue->
getType();
3678 llvm::APFloat Probability(0.0);
3679 const Expr *ProbArg =
E->getArg(2);
3681 assert(EvalSucceed &&
"probability should be able to evaluate as float");
3683 bool LoseInfo =
false;
3684 Probability.convert(llvm::APFloat::IEEEdouble(),
3685 llvm::RoundingMode::Dynamic, &LoseInfo);
3687 Constant *Confidence = ConstantFP::get(Ty, Probability);
3697 FnExpect, {ArgValue, ExpectedValue, Confidence},
"expval");
3700 case Builtin::BI__builtin_assume_aligned: {
3701 const Expr *Ptr =
E->getArg(0);
3703 Value *OffsetValue =
3707 ConstantInt *AlignmentCI = cast<ConstantInt>(AlignmentValue);
3708 if (AlignmentCI->getValue().ugt(llvm::Value::MaximumAlignment))
3709 AlignmentCI = ConstantInt::get(AlignmentCI->getIntegerType(),
3710 llvm::Value::MaximumAlignment);
3714 AlignmentCI, OffsetValue);
3717 case Builtin::BI__assume:
3718 case Builtin::BI__builtin_assume: {
3724 Builder.CreateCall(FnAssume, ArgValue);
3727 case Builtin::BI__builtin_assume_separate_storage: {
3728 const Expr *Arg0 =
E->getArg(0);
3729 const Expr *Arg1 =
E->getArg(1);
3734 Value *Values[] = {Value0, Value1};
3735 OperandBundleDefT<Value *> OBD(
"separate_storage", Values);
3739 case Builtin::BI__builtin_allow_runtime_check: {
3743 llvm::Value *Allow =
Builder.CreateCall(
3745 llvm::MetadataAsValue::get(Ctx, llvm::MDString::get(Ctx, Kind)));
3748 case Builtin::BI__arithmetic_fence: {
3751 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
3752 llvm::FastMathFlags FMF =
Builder.getFastMathFlags();
3753 bool isArithmeticFenceEnabled =
3754 FMF.allowReassoc() &&
3758 if (isArithmeticFenceEnabled) {
3761 Value *Real =
Builder.CreateArithmeticFence(ComplexVal.first,
3763 Value *Imag =
Builder.CreateArithmeticFence(ComplexVal.second,
3768 Value *Real = ComplexVal.first;
3769 Value *Imag = ComplexVal.second;
3773 if (isArithmeticFenceEnabled)
3778 case Builtin::BI__builtin_bswap16:
3779 case Builtin::BI__builtin_bswap32:
3780 case Builtin::BI__builtin_bswap64:
3781 case Builtin::BI_byteswap_ushort:
3782 case Builtin::BI_byteswap_ulong:
3783 case Builtin::BI_byteswap_uint64: {
3785 emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::bswap));
3787 case Builtin::BI__builtin_bitreverse8:
3788 case Builtin::BI__builtin_bitreverse16:
3789 case Builtin::BI__builtin_bitreverse32:
3790 case Builtin::BI__builtin_bitreverse64: {
3792 emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::bitreverse));
3794 case Builtin::BI__builtin_rotateleft8:
3795 case Builtin::BI__builtin_rotateleft16:
3796 case Builtin::BI__builtin_rotateleft32:
3797 case Builtin::BI__builtin_rotateleft64:
3798 case Builtin::BI_rotl8:
3799 case Builtin::BI_rotl16:
3800 case Builtin::BI_rotl:
3801 case Builtin::BI_lrotl:
3802 case Builtin::BI_rotl64:
3805 case Builtin::BI__builtin_rotateright8:
3806 case Builtin::BI__builtin_rotateright16:
3807 case Builtin::BI__builtin_rotateright32:
3808 case Builtin::BI__builtin_rotateright64:
3809 case Builtin::BI_rotr8:
3810 case Builtin::BI_rotr16:
3811 case Builtin::BI_rotr:
3812 case Builtin::BI_lrotr:
3813 case Builtin::BI_rotr64:
3816 case Builtin::BI__builtin_constant_p: {
3819 const Expr *Arg =
E->getArg(0);
3827 return RValue::get(ConstantInt::get(ResultType, 0));
3832 return RValue::get(ConstantInt::get(ResultType, 0));
3844 if (
Result->getType() != ResultType)
3848 case Builtin::BI__builtin_dynamic_object_size:
3849 case Builtin::BI__builtin_object_size: {
3856 bool IsDynamic = BuiltinID == Builtin::BI__builtin_dynamic_object_size;
3858 nullptr, IsDynamic));
3860 case Builtin::BI__builtin_counted_by_ref: {
3862 llvm::Value *
Result = llvm::ConstantPointerNull::get(
3867 if (
auto *UO = dyn_cast<UnaryOperator>(Arg);
3868 UO && UO->getOpcode() == UO_AddrOf) {
3871 if (
auto *ASE = dyn_cast<ArraySubscriptExpr>(Arg))
3875 if (
const MemberExpr *ME = dyn_cast_if_present<MemberExpr>(Arg)) {
3879 const auto *FAMDecl = cast<FieldDecl>(ME->getMemberDecl());
3883 llvm::report_fatal_error(
"Cannot find the counted_by 'count' field");
3889 case Builtin::BI__builtin_prefetch: {
3893 llvm::ConstantInt::get(
Int32Ty, 0);
3895 llvm::ConstantInt::get(
Int32Ty, 3);
3901 case Builtin::BI__builtin_readcyclecounter: {
3905 case Builtin::BI__builtin_readsteadycounter: {
3909 case Builtin::BI__builtin___clear_cache: {
3915 case Builtin::BI__builtin_trap:
3918 case Builtin::BI__builtin_verbose_trap: {
3919 llvm::DILocation *TrapLocation =
Builder.getCurrentDebugLocation();
3930 case Builtin::BI__debugbreak:
3933 case Builtin::BI__builtin_unreachable: {
3942 case Builtin::BI__builtin_powi:
3943 case Builtin::BI__builtin_powif:
3944 case Builtin::BI__builtin_powil: {
3948 if (
Builder.getIsFPConstrained()) {
3951 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
3958 { Src0->getType(), Src1->getType() });
3961 case Builtin::BI__builtin_frexpl: {
3965 if (&
getTarget().getLongDoubleFormat() == &llvm::APFloat::PPCDoubleDouble())
3969 case Builtin::BI__builtin_frexp:
3970 case Builtin::BI__builtin_frexpf:
3971 case Builtin::BI__builtin_frexpf128:
3972 case Builtin::BI__builtin_frexpf16:
3974 case Builtin::BI__builtin_isgreater:
3975 case Builtin::BI__builtin_isgreaterequal:
3976 case Builtin::BI__builtin_isless:
3977 case Builtin::BI__builtin_islessequal:
3978 case Builtin::BI__builtin_islessgreater:
3979 case Builtin::BI__builtin_isunordered: {
3982 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
3986 switch (BuiltinID) {
3987 default: llvm_unreachable(
"Unknown ordered comparison");
3988 case Builtin::BI__builtin_isgreater:
3989 LHS =
Builder.CreateFCmpOGT(LHS, RHS,
"cmp");
3991 case Builtin::BI__builtin_isgreaterequal:
3992 LHS =
Builder.CreateFCmpOGE(LHS, RHS,
"cmp");
3994 case Builtin::BI__builtin_isless:
3995 LHS =
Builder.CreateFCmpOLT(LHS, RHS,
"cmp");
3997 case Builtin::BI__builtin_islessequal:
3998 LHS =
Builder.CreateFCmpOLE(LHS, RHS,
"cmp");
4000 case Builtin::BI__builtin_islessgreater:
4001 LHS =
Builder.CreateFCmpONE(LHS, RHS,
"cmp");
4003 case Builtin::BI__builtin_isunordered:
4004 LHS =
Builder.CreateFCmpUNO(LHS, RHS,
"cmp");
4011 case Builtin::BI__builtin_isnan: {
4012 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4021 case Builtin::BI__builtin_issignaling: {
4022 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4029 case Builtin::BI__builtin_isinf: {
4030 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4039 case Builtin::BIfinite:
4040 case Builtin::BI__finite:
4041 case Builtin::BIfinitef:
4042 case Builtin::BI__finitef:
4043 case Builtin::BIfinitel:
4044 case Builtin::BI__finitel:
4045 case Builtin::BI__builtin_isfinite: {
4046 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4055 case Builtin::BI__builtin_isnormal: {
4056 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4063 case Builtin::BI__builtin_issubnormal: {
4064 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4067 Builder.CreateZExt(
Builder.createIsFPClass(
V, FPClassTest::fcSubnormal),
4071 case Builtin::BI__builtin_iszero: {
4072 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4079 case Builtin::BI__builtin_isfpclass: {
4084 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4090 case Builtin::BI__builtin_nondeterministic_value: {
4099 case Builtin::BI__builtin_elementwise_abs: {
4104 QT = VecTy->getElementType();
4108 Builder.getFalse(),
nullptr,
"elt.abs");
4110 Result = emitBuiltinWithOneOverloadedType<1>(
4111 *
this,
E, llvm::Intrinsic::fabs,
"elt.abs");
4115 case Builtin::BI__builtin_elementwise_acos:
4116 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4117 *
this,
E, llvm::Intrinsic::acos,
"elt.acos"));
4118 case Builtin::BI__builtin_elementwise_asin:
4119 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4120 *
this,
E, llvm::Intrinsic::asin,
"elt.asin"));
4121 case Builtin::BI__builtin_elementwise_atan:
4122 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4123 *
this,
E, llvm::Intrinsic::atan,
"elt.atan"));
4124 case Builtin::BI__builtin_elementwise_atan2:
4125 return RValue::get(emitBuiltinWithOneOverloadedType<2>(
4126 *
this,
E, llvm::Intrinsic::atan2,
"elt.atan2"));
4127 case Builtin::BI__builtin_elementwise_ceil:
4128 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4129 *
this,
E, llvm::Intrinsic::ceil,
"elt.ceil"));
4130 case Builtin::BI__builtin_elementwise_exp:
4131 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4132 *
this,
E, llvm::Intrinsic::exp,
"elt.exp"));
4133 case Builtin::BI__builtin_elementwise_exp2:
4134 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4135 *
this,
E, llvm::Intrinsic::exp2,
"elt.exp2"));
4136 case Builtin::BI__builtin_elementwise_log:
4137 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4138 *
this,
E, llvm::Intrinsic::log,
"elt.log"));
4139 case Builtin::BI__builtin_elementwise_log2:
4140 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4141 *
this,
E, llvm::Intrinsic::log2,
"elt.log2"));
4142 case Builtin::BI__builtin_elementwise_log10:
4143 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4144 *
this,
E, llvm::Intrinsic::log10,
"elt.log10"));
4145 case Builtin::BI__builtin_elementwise_pow: {
4147 emitBuiltinWithOneOverloadedType<2>(*
this,
E, llvm::Intrinsic::pow));
4149 case Builtin::BI__builtin_elementwise_bitreverse:
4150 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4151 *
this,
E, llvm::Intrinsic::bitreverse,
"elt.bitreverse"));
4152 case Builtin::BI__builtin_elementwise_cos:
4153 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4154 *
this,
E, llvm::Intrinsic::cos,
"elt.cos"));
4155 case Builtin::BI__builtin_elementwise_cosh:
4156 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4157 *
this,
E, llvm::Intrinsic::cosh,
"elt.cosh"));
4158 case Builtin::BI__builtin_elementwise_floor:
4159 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4160 *
this,
E, llvm::Intrinsic::floor,
"elt.floor"));
4161 case Builtin::BI__builtin_elementwise_popcount:
4162 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4163 *
this,
E, llvm::Intrinsic::ctpop,
"elt.ctpop"));
4164 case Builtin::BI__builtin_elementwise_roundeven:
4165 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4166 *
this,
E, llvm::Intrinsic::roundeven,
"elt.roundeven"));
4167 case Builtin::BI__builtin_elementwise_round:
4168 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4169 *
this,
E, llvm::Intrinsic::round,
"elt.round"));
4170 case Builtin::BI__builtin_elementwise_rint:
4171 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4172 *
this,
E, llvm::Intrinsic::rint,
"elt.rint"));
4173 case Builtin::BI__builtin_elementwise_nearbyint:
4174 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4175 *
this,
E, llvm::Intrinsic::nearbyint,
"elt.nearbyint"));
4176 case Builtin::BI__builtin_elementwise_sin:
4177 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4178 *
this,
E, llvm::Intrinsic::sin,
"elt.sin"));
4179 case Builtin::BI__builtin_elementwise_sinh:
4180 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4181 *
this,
E, llvm::Intrinsic::sinh,
"elt.sinh"));
4182 case Builtin::BI__builtin_elementwise_tan:
4183 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4184 *
this,
E, llvm::Intrinsic::tan,
"elt.tan"));
4185 case Builtin::BI__builtin_elementwise_tanh:
4186 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4187 *
this,
E, llvm::Intrinsic::tanh,
"elt.tanh"));
4188 case Builtin::BI__builtin_elementwise_trunc:
4189 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4190 *
this,
E, llvm::Intrinsic::trunc,
"elt.trunc"));
4191 case Builtin::BI__builtin_elementwise_canonicalize:
4192 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4193 *
this,
E, llvm::Intrinsic::canonicalize,
"elt.canonicalize"));
4194 case Builtin::BI__builtin_elementwise_copysign:
4195 return RValue::get(emitBuiltinWithOneOverloadedType<2>(
4196 *
this,
E, llvm::Intrinsic::copysign));
4197 case Builtin::BI__builtin_elementwise_fma:
4199 emitBuiltinWithOneOverloadedType<3>(*
this,
E, llvm::Intrinsic::fma));
4200 case Builtin::BI__builtin_elementwise_add_sat:
4201 case Builtin::BI__builtin_elementwise_sub_sat: {
4205 assert(Op0->
getType()->isIntOrIntVectorTy() &&
"integer type expected");
4208 Ty = VecTy->getElementType();
4211 if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_add_sat)
4212 Opc = IsSigned ? llvm::Intrinsic::sadd_sat : llvm::Intrinsic::uadd_sat;
4214 Opc = IsSigned ? llvm::Intrinsic::ssub_sat : llvm::Intrinsic::usub_sat;
4215 Result =
Builder.CreateBinaryIntrinsic(Opc, Op0, Op1,
nullptr,
"elt.sat");
4219 case Builtin::BI__builtin_elementwise_max: {
4223 if (Op0->
getType()->isIntOrIntVectorTy()) {
4226 Ty = VecTy->getElementType();
4228 ? llvm::Intrinsic::smax
4229 : llvm::Intrinsic::umax,
4230 Op0, Op1,
nullptr,
"elt.max");
4235 case Builtin::BI__builtin_elementwise_min: {
4239 if (Op0->
getType()->isIntOrIntVectorTy()) {
4242 Ty = VecTy->getElementType();
4244 ? llvm::Intrinsic::smin
4245 : llvm::Intrinsic::umin,
4246 Op0, Op1,
nullptr,
"elt.min");
4252 case Builtin::BI__builtin_elementwise_maximum: {
4256 Op1,
nullptr,
"elt.maximum");
4260 case Builtin::BI__builtin_elementwise_minimum: {
4264 Op1,
nullptr,
"elt.minimum");
4268 case Builtin::BI__builtin_reduce_max: {
4269 auto GetIntrinsicID = [
this](
QualType QT) {
4271 QT = VecTy->getElementType();
4276 return llvm::Intrinsic::vector_reduce_smax;
4278 return llvm::Intrinsic::vector_reduce_umax;
4280 return llvm::Intrinsic::vector_reduce_fmax;
4282 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4283 *
this,
E, GetIntrinsicID(
E->getArg(0)->
getType()),
"rdx.min"));
4286 case Builtin::BI__builtin_reduce_min: {
4287 auto GetIntrinsicID = [
this](
QualType QT) {
4289 QT = VecTy->getElementType();
4294 return llvm::Intrinsic::vector_reduce_smin;
4296 return llvm::Intrinsic::vector_reduce_umin;
4298 return llvm::Intrinsic::vector_reduce_fmin;
4301 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4302 *
this,
E, GetIntrinsicID(
E->getArg(0)->
getType()),
"rdx.min"));
4305 case Builtin::BI__builtin_reduce_add:
4306 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4307 *
this,
E, llvm::Intrinsic::vector_reduce_add,
"rdx.add"));
4308 case Builtin::BI__builtin_reduce_mul:
4309 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4310 *
this,
E, llvm::Intrinsic::vector_reduce_mul,
"rdx.mul"));
4311 case Builtin::BI__builtin_reduce_xor:
4312 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4313 *
this,
E, llvm::Intrinsic::vector_reduce_xor,
"rdx.xor"));
4314 case Builtin::BI__builtin_reduce_or:
4315 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4316 *
this,
E, llvm::Intrinsic::vector_reduce_or,
"rdx.or"));
4317 case Builtin::BI__builtin_reduce_and:
4318 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4319 *
this,
E, llvm::Intrinsic::vector_reduce_and,
"rdx.and"));
4320 case Builtin::BI__builtin_reduce_maximum:
4321 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4322 *
this,
E, llvm::Intrinsic::vector_reduce_fmaximum,
"rdx.maximum"));
4323 case Builtin::BI__builtin_reduce_minimum:
4324 return RValue::get(emitBuiltinWithOneOverloadedType<1>(
4325 *
this,
E, llvm::Intrinsic::vector_reduce_fminimum,
"rdx.minimum"));
4327 case Builtin::BI__builtin_matrix_transpose: {
4331 Value *
Result = MB.CreateMatrixTranspose(MatValue, MatrixTy->getNumRows(),
4332 MatrixTy->getNumColumns());
4336 case Builtin::BI__builtin_matrix_column_major_load: {
4342 assert(PtrTy &&
"arg0 must be of pointer type");
4352 ResultTy->getNumRows(), ResultTy->getNumColumns(),
"matrix");
4356 case Builtin::BI__builtin_matrix_column_major_store: {
4364 assert(PtrTy &&
"arg1 must be of pointer type");
4373 MatrixTy->getNumRows(), MatrixTy->getNumColumns());
4377 case Builtin::BI__builtin_isinf_sign: {
4379 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4384 AbsArg, ConstantFP::getInfinity(Arg->
getType()),
"isinf");
4390 Value *NegativeOne = ConstantInt::get(
IntTy, -1);
4391 Value *SignResult =
Builder.CreateSelect(IsNeg, NegativeOne, One);
4396 case Builtin::BI__builtin_flt_rounds: {
4401 if (
Result->getType() != ResultType)
4407 case Builtin::BI__builtin_set_flt_rounds: {
4415 case Builtin::BI__builtin_fpclassify: {
4416 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
4427 "fpclassify_result");
4431 Value *IsZero =
Builder.CreateFCmpOEQ(
V, Constant::getNullValue(Ty),
4435 Builder.CreateCondBr(IsZero, End, NotZero);
4439 Builder.SetInsertPoint(NotZero);
4443 Builder.CreateCondBr(IsNan, End, NotNan);
4444 Result->addIncoming(NanLiteral, NotZero);
4447 Builder.SetInsertPoint(NotNan);
4450 Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(
V->getType()),
4454 Builder.CreateCondBr(IsInf, End, NotInf);
4455 Result->addIncoming(InfLiteral, NotNan);
4458 Builder.SetInsertPoint(NotInf);
4459 APFloat Smallest = APFloat::getSmallestNormalized(
4462 Builder.CreateFCmpUGE(VAbs, ConstantFP::get(
V->getContext(), Smallest),
4464 Value *NormalResult =
4468 Result->addIncoming(NormalResult, NotInf);
4481 case Builtin::BIalloca:
4482 case Builtin::BI_alloca:
4483 case Builtin::BI__builtin_alloca_uninitialized:
4484 case Builtin::BI__builtin_alloca: {
4488 const Align SuitableAlignmentInBytes =
4492 AllocaInst *AI =
Builder.CreateAlloca(
Builder.getInt8Ty(), Size);
4493 AI->setAlignment(SuitableAlignmentInBytes);
4494 if (BuiltinID != Builtin::BI__builtin_alloca_uninitialized)
4506 case Builtin::BI__builtin_alloca_with_align_uninitialized:
4507 case Builtin::BI__builtin_alloca_with_align: {
4510 auto *AlignmentInBitsCI = cast<ConstantInt>(AlignmentInBitsValue);
4511 unsigned AlignmentInBits = AlignmentInBitsCI->getZExtValue();
4512 const Align AlignmentInBytes =
4514 AllocaInst *AI =
Builder.CreateAlloca(
Builder.getInt8Ty(), Size);
4515 AI->setAlignment(AlignmentInBytes);
4516 if (BuiltinID != Builtin::BI__builtin_alloca_with_align_uninitialized)
4528 case Builtin::BIbzero:
4529 case Builtin::BI__builtin_bzero: {
4538 case Builtin::BIbcopy:
4539 case Builtin::BI__builtin_bcopy: {
4553 case Builtin::BImemcpy:
4554 case Builtin::BI__builtin_memcpy:
4555 case Builtin::BImempcpy:
4556 case Builtin::BI__builtin_mempcpy: {
4560 EmitArgCheck(
TCK_Store, Dest,
E->getArg(0), 0);
4561 EmitArgCheck(
TCK_Load, Src,
E->getArg(1), 1);
4563 if (BuiltinID == Builtin::BImempcpy ||
4564 BuiltinID == Builtin::BI__builtin_mempcpy)
4571 case Builtin::BI__builtin_memcpy_inline: {
4576 EmitArgCheck(
TCK_Store, Dest,
E->getArg(0), 0);
4577 EmitArgCheck(
TCK_Load, Src,
E->getArg(1), 1);
4582 case Builtin::BI__builtin_char_memchr:
4583 BuiltinID = Builtin::BI__builtin_memchr;
4586 case Builtin::BI__builtin___memcpy_chk: {
4593 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
4594 if (
Size.ugt(DstSize))
4598 Value *SizeVal = llvm::ConstantInt::get(
Builder.getContext(), Size);
4603 case Builtin::BI__builtin_objc_memmove_collectable: {
4608 DestAddr, SrcAddr, SizeVal);
4612 case Builtin::BI__builtin___memmove_chk: {
4619 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
4620 if (
Size.ugt(DstSize))
4624 Value *SizeVal = llvm::ConstantInt::get(
Builder.getContext(), Size);
4629 case Builtin::BImemmove:
4630 case Builtin::BI__builtin_memmove: {
4634 EmitArgCheck(
TCK_Store, Dest,
E->getArg(0), 0);
4635 EmitArgCheck(
TCK_Load, Src,
E->getArg(1), 1);
4639 case Builtin::BImemset:
4640 case Builtin::BI__builtin_memset: {
4650 case Builtin::BI__builtin_memset_inline: {
4662 case Builtin::BI__builtin___memset_chk: {
4669 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
4670 if (
Size.ugt(DstSize))
4675 Value *SizeVal = llvm::ConstantInt::get(
Builder.getContext(), Size);
4679 case Builtin::BI__builtin_wmemchr: {
4682 if (!
getTarget().getTriple().isOSMSVCRT())
4690 BasicBlock *Entry =
Builder.GetInsertBlock();
4695 Builder.CreateCondBr(SizeEq0, Exit, CmpEq);
4699 StrPhi->addIncoming(Str, Entry);
4701 SizePhi->addIncoming(Size, Entry);
4705 Value *FoundChr =
Builder.CreateConstInBoundsGEP1_32(WCharTy, StrPhi, 0);
4707 Builder.CreateCondBr(StrEqChr, Exit, Next);
4710 Value *NextStr =
Builder.CreateConstInBoundsGEP1_32(WCharTy, StrPhi, 1);
4712 Value *NextSizeEq0 =
4713 Builder.CreateICmpEQ(NextSize, ConstantInt::get(
SizeTy, 0));
4714 Builder.CreateCondBr(NextSizeEq0, Exit, CmpEq);
4715 StrPhi->addIncoming(NextStr, Next);
4716 SizePhi->addIncoming(NextSize, Next);
4720 Ret->addIncoming(llvm::Constant::getNullValue(Str->
getType()), Entry);
4721 Ret->addIncoming(llvm::Constant::getNullValue(Str->
getType()), Next);
4722 Ret->addIncoming(FoundChr, CmpEq);
4725 case Builtin::BI__builtin_wmemcmp: {
4728 if (!
getTarget().getTriple().isOSMSVCRT())
4737 BasicBlock *Entry =
Builder.GetInsertBlock();
4743 Builder.CreateCondBr(SizeEq0, Exit, CmpGT);
4747 DstPhi->addIncoming(Dst, Entry);
4749 SrcPhi->addIncoming(Src, Entry);
4751 SizePhi->addIncoming(Size, Entry);
4757 Builder.CreateCondBr(DstGtSrc, Exit, CmpLT);
4761 Builder.CreateCondBr(DstLtSrc, Exit, Next);
4764 Value *NextDst =
Builder.CreateConstInBoundsGEP1_32(WCharTy, DstPhi, 1);
4765 Value *NextSrc =
Builder.CreateConstInBoundsGEP1_32(WCharTy, SrcPhi, 1);
4767 Value *NextSizeEq0 =
4768 Builder.CreateICmpEQ(NextSize, ConstantInt::get(
SizeTy, 0));
4769 Builder.CreateCondBr(NextSizeEq0, Exit, CmpGT);
4770 DstPhi->addIncoming(NextDst, Next);
4771 SrcPhi->addIncoming(NextSrc, Next);
4772 SizePhi->addIncoming(NextSize, Next);
4776 Ret->addIncoming(ConstantInt::get(
IntTy, 0), Entry);
4777 Ret->addIncoming(ConstantInt::get(
IntTy, 1), CmpGT);
4778 Ret->addIncoming(ConstantInt::get(
IntTy, -1), CmpLT);
4779 Ret->addIncoming(ConstantInt::get(
IntTy, 0), Next);
4782 case Builtin::BI__builtin_dwarf_cfa: {
4795 llvm::ConstantInt::get(
Int32Ty, Offset)));
4797 case Builtin::BI__builtin_return_address: {
4803 case Builtin::BI_ReturnAddress: {
4807 case Builtin::BI__builtin_frame_address: {
4813 case Builtin::BI__builtin_extract_return_addr: {
4818 case Builtin::BI__builtin_frob_return_addr: {
4823 case Builtin::BI__builtin_dwarf_sp_column: {
4824 llvm::IntegerType *Ty
4833 case Builtin::BI__builtin_init_dwarf_reg_size_table: {
4839 case Builtin::BI__builtin_eh_return: {
4843 llvm::IntegerType *
IntTy = cast<llvm::IntegerType>(
Int->getType());
4844 assert((
IntTy->getBitWidth() == 32 ||
IntTy->getBitWidth() == 64) &&
4845 "LLVM's __builtin_eh_return only supports 32- and 64-bit variants");
4848 : Intrinsic::eh_return_i64);
4857 case Builtin::BI__builtin_unwind_init: {
4862 case Builtin::BI__builtin_extend_pointer: {
4887 case Builtin::BI__builtin_setjmp: {
4891 if (
getTarget().getTriple().getArch() == llvm::Triple::systemz) {
4902 ConstantInt::get(
Int32Ty, 0));
4916 case Builtin::BI__builtin_longjmp: {
4930 case Builtin::BI__builtin_launder: {
4931 const Expr *Arg =
E->getArg(0);
4939 case Builtin::BI__sync_fetch_and_add:
4940 case Builtin::BI__sync_fetch_and_sub:
4941 case Builtin::BI__sync_fetch_and_or:
4942 case Builtin::BI__sync_fetch_and_and:
4943 case Builtin::BI__sync_fetch_and_xor:
4944 case Builtin::BI__sync_fetch_and_nand:
4945 case Builtin::BI__sync_add_and_fetch:
4946 case Builtin::BI__sync_sub_and_fetch:
4947 case Builtin::BI__sync_and_and_fetch:
4948 case Builtin::BI__sync_or_and_fetch:
4949 case Builtin::BI__sync_xor_and_fetch:
4950 case Builtin::BI__sync_nand_and_fetch:
4951 case Builtin::BI__sync_val_compare_and_swap:
4952 case Builtin::BI__sync_bool_compare_and_swap:
4953 case Builtin::BI__sync_lock_test_and_set:
4954 case Builtin::BI__sync_lock_release:
4955 case Builtin::BI__sync_swap:
4956 llvm_unreachable(
"Shouldn't make it through sema");
4957 case Builtin::BI__sync_fetch_and_add_1:
4958 case Builtin::BI__sync_fetch_and_add_2:
4959 case Builtin::BI__sync_fetch_and_add_4:
4960 case Builtin::BI__sync_fetch_and_add_8:
4961 case Builtin::BI__sync_fetch_and_add_16:
4963 case Builtin::BI__sync_fetch_and_sub_1:
4964 case Builtin::BI__sync_fetch_and_sub_2:
4965 case Builtin::BI__sync_fetch_and_sub_4:
4966 case Builtin::BI__sync_fetch_and_sub_8:
4967 case Builtin::BI__sync_fetch_and_sub_16:
4969 case Builtin::BI__sync_fetch_and_or_1:
4970 case Builtin::BI__sync_fetch_and_or_2:
4971 case Builtin::BI__sync_fetch_and_or_4:
4972 case Builtin::BI__sync_fetch_and_or_8:
4973 case Builtin::BI__sync_fetch_and_or_16:
4975 case Builtin::BI__sync_fetch_and_and_1:
4976 case Builtin::BI__sync_fetch_and_and_2:
4977 case Builtin::BI__sync_fetch_and_and_4:
4978 case Builtin::BI__sync_fetch_and_and_8:
4979 case Builtin::BI__sync_fetch_and_and_16:
4981 case Builtin::BI__sync_fetch_and_xor_1:
4982 case Builtin::BI__sync_fetch_and_xor_2:
4983 case Builtin::BI__sync_fetch_and_xor_4:
4984 case Builtin::BI__sync_fetch_and_xor_8:
4985 case Builtin::BI__sync_fetch_and_xor_16:
4987 case Builtin::BI__sync_fetch_and_nand_1:
4988 case Builtin::BI__sync_fetch_and_nand_2:
4989 case Builtin::BI__sync_fetch_and_nand_4:
4990 case Builtin::BI__sync_fetch_and_nand_8:
4991 case Builtin::BI__sync_fetch_and_nand_16:
4995 case Builtin::BI__sync_fetch_and_min:
4997 case Builtin::BI__sync_fetch_and_max:
4999 case Builtin::BI__sync_fetch_and_umin:
5001 case Builtin::BI__sync_fetch_and_umax:
5004 case Builtin::BI__sync_add_and_fetch_1:
5005 case Builtin::BI__sync_add_and_fetch_2:
5006 case Builtin::BI__sync_add_and_fetch_4:
5007 case Builtin::BI__sync_add_and_fetch_8:
5008 case Builtin::BI__sync_add_and_fetch_16:
5010 llvm::Instruction::Add);
5011 case Builtin::BI__sync_sub_and_fetch_1:
5012 case Builtin::BI__sync_sub_and_fetch_2:
5013 case Builtin::BI__sync_sub_and_fetch_4:
5014 case Builtin::BI__sync_sub_and_fetch_8:
5015 case Builtin::BI__sync_sub_and_fetch_16:
5017 llvm::Instruction::Sub);
5018 case Builtin::BI__sync_and_and_fetch_1:
5019 case Builtin::BI__sync_and_and_fetch_2:
5020 case Builtin::BI__sync_and_and_fetch_4:
5021 case Builtin::BI__sync_and_and_fetch_8:
5022 case Builtin::BI__sync_and_and_fetch_16:
5024 llvm::Instruction::And);
5025 case Builtin::BI__sync_or_and_fetch_1:
5026 case Builtin::BI__sync_or_and_fetch_2:
5027 case Builtin::BI__sync_or_and_fetch_4:
5028 case Builtin::BI__sync_or_and_fetch_8:
5029 case Builtin::BI__sync_or_and_fetch_16:
5031 llvm::Instruction::Or);
5032 case Builtin::BI__sync_xor_and_fetch_1:
5033 case Builtin::BI__sync_xor_and_fetch_2:
5034 case Builtin::BI__sync_xor_and_fetch_4:
5035 case Builtin::BI__sync_xor_and_fetch_8:
5036 case Builtin::BI__sync_xor_and_fetch_16:
5038 llvm::Instruction::Xor);
5039 case Builtin::BI__sync_nand_and_fetch_1:
5040 case Builtin::BI__sync_nand_and_fetch_2:
5041 case Builtin::BI__sync_nand_and_fetch_4:
5042 case Builtin::BI__sync_nand_and_fetch_8:
5043 case Builtin::BI__sync_nand_and_fetch_16:
5045 llvm::Instruction::And,
true);
5047 case Builtin::BI__sync_val_compare_and_swap_1:
5048 case Builtin::BI__sync_val_compare_and_swap_2:
5049 case Builtin::BI__sync_val_compare_and_swap_4:
5050 case Builtin::BI__sync_val_compare_and_swap_8:
5051 case Builtin::BI__sync_val_compare_and_swap_16:
5054 case Builtin::BI__sync_bool_compare_and_swap_1:
5055 case Builtin::BI__sync_bool_compare_and_swap_2:
5056 case Builtin::BI__sync_bool_compare_and_swap_4:
5057 case Builtin::BI__sync_bool_compare_and_swap_8:
5058 case Builtin::BI__sync_bool_compare_and_swap_16:
5061 case Builtin::BI__sync_swap_1:
5062 case Builtin::BI__sync_swap_2:
5063 case Builtin::BI__sync_swap_4:
5064 case Builtin::BI__sync_swap_8:
5065 case Builtin::BI__sync_swap_16:
5068 case Builtin::BI__sync_lock_test_and_set_1:
5069 case Builtin::BI__sync_lock_test_and_set_2:
5070 case Builtin::BI__sync_lock_test_and_set_4:
5071 case Builtin::BI__sync_lock_test_and_set_8:
5072 case Builtin::BI__sync_lock_test_and_set_16:
5075 case Builtin::BI__sync_lock_release_1:
5076 case Builtin::BI__sync_lock_release_2:
5077 case Builtin::BI__sync_lock_release_4:
5078 case Builtin::BI__sync_lock_release_8:
5079 case Builtin::BI__sync_lock_release_16: {
5085 llvm::StoreInst *
Store =
5087 Store->setAtomic(llvm::AtomicOrdering::Release);
5091 case Builtin::BI__sync_synchronize: {
5099 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent);
5103 case Builtin::BI__builtin_nontemporal_load:
5105 case Builtin::BI__builtin_nontemporal_store:
5107 case Builtin::BI__c11_atomic_is_lock_free:
5108 case Builtin::BI__atomic_is_lock_free: {
5112 const char *LibCallName =
"__atomic_is_lock_free";
5116 if (BuiltinID == Builtin::BI__atomic_is_lock_free)
5130 case Builtin::BI__atomic_test_and_set: {
5142 if (isa<llvm::ConstantInt>(Order)) {
5143 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
5144 AtomicRMWInst *
Result =
nullptr;
5149 llvm::AtomicOrdering::Monotonic);
5154 llvm::AtomicOrdering::Acquire);
5158 llvm::AtomicOrdering::Release);
5163 llvm::AtomicOrdering::AcquireRelease);
5167 llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
5168 llvm::AtomicOrdering::SequentiallyConsistent);
5171 Result->setVolatile(Volatile);
5177 llvm::BasicBlock *BBs[5] = {
5184 llvm::AtomicOrdering Orders[5] = {
5185 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Acquire,
5186 llvm::AtomicOrdering::Release, llvm::AtomicOrdering::AcquireRelease,
5187 llvm::AtomicOrdering::SequentiallyConsistent};
5189 Order =
Builder.CreateIntCast(Order,
Builder.getInt32Ty(),
false);
5190 llvm::SwitchInst *SI =
Builder.CreateSwitch(Order, BBs[0]);
5192 Builder.SetInsertPoint(ContBB);
5195 for (
unsigned i = 0; i < 5; ++i) {
5196 Builder.SetInsertPoint(BBs[i]);
5198 Ptr, NewVal, Orders[i]);
5199 RMW->setVolatile(Volatile);
5200 Result->addIncoming(RMW, BBs[i]);
5204 SI->addCase(
Builder.getInt32(0), BBs[0]);
5205 SI->addCase(
Builder.getInt32(1), BBs[1]);
5206 SI->addCase(
Builder.getInt32(2), BBs[1]);
5207 SI->addCase(
Builder.getInt32(3), BBs[2]);
5208 SI->addCase(
Builder.getInt32(4), BBs[3]);
5209 SI->addCase(
Builder.getInt32(5), BBs[4]);
5211 Builder.SetInsertPoint(ContBB);
5215 case Builtin::BI__atomic_clear: {
5224 if (isa<llvm::ConstantInt>(Order)) {
5225 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
5230 Store->setOrdering(llvm::AtomicOrdering::Monotonic);
5233 Store->setOrdering(llvm::AtomicOrdering::Release);
5236 Store->setOrdering(llvm::AtomicOrdering::SequentiallyConsistent);
5244 llvm::BasicBlock *BBs[3] = {
5249 llvm::AtomicOrdering Orders[3] = {
5250 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Release,
5251 llvm::AtomicOrdering::SequentiallyConsistent};
5253 Order =
Builder.CreateIntCast(Order,
Builder.getInt32Ty(),
false);
5254 llvm::SwitchInst *SI =
Builder.CreateSwitch(Order, BBs[0]);
5256 for (
unsigned i = 0; i < 3; ++i) {
5257 Builder.SetInsertPoint(BBs[i]);
5259 Store->setOrdering(Orders[i]);
5263 SI->addCase(
Builder.getInt32(0), BBs[0]);
5264 SI->addCase(
Builder.getInt32(3), BBs[1]);
5265 SI->addCase(
Builder.getInt32(5), BBs[2]);
5267 Builder.SetInsertPoint(ContBB);
5271 case Builtin::BI__atomic_thread_fence:
5272 case Builtin::BI__atomic_signal_fence:
5273 case Builtin::BI__c11_atomic_thread_fence:
5274 case Builtin::BI__c11_atomic_signal_fence: {
5275 llvm::SyncScope::ID SSID;
5276 if (BuiltinID == Builtin::BI__atomic_signal_fence ||
5277 BuiltinID == Builtin::BI__c11_atomic_signal_fence)
5278 SSID = llvm::SyncScope::SingleThread;
5280 SSID = llvm::SyncScope::System;
5282 if (isa<llvm::ConstantInt>(Order)) {
5283 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
5290 Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
5293 Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
5296 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
5299 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
5305 llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB;
5312 Order =
Builder.CreateIntCast(Order,
Builder.getInt32Ty(),
false);
5313 llvm::SwitchInst *SI =
Builder.CreateSwitch(Order, ContBB);
5315 Builder.SetInsertPoint(AcquireBB);
5316 Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
5318 SI->addCase(
Builder.getInt32(1), AcquireBB);
5319 SI->addCase(
Builder.getInt32(2), AcquireBB);
5321 Builder.SetInsertPoint(ReleaseBB);
5322 Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
5324 SI->addCase(
Builder.getInt32(3), ReleaseBB);
5326 Builder.SetInsertPoint(AcqRelBB);
5327 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
5329 SI->addCase(
Builder.getInt32(4), AcqRelBB);
5331 Builder.SetInsertPoint(SeqCstBB);
5332 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
5334 SI->addCase(
Builder.getInt32(5), SeqCstBB);
5336 Builder.SetInsertPoint(ContBB);
5339 case Builtin::BI__scoped_atomic_thread_fence: {
5344 auto Ord = dyn_cast<llvm::ConstantInt>(Order);
5345 auto Scp = dyn_cast<llvm::ConstantInt>(
Scope);
5347 SyncScope SS = ScopeModel->isValid(Scp->getZExtValue())
5348 ? ScopeModel->map(Scp->getZExtValue())
5349 : ScopeModel->map(ScopeModel->getFallBackValue());
5350 switch (Ord->getZExtValue()) {
5357 llvm::AtomicOrdering::Acquire,
5359 llvm::AtomicOrdering::Acquire,
5364 llvm::AtomicOrdering::Release,
5366 llvm::AtomicOrdering::Release,
5370 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease,
5373 llvm::AtomicOrdering::AcquireRelease,
5377 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
5380 llvm::AtomicOrdering::SequentiallyConsistent,
5392 switch (Ord->getZExtValue()) {
5395 ContBB->eraseFromParent();
5399 OrderBBs.emplace_back(
Builder.GetInsertBlock(),
5400 llvm::AtomicOrdering::Acquire);
5403 OrderBBs.emplace_back(
Builder.GetInsertBlock(),
5404 llvm::AtomicOrdering::Release);
5407 OrderBBs.emplace_back(
Builder.GetInsertBlock(),
5408 llvm::AtomicOrdering::AcquireRelease);
5411 OrderBBs.emplace_back(
Builder.GetInsertBlock(),
5412 llvm::AtomicOrdering::SequentiallyConsistent);
5421 Order =
Builder.CreateIntCast(Order,
Builder.getInt32Ty(),
false);
5422 llvm::SwitchInst *SI =
Builder.CreateSwitch(Order, ContBB);
5423 SI->addCase(
Builder.getInt32(1), AcquireBB);
5424 SI->addCase(
Builder.getInt32(2), AcquireBB);
5425 SI->addCase(
Builder.getInt32(3), ReleaseBB);
5426 SI->addCase(
Builder.getInt32(4), AcqRelBB);
5427 SI->addCase(
Builder.getInt32(5), SeqCstBB);
5429 OrderBBs.emplace_back(AcquireBB, llvm::AtomicOrdering::Acquire);
5430 OrderBBs.emplace_back(ReleaseBB, llvm::AtomicOrdering::Release);
5431 OrderBBs.emplace_back(AcqRelBB, llvm::AtomicOrdering::AcquireRelease);
5432 OrderBBs.emplace_back(SeqCstBB,
5433 llvm::AtomicOrdering::SequentiallyConsistent);
5436 for (
auto &[OrderBB, Ordering] : OrderBBs) {
5437 Builder.SetInsertPoint(OrderBB);
5439 SyncScope SS = ScopeModel->isValid(Scp->getZExtValue())
5440 ? ScopeModel->map(Scp->getZExtValue())
5441 : ScopeModel->map(ScopeModel->getFallBackValue());
5447 llvm::DenseMap<unsigned, llvm::BasicBlock *> BBs;
5448 for (
unsigned Scp : ScopeModel->getRuntimeValues())
5452 llvm::SwitchInst *SI =
Builder.CreateSwitch(SC, ContBB);
5453 for (
unsigned Scp : ScopeModel->getRuntimeValues()) {
5455 SI->addCase(
Builder.getInt32(Scp), B);
5466 Builder.SetInsertPoint(ContBB);
5470 case Builtin::BI__builtin_signbit:
5471 case Builtin::BI__builtin_signbitf:
5472 case Builtin::BI__builtin_signbitl: {
5477 case Builtin::BI__warn_memset_zero_len:
5479 case Builtin::BI__annotation: {
5482 for (
const Expr *Arg :
E->arguments()) {
5484 assert(Str->getCharByteWidth() == 2);
5485 StringRef WideBytes = Str->getBytes();
5486 std::string StrUtf8;
5487 if (!convertUTF16ToUTF8String(
5488 ArrayRef(WideBytes.data(), WideBytes.size()), StrUtf8)) {
5492 Strings.push_back(llvm::MDString::get(
getLLVMContext(), StrUtf8));
5502 case Builtin::BI__builtin_annotation: {
5511 StringRef Str = cast<StringLiteral>(AnnotationStrExpr)->getString();
5515 case Builtin::BI__builtin_addcb:
5516 case Builtin::BI__builtin_addcs:
5517 case Builtin::BI__builtin_addc:
5518 case Builtin::BI__builtin_addcl:
5519 case Builtin::BI__builtin_addcll:
5520 case Builtin::BI__builtin_subcb:
5521 case Builtin::BI__builtin_subcs:
5522 case Builtin::BI__builtin_subc:
5523 case Builtin::BI__builtin_subcl:
5524 case Builtin::BI__builtin_subcll: {
5550 llvm::Intrinsic::ID IntrinsicId;
5551 switch (BuiltinID) {
5552 default: llvm_unreachable(
"Unknown multiprecision builtin id.");
5553 case Builtin::BI__builtin_addcb:
5554 case Builtin::BI__builtin_addcs:
5555 case Builtin::BI__builtin_addc:
5556 case Builtin::BI__builtin_addcl:
5557 case Builtin::BI__builtin_addcll:
5558 IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
5560 case Builtin::BI__builtin_subcb:
5561 case Builtin::BI__builtin_subcs:
5562 case Builtin::BI__builtin_subc:
5563 case Builtin::BI__builtin_subcl:
5564 case Builtin::BI__builtin_subcll:
5565 IntrinsicId = llvm::Intrinsic::usub_with_overflow;
5570 llvm::Value *Carry1;
5573 llvm::Value *Carry2;
5575 Sum1, Carryin, Carry2);
5576 llvm::Value *CarryOut =
Builder.CreateZExt(
Builder.CreateOr(Carry1, Carry2),
5582 case Builtin::BI__builtin_add_overflow:
5583 case Builtin::BI__builtin_sub_overflow:
5584 case Builtin::BI__builtin_mul_overflow: {
5592 WidthAndSignedness LeftInfo =
5594 WidthAndSignedness RightInfo =
5596 WidthAndSignedness ResultInfo =
5603 RightInfo, ResultArg, ResultQTy,
5609 *
this, LeftArg, LeftInfo, RightArg, RightInfo, ResultArg, ResultQTy,
5612 WidthAndSignedness EncompassingInfo =
5615 llvm::Type *EncompassingLLVMTy =
5620 llvm::Intrinsic::ID IntrinsicId;
5621 switch (BuiltinID) {
5623 llvm_unreachable(
"Unknown overflow builtin id.");
5624 case Builtin::BI__builtin_add_overflow:
5625 IntrinsicId = EncompassingInfo.Signed
5626 ? llvm::Intrinsic::sadd_with_overflow
5627 : llvm::Intrinsic::uadd_with_overflow;
5629 case Builtin::BI__builtin_sub_overflow:
5630 IntrinsicId = EncompassingInfo.Signed
5631 ? llvm::Intrinsic::ssub_with_overflow
5632 : llvm::Intrinsic::usub_with_overflow;
5634 case Builtin::BI__builtin_mul_overflow:
5635 IntrinsicId = EncompassingInfo.Signed
5636 ? llvm::Intrinsic::smul_with_overflow
5637 : llvm::Intrinsic::umul_with_overflow;
5646 Left =
Builder.CreateIntCast(Left, EncompassingLLVMTy, LeftInfo.Signed);
5647 Right =
Builder.CreateIntCast(Right, EncompassingLLVMTy, RightInfo.Signed);
5650 llvm::Value *Overflow, *
Result;
5653 if (EncompassingInfo.Width > ResultInfo.Width) {
5656 llvm::Value *ResultTrunc =
Builder.CreateTrunc(
Result, ResultLLVMTy);
5660 llvm::Value *ResultTruncExt =
Builder.CreateIntCast(
5661 ResultTrunc, EncompassingLLVMTy, ResultInfo.Signed);
5662 llvm::Value *TruncationOverflow =
5665 Overflow =
Builder.CreateOr(Overflow, TruncationOverflow);
5677 case Builtin::BI__builtin_uadd_overflow:
5678 case Builtin::BI__builtin_uaddl_overflow:
5679 case Builtin::BI__builtin_uaddll_overflow:
5680 case Builtin::BI__builtin_usub_overflow:
5681 case Builtin::BI__builtin_usubl_overflow:
5682 case Builtin::BI__builtin_usubll_overflow:
5683 case Builtin::BI__builtin_umul_overflow:
5684 case Builtin::BI__builtin_umull_overflow:
5685 case Builtin::BI__builtin_umulll_overflow:
5686 case Builtin::BI__builtin_sadd_overflow:
5687 case Builtin::BI__builtin_saddl_overflow:
5688 case Builtin::BI__builtin_saddll_overflow:
5689 case Builtin::BI__builtin_ssub_overflow:
5690 case Builtin::BI__builtin_ssubl_overflow:
5691 case Builtin::BI__builtin_ssubll_overflow:
5692 case Builtin::BI__builtin_smul_overflow:
5693 case Builtin::BI__builtin_smull_overflow:
5694 case Builtin::BI__builtin_smulll_overflow: {
5704 llvm::Intrinsic::ID IntrinsicId;
5705 switch (BuiltinID) {
5706 default: llvm_unreachable(
"Unknown overflow builtin id.");
5707 case Builtin::BI__builtin_uadd_overflow:
5708 case Builtin::BI__builtin_uaddl_overflow:
5709 case Builtin::BI__builtin_uaddll_overflow:
5710 IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
5712 case Builtin::BI__builtin_usub_overflow:
5713 case Builtin::BI__builtin_usubl_overflow:
5714 case Builtin::BI__builtin_usubll_overflow:
5715 IntrinsicId = llvm::Intrinsic::usub_with_overflow;
5717 case Builtin::BI__builtin_umul_overflow:
5718 case Builtin::BI__builtin_umull_overflow:
5719 case Builtin::BI__builtin_umulll_overflow:
5720 IntrinsicId = llvm::Intrinsic::umul_with_overflow;
5722 case Builtin::BI__builtin_sadd_overflow:
5723 case Builtin::BI__builtin_saddl_overflow:
5724 case Builtin::BI__builtin_saddll_overflow:
5725 IntrinsicId = llvm::Intrinsic::sadd_with_overflow;
5727 case Builtin::BI__builtin_ssub_overflow:
5728 case Builtin::BI__builtin_ssubl_overflow:
5729 case Builtin::BI__builtin_ssubll_overflow:
5730 IntrinsicId = llvm::Intrinsic::ssub_with_overflow;
5732 case Builtin::BI__builtin_smul_overflow:
5733 case Builtin::BI__builtin_smull_overflow:
5734 case Builtin::BI__builtin_smulll_overflow:
5735 IntrinsicId = llvm::Intrinsic::smul_with_overflow;
5746 case Builtin::BIaddressof:
5747 case Builtin::BI__addressof:
5748 case Builtin::BI__builtin_addressof:
5750 case Builtin::BI__builtin_function_start:
5753 case Builtin::BI__builtin_operator_new:
5756 case Builtin::BI__builtin_operator_delete:
5761 case Builtin::BI__builtin_is_aligned:
5763 case Builtin::BI__builtin_align_up:
5765 case Builtin::BI__builtin_align_down:
5768 case Builtin::BI__noop:
5771 case Builtin::BI__builtin_call_with_static_chain: {
5773 const Expr *Chain =
E->getArg(1);
5778 case Builtin::BI_InterlockedExchange8:
5779 case Builtin::BI_InterlockedExchange16:
5780 case Builtin::BI_InterlockedExchange:
5781 case Builtin::BI_InterlockedExchangePointer:
5784 case Builtin::BI_InterlockedCompareExchangePointer:
5787 case Builtin::BI_InterlockedCompareExchangePointer_nf:
5790 case Builtin::BI_InterlockedCompareExchange8:
5791 case Builtin::BI_InterlockedCompareExchange16:
5792 case Builtin::BI_InterlockedCompareExchange:
5793 case Builtin::BI_InterlockedCompareExchange64:
5795 case Builtin::BI_InterlockedIncrement16:
5796 case Builtin::BI_InterlockedIncrement:
5799 case Builtin::BI_InterlockedDecrement16:
5800 case Builtin::BI_InterlockedDecrement:
5803 case Builtin::BI_InterlockedAnd8:
5804 case Builtin::BI_InterlockedAnd16:
5805 case Builtin::BI_InterlockedAnd:
5807 case Builtin::BI_InterlockedExchangeAdd8:
5808 case Builtin::BI_InterlockedExchangeAdd16:
5809 case Builtin::BI_InterlockedExchangeAdd:
5812 case Builtin::BI_InterlockedExchangeSub8:
5813 case Builtin::BI_InterlockedExchangeSub16:
5814 case Builtin::BI_InterlockedExchangeSub:
5817 case Builtin::BI_InterlockedOr8:
5818 case Builtin::BI_InterlockedOr16:
5819 case Builtin::BI_InterlockedOr:
5821 case Builtin::BI_InterlockedXor8:
5822 case Builtin::BI_InterlockedXor16:
5823 case Builtin::BI_InterlockedXor:
5826 case Builtin::BI_bittest64:
5827 case Builtin::BI_bittest:
5828 case Builtin::BI_bittestandcomplement64:
5829 case Builtin::BI_bittestandcomplement:
5830 case Builtin::BI_bittestandreset64:
5831 case Builtin::BI_bittestandreset:
5832 case Builtin::BI_bittestandset64:
5833 case Builtin::BI_bittestandset:
5834 case Builtin::BI_interlockedbittestandreset:
5835 case Builtin::BI_interlockedbittestandreset64:
5836 case Builtin::BI_interlockedbittestandset64:
5837 case Builtin::BI_interlockedbittestandset:
5838 case Builtin::BI_interlockedbittestandset_acq:
5839 case Builtin::BI_interlockedbittestandset_rel:
5840 case Builtin::BI_interlockedbittestandset_nf:
5841 case Builtin::BI_interlockedbittestandreset_acq:
5842 case Builtin::BI_interlockedbittestandreset_rel:
5843 case Builtin::BI_interlockedbittestandreset_nf:
5848 case Builtin::BI__iso_volatile_load8:
5849 case Builtin::BI__iso_volatile_load16:
5850 case Builtin::BI__iso_volatile_load32:
5851 case Builtin::BI__iso_volatile_load64:
5853 case Builtin::BI__iso_volatile_store8:
5854 case Builtin::BI__iso_volatile_store16:
5855 case Builtin::BI__iso_volatile_store32:
5856 case Builtin::BI__iso_volatile_store64:
5859 case Builtin::BI__builtin_ptrauth_sign_constant:
5862 case Builtin::BI__builtin_ptrauth_auth:
5863 case Builtin::BI__builtin_ptrauth_auth_and_resign:
5864 case Builtin::BI__builtin_ptrauth_blend_discriminator:
5865 case Builtin::BI__builtin_ptrauth_sign_generic_data:
5866 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
5867 case Builtin::BI__builtin_ptrauth_strip: {
5870 for (
auto argExpr :
E->arguments())
5874 llvm::Type *OrigValueType = Args[0]->getType();
5875 if (OrigValueType->isPointerTy())
5878 switch (BuiltinID) {
5879 case Builtin::BI__builtin_ptrauth_auth_and_resign:
5880 if (Args[4]->getType()->isPointerTy())
5884 case Builtin::BI__builtin_ptrauth_auth:
5885 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
5886 if (Args[2]->getType()->isPointerTy())
5890 case Builtin::BI__builtin_ptrauth_sign_generic_data:
5891 if (Args[1]->getType()->isPointerTy())
5895 case Builtin::BI__builtin_ptrauth_blend_discriminator:
5896 case Builtin::BI__builtin_ptrauth_strip:
5901 auto IntrinsicID = [&]() ->
unsigned {
5902 switch (BuiltinID) {
5903 case Builtin::BI__builtin_ptrauth_auth:
5904 return llvm::Intrinsic::ptrauth_auth;
5905 case Builtin::BI__builtin_ptrauth_auth_and_resign:
5906 return llvm::Intrinsic::ptrauth_resign;
5907 case Builtin::BI__builtin_ptrauth_blend_discriminator:
5908 return llvm::Intrinsic::ptrauth_blend;
5909 case Builtin::BI__builtin_ptrauth_sign_generic_data:
5910 return llvm::Intrinsic::ptrauth_sign_generic;
5911 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
5912 return llvm::Intrinsic::ptrauth_sign;
5913 case Builtin::BI__builtin_ptrauth_strip:
5914 return llvm::Intrinsic::ptrauth_strip;
5916 llvm_unreachable(
"bad ptrauth intrinsic");
5921 if (BuiltinID != Builtin::BI__builtin_ptrauth_sign_generic_data &&
5922 BuiltinID != Builtin::BI__builtin_ptrauth_blend_discriminator &&
5923 OrigValueType->isPointerTy()) {
5929 case Builtin::BI__exception_code:
5930 case Builtin::BI_exception_code:
5932 case Builtin::BI__exception_info:
5933 case Builtin::BI_exception_info:
5935 case Builtin::BI__abnormal_termination:
5936 case Builtin::BI_abnormal_termination:
5938 case Builtin::BI_setjmpex:
5939 if (
getTarget().getTriple().isOSMSVCRT() &&
E->getNumArgs() == 1 &&
5943 case Builtin::BI_setjmp:
5944 if (
getTarget().getTriple().isOSMSVCRT() &&
E->getNumArgs() == 1 &&
5946 if (
getTarget().getTriple().getArch() == llvm::Triple::x86)
5948 else if (
getTarget().getTriple().getArch() == llvm::Triple::aarch64)
5955 case Builtin::BImove:
5956 case Builtin::BImove_if_noexcept:
5957 case Builtin::BIforward:
5958 case Builtin::BIforward_like:
5959 case Builtin::BIas_const:
5961 case Builtin::BI__GetExceptionInfo: {
5962 if (llvm::GlobalVariable *GV =
5968 case Builtin::BI__fastfail:
5971 case Builtin::BI__builtin_coro_id:
5973 case Builtin::BI__builtin_coro_promise:
5975 case Builtin::BI__builtin_coro_resume:
5978 case Builtin::BI__builtin_coro_frame:
5980 case Builtin::BI__builtin_coro_noop:
5982 case Builtin::BI__builtin_coro_free:
5984 case Builtin::BI__builtin_coro_destroy:
5987 case Builtin::BI__builtin_coro_done:
5989 case Builtin::BI__builtin_coro_alloc:
5991 case Builtin::BI__builtin_coro_begin:
5993 case Builtin::BI__builtin_coro_end:
5995 case Builtin::BI__builtin_coro_suspend:
5997 case Builtin::BI__builtin_coro_size:
5999 case Builtin::BI__builtin_coro_align:
6003 case Builtin::BIread_pipe:
6004 case Builtin::BIwrite_pipe: {
6008 Value *PacketSize = OpenCLRT.getPipeElemSize(
E->getArg(0));
6009 Value *PacketAlign = OpenCLRT.getPipeElemAlign(
E->getArg(0));
6012 unsigned GenericAS =
6014 llvm::Type *I8PTy = llvm::PointerType::get(
getLLVMContext(), GenericAS);
6017 if (2U ==
E->getNumArgs()) {
6018 const char *Name = (BuiltinID == Builtin::BIread_pipe) ?
"__read_pipe_2"
6023 llvm::FunctionType *FTy = llvm::FunctionType::get(
6028 {Arg0, ACast, PacketSize, PacketAlign}));
6030 assert(4 ==
E->getNumArgs() &&
6031 "Illegal number of parameters to pipe function");
6032 const char *Name = (BuiltinID == Builtin::BIread_pipe) ?
"__read_pipe_4"
6039 llvm::FunctionType *FTy = llvm::FunctionType::get(
6048 {Arg0, Arg1, Arg2, ACast, PacketSize, PacketAlign}));
6053 case Builtin::BIreserve_read_pipe:
6054 case Builtin::BIreserve_write_pipe:
6055 case Builtin::BIwork_group_reserve_read_pipe:
6056 case Builtin::BIwork_group_reserve_write_pipe:
6057 case Builtin::BIsub_group_reserve_read_pipe:
6058 case Builtin::BIsub_group_reserve_write_pipe: {
6061 if (BuiltinID == Builtin::BIreserve_read_pipe)
6062 Name =
"__reserve_read_pipe";
6063 else if (BuiltinID == Builtin::BIreserve_write_pipe)
6064 Name =
"__reserve_write_pipe";
6065 else if (BuiltinID == Builtin::BIwork_group_reserve_read_pipe)
6066 Name =
"__work_group_reserve_read_pipe";
6067 else if (BuiltinID == Builtin::BIwork_group_reserve_write_pipe)
6068 Name =
"__work_group_reserve_write_pipe";
6069 else if (BuiltinID == Builtin::BIsub_group_reserve_read_pipe)
6070 Name =
"__sub_group_reserve_read_pipe";
6072 Name =
"__sub_group_reserve_write_pipe";
6078 Value *PacketSize = OpenCLRT.getPipeElemSize(
E->getArg(0));
6079 Value *PacketAlign = OpenCLRT.getPipeElemAlign(
E->getArg(0));
6083 llvm::FunctionType *FTy = llvm::FunctionType::get(
6090 {Arg0, Arg1, PacketSize, PacketAlign}));
6094 case Builtin::BIcommit_read_pipe:
6095 case Builtin::BIcommit_write_pipe:
6096 case Builtin::BIwork_group_commit_read_pipe:
6097 case Builtin::BIwork_group_commit_write_pipe:
6098 case Builtin::BIsub_group_commit_read_pipe:
6099 case Builtin::BIsub_group_commit_write_pipe: {
6101 if (BuiltinID == Builtin::BIcommit_read_pipe)
6102 Name =
"__commit_read_pipe";
6103 else if (BuiltinID == Builtin::BIcommit_write_pipe)
6104 Name =
"__commit_write_pipe";
6105 else if (BuiltinID == Builtin::BIwork_group_commit_read_pipe)
6106 Name =
"__work_group_commit_read_pipe";
6107 else if (BuiltinID == Builtin::BIwork_group_commit_write_pipe)
6108 Name =
"__work_group_commit_write_pipe";
6109 else if (BuiltinID == Builtin::BIsub_group_commit_read_pipe)
6110 Name =
"__sub_group_commit_read_pipe";
6112 Name =
"__sub_group_commit_write_pipe";
6117 Value *PacketSize = OpenCLRT.getPipeElemSize(
E->getArg(0));
6118 Value *PacketAlign = OpenCLRT.getPipeElemAlign(
E->getArg(0));
6122 llvm::FunctionType *FTy =
6127 {Arg0, Arg1, PacketSize, PacketAlign}));
6130 case Builtin::BIget_pipe_num_packets:
6131 case Builtin::BIget_pipe_max_packets: {
6132 const char *BaseName;
6134 if (BuiltinID == Builtin::BIget_pipe_num_packets)
6135 BaseName =
"__get_pipe_num_packets";
6137 BaseName =
"__get_pipe_max_packets";
6138 std::string Name = std::string(BaseName) +
6139 std::string(PipeTy->isReadOnly() ?
"_ro" :
"_wo");
6144 Value *PacketSize = OpenCLRT.getPipeElemSize(
E->getArg(0));
6145 Value *PacketAlign = OpenCLRT.getPipeElemAlign(
E->getArg(0));
6147 llvm::FunctionType *FTy = llvm::FunctionType::get(
6151 {Arg0, PacketSize, PacketAlign}));
6155 case Builtin::BIto_global:
6156 case Builtin::BIto_local:
6157 case Builtin::BIto_private: {
6159 auto NewArgT = llvm::PointerType::get(
6162 auto NewRetT = llvm::PointerType::get(
6166 auto FTy = llvm::FunctionType::get(NewRetT, {NewArgT},
false);
6167 llvm::Value *NewArg;
6168 if (Arg0->
getType()->getPointerAddressSpace() !=
6169 NewArgT->getPointerAddressSpace())
6172 NewArg =
Builder.CreateBitOrPointerCast(Arg0, NewArgT);
6173 auto NewName = std::string(
"__") +
E->getDirectCallee()->getName().str();
6188 case Builtin::BIenqueue_kernel: {
6190 unsigned NumArgs =
E->getNumArgs();
6193 llvm::Type *GenericVoidPtrTy =
Builder.getPtrTy(
6205 Name =
"__enqueue_kernel_basic";
6206 llvm::Type *ArgTys[] = {QueueTy,
Int32Ty, RangeTy, GenericVoidPtrTy,
6208 llvm::FunctionType *FTy = llvm::FunctionType::get(
6214 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
6215 llvm::Value *
Block =
6216 Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
6219 {Queue, Flags, Range, Kernel, Block});
6222 assert(NumArgs >= 5 &&
"Invalid enqueue_kernel signature");
6226 auto CreateArrayForSizeVar = [=](
unsigned First)
6227 -> std::tuple<llvm::Value *, llvm::Value *, llvm::Value *> {
6228 llvm::APInt ArraySize(32, NumArgs -
First);
6230 getContext().getSizeType(), ArraySize,
nullptr,
6234 llvm::Value *TmpPtr = Tmp.getPointer();
6239 llvm::Value *Alloca = TmpPtr->stripPointerCasts();
6242 llvm::Value *ElemPtr;
6245 auto *
Zero = llvm::ConstantInt::get(
IntTy, 0);
6246 for (
unsigned I =
First; I < NumArgs; ++I) {
6247 auto *Index = llvm::ConstantInt::get(
IntTy, I -
First);
6259 return std::tie(ElemPtr, TmpSize, Alloca);
6265 Name =
"__enqueue_kernel_varargs";
6269 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
6270 auto *
Block =
Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
6271 llvm::Value *ElemPtr, *TmpSize, *TmpPtr;
6272 std::tie(ElemPtr, TmpSize, TmpPtr) = CreateArrayForSizeVar(4);
6276 llvm::Value *
const Args[] = {Queue, Flags,
6280 llvm::Type *
const ArgTys[] = {
6281 QueueTy,
IntTy, RangeTy, GenericVoidPtrTy,
6282 GenericVoidPtrTy,
IntTy, ElemPtr->getType()};
6284 llvm::FunctionType *FTy = llvm::FunctionType::get(
Int32Ty, ArgTys,
false);
6293 llvm::PointerType *PtrTy = llvm::PointerType::get(
6297 llvm::Value *NumEvents =
6303 llvm::Value *EventWaitList =
nullptr;
6306 EventWaitList = llvm::ConstantPointerNull::get(PtrTy);
6313 EventWaitList =
Builder.CreatePointerCast(EventWaitList, PtrTy);
6315 llvm::Value *EventRet =
nullptr;
6318 EventRet = llvm::ConstantPointerNull::get(PtrTy);
6327 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
6328 llvm::Value *
Block =
6329 Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
6331 std::vector<llvm::Type *> ArgTys = {
6333 PtrTy, PtrTy, GenericVoidPtrTy, GenericVoidPtrTy};
6335 std::vector<llvm::Value *> Args = {Queue, Flags,
Range,
6336 NumEvents, EventWaitList, EventRet,
6341 Name =
"__enqueue_kernel_basic_events";
6342 llvm::FunctionType *FTy = llvm::FunctionType::get(
6350 Args.push_back(ConstantInt::get(
Int32Ty, NumArgs - 7));
6352 Name =
"__enqueue_kernel_events_varargs";
6354 llvm::Value *ElemPtr, *TmpSize, *TmpPtr;
6355 std::tie(ElemPtr, TmpSize, TmpPtr) = CreateArrayForSizeVar(7);
6356 Args.push_back(ElemPtr);
6357 ArgTys.push_back(ElemPtr->getType());
6359 llvm::FunctionType *FTy = llvm::FunctionType::get(
6368 llvm_unreachable(
"Unexpected enqueue_kernel signature");
6372 case Builtin::BIget_kernel_work_group_size: {
6373 llvm::Type *GenericVoidPtrTy =
Builder.getPtrTy(
6378 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
6379 Value *Arg =
Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
6382 llvm::FunctionType::get(
IntTy, {GenericVoidPtrTy, GenericVoidPtrTy},
6384 "__get_kernel_work_group_size_impl"),
6387 case Builtin::BIget_kernel_preferred_work_group_size_multiple: {
6388 llvm::Type *GenericVoidPtrTy =
Builder.getPtrTy(
6393 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
6394 Value *Arg =
Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
6397 llvm::FunctionType::get(
IntTy, {GenericVoidPtrTy, GenericVoidPtrTy},
6399 "__get_kernel_preferred_work_group_size_multiple_impl"),
6402 case Builtin::BIget_kernel_max_sub_group_size_for_ndrange:
6403 case Builtin::BIget_kernel_sub_group_count_for_ndrange: {
6404 llvm::Type *GenericVoidPtrTy =
Builder.getPtrTy(
6411 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
6414 BuiltinID == Builtin::BIget_kernel_max_sub_group_size_for_ndrange
6415 ?
"__get_kernel_max_sub_group_size_for_ndrange_impl"
6416 :
"__get_kernel_sub_group_count_for_ndrange_impl";
6419 llvm::FunctionType::get(
6420 IntTy, {NDRange->getType(), GenericVoidPtrTy, GenericVoidPtrTy},
6423 {NDRange, Kernel, Block}));
6425 case Builtin::BI__builtin_store_half:
6426 case Builtin::BI__builtin_store_halff: {
6433 case Builtin::BI__builtin_load_half: {
6438 case Builtin::BI__builtin_load_halff: {
6443 case Builtin::BI__builtin_printf:
6444 case Builtin::BIprintf:
6445 if (
getTarget().getTriple().isNVPTX() ||
6448 getTarget().getTriple().getVendor() == Triple::VendorType::AMD)) {
6451 if ((
getTarget().getTriple().isAMDGCN() ||
6458 case Builtin::BI__builtin_canonicalize:
6459 case Builtin::BI__builtin_canonicalizef:
6460 case Builtin::BI__builtin_canonicalizef16:
6461 case Builtin::BI__builtin_canonicalizel:
6463 emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::canonicalize));
6465 case Builtin::BI__builtin_thread_pointer: {
6466 if (!
getContext().getTargetInfo().isTLSSupported())
6471 case Builtin::BI__builtin_os_log_format:
6474 case Builtin::BI__xray_customevent: {
6487 auto FTy = F->getFunctionType();
6488 auto Arg0 =
E->getArg(0);
6490 auto Arg0Ty = Arg0->
getType();
6491 auto PTy0 = FTy->getParamType(0);
6492 if (PTy0 != Arg0Val->getType()) {
6493 if (Arg0Ty->isArrayType())
6496 Arg0Val =
Builder.CreatePointerCast(Arg0Val, PTy0);
6499 auto PTy1 = FTy->getParamType(1);
6501 Arg1 =
Builder.CreateTruncOrBitCast(Arg1, PTy1);
6505 case Builtin::BI__xray_typedevent: {
6521 auto FTy = F->getFunctionType();
6523 auto PTy0 = FTy->getParamType(0);
6525 Arg0 =
Builder.CreateTruncOrBitCast(Arg0, PTy0);
6526 auto Arg1 =
E->getArg(1);
6528 auto Arg1Ty = Arg1->
getType();
6529 auto PTy1 = FTy->getParamType(1);
6530 if (PTy1 != Arg1Val->getType()) {
6531 if (Arg1Ty->isArrayType())
6534 Arg1Val =
Builder.CreatePointerCast(Arg1Val, PTy1);
6537 auto PTy2 = FTy->getParamType(2);
6539 Arg2 =
Builder.CreateTruncOrBitCast(Arg2, PTy2);
6543 case Builtin::BI__builtin_ms_va_start:
6544 case Builtin::BI__builtin_ms_va_end:
6547 BuiltinID == Builtin::BI__builtin_ms_va_start));
6549 case Builtin::BI__builtin_ms_va_copy: {
6566 case Builtin::BI__builtin_get_device_side_mangled_name: {
6594 LargestVectorWidth = std::max(LargestVectorWidth, VectorWidth);
6598 Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic;
6600 llvm::Triple::getArchTypePrefix(
getTarget().getTriple().getArch());
6601 if (!Prefix.empty()) {
6602 IntrinsicID = Intrinsic::getIntrinsicForClangBuiltin(Prefix.data(), Name);
6603 if (IntrinsicID == Intrinsic::not_intrinsic && Prefix ==
"spv" &&
6604 getTarget().getTriple().getOS() == llvm::Triple::OSType::AMDHSA)
6605 IntrinsicID = Intrinsic::getIntrinsicForClangBuiltin(
"amdgcn", Name);
6609 if (IntrinsicID == Intrinsic::not_intrinsic)
6610 IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(Prefix.data(), Name);
6613 if (IntrinsicID != Intrinsic::not_intrinsic) {
6618 unsigned ICEArguments = 0;
6624 llvm::FunctionType *FTy = F->getFunctionType();
6626 for (
unsigned i = 0, e =
E->getNumArgs(); i != e; ++i) {
6630 llvm::Type *PTy = FTy->getParamType(i);
6631 if (PTy != ArgValue->
getType()) {
6633 if (
auto *PtrTy = dyn_cast<llvm::PointerType>(PTy)) {
6634 if (PtrTy->getAddressSpace() !=
6635 ArgValue->
getType()->getPointerAddressSpace()) {
6638 PtrTy->getAddressSpace()));
6644 if (PTy->isX86_AMXTy())
6645 ArgValue =
Builder.CreateIntrinsic(Intrinsic::x86_cast_vector_to_tile,
6646 {ArgValue->
getType()}, {ArgValue});
6648 ArgValue =
Builder.CreateBitCast(ArgValue, PTy);
6651 Args.push_back(ArgValue);
6657 llvm::Type *RetTy =
VoidTy;
6661 if (RetTy !=
V->getType()) {
6663 if (
auto *PtrTy = dyn_cast<llvm::PointerType>(RetTy)) {
6664 if (PtrTy->getAddressSpace() !=
V->getType()->getPointerAddressSpace()) {
6667 PtrTy->getAddressSpace()));
6673 if (
V->getType()->isX86_AMXTy())
6674 V =
Builder.CreateIntrinsic(Intrinsic::x86_cast_tile_to_vector, {RetTy},
6680 if (RetTy->isVoidTy())
6700 if (
V->getType()->isVoidTy())
6707 llvm_unreachable(
"No current target builtin returns complex");
6709 llvm_unreachable(
"Bad evaluation kind in EmitBuiltinExpr");
6716 if (
V->getType()->isVoidTy())
6723 llvm_unreachable(
"No current hlsl builtin returns complex");
6725 llvm_unreachable(
"Bad evaluation kind in EmitBuiltinExpr");
6740 llvm::Triple::ArchType Arch) {
6752 case llvm::Triple::arm:
6753 case llvm::Triple::armeb:
6754 case llvm::Triple::thumb:
6755 case llvm::Triple::thumbeb:
6757 case llvm::Triple::aarch64:
6758 case llvm::Triple::aarch64_32:
6759 case llvm::Triple::aarch64_be:
6761 case llvm::Triple::bpfeb:
6762 case llvm::Triple::bpfel:
6764 case llvm::Triple::x86:
6765 case llvm::Triple::x86_64:
6767 case llvm::Triple::ppc:
6768 case llvm::Triple::ppcle:
6769 case llvm::Triple::ppc64:
6770 case llvm::Triple::ppc64le:
6772 case llvm::Triple::r600:
6773 case llvm::Triple::amdgcn:
6775 case llvm::Triple::systemz:
6777 case llvm::Triple::nvptx:
6778 case llvm::Triple::nvptx64:
6780 case llvm::Triple::wasm32:
6781 case llvm::Triple::wasm64:
6783 case llvm::Triple::hexagon:
6785 case llvm::Triple::riscv32:
6786 case llvm::Triple::riscv64:
6788 case llvm::Triple::spirv:
6790 case llvm::Triple::spirv64:
6803 assert(
getContext().getAuxTargetInfo() &&
"Missing aux target info");
6815 bool HasLegalHalfType =
true,
6817 bool AllowBFloatArgsAndRet =
true) {
6818 int IsQuad = TypeFlags.
isQuad();
6822 return llvm::FixedVectorType::get(CGF->
Int8Ty, V1Ty ? 1 : (8 << IsQuad));
6825 return llvm::FixedVectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
6827 if (AllowBFloatArgsAndRet)
6828 return llvm::FixedVectorType::get(CGF->
BFloatTy, V1Ty ? 1 : (4 << IsQuad));
6830 return llvm::FixedVectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
6832 if (HasLegalHalfType)
6833 return llvm::FixedVectorType::get(CGF->
HalfTy, V1Ty ? 1 : (4 << IsQuad));
6835 return llvm::FixedVectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
6837 return llvm::FixedVectorType::get(CGF->
Int32Ty, V1Ty ? 1 : (2 << IsQuad));
6840 return llvm::FixedVectorType::get(CGF->
Int64Ty, V1Ty ? 1 : (1 << IsQuad));
6845 return llvm::FixedVectorType::get(CGF->
Int8Ty, 16);
6847 return llvm::FixedVectorType::get(CGF->
FloatTy, V1Ty ? 1 : (2 << IsQuad));
6849 return llvm::FixedVectorType::get(CGF->
DoubleTy, V1Ty ? 1 : (1 << IsQuad));
6851 llvm_unreachable(
"Unknown vector element type!");
6856 int IsQuad = IntTypeFlags.
isQuad();
6859 return llvm::FixedVectorType::get(CGF->
HalfTy, (4 << IsQuad));
6861 return llvm::FixedVectorType::get(CGF->
FloatTy, (2 << IsQuad));
6863 return llvm::FixedVectorType::get(CGF->
DoubleTy, (1 << IsQuad));
6865 llvm_unreachable(
"Type can't be converted to floating-point!");
6870 const ElementCount &Count) {
6871 Value *SV = llvm::ConstantVector::getSplat(Count,
C);
6872 return Builder.CreateShuffleVector(
V,
V, SV,
"lane");
6876 ElementCount EC = cast<llvm::VectorType>(
V->getType())->getElementCount();
6882 unsigned shift,
bool rightshift) {
6884 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
6885 ai != ae; ++ai, ++j) {
6886 if (F->isConstrainedFPIntrinsic())
6887 if (ai->getType()->isMetadataTy())
6889 if (shift > 0 && shift == j)
6892 Ops[j] =
Builder.CreateBitCast(Ops[j], ai->getType(), name);
6895 if (F->isConstrainedFPIntrinsic())
6896 return Builder.CreateConstrainedFPCall(F, Ops, name);
6898 return Builder.CreateCall(F, Ops, name);
6903 int SV = cast<ConstantInt>(
V)->getSExtValue();
6904 return ConstantInt::get(Ty, neg ? -SV : SV);
6909 llvm::Type *Ty,
bool usgn,
6911 llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
6913 int ShiftAmt = cast<ConstantInt>(Shift)->getSExtValue();
6914 int EltSize = VTy->getScalarSizeInBits();
6916 Vec =
Builder.CreateBitCast(Vec, Ty);
6920 if (ShiftAmt == EltSize) {
6923 return llvm::ConstantAggregateZero::get(VTy);
6928 Shift = ConstantInt::get(VTy->getElementType(), ShiftAmt);
6934 return Builder.CreateLShr(Vec, Shift, name);
6936 return Builder.CreateAShr(Vec, Shift, name);
6962struct ARMVectorIntrinsicInfo {
6963 const char *NameHint;
6965 unsigned LLVMIntrinsic;
6966 unsigned AltLLVMIntrinsic;
6969 bool operator<(
unsigned RHSBuiltinID)
const {
6970 return BuiltinID < RHSBuiltinID;
6972 bool operator<(
const ARMVectorIntrinsicInfo &TE)
const {
6973 return BuiltinID < TE.BuiltinID;
6978#define NEONMAP0(NameBase) \
6979 { #NameBase, NEON::BI__builtin_neon_ ## NameBase, 0, 0, 0 }
6981#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
6982 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
6983 Intrinsic::LLVMIntrinsic, 0, TypeModifier }
6985#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier) \
6986 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
6987 Intrinsic::LLVMIntrinsic, Intrinsic::AltLLVMIntrinsic, \
6991 NEONMAP1(__a32_vcvt_bf16_f32, arm_neon_vcvtfp2bf, 0),
6998 NEONMAP1(vabs_v, arm_neon_vabs, 0),
6999 NEONMAP1(vabsq_v, arm_neon_vabs, 0),
7003 NEONMAP1(vaesdq_u8, arm_neon_aesd, 0),
7004 NEONMAP1(vaeseq_u8, arm_neon_aese, 0),
7005 NEONMAP1(vaesimcq_u8, arm_neon_aesimc, 0),
7006 NEONMAP1(vaesmcq_u8, arm_neon_aesmc, 0),
7007 NEONMAP1(vbfdot_f32, arm_neon_bfdot, 0),
7008 NEONMAP1(vbfdotq_f32, arm_neon_bfdot, 0),
7009 NEONMAP1(vbfmlalbq_f32, arm_neon_bfmlalb, 0),
7010 NEONMAP1(vbfmlaltq_f32, arm_neon_bfmlalt, 0),
7011 NEONMAP1(vbfmmlaq_f32, arm_neon_bfmmla, 0),
7024 NEONMAP1(vcage_v, arm_neon_vacge, 0),
7025 NEONMAP1(vcageq_v, arm_neon_vacge, 0),
7026 NEONMAP1(vcagt_v, arm_neon_vacgt, 0),
7027 NEONMAP1(vcagtq_v, arm_neon_vacgt, 0),
7028 NEONMAP1(vcale_v, arm_neon_vacge, 0),
7029 NEONMAP1(vcaleq_v, arm_neon_vacge, 0),
7030 NEONMAP1(vcalt_v, arm_neon_vacgt, 0),
7031 NEONMAP1(vcaltq_v, arm_neon_vacgt, 0),
7048 NEONMAP1(vcvt_f16_f32, arm_neon_vcvtfp2hf, 0),
7051 NEONMAP1(vcvt_f32_f16, arm_neon_vcvthf2fp, 0),
7053 NEONMAP1(vcvt_n_f16_s16, arm_neon_vcvtfxs2fp, 0),
7054 NEONMAP1(vcvt_n_f16_u16, arm_neon_vcvtfxu2fp, 0),
7055 NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
7056 NEONMAP1(vcvt_n_s16_f16, arm_neon_vcvtfp2fxs, 0),
7057 NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0),
7058 NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0),
7059 NEONMAP1(vcvt_n_u16_f16, arm_neon_vcvtfp2fxu, 0),
7060 NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0),
7061 NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0),
7068 NEONMAP1(vcvta_s16_f16, arm_neon_vcvtas, 0),
7069 NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0),
7070 NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0),
7071 NEONMAP1(vcvta_u16_f16, arm_neon_vcvtau, 0),
7072 NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0),
7073 NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0),
7074 NEONMAP1(vcvtaq_s16_f16, arm_neon_vcvtas, 0),
7075 NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0),
7076 NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0),
7077 NEONMAP1(vcvtaq_u16_f16, arm_neon_vcvtau, 0),
7078 NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0),
7079 NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0),
7080 NEONMAP1(vcvth_bf16_f32, arm_neon_vcvtbfp2bf, 0),
7081 NEONMAP1(vcvtm_s16_f16, arm_neon_vcvtms, 0),
7082 NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0),
7083 NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0),
7084 NEONMAP1(vcvtm_u16_f16, arm_neon_vcvtmu, 0),
7085 NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0),
7086 NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0),
7087 NEONMAP1(vcvtmq_s16_f16, arm_neon_vcvtms, 0),
7088 NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0),
7089 NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0),
7090 NEONMAP1(vcvtmq_u16_f16, arm_neon_vcvtmu, 0),
7091 NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0),
7092 NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0),
7093 NEONMAP1(vcvtn_s16_f16, arm_neon_vcvtns, 0),
7094 NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0),
7095 NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0),
7096 NEONMAP1(vcvtn_u16_f16, arm_neon_vcvtnu, 0),
7097 NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0),
7098 NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0),
7099 NEONMAP1(vcvtnq_s16_f16, arm_neon_vcvtns, 0),
7100 NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0),
7101 NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0),
7102 NEONMAP1(vcvtnq_u16_f16, arm_neon_vcvtnu, 0),
7103 NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0),
7104 NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0),
7105 NEONMAP1(vcvtp_s16_f16, arm_neon_vcvtps, 0),
7106 NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0),
7107 NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0),
7108 NEONMAP1(vcvtp_u16_f16, arm_neon_vcvtpu, 0),
7109 NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0),
7110 NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0),
7111 NEONMAP1(vcvtpq_s16_f16, arm_neon_vcvtps, 0),
7112 NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0),
7113 NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0),
7114 NEONMAP1(vcvtpq_u16_f16, arm_neon_vcvtpu, 0),
7115 NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0),
7116 NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0),
7120 NEONMAP1(vcvtq_n_f16_s16, arm_neon_vcvtfxs2fp, 0),
7121 NEONMAP1(vcvtq_n_f16_u16, arm_neon_vcvtfxu2fp, 0),
7122 NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
7123 NEONMAP1(vcvtq_n_s16_f16, arm_neon_vcvtfp2fxs, 0),
7124 NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0),
7125 NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0),
7126 NEONMAP1(vcvtq_n_u16_f16, arm_neon_vcvtfp2fxu, 0),
7127 NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0),
7128 NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0),
7135 NEONMAP1(vdot_s32, arm_neon_sdot, 0),
7136 NEONMAP1(vdot_u32, arm_neon_udot, 0),
7137 NEONMAP1(vdotq_s32, arm_neon_sdot, 0),
7138 NEONMAP1(vdotq_u32, arm_neon_udot, 0),
7148 NEONMAP1(vld1_v, arm_neon_vld1, 0),
7149 NEONMAP1(vld1_x2_v, arm_neon_vld1x2, 0),
7150 NEONMAP1(vld1_x3_v, arm_neon_vld1x3, 0),
7151 NEONMAP1(vld1_x4_v, arm_neon_vld1x4, 0),
7153 NEONMAP1(vld1q_v, arm_neon_vld1, 0),
7154 NEONMAP1(vld1q_x2_v, arm_neon_vld1x2, 0),
7155 NEONMAP1(vld1q_x3_v, arm_neon_vld1x3, 0),
7156 NEONMAP1(vld1q_x4_v, arm_neon_vld1x4, 0),
7157 NEONMAP1(vld2_dup_v, arm_neon_vld2dup, 0),
7158 NEONMAP1(vld2_lane_v, arm_neon_vld2lane, 0),
7159 NEONMAP1(vld2_v, arm_neon_vld2, 0),
7160 NEONMAP1(vld2q_dup_v, arm_neon_vld2dup, 0),
7161 NEONMAP1(vld2q_lane_v, arm_neon_vld2lane, 0),
7162 NEONMAP1(vld2q_v, arm_neon_vld2, 0),
7163 NEONMAP1(vld3_dup_v, arm_neon_vld3dup, 0),
7164 NEONMAP1(vld3_lane_v, arm_neon_vld3lane, 0),
7165 NEONMAP1(vld3_v, arm_neon_vld3, 0),
7166 NEONMAP1(vld3q_dup_v, arm_neon_vld3dup, 0),
7167 NEONMAP1(vld3q_lane_v, arm_neon_vld3lane, 0),
7168 NEONMAP1(vld3q_v, arm_neon_vld3, 0),
7169 NEONMAP1(vld4_dup_v, arm_neon_vld4dup, 0),
7170 NEONMAP1(vld4_lane_v, arm_neon_vld4lane, 0),
7171 NEONMAP1(vld4_v, arm_neon_vld4, 0),
7172 NEONMAP1(vld4q_dup_v, arm_neon_vld4dup, 0),
7173 NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0),
7174 NEONMAP1(vld4q_v, arm_neon_vld4, 0),
7183 NEONMAP1(vmmlaq_s32, arm_neon_smmla, 0),
7184 NEONMAP1(vmmlaq_u32, arm_neon_ummla, 0),
7202 NEONMAP2(vqdmlal_v, arm_neon_vqdmull, sadd_sat, 0),
7203 NEONMAP2(vqdmlsl_v, arm_neon_vqdmull, ssub_sat, 0),
7227 NEONMAP1(vqshlu_n_v, arm_neon_vqshiftsu, 0),
7228 NEONMAP1(vqshluq_n_v, arm_neon_vqshiftsu, 0),
7232 NEONMAP2(vrecpe_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
7233 NEONMAP2(vrecpeq_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
7256 NEONMAP2(vrsqrte_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
7257 NEONMAP2(vrsqrteq_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
7261 NEONMAP1(vsha1su0q_u32, arm_neon_sha1su0, 0),
7262 NEONMAP1(vsha1su1q_u32, arm_neon_sha1su1, 0),
7263 NEONMAP1(vsha256h2q_u32, arm_neon_sha256h2, 0),
7264 NEONMAP1(vsha256hq_u32, arm_neon_sha256h, 0),
7265 NEONMAP1(vsha256su0q_u32, arm_neon_sha256su0, 0),
7266 NEONMAP1(vsha256su1q_u32, arm_neon_sha256su1, 0),
7275 NEONMAP1(vst1_v, arm_neon_vst1, 0),
7276 NEONMAP1(vst1_x2_v, arm_neon_vst1x2, 0),
7277 NEONMAP1(vst1_x3_v, arm_neon_vst1x3, 0),
7278 NEONMAP1(vst1_x4_v, arm_neon_vst1x4, 0),
7279 NEONMAP1(vst1q_v, arm_neon_vst1, 0),
7280 NEONMAP1(vst1q_x2_v, arm_neon_vst1x2, 0),
7281 NEONMAP1(vst1q_x3_v, arm_neon_vst1x3, 0),
7282 NEONMAP1(vst1q_x4_v, arm_neon_vst1x4, 0),
7283 NEONMAP1(vst2_lane_v, arm_neon_vst2lane, 0),
7284 NEONMAP1(vst2_v, arm_neon_vst2, 0),
7285 NEONMAP1(vst2q_lane_v, arm_neon_vst2lane, 0),
7286 NEONMAP1(vst2q_v, arm_neon_vst2, 0),
7287 NEONMAP1(vst3_lane_v, arm_neon_vst3lane, 0),
7288 NEONMAP1(vst3_v, arm_neon_vst3, 0),
7289 NEONMAP1(vst3q_lane_v, arm_neon_vst3lane, 0),
7290 NEONMAP1(vst3q_v, arm_neon_vst3, 0),
7291 NEONMAP1(vst4_lane_v, arm_neon_vst4lane, 0),
7292 NEONMAP1(vst4_v, arm_neon_vst4, 0),
7293 NEONMAP1(vst4q_lane_v, arm_neon_vst4lane, 0),
7294 NEONMAP1(vst4q_v, arm_neon_vst4, 0),
7300 NEONMAP1(vusdot_s32, arm_neon_usdot, 0),
7301 NEONMAP1(vusdotq_s32, arm_neon_usdot, 0),
7302 NEONMAP1(vusmmlaq_s32, arm_neon_usmmla, 0),
7310 NEONMAP1(__a64_vcvtq_low_bf16_f32, aarch64_neon_bfcvtn, 0),
7315 NEONMAP1(vabs_v, aarch64_neon_abs, 0),
7316 NEONMAP1(vabsq_v, aarch64_neon_abs, 0),
7321 NEONMAP1(vaesdq_u8, aarch64_crypto_aesd, 0),
7322 NEONMAP1(vaeseq_u8, aarch64_crypto_aese, 0),
7323 NEONMAP1(vaesimcq_u8, aarch64_crypto_aesimc, 0),
7324 NEONMAP1(vaesmcq_u8, aarch64_crypto_aesmc, 0),
7333 NEONMAP1(vbfdot_f32, aarch64_neon_bfdot, 0),
7334 NEONMAP1(vbfdotq_f32, aarch64_neon_bfdot, 0),
7335 NEONMAP1(vbfmlalbq_f32, aarch64_neon_bfmlalb, 0),
7336 NEONMAP1(vbfmlaltq_f32, aarch64_neon_bfmlalt, 0),
7337 NEONMAP1(vbfmmlaq_f32, aarch64_neon_bfmmla, 0),
7348 NEONMAP1(vcage_v, aarch64_neon_facge, 0),
7349 NEONMAP1(vcageq_v, aarch64_neon_facge, 0),
7350 NEONMAP1(vcagt_v, aarch64_neon_facgt, 0),
7351 NEONMAP1(vcagtq_v, aarch64_neon_facgt, 0),
7352 NEONMAP1(vcale_v, aarch64_neon_facge, 0),
7353 NEONMAP1(vcaleq_v, aarch64_neon_facge, 0),
7354 NEONMAP1(vcalt_v, aarch64_neon_facgt, 0),
7355 NEONMAP1(vcaltq_v, aarch64_neon_facgt, 0),
7392 NEONMAP1(vcvt_f16_f32, aarch64_neon_vcvtfp2hf, 0),
7395 NEONMAP1(vcvt_f32_f16, aarch64_neon_vcvthf2fp, 0),
7397 NEONMAP1(vcvt_n_f16_s16, aarch64_neon_vcvtfxs2fp, 0),
7398 NEONMAP1(vcvt_n_f16_u16, aarch64_neon_vcvtfxu2fp, 0),
7399 NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
7400 NEONMAP2(vcvt_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
7401 NEONMAP1(vcvt_n_s16_f16, aarch64_neon_vcvtfp2fxs, 0),
7402 NEONMAP1(vcvt_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
7403 NEONMAP1(vcvt_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
7404 NEONMAP1(vcvt_n_u16_f16, aarch64_neon_vcvtfp2fxu, 0),
7405 NEONMAP1(vcvt_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
7406 NEONMAP1(vcvt_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
7410 NEONMAP1(vcvtq_high_bf16_f32, aarch64_neon_bfcvtn2, 0),
7411 NEONMAP1(vcvtq_n_f16_s16, aarch64_neon_vcvtfxs2fp, 0),
7412 NEONMAP1(vcvtq_n_f16_u16, aarch64_neon_vcvtfxu2fp, 0),
7413 NEONMAP2(vcvtq_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
7414 NEONMAP2(vcvtq_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
7415 NEONMAP1(vcvtq_n_s16_f16, aarch64_neon_vcvtfp2fxs, 0),
7416 NEONMAP1(vcvtq_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
7417 NEONMAP1(vcvtq_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
7418 NEONMAP1(vcvtq_n_u16_f16, aarch64_neon_vcvtfp2fxu, 0),
7419 NEONMAP1(vcvtq_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
7420 NEONMAP1(vcvtq_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
7422 NEONMAP1(vdot_s32, aarch64_neon_sdot, 0),
7423 NEONMAP1(vdot_u32, aarch64_neon_udot, 0),
7424 NEONMAP1(vdotq_s32, aarch64_neon_sdot, 0),
7425 NEONMAP1(vdotq_u32, aarch64_neon_udot, 0),
7438 NEONMAP1(vfmlal_high_f16, aarch64_neon_fmlal2, 0),
7439 NEONMAP1(vfmlal_low_f16, aarch64_neon_fmlal, 0),
7440 NEONMAP1(vfmlalq_high_f16, aarch64_neon_fmlal2, 0),
7441 NEONMAP1(vfmlalq_low_f16, aarch64_neon_fmlal, 0),
7442 NEONMAP1(vfmlsl_high_f16, aarch64_neon_fmlsl2, 0),
7443 NEONMAP1(vfmlsl_low_f16, aarch64_neon_fmlsl, 0),
7444 NEONMAP1(vfmlslq_high_f16, aarch64_neon_fmlsl2, 0),
7445 NEONMAP1(vfmlslq_low_f16, aarch64_neon_fmlsl, 0),
7450 NEONMAP1(vld1_x2_v, aarch64_neon_ld1x2, 0),
7451 NEONMAP1(vld1_x3_v, aarch64_neon_ld1x3, 0),
7452 NEONMAP1(vld1_x4_v, aarch64_neon_ld1x4, 0),
7453 NEONMAP1(vld1q_x2_v, aarch64_neon_ld1x2, 0),
7454 NEONMAP1(vld1q_x3_v, aarch64_neon_ld1x3, 0),
7455 NEONMAP1(vld1q_x4_v, aarch64_neon_ld1x4, 0),
7456 NEONMAP1(vmmlaq_s32, aarch64_neon_smmla, 0),
7457 NEONMAP1(vmmlaq_u32, aarch64_neon_ummla, 0),
7470 NEONMAP2(vqdmlal_v, aarch64_neon_sqdmull, aarch64_neon_sqadd, 0),
7471 NEONMAP2(vqdmlsl_v, aarch64_neon_sqdmull, aarch64_neon_sqsub, 0),
7472 NEONMAP1(vqdmulh_lane_v, aarch64_neon_sqdmulh_lane, 0),
7473 NEONMAP1(vqdmulh_laneq_v, aarch64_neon_sqdmulh_laneq, 0),
7475 NEONMAP1(vqdmulhq_lane_v, aarch64_neon_sqdmulh_lane, 0),
7476 NEONMAP1(vqdmulhq_laneq_v, aarch64_neon_sqdmulh_laneq, 0),
7491 NEONMAP1(vqrdmulh_lane_v, aarch64_neon_sqrdmulh_lane, 0),
7492 NEONMAP1(vqrdmulh_laneq_v, aarch64_neon_sqrdmulh_laneq, 0),
7494 NEONMAP1(vqrdmulhq_lane_v, aarch64_neon_sqrdmulh_lane, 0),
7495 NEONMAP1(vqrdmulhq_laneq_v, aarch64_neon_sqrdmulh_laneq, 0),
7503 NEONMAP1(vqshlu_n_v, aarch64_neon_sqshlu, 0),
7504 NEONMAP1(vqshluq_n_v, aarch64_neon_sqshlu, 0),
7508 NEONMAP1(vrax1q_u64, aarch64_crypto_rax1, 0),
7509 NEONMAP2(vrecpe_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
7510 NEONMAP2(vrecpeq_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
7537 NEONMAP2(vrsqrte_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
7538 NEONMAP2(vrsqrteq_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
7542 NEONMAP1(vsha1su0q_u32, aarch64_crypto_sha1su0, 0),
7543 NEONMAP1(vsha1su1q_u32, aarch64_crypto_sha1su1, 0),
7544 NEONMAP1(vsha256h2q_u32, aarch64_crypto_sha256h2, 0),
7545 NEONMAP1(vsha256hq_u32, aarch64_crypto_sha256h, 0),
7546 NEONMAP1(vsha256su0q_u32, aarch64_crypto_sha256su0, 0),
7547 NEONMAP1(vsha256su1q_u32, aarch64_crypto_sha256su1, 0),
7548 NEONMAP1(vsha512h2q_u64, aarch64_crypto_sha512h2, 0),
7549 NEONMAP1(vsha512hq_u64, aarch64_crypto_sha512h, 0),
7550 NEONMAP1(vsha512su0q_u64, aarch64_crypto_sha512su0, 0),
7551 NEONMAP1(vsha512su1q_u64, aarch64_crypto_sha512su1, 0),
7560 NEONMAP1(vsm3partw1q_u32, aarch64_crypto_sm3partw1, 0),
7561 NEONMAP1(vsm3partw2q_u32, aarch64_crypto_sm3partw2, 0),
7562 NEONMAP1(vsm3ss1q_u32, aarch64_crypto_sm3ss1, 0),
7563 NEONMAP1(vsm3tt1aq_u32, aarch64_crypto_sm3tt1a, 0),
7564 NEONMAP1(vsm3tt1bq_u32, aarch64_crypto_sm3tt1b, 0),
7565 NEONMAP1(vsm3tt2aq_u32, aarch64_crypto_sm3tt2a, 0),
7566 NEONMAP1(vsm3tt2bq_u32, aarch64_crypto_sm3tt2b, 0),
7567 NEONMAP1(vsm4ekeyq_u32, aarch64_crypto_sm4ekey, 0),
7568 NEONMAP1(vsm4eq_u32, aarch64_crypto_sm4e, 0),
7569 NEONMAP1(vst1_x2_v, aarch64_neon_st1x2, 0),
7570 NEONMAP1(vst1_x3_v, aarch64_neon_st1x3, 0),
7571 NEONMAP1(vst1_x4_v, aarch64_neon_st1x4, 0),
7572 NEONMAP1(vst1q_x2_v, aarch64_neon_st1x2, 0),
7573 NEONMAP1(vst1q_x3_v, aarch64_neon_st1x3, 0),
7574 NEONMAP1(vst1q_x4_v, aarch64_neon_st1x4, 0),
7578 NEONMAP1(vusdot_s32, aarch64_neon_usdot, 0),
7579 NEONMAP1(vusdotq_s32, aarch64_neon_usdot, 0),
7580 NEONMAP1(vusmmlaq_s32, aarch64_neon_usmmla, 0),
7581 NEONMAP1(vxarq_u64, aarch64_crypto_xar, 0),
7619 NEONMAP1(vcvth_bf16_f32, aarch64_neon_bfcvt, 0),
7638 NEONMAP1(vcvtxd_f32_f64, aarch64_sisd_fcvtxn, 0),
7659 NEONMAP1(vmull_p64, aarch64_neon_pmull64, 0),
7687 NEONMAP1(vqdmulls_s32, aarch64_neon_sqdmulls_scalar, 0),
7768 NEONMAP1(vsha1cq_u32, aarch64_crypto_sha1c, 0),
7769 NEONMAP1(vsha1h_u32, aarch64_crypto_sha1h, 0),
7770 NEONMAP1(vsha1mq_u32, aarch64_crypto_sha1m, 0),
7771 NEONMAP1(vsha1pq_u32, aarch64_crypto_sha1p, 0),
7825 { NEON::BI__builtin_neon_splat_lane_bf16, NEON::BI__builtin_neon_splat_lane_v, },
7826 { NEON::BI__builtin_neon_splat_laneq_bf16, NEON::BI__builtin_neon_splat_laneq_v, },
7827 { NEON::BI__builtin_neon_splatq_lane_bf16, NEON::BI__builtin_neon_splatq_lane_v, },
7828 { NEON::BI__builtin_neon_splatq_laneq_bf16, NEON::BI__builtin_neon_splatq_laneq_v, },
7829 { NEON::BI__builtin_neon_vabd_f16, NEON::BI__builtin_neon_vabd_v, },
7830 { NEON::BI__builtin_neon_vabdq_f16, NEON::BI__builtin_neon_vabdq_v, },
7831 { NEON::BI__builtin_neon_vabs_f16, NEON::BI__builtin_neon_vabs_v, },
7832 { NEON::BI__builtin_neon_vabsq_f16, NEON::BI__builtin_neon_vabsq_v, },
7833 { NEON::BI__builtin_neon_vcage_f16, NEON::BI__builtin_neon_vcage_v, },
7834 { NEON::BI__builtin_neon_vcageq_f16, NEON::BI__builtin_neon_vcageq_v, },
7835 { NEON::BI__builtin_neon_vcagt_f16, NEON::BI__builtin_neon_vcagt_v, },
7836 { NEON::BI__builtin_neon_vcagtq_f16, NEON::BI__builtin_neon_vcagtq_v, },
7837 { NEON::BI__builtin_neon_vcale_f16, NEON::BI__builtin_neon_vcale_v, },
7838 { NEON::BI__builtin_neon_vcaleq_f16, NEON::BI__builtin_neon_vcaleq_v, },
7839 { NEON::BI__builtin_neon_vcalt_f16, NEON::BI__builtin_neon_vcalt_v, },
7840 { NEON::BI__builtin_neon_vcaltq_f16, NEON::BI__builtin_neon_vcaltq_v, },
7841 { NEON::BI__builtin_neon_vceqz_f16, NEON::BI__builtin_neon_vceqz_v, },
7842 { NEON::BI__builtin_neon_vceqzq_f16, NEON::BI__builtin_neon_vceqzq_v, },
7843 { NEON::BI__builtin_neon_vcgez_f16, NEON::BI__builtin_neon_vcgez_v, },
7844 { NEON::BI__builtin_neon_vcgezq_f16, NEON::BI__builtin_neon_vcgezq_v, },
7845 { NEON::BI__builtin_neon_vcgtz_f16, NEON::BI__builtin_neon_vcgtz_v, },
7846 { NEON::BI__builtin_neon_vcgtzq_f16, NEON::BI__builtin_neon_vcgtzq_v, },
7847 { NEON::BI__builtin_neon_vclez_f16, NEON::BI__builtin_neon_vclez_v, },
7848 { NEON::BI__builtin_neon_vclezq_f16, NEON::BI__builtin_neon_vclezq_v, },
7849 { NEON::BI__builtin_neon_vcltz_f16, NEON::BI__builtin_neon_vcltz_v, },
7850 { NEON::BI__builtin_neon_vcltzq_f16, NEON::BI__builtin_neon_vcltzq_v, },
7851 { NEON::BI__builtin_neon_vfma_f16, NEON::BI__builtin_neon_vfma_v, },
7852 { NEON::BI__builtin_neon_vfma_lane_f16, NEON::BI__builtin_neon_vfma_lane_v, },
7853 { NEON::BI__builtin_neon_vfma_laneq_f16, NEON::BI__builtin_neon_vfma_laneq_v, },
7854 { NEON::BI__builtin_neon_vfmaq_f16, NEON::BI__builtin_neon_vfmaq_v, },
7855 { NEON::BI__builtin_neon_vfmaq_lane_f16, NEON::BI__builtin_neon_vfmaq_lane_v, },
7856 { NEON::BI__builtin_neon_vfmaq_laneq_f16, NEON::BI__builtin_neon_vfmaq_laneq_v, },
7857 { NEON::BI__builtin_neon_vld1_bf16_x2, NEON::BI__builtin_neon_vld1_x2_v },
7858 { NEON::BI__builtin_neon_vld1_bf16_x3, NEON::BI__builtin_neon_vld1_x3_v },
7859 { NEON::BI__builtin_neon_vld1_bf16_x4, NEON::BI__builtin_neon_vld1_x4_v },
7860 { NEON::BI__builtin_neon_vld1_bf16, NEON::BI__builtin_neon_vld1_v },
7861 { NEON::BI__builtin_neon_vld1_dup_bf16, NEON::BI__builtin_neon_vld1_dup_v },
7862 { NEON::BI__builtin_neon_vld1_lane_bf16, NEON::BI__builtin_neon_vld1_lane_v },
7863 { NEON::BI__builtin_neon_vld1q_bf16_x2, NEON::BI__builtin_neon_vld1q_x2_v },
7864 { NEON::BI__builtin_neon_vld1q_bf16_x3, NEON::BI__builtin_neon_vld1q_x3_v },
7865 { NEON::BI__builtin_neon_vld1q_bf16_x4, NEON::BI__builtin_neon_vld1q_x4_v },
7866 { NEON::BI__builtin_neon_vld1q_bf16, NEON::BI__builtin_neon_vld1q_v },
7867 { NEON::BI__builtin_neon_vld1q_dup_bf16, NEON::BI__builtin_neon_vld1q_dup_v },
7868 { NEON::BI__builtin_neon_vld1q_lane_bf16, NEON::BI__builtin_neon_vld1q_lane_v },
7869 { NEON::BI__builtin_neon_vld2_bf16, NEON::BI__builtin_neon_vld2_v },
7870 { NEON::BI__builtin_neon_vld2_dup_bf16, NEON::BI__builtin_neon_vld2_dup_v },
7871 { NEON::BI__builtin_neon_vld2_lane_bf16, NEON::BI__builtin_neon_vld2_lane_v },
7872 { NEON::BI__builtin_neon_vld2q_bf16, NEON::BI__builtin_neon_vld2q_v },
7873 { NEON::BI__builtin_neon_vld2q_dup_bf16, NEON::BI__builtin_neon_vld2q_dup_v },
7874 { NEON::BI__builtin_neon_vld2q_lane_bf16, NEON::BI__builtin_neon_vld2q_lane_v },
7875 { NEON::BI__builtin_neon_vld3_bf16, NEON::BI__builtin_neon_vld3_v },
7876 { NEON::BI__builtin_neon_vld3_dup_bf16, NEON::BI__builtin_neon_vld3_dup_v },
7877 { NEON::BI__builtin_neon_vld3_lane_bf16, NEON::BI__builtin_neon_vld3_lane_v },
7878 { NEON::BI__builtin_neon_vld3q_bf16, NEON::BI__builtin_neon_vld3q_v },
7879 { NEON::BI__builtin_neon_vld3q_dup_bf16, NEON::BI__builtin_neon_vld3q_dup_v },
7880 { NEON::BI__builtin_neon_vld3q_lane_bf16, NEON::BI__builtin_neon_vld3q_lane_v },
7881 { NEON::BI__builtin_neon_vld4_bf16, NEON::BI__builtin_neon_vld4_v },
7882 { NEON::BI__builtin_neon_vld4_dup_bf16, NEON::BI__builtin_neon_vld4_dup_v },
7883 { NEON::BI__builtin_neon_vld4_lane_bf16, NEON::BI__builtin_neon_vld4_lane_v },
7884 { NEON::BI__builtin_neon_vld4q_bf16, NEON::BI__builtin_neon_vld4q_v },
7885 { NEON::BI__builtin_neon_vld4q_dup_bf16, NEON::BI__builtin_neon_vld4q_dup_v },
7886 { NEON::BI__builtin_neon_vld4q_lane_bf16, NEON::BI__builtin_neon_vld4q_lane_v },
7887 { NEON::BI__builtin_neon_vmax_f16, NEON::BI__builtin_neon_vmax_v, },
7888 { NEON::BI__builtin_neon_vmaxnm_f16, NEON::BI__builtin_neon_vmaxnm_v, },
7889 { NEON::BI__builtin_neon_vmaxnmq_f16, NEON::BI__builtin_neon_vmaxnmq_v, },
7890 { NEON::BI__builtin_neon_vmaxq_f16, NEON::BI__builtin_neon_vmaxq_v, },
7891 { NEON::BI__builtin_neon_vmin_f16, NEON::BI__builtin_neon_vmin_v, },
7892 { NEON::BI__builtin_neon_vminnm_f16, NEON::BI__builtin_neon_vminnm_v, },
7893 { NEON::BI__builtin_neon_vminnmq_f16, NEON::BI__builtin_neon_vminnmq_v, },
7894 { NEON::BI__builtin_neon_vminq_f16, NEON::BI__builtin_neon_vminq_v, },
7895 { NEON::BI__builtin_neon_vmulx_f16, NEON::BI__builtin_neon_vmulx_v, },
7896 { NEON::BI__builtin_neon_vmulxq_f16, NEON::BI__builtin_neon_vmulxq_v, },
7897 { NEON::BI__builtin_neon_vpadd_f16, NEON::BI__builtin_neon_vpadd_v, },
7898 { NEON::BI__builtin_neon_vpaddq_f16, NEON::BI__builtin_neon_vpaddq_v, },
7899 { NEON::BI__builtin_neon_vpmax_f16, NEON::BI__builtin_neon_vpmax_v, },
7900 { NEON::BI__builtin_neon_vpmaxnm_f16, NEON::BI__builtin_neon_vpmaxnm_v, },
7901 { NEON::BI__builtin_neon_vpmaxnmq_f16, NEON::BI__builtin_neon_vpmaxnmq_v, },
7902 { NEON::BI__builtin_neon_vpmaxq_f16, NEON::BI__builtin_neon_vpmaxq_v, },
7903 { NEON::BI__builtin_neon_vpmin_f16, NEON::BI__builtin_neon_vpmin_v, },
7904 { NEON::BI__builtin_neon_vpminnm_f16, NEON::BI__builtin_neon_vpminnm_v, },
7905 { NEON::BI__builtin_neon_vpminnmq_f16, NEON::BI__builtin_neon_vpminnmq_v, },
7906 { NEON::BI__builtin_neon_vpminq_f16, NEON::BI__builtin_neon_vpminq_v, },
7907 { NEON::BI__builtin_neon_vrecpe_f16, NEON::BI__builtin_neon_vrecpe_v, },
7908 { NEON::BI__builtin_neon_vrecpeq_f16, NEON::BI__builtin_neon_vrecpeq_v, },
7909 { NEON::BI__builtin_neon_vrecps_f16, NEON::BI__builtin_neon_vrecps_v, },
7910 { NEON::BI__builtin_neon_vrecpsq_f16, NEON::BI__builtin_neon_vrecpsq_v, },
7911 { NEON::BI__builtin_neon_vrnd_f16, NEON::BI__builtin_neon_vrnd_v, },
7912 { NEON::BI__builtin_neon_vrnda_f16, NEON::BI__builtin_neon_vrnda_v, },
7913 { NEON::BI__builtin_neon_vrndaq_f16, NEON::BI__builtin_neon_vrndaq_v, },
7914 { NEON::BI__builtin_neon_vrndi_f16, NEON::BI__builtin_neon_vrndi_v, },
7915 { NEON::BI__builtin_neon_vrndiq_f16, NEON::BI__builtin_neon_vrndiq_v, },
7916 { NEON::BI__builtin_neon_vrndm_f16, NEON::BI__builtin_neon_vrndm_v, },
7917 { NEON::BI__builtin_neon_vrndmq_f16, NEON::BI__builtin_neon_vrndmq_v, },
7918 { NEON::BI__builtin_neon_vrndn_f16, NEON::BI__builtin_neon_vrndn_v, },
7919 { NEON::BI__builtin_neon_vrndnq_f16, NEON::BI__builtin_neon_vrndnq_v, },
7920 { NEON::BI__builtin_neon_vrndp_f16, NEON::BI__builtin_neon_vrndp_v, },
7921 { NEON::BI__builtin_neon_vrndpq_f16, NEON::BI__builtin_neon_vrndpq_v, },
7922 { NEON::BI__builtin_neon_vrndq_f16, NEON::BI__builtin_neon_vrndq_v, },
7923 { NEON::BI__builtin_neon_vrndx_f16, NEON::BI__builtin_neon_vrndx_v, },
7924 { NEON::BI__builtin_neon_vrndxq_f16, NEON::BI__builtin_neon_vrndxq_v, },
7925 { NEON::BI__builtin_neon_vrsqrte_f16, NEON::BI__builtin_neon_vrsqrte_v, },
7926 { NEON::BI__builtin_neon_vrsqrteq_f16, NEON::BI__builtin_neon_vrsqrteq_v, },
7927 { NEON::BI__builtin_neon_vrsqrts_f16, NEON::BI__builtin_neon_vrsqrts_v, },
7928 { NEON::BI__builtin_neon_vrsqrtsq_f16, NEON::BI__builtin_neon_vrsqrtsq_v, },
7929 { NEON::BI__builtin_neon_vsqrt_f16, NEON::BI__builtin_neon_vsqrt_v, },
7930 { NEON::BI__builtin_neon_vsqrtq_f16, NEON::BI__builtin_neon_vsqrtq_v, },
7931 { NEON::BI__builtin_neon_vst1_bf16_x2, NEON::BI__builtin_neon_vst1_x2_v },
7932 { NEON::BI__builtin_neon_vst1_bf16_x3, NEON::BI__builtin_neon_vst1_x3_v },
7933 { NEON::BI__builtin_neon_vst1_bf16_x4, NEON::BI__builtin_neon_vst1_x4_v },
7934 { NEON::BI__builtin_neon_vst1_bf16, NEON::BI__builtin_neon_vst1_v },
7935 { NEON::BI__builtin_neon_vst1_lane_bf16, NEON::BI__builtin_neon_vst1_lane_v },
7936 { NEON::BI__builtin_neon_vst1q_bf16_x2, NEON::BI__builtin_neon_vst1q_x2_v },
7937 { NEON::BI__builtin_neon_vst1q_bf16_x3, NEON::BI__builtin_neon_vst1q_x3_v },
7938 { NEON::BI__builtin_neon_vst1q_bf16_x4, NEON::BI__builtin_neon_vst1q_x4_v },
7939 { NEON::BI__builtin_neon_vst1q_bf16, NEON::BI__builtin_neon_vst1q_v },
7940 { NEON::BI__builtin_neon_vst1q_lane_bf16, NEON::BI__builtin_neon_vst1q_lane_v },
7941 { NEON::BI__builtin_neon_vst2_bf16, NEON::BI__builtin_neon_vst2_v },
7942 { NEON::BI__builtin_neon_vst2_lane_bf16, NEON::BI__builtin_neon_vst2_lane_v },
7943 { NEON::BI__builtin_neon_vst2q_bf16, NEON::BI__builtin_neon_vst2q_v },
7944 { NEON::BI__builtin_neon_vst2q_lane_bf16, NEON::BI__builtin_neon_vst2q_lane_v },
7945 { NEON::BI__builtin_neon_vst3_bf16, NEON::BI__builtin_neon_vst3_v },
7946 { NEON::BI__builtin_neon_vst3_lane_bf16, NEON::BI__builtin_neon_vst3_lane_v },
7947 { NEON::BI__builtin_neon_vst3q_bf16, NEON::BI__builtin_neon_vst3q_v },
7948 { NEON::BI__builtin_neon_vst3q_lane_bf16, NEON::BI__builtin_neon_vst3q_lane_v },
7949 { NEON::BI__builtin_neon_vst4_bf16, NEON::BI__builtin_neon_vst4_v },
7950 { NEON::BI__builtin_neon_vst4_lane_bf16, NEON::BI__builtin_neon_vst4_lane_v },
7951 { NEON::BI__builtin_neon_vst4q_bf16, NEON::BI__builtin_neon_vst4q_v },
7952 { NEON::BI__builtin_neon_vst4q_lane_bf16, NEON::BI__builtin_neon_vst4q_lane_v },
7956 { NEON::BI__builtin_neon_vldap1_lane_u64, NEON::BI__builtin_neon_vldap1_lane_s64 },
7957 { NEON::BI__builtin_neon_vldap1_lane_f64, NEON::BI__builtin_neon_vldap1_lane_s64 },
7958 { NEON::BI__builtin_neon_vldap1_lane_p64, NEON::BI__builtin_neon_vldap1_lane_s64 },
7959 { NEON::BI__builtin_neon_vldap1q_lane_u64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
7960 { NEON::BI__builtin_neon_vldap1q_lane_f64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
7961 { NEON::BI__builtin_neon_vldap1q_lane_p64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
7962 { NEON::BI__builtin_neon_vstl1_lane_u64, NEON::BI__builtin_neon_vstl1_lane_s64 },
7963 { NEON::BI__builtin_neon_vstl1_lane_f64, NEON::BI__builtin_neon_vstl1_lane_s64 },
7964 { NEON::BI__builtin_neon_vstl1_lane_p64, NEON::BI__builtin_neon_vstl1_lane_s64 },
7965 { NEON::BI__builtin_neon_vstl1q_lane_u64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
7966 { NEON::BI__builtin_neon_vstl1q_lane_f64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
7967 { NEON::BI__builtin_neon_vstl1q_lane_p64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
7974#define SVEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
7976 #NameBase, SVE::BI__builtin_sve_##NameBase, Intrinsic::LLVMIntrinsic, 0, \
7980#define SVEMAP2(NameBase, TypeModifier) \
7981 { #NameBase, SVE::BI__builtin_sve_##NameBase, 0, 0, TypeModifier }
7983#define GET_SVE_LLVM_INTRINSIC_MAP
7984#include "clang/Basic/arm_sve_builtin_cg.inc"
7985#include "clang/Basic/BuiltinsAArch64NeonSVEBridge_cg.def"
7986#undef GET_SVE_LLVM_INTRINSIC_MAP
7992#define SMEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
7994 #NameBase, SME::BI__builtin_sme_##NameBase, Intrinsic::LLVMIntrinsic, 0, \
7998#define SMEMAP2(NameBase, TypeModifier) \
7999 { #NameBase, SME::BI__builtin_sme_##NameBase, 0, 0, TypeModifier }
8001#define GET_SME_LLVM_INTRINSIC_MAP
8002#include "clang/Basic/arm_sme_builtin_cg.inc"
8003#undef GET_SME_LLVM_INTRINSIC_MAP
8016static const ARMVectorIntrinsicInfo *
8018 unsigned BuiltinID,
bool &MapProvenSorted) {
8021 if (!MapProvenSorted) {
8022 assert(llvm::is_sorted(IntrinsicMap));
8023 MapProvenSorted =
true;
8027 const ARMVectorIntrinsicInfo *Builtin =
8028 llvm::lower_bound(IntrinsicMap, BuiltinID);
8030 if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID)
8038 llvm::Type *ArgType,
8051 Ty = llvm::FixedVectorType::get(
8052 Ty, VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1);
8059 int Elts = VectorSize ? VectorSize / ArgType->getPrimitiveSizeInBits() : 1;
8060 ArgType = llvm::FixedVectorType::get(ArgType, Elts);
8064 Tys.push_back(ArgType);
8067 Tys.push_back(ArgType);
8078 unsigned BuiltinID = SISDInfo.BuiltinID;
8079 unsigned int Int = SISDInfo.LLVMIntrinsic;
8080 unsigned Modifier = SISDInfo.TypeModifier;
8081 const char *
s = SISDInfo.NameHint;
8083 switch (BuiltinID) {
8084 case NEON::BI__builtin_neon_vcled_s64:
8085 case NEON::BI__builtin_neon_vcled_u64:
8086 case NEON::BI__builtin_neon_vcles_f32:
8087 case NEON::BI__builtin_neon_vcled_f64:
8088 case NEON::BI__builtin_neon_vcltd_s64:
8089 case NEON::BI__builtin_neon_vcltd_u64:
8090 case NEON::BI__builtin_neon_vclts_f32:
8091 case NEON::BI__builtin_neon_vcltd_f64:
8092 case NEON::BI__builtin_neon_vcales_f32:
8093 case NEON::BI__builtin_neon_vcaled_f64:
8094 case NEON::BI__builtin_neon_vcalts_f32:
8095 case NEON::BI__builtin_neon_vcaltd_f64:
8099 std::swap(Ops[0], Ops[1]);
8103 assert(Int &&
"Generic code assumes a valid intrinsic");
8106 const Expr *Arg =
E->getArg(0);
8111 ConstantInt *C0 = ConstantInt::get(CGF.
SizeTy, 0);
8112 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
8113 ai != ae; ++ai, ++j) {
8114 llvm::Type *ArgTy = ai->getType();
8115 if (Ops[j]->getType()->getPrimitiveSizeInBits() ==
8116 ArgTy->getPrimitiveSizeInBits())
8119 assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy());
8122 Ops[j] = CGF.
Builder.CreateTruncOrBitCast(
8123 Ops[j], cast<llvm::VectorType>(ArgTy)->getElementType());
8125 CGF.
Builder.CreateInsertElement(PoisonValue::get(ArgTy), Ops[j], C0);
8130 if (ResultType->getPrimitiveSizeInBits().getFixedValue() <
8131 Result->getType()->getPrimitiveSizeInBits().getFixedValue())
8138 unsigned BuiltinID,
unsigned LLVMIntrinsic,
unsigned AltLLVMIntrinsic,
8139 const char *NameHint,
unsigned Modifier,
const CallExpr *
E,
8141 llvm::Triple::ArchType Arch) {
8143 const Expr *Arg =
E->getArg(
E->getNumArgs() - 1);
8144 std::optional<llvm::APSInt> NeonTypeConst =
8151 bool Usgn =
Type.isUnsigned();
8152 bool Quad =
Type.isQuad();
8154 const bool AllowBFloatArgsAndRet =
8157 llvm::FixedVectorType *VTy =
8158 GetNeonType(
this,
Type, HasLegalHalfType,
false, AllowBFloatArgsAndRet);
8159 llvm::Type *Ty = VTy;
8163 auto getAlignmentValue32 = [&](
Address addr) ->
Value* {
8164 return Builder.getInt32(addr.getAlignment().getQuantity());
8167 unsigned Int = LLVMIntrinsic;
8169 Int = AltLLVMIntrinsic;
8171 switch (BuiltinID) {
8173 case NEON::BI__builtin_neon_splat_lane_v:
8174 case NEON::BI__builtin_neon_splat_laneq_v:
8175 case NEON::BI__builtin_neon_splatq_lane_v:
8176 case NEON::BI__builtin_neon_splatq_laneq_v: {
8177 auto NumElements = VTy->getElementCount();
8178 if (BuiltinID == NEON::BI__builtin_neon_splatq_lane_v)
8179 NumElements = NumElements * 2;
8180 if (BuiltinID == NEON::BI__builtin_neon_splat_laneq_v)
8181 NumElements = NumElements.divideCoefficientBy(2);
8183 Ops[0] =
Builder.CreateBitCast(Ops[0], VTy);
8184 return EmitNeonSplat(Ops[0], cast<ConstantInt>(Ops[1]), NumElements);
8186 case NEON::BI__builtin_neon_vpadd_v:
8187 case NEON::BI__builtin_neon_vpaddq_v:
8189 if (VTy->getElementType()->isFloatingPointTy() &&
8190 Int == Intrinsic::aarch64_neon_addp)
8191 Int = Intrinsic::aarch64_neon_faddp;
8193 case NEON::BI__builtin_neon_vabs_v:
8194 case NEON::BI__builtin_neon_vabsq_v:
8195 if (VTy->getElementType()->isFloatingPointTy())
8198 case NEON::BI__builtin_neon_vadd_v:
8199 case NEON::BI__builtin_neon_vaddq_v: {
8200 llvm::Type *VTy = llvm::FixedVectorType::get(
Int8Ty, Quad ? 16 : 8);
8201 Ops[0] =
Builder.CreateBitCast(Ops[0], VTy);
8202 Ops[1] =
Builder.CreateBitCast(Ops[1], VTy);
8203 Ops[0] =
Builder.CreateXor(Ops[0], Ops[1]);
8204 return Builder.CreateBitCast(Ops[0], Ty);
8206 case NEON::BI__builtin_neon_vaddhn_v: {
8207 llvm::FixedVectorType *SrcTy =
8208 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
8211 Ops[0] =
Builder.CreateBitCast(Ops[0], SrcTy);
8212 Ops[1] =
Builder.CreateBitCast(Ops[1], SrcTy);
8213 Ops[0] =
Builder.CreateAdd(Ops[0], Ops[1],
"vaddhn");
8216 Constant *ShiftAmt =
8217 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
8218 Ops[0] =
Builder.CreateLShr(Ops[0], ShiftAmt,
"vaddhn");
8221 return Builder.CreateTrunc(Ops[0], VTy,
"vaddhn");
8223 case NEON::BI__builtin_neon_vcale_v:
8224 case NEON::BI__builtin_neon_vcaleq_v:
8225 case NEON::BI__builtin_neon_vcalt_v:
8226 case NEON::BI__builtin_neon_vcaltq_v:
8227 std::swap(Ops[0], Ops[1]);
8229 case NEON::BI__builtin_neon_vcage_v:
8230 case NEON::BI__builtin_neon_vcageq_v:
8231 case NEON::BI__builtin_neon_vcagt_v:
8232 case NEON::BI__builtin_neon_vcagtq_v: {
8234 switch (VTy->getScalarSizeInBits()) {
8235 default: llvm_unreachable(
"unexpected type");
8246 auto *VecFlt = llvm::FixedVectorType::get(Ty, VTy->getNumElements());
8247 llvm::Type *Tys[] = { VTy, VecFlt };
8251 case NEON::BI__builtin_neon_vceqz_v:
8252 case NEON::BI__builtin_neon_vceqzq_v:
8254 ICmpInst::ICMP_EQ,
"vceqz");
8255 case NEON::BI__builtin_neon_vcgez_v:
8256 case NEON::BI__builtin_neon_vcgezq_v:
8258 ICmpInst::ICMP_SGE,
"vcgez");
8259 case NEON::BI__builtin_neon_vclez_v:
8260 case NEON::BI__builtin_neon_vclezq_v:
8262 ICmpInst::ICMP_SLE,
"vclez");
8263 case NEON::BI__builtin_neon_vcgtz_v:
8264 case NEON::BI__builtin_neon_vcgtzq_v:
8266 ICmpInst::ICMP_SGT,
"vcgtz");
8267 case NEON::BI__builtin_neon_vcltz_v:
8268 case NEON::BI__builtin_neon_vcltzq_v:
8270 ICmpInst::ICMP_SLT,
"vcltz");
8271 case NEON::BI__builtin_neon_vclz_v:
8272 case NEON::BI__builtin_neon_vclzq_v:
8277 case NEON::BI__builtin_neon_vcvt_f32_v:
8278 case NEON::BI__builtin_neon_vcvtq_f32_v:
8279 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
8282 return Usgn ?
Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
8283 :
Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
8284 case NEON::BI__builtin_neon_vcvt_f16_s16:
8285 case NEON::BI__builtin_neon_vcvt_f16_u16:
8286 case NEON::BI__builtin_neon_vcvtq_f16_s16:
8287 case NEON::BI__builtin_neon_vcvtq_f16_u16:
8288 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
8291 return Usgn ?
Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
8292 :
Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
8293 case NEON::BI__builtin_neon_vcvt_n_f16_s16:
8294 case NEON::BI__builtin_neon_vcvt_n_f16_u16:
8295 case NEON::BI__builtin_neon_vcvtq_n_f16_s16:
8296 case NEON::BI__builtin_neon_vcvtq_n_f16_u16: {
8301 case NEON::BI__builtin_neon_vcvt_n_f32_v:
8302 case NEON::BI__builtin_neon_vcvt_n_f64_v:
8303 case NEON::BI__builtin_neon_vcvtq_n_f32_v:
8304 case NEON::BI__builtin_neon_vcvtq_n_f64_v: {
8306 Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
8310 case NEON::BI__builtin_neon_vcvt_n_s16_f16:
8311 case NEON::BI__builtin_neon_vcvt_n_s32_v:
8312 case NEON::BI__builtin_neon_vcvt_n_u16_f16:
8313 case NEON::BI__builtin_neon_vcvt_n_u32_v:
8314 case NEON::BI__builtin_neon_vcvt_n_s64_v:
8315 case NEON::BI__builtin_neon_vcvt_n_u64_v:
8316 case NEON::BI__builtin_neon_vcvtq_n_s16_f16:
8317 case NEON::BI__builtin_neon_vcvtq_n_s32_v:
8318 case NEON::BI__builtin_neon_vcvtq_n_u16_f16:
8319 case NEON::BI__builtin_neon_vcvtq_n_u32_v:
8320 case NEON::BI__builtin_neon_vcvtq_n_s64_v:
8321 case NEON::BI__builtin_neon_vcvtq_n_u64_v: {
8326 case NEON::BI__builtin_neon_vcvt_s32_v:
8327 case NEON::BI__builtin_neon_vcvt_u32_v:
8328 case NEON::BI__builtin_neon_vcvt_s64_v:
8329 case NEON::BI__builtin_neon_vcvt_u64_v:
8330 case NEON::BI__builtin_neon_vcvt_s16_f16:
8331 case NEON::BI__builtin_neon_vcvt_u16_f16:
8332 case NEON::BI__builtin_neon_vcvtq_s32_v:
8333 case NEON::BI__builtin_neon_vcvtq_u32_v:
8334 case NEON::BI__builtin_neon_vcvtq_s64_v:
8335 case NEON::BI__builtin_neon_vcvtq_u64_v:
8336 case NEON::BI__builtin_neon_vcvtq_s16_f16:
8337 case NEON::BI__builtin_neon_vcvtq_u16_f16: {
8339 return Usgn ?
Builder.CreateFPToUI(Ops[0], Ty,
"vcvt")
8340 :
Builder.CreateFPToSI(Ops[0], Ty,
"vcvt");
8342 case NEON::BI__builtin_neon_vcvta_s16_f16:
8343 case NEON::BI__builtin_neon_vcvta_s32_v:
8344 case NEON::BI__builtin_neon_vcvta_s64_v:
8345 case NEON::BI__builtin_neon_vcvta_u16_f16:
8346 case NEON::BI__builtin_neon_vcvta_u32_v:
8347 case NEON::BI__builtin_neon_vcvta_u64_v:
8348 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
8349 case NEON::BI__builtin_neon_vcvtaq_s32_v:
8350 case NEON::BI__builtin_neon_vcvtaq_s64_v:
8351 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
8352 case NEON::BI__builtin_neon_vcvtaq_u32_v:
8353 case NEON::BI__builtin_neon_vcvtaq_u64_v:
8354 case NEON::BI__builtin_neon_vcvtn_s16_f16:
8355 case NEON::BI__builtin_neon_vcvtn_s32_v:
8356 case NEON::BI__builtin_neon_vcvtn_s64_v:
8357 case NEON::BI__builtin_neon_vcvtn_u16_f16:
8358 case NEON::BI__builtin_neon_vcvtn_u32_v:
8359 case NEON::BI__builtin_neon_vcvtn_u64_v:
8360 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
8361 case NEON::BI__builtin_neon_vcvtnq_s32_v:
8362 case NEON::BI__builtin_neon_vcvtnq_s64_v:
8363 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
8364 case NEON::BI__builtin_neon_vcvtnq_u32_v:
8365 case NEON::BI__builtin_neon_vcvtnq_u64_v:
8366 case NEON::BI__builtin_neon_vcvtp_s16_f16:
8367 case NEON::BI__builtin_neon_vcvtp_s32_v:
8368 case NEON::BI__builtin_neon_vcvtp_s64_v:
8369 case NEON::BI__builtin_neon_vcvtp_u16_f16:
8370 case NEON::BI__builtin_neon_vcvtp_u32_v:
8371 case NEON::BI__builtin_neon_vcvtp_u64_v:
8372 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
8373 case NEON::BI__builtin_neon_vcvtpq_s32_v:
8374 case NEON::BI__builtin_neon_vcvtpq_s64_v:
8375 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
8376 case NEON::BI__builtin_neon_vcvtpq_u32_v:
8377 case NEON::BI__builtin_neon_vcvtpq_u64_v:
8378 case NEON::BI__builtin_neon_vcvtm_s16_f16:
8379 case NEON::BI__builtin_neon_vcvtm_s32_v:
8380 case NEON::BI__builtin_neon_vcvtm_s64_v:
8381 case NEON::BI__builtin_neon_vcvtm_u16_f16:
8382 case NEON::BI__builtin_neon_vcvtm_u32_v:
8383 case NEON::BI__builtin_neon_vcvtm_u64_v:
8384 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
8385 case NEON::BI__builtin_neon_vcvtmq_s32_v:
8386 case NEON::BI__builtin_neon_vcvtmq_s64_v:
8387 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
8388 case NEON::BI__builtin_neon_vcvtmq_u32_v:
8389 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
8393 case NEON::BI__builtin_neon_vcvtx_f32_v: {
8394 llvm::Type *Tys[2] = { VTy->getTruncatedElementVectorType(VTy), Ty};
8398 case NEON::BI__builtin_neon_vext_v:
8399 case NEON::BI__builtin_neon_vextq_v: {
8400 int CV = cast<ConstantInt>(Ops[2])->getSExtValue();
8402 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
8403 Indices.push_back(i+CV);
8405 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
8406 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
8407 return Builder.CreateShuffleVector(Ops[0], Ops[1], Indices,
"vext");
8409 case NEON::BI__builtin_neon_vfma_v:
8410 case NEON::BI__builtin_neon_vfmaq_v: {
8411 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
8412 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
8413 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
8417 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
8418 {Ops[1], Ops[2], Ops[0]});
8420 case NEON::BI__builtin_neon_vld1_v:
8421 case NEON::BI__builtin_neon_vld1q_v: {
8423 Ops.push_back(getAlignmentValue32(PtrOp0));
8426 case NEON::BI__builtin_neon_vld1_x2_v:
8427 case NEON::BI__builtin_neon_vld1q_x2_v:
8428 case NEON::BI__builtin_neon_vld1_x3_v:
8429 case NEON::BI__builtin_neon_vld1q_x3_v:
8430 case NEON::BI__builtin_neon_vld1_x4_v:
8431 case NEON::BI__builtin_neon_vld1q_x4_v: {
8434 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld1xN");
8437 case NEON::BI__builtin_neon_vld2_v:
8438 case NEON::BI__builtin_neon_vld2q_v:
8439 case NEON::BI__builtin_neon_vld3_v:
8440 case NEON::BI__builtin_neon_vld3q_v:
8441 case NEON::BI__builtin_neon_vld4_v:
8442 case NEON::BI__builtin_neon_vld4q_v:
8443 case NEON::BI__builtin_neon_vld2_dup_v:
8444 case NEON::BI__builtin_neon_vld2q_dup_v:
8445 case NEON::BI__builtin_neon_vld3_dup_v:
8446 case NEON::BI__builtin_neon_vld3q_dup_v:
8447 case NEON::BI__builtin_neon_vld4_dup_v:
8448 case NEON::BI__builtin_neon_vld4q_dup_v: {
8451 Value *Align = getAlignmentValue32(PtrOp1);
8452 Ops[1] =
Builder.CreateCall(F, {Ops[1], Align}, NameHint);
8455 case NEON::BI__builtin_neon_vld1_dup_v:
8456 case NEON::BI__builtin_neon_vld1q_dup_v: {
8457 Value *
V = PoisonValue::get(Ty);
8460 llvm::Constant *CI = ConstantInt::get(
SizeTy, 0);
8461 Ops[0] =
Builder.CreateInsertElement(
V, Ld, CI);
8464 case NEON::BI__builtin_neon_vld2_lane_v:
8465 case NEON::BI__builtin_neon_vld2q_lane_v:
8466 case NEON::BI__builtin_neon_vld3_lane_v:
8467 case NEON::BI__builtin_neon_vld3q_lane_v:
8468 case NEON::BI__builtin_neon_vld4_lane_v:
8469 case NEON::BI__builtin_neon_vld4q_lane_v: {
8472 for (
unsigned I = 2; I < Ops.size() - 1; ++I)
8473 Ops[I] =
Builder.CreateBitCast(Ops[I], Ty);
8474 Ops.push_back(getAlignmentValue32(PtrOp1));
8478 case NEON::BI__builtin_neon_vmovl_v: {
8479 llvm::FixedVectorType *DTy =
8480 llvm::FixedVectorType::getTruncatedElementVectorType(VTy);
8481 Ops[0] =
Builder.CreateBitCast(Ops[0], DTy);
8483 return Builder.CreateZExt(Ops[0], Ty,
"vmovl");
8484 return Builder.CreateSExt(Ops[0], Ty,
"vmovl");
8486 case NEON::BI__builtin_neon_vmovn_v: {
8487 llvm::FixedVectorType *QTy =
8488 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
8489 Ops[0] =
Builder.CreateBitCast(Ops[0], QTy);
8490 return Builder.CreateTrunc(Ops[0], Ty,
"vmovn");
8492 case NEON::BI__builtin_neon_vmull_v:
8498 Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls;
8501 case NEON::BI__builtin_neon_vpadal_v:
8502 case NEON::BI__builtin_neon_vpadalq_v: {
8504 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
8508 llvm::FixedVectorType::get(EltTy, VTy->getNumElements() * 2);
8509 llvm::Type *Tys[2] = { Ty, NarrowTy };
8512 case NEON::BI__builtin_neon_vpaddl_v:
8513 case NEON::BI__builtin_neon_vpaddlq_v: {
8515 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
8516 llvm::Type *EltTy = llvm::IntegerType::get(
getLLVMContext(), EltBits / 2);
8518 llvm::FixedVectorType::get(EltTy, VTy->getNumElements() * 2);
8519 llvm::Type *Tys[2] = { Ty, NarrowTy };
8522 case NEON::BI__builtin_neon_vqdmlal_v:
8523 case NEON::BI__builtin_neon_vqdmlsl_v: {
8530 case NEON::BI__builtin_neon_vqdmulhq_lane_v:
8531 case NEON::BI__builtin_neon_vqdmulh_lane_v:
8532 case NEON::BI__builtin_neon_vqrdmulhq_lane_v:
8533 case NEON::BI__builtin_neon_vqrdmulh_lane_v: {
8534 auto *RTy = cast<llvm::FixedVectorType>(Ty);
8535 if (BuiltinID == NEON::BI__builtin_neon_vqdmulhq_lane_v ||
8536 BuiltinID == NEON::BI__builtin_neon_vqrdmulhq_lane_v)
8537 RTy = llvm::FixedVectorType::get(RTy->getElementType(),
8538 RTy->getNumElements() * 2);
8539 llvm::Type *Tys[2] = {
8544 case NEON::BI__builtin_neon_vqdmulhq_laneq_v:
8545 case NEON::BI__builtin_neon_vqdmulh_laneq_v:
8546 case NEON::BI__builtin_neon_vqrdmulhq_laneq_v:
8547 case NEON::BI__builtin_neon_vqrdmulh_laneq_v: {
8548 llvm::Type *Tys[2] = {
8553 case NEON::BI__builtin_neon_vqshl_n_v:
8554 case NEON::BI__builtin_neon_vqshlq_n_v:
8557 case NEON::BI__builtin_neon_vqshlu_n_v:
8558 case NEON::BI__builtin_neon_vqshluq_n_v:
8561 case NEON::BI__builtin_neon_vrecpe_v:
8562 case NEON::BI__builtin_neon_vrecpeq_v:
8563 case NEON::BI__builtin_neon_vrsqrte_v:
8564 case NEON::BI__builtin_neon_vrsqrteq_v:
8565 Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic;
8567 case NEON::BI__builtin_neon_vrndi_v:
8568 case NEON::BI__builtin_neon_vrndiq_v:
8570 ? Intrinsic::experimental_constrained_nearbyint
8571 : Intrinsic::nearbyint;
8573 case NEON::BI__builtin_neon_vrshr_n_v:
8574 case NEON::BI__builtin_neon_vrshrq_n_v:
8577 case NEON::BI__builtin_neon_vsha512hq_u64:
8578 case NEON::BI__builtin_neon_vsha512h2q_u64:
8579 case NEON::BI__builtin_neon_vsha512su0q_u64:
8580 case NEON::BI__builtin_neon_vsha512su1q_u64: {
8584 case NEON::BI__builtin_neon_vshl_n_v:
8585 case NEON::BI__builtin_neon_vshlq_n_v:
8587 return Builder.CreateShl(
Builder.CreateBitCast(Ops[0],Ty), Ops[1],
8589 case NEON::BI__builtin_neon_vshll_n_v: {
8590 llvm::FixedVectorType *SrcTy =
8591 llvm::FixedVectorType::getTruncatedElementVectorType(VTy);
8592 Ops[0] =
Builder.CreateBitCast(Ops[0], SrcTy);
8594 Ops[0] =
Builder.CreateZExt(Ops[0], VTy);
8596 Ops[0] =
Builder.CreateSExt(Ops[0], VTy);
8598 return Builder.CreateShl(Ops[0], Ops[1],
"vshll_n");
8600 case NEON::BI__builtin_neon_vshrn_n_v: {
8601 llvm::FixedVectorType *SrcTy =
8602 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
8603 Ops[0] =
Builder.CreateBitCast(Ops[0], SrcTy);
8606 Ops[0] =
Builder.CreateLShr(Ops[0], Ops[1]);
8608 Ops[0] =
Builder.CreateAShr(Ops[0], Ops[1]);
8609 return Builder.CreateTrunc(Ops[0], Ty,
"vshrn_n");
8611 case NEON::BI__builtin_neon_vshr_n_v:
8612 case NEON::BI__builtin_neon_vshrq_n_v:
8614 case NEON::BI__builtin_neon_vst1_v:
8615 case NEON::BI__builtin_neon_vst1q_v:
8616 case NEON::BI__builtin_neon_vst2_v:
8617 case NEON::BI__builtin_neon_vst2q_v:
8618 case NEON::BI__builtin_neon_vst3_v:
8619 case NEON::BI__builtin_neon_vst3q_v:
8620 case NEON::BI__builtin_neon_vst4_v:
8621 case NEON::BI__builtin_neon_vst4q_v:
8622 case NEON::BI__builtin_neon_vst2_lane_v:
8623 case NEON::BI__builtin_neon_vst2q_lane_v:
8624 case NEON::BI__builtin_neon_vst3_lane_v:
8625 case NEON::BI__builtin_neon_vst3q_lane_v:
8626 case NEON::BI__builtin_neon_vst4_lane_v:
8627 case NEON::BI__builtin_neon_vst4q_lane_v: {
8629 Ops.push_back(getAlignmentValue32(PtrOp0));
8632 case NEON::BI__builtin_neon_vsm3partw1q_u32:
8633 case NEON::BI__builtin_neon_vsm3partw2q_u32:
8634 case NEON::BI__builtin_neon_vsm3ss1q_u32:
8635 case NEON::BI__builtin_neon_vsm4ekeyq_u32:
8636 case NEON::BI__builtin_neon_vsm4eq_u32: {
8640 case NEON::BI__builtin_neon_vsm3tt1aq_u32:
8641 case NEON::BI__builtin_neon_vsm3tt1bq_u32:
8642 case NEON::BI__builtin_neon_vsm3tt2aq_u32:
8643 case NEON::BI__builtin_neon_vsm3tt2bq_u32: {
8648 case NEON::BI__builtin_neon_vst1_x2_v:
8649 case NEON::BI__builtin_neon_vst1q_x2_v:
8650 case NEON::BI__builtin_neon_vst1_x3_v:
8651 case NEON::BI__builtin_neon_vst1q_x3_v:
8652 case NEON::BI__builtin_neon_vst1_x4_v:
8653 case NEON::BI__builtin_neon_vst1q_x4_v: {
8656 if (Arch == llvm::Triple::aarch64 || Arch == llvm::Triple::aarch64_be ||
8657 Arch == llvm::Triple::aarch64_32) {
8659 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
8665 case NEON::BI__builtin_neon_vsubhn_v: {
8666 llvm::FixedVectorType *SrcTy =
8667 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
8670 Ops[0] =
Builder.CreateBitCast(Ops[0], SrcTy);
8671 Ops[1] =
Builder.CreateBitCast(Ops[1], SrcTy);
8672 Ops[0] =
Builder.CreateSub(Ops[0], Ops[1],
"vsubhn");
8675 Constant *ShiftAmt =
8676 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
8677 Ops[0] =
Builder.CreateLShr(Ops[0], ShiftAmt,
"vsubhn");
8680 return Builder.CreateTrunc(Ops[0], VTy,
"vsubhn");
8682 case NEON::BI__builtin_neon_vtrn_v:
8683 case NEON::BI__builtin_neon_vtrnq_v: {
8684 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
8685 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
8686 Value *SV =
nullptr;
8688 for (
unsigned vi = 0; vi != 2; ++vi) {
8690 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
8691 Indices.push_back(i+vi);
8692 Indices.push_back(i+e+vi);
8694 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8695 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vtrn");
8700 case NEON::BI__builtin_neon_vtst_v:
8701 case NEON::BI__builtin_neon_vtstq_v: {
8702 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
8703 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
8704 Ops[0] =
Builder.CreateAnd(Ops[0], Ops[1]);
8705 Ops[0] =
Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
8706 ConstantAggregateZero::get(Ty));
8707 return Builder.CreateSExt(Ops[0], Ty,
"vtst");
8709 case NEON::BI__builtin_neon_vuzp_v:
8710 case NEON::BI__builtin_neon_vuzpq_v: {
8711 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
8712 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
8713 Value *SV =
nullptr;
8715 for (
unsigned vi = 0; vi != 2; ++vi) {
8717 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
8718 Indices.push_back(2*i+vi);
8720 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8721 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vuzp");
8726 case NEON::BI__builtin_neon_vxarq_u64: {
8731 case NEON::BI__builtin_neon_vzip_v:
8732 case NEON::BI__builtin_neon_vzipq_v: {
8733 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
8734 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
8735 Value *SV =
nullptr;
8737 for (
unsigned vi = 0; vi != 2; ++vi) {
8739 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
8740 Indices.push_back((i + vi*e) >> 1);
8741 Indices.push_back(((i + vi*e) >> 1)+e);
8743 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8744 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vzip");
8749 case NEON::BI__builtin_neon_vdot_s32:
8750 case NEON::BI__builtin_neon_vdot_u32:
8751 case NEON::BI__builtin_neon_vdotq_s32:
8752 case NEON::BI__builtin_neon_vdotq_u32: {
8754 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8755 llvm::Type *Tys[2] = { Ty, InputTy };
8758 case NEON::BI__builtin_neon_vfmlal_low_f16:
8759 case NEON::BI__builtin_neon_vfmlalq_low_f16: {
8761 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8762 llvm::Type *Tys[2] = { Ty, InputTy };
8765 case NEON::BI__builtin_neon_vfmlsl_low_f16:
8766 case NEON::BI__builtin_neon_vfmlslq_low_f16: {
8768 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8769 llvm::Type *Tys[2] = { Ty, InputTy };
8772 case NEON::BI__builtin_neon_vfmlal_high_f16:
8773 case NEON::BI__builtin_neon_vfmlalq_high_f16: {
8775 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8776 llvm::Type *Tys[2] = { Ty, InputTy };
8779 case NEON::BI__builtin_neon_vfmlsl_high_f16:
8780 case NEON::BI__builtin_neon_vfmlslq_high_f16: {
8782 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8783 llvm::Type *Tys[2] = { Ty, InputTy };
8786 case NEON::BI__builtin_neon_vmmlaq_s32:
8787 case NEON::BI__builtin_neon_vmmlaq_u32: {
8789 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8790 llvm::Type *Tys[2] = { Ty, InputTy };
8793 case NEON::BI__builtin_neon_vusmmlaq_s32: {
8795 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8796 llvm::Type *Tys[2] = { Ty, InputTy };
8799 case NEON::BI__builtin_neon_vusdot_s32:
8800 case NEON::BI__builtin_neon_vusdotq_s32: {
8802 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8803 llvm::Type *Tys[2] = { Ty, InputTy };
8806 case NEON::BI__builtin_neon_vbfdot_f32:
8807 case NEON::BI__builtin_neon_vbfdotq_f32: {
8808 llvm::Type *InputTy =
8809 llvm::FixedVectorType::get(
BFloatTy, Ty->getPrimitiveSizeInBits() / 16);
8810 llvm::Type *Tys[2] = { Ty, InputTy };
8813 case NEON::BI__builtin_neon___a32_vcvt_bf16_f32: {
8814 llvm::Type *Tys[1] = { Ty };
8821 assert(Int &&
"Expected valid intrinsic number");
8834 Value *Op, llvm::Type *Ty,
const CmpInst::Predicate Fp,
8835 const CmpInst::Predicate Ip,
const Twine &Name) {
8836 llvm::Type *OTy = Op->
getType();
8842 if (BitCastInst *BI = dyn_cast<BitCastInst>(Op))
8843 OTy = BI->getOperand(0)->getType();
8845 Op =
Builder.CreateBitCast(Op, OTy);
8846 if (OTy->getScalarType()->isFloatingPointTy()) {
8847 if (Fp == CmpInst::FCMP_OEQ)
8848 Op =
Builder.CreateFCmp(Fp, Op, Constant::getNullValue(OTy));
8850 Op =
Builder.CreateFCmpS(Fp, Op, Constant::getNullValue(OTy));
8852 Op =
Builder.CreateICmp(Ip, Op, Constant::getNullValue(OTy));
8854 return Builder.CreateSExt(Op, Ty, Name);
8859 llvm::Type *ResTy,
unsigned IntID,
8863 TblOps.push_back(ExtOp);
8867 auto *TblTy = cast<llvm::FixedVectorType>(Ops[0]->getType());
8868 for (
unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) {
8869 Indices.push_back(2*i);
8870 Indices.push_back(2*i+1);
8873 int PairPos = 0, End = Ops.size() - 1;
8874 while (PairPos < End) {
8875 TblOps.push_back(CGF.
Builder.CreateShuffleVector(Ops[PairPos],
8876 Ops[PairPos+1], Indices,
8883 if (PairPos == End) {
8884 Value *ZeroTbl = ConstantAggregateZero::get(TblTy);
8885 TblOps.push_back(CGF.
Builder.CreateShuffleVector(Ops[PairPos],
8886 ZeroTbl, Indices, Name));
8890 TblOps.push_back(IndexOp);
8896Value *CodeGenFunction::GetValueForARMHint(
unsigned BuiltinID) {
8898 switch (BuiltinID) {
8901 case clang::ARM::BI__builtin_arm_nop:
8904 case clang::ARM::BI__builtin_arm_yield:
8905 case clang::ARM::BI__yield:
8908 case clang::ARM::BI__builtin_arm_wfe:
8909 case clang::ARM::BI__wfe:
8912 case clang::ARM::BI__builtin_arm_wfi:
8913 case clang::ARM::BI__wfi:
8916 case clang::ARM::BI__builtin_arm_sev:
8917 case clang::ARM::BI__sev:
8920 case clang::ARM::BI__builtin_arm_sevl:
8921 case clang::ARM::BI__sevl:
8940 llvm::Type *ValueType,
bool isExecHi) {
8945 llvm::Value *
Call = Builder.CreateCall(F, {Builder.getInt1(
true)});
8948 Value *Rt2 = Builder.CreateLShr(
Call, 32);
8949 Rt2 = Builder.CreateTrunc(Rt2, CGF.
Int32Ty);
8962 llvm::Type *ValueType,
8964 StringRef SysReg =
"") {
8968 "Unsupported size for register.");
8974 if (SysReg.empty()) {
8976 SysReg = cast<clang::StringLiteral>(SysRegStrExpr)->getString();
8979 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) };
8980 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
8981 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
8985 bool MixedTypes =
RegisterType->isIntegerTy(64) && ValueType->isIntegerTy(32);
8986 assert(!(
RegisterType->isIntegerTy(32) && ValueType->isIntegerTy(64))
8987 &&
"Can't fit 64-bit value in 32-bit register");
8989 if (AccessKind !=
Write) {
8992 AccessKind ==
VolatileRead ? llvm::Intrinsic::read_volatile_register
8993 : llvm::Intrinsic::read_register,
8995 llvm::Value *
Call = Builder.CreateCall(F, Metadata);
8999 return Builder.CreateTrunc(
Call, ValueType);
9001 if (ValueType->isPointerTy())
9003 return Builder.CreateIntToPtr(
Call, ValueType);
9008 llvm::Function *F = CGM.
getIntrinsic(llvm::Intrinsic::write_register, Types);
9013 return Builder.CreateCall(F, { Metadata, ArgValue });
9016 if (ValueType->isPointerTy()) {
9018 ArgValue = Builder.CreatePtrToInt(ArgValue,
RegisterType);
9019 return Builder.CreateCall(F, { Metadata, ArgValue });
9022 return Builder.CreateCall(F, { Metadata, ArgValue });
9028 switch (BuiltinID) {
9030 case NEON::BI__builtin_neon_vget_lane_i8:
9031 case NEON::BI__builtin_neon_vget_lane_i16:
9032 case NEON::BI__builtin_neon_vget_lane_bf16:
9033 case NEON::BI__builtin_neon_vget_lane_i32:
9034 case NEON::BI__builtin_neon_vget_lane_i64:
9035 case NEON::BI__builtin_neon_vget_lane_f32:
9036 case NEON::BI__builtin_neon_vgetq_lane_i8:
9037 case NEON::BI__builtin_neon_vgetq_lane_i16:
9038 case NEON::BI__builtin_neon_vgetq_lane_bf16:
9039 case NEON::BI__builtin_neon_vgetq_lane_i32:
9040 case NEON::BI__builtin_neon_vgetq_lane_i64:
9041 case NEON::BI__builtin_neon_vgetq_lane_f32:
9042 case NEON::BI__builtin_neon_vduph_lane_bf16:
9043 case NEON::BI__builtin_neon_vduph_laneq_bf16:
9044 case NEON::BI__builtin_neon_vset_lane_i8:
9045 case NEON::BI__builtin_neon_vset_lane_i16:
9046 case NEON::BI__builtin_neon_vset_lane_bf16:
9047 case NEON::BI__builtin_neon_vset_lane_i32:
9048 case NEON::BI__builtin_neon_vset_lane_i64:
9049 case NEON::BI__builtin_neon_vset_lane_f32:
9050 case NEON::BI__builtin_neon_vsetq_lane_i8:
9051 case NEON::BI__builtin_neon_vsetq_lane_i16:
9052 case NEON::BI__builtin_neon_vsetq_lane_bf16:
9053 case NEON::BI__builtin_neon_vsetq_lane_i32:
9054 case NEON::BI__builtin_neon_vsetq_lane_i64:
9055 case NEON::BI__builtin_neon_vsetq_lane_f32:
9056 case NEON::BI__builtin_neon_vsha1h_u32:
9057 case NEON::BI__builtin_neon_vsha1cq_u32:
9058 case NEON::BI__builtin_neon_vsha1pq_u32:
9059 case NEON::BI__builtin_neon_vsha1mq_u32:
9060 case NEON::BI__builtin_neon_vcvth_bf16_f32:
9061 case clang::ARM::BI_MoveToCoprocessor:
9062 case clang::ARM::BI_MoveToCoprocessor2:
9071 llvm::Triple::ArchType Arch) {
9072 if (
auto Hint = GetValueForARMHint(BuiltinID))
9075 if (BuiltinID == clang::ARM::BI__emit) {
9077 llvm::FunctionType *FTy =
9078 llvm::FunctionType::get(
VoidTy,
false);
9082 llvm_unreachable(
"Sema will ensure that the parameter is constant");
9085 uint64_t ZExtValue =
Value.zextOrTrunc(IsThumb ? 16 : 32).getZExtValue();
9087 llvm::InlineAsm *Emit =
9088 IsThumb ? InlineAsm::get(FTy,
".inst.n 0x" + utohexstr(ZExtValue),
"",
9090 : InlineAsm::get(FTy,
".inst 0x" + utohexstr(ZExtValue),
"",
9093 return Builder.CreateCall(Emit);
9096 if (BuiltinID == clang::ARM::BI__builtin_arm_dbg) {
9101 if (BuiltinID == clang::ARM::BI__builtin_arm_prefetch) {
9113 if (BuiltinID == clang::ARM::BI__builtin_arm_rbit) {
9116 CGM.
getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
9119 if (BuiltinID == clang::ARM::BI__builtin_arm_clz ||
9120 BuiltinID == clang::ARM::BI__builtin_arm_clz64) {
9124 if (BuiltinID == clang::ARM::BI__builtin_arm_clz64)
9130 if (BuiltinID == clang::ARM::BI__builtin_arm_cls) {
9134 if (BuiltinID == clang::ARM::BI__builtin_arm_cls64) {
9140 if (BuiltinID == clang::ARM::BI__clear_cache) {
9141 assert(
E->getNumArgs() == 2 &&
"__clear_cache takes 2 arguments");
9144 for (
unsigned i = 0; i < 2; i++)
9147 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
9148 StringRef Name = FD->
getName();
9152 if (BuiltinID == clang::ARM::BI__builtin_arm_mcrr ||
9153 BuiltinID == clang::ARM::BI__builtin_arm_mcrr2) {
9156 switch (BuiltinID) {
9157 default: llvm_unreachable(
"unexpected builtin");
9158 case clang::ARM::BI__builtin_arm_mcrr:
9161 case clang::ARM::BI__builtin_arm_mcrr2:
9183 return Builder.CreateCall(F, {Coproc, Opc1, Rt, Rt2, CRm});
9186 if (BuiltinID == clang::ARM::BI__builtin_arm_mrrc ||
9187 BuiltinID == clang::ARM::BI__builtin_arm_mrrc2) {
9190 switch (BuiltinID) {
9191 default: llvm_unreachable(
"unexpected builtin");
9192 case clang::ARM::BI__builtin_arm_mrrc:
9195 case clang::ARM::BI__builtin_arm_mrrc2:
9203 Value *RtAndRt2 =
Builder.CreateCall(F, {Coproc, Opc1, CRm});
9213 Value *ShiftCast = llvm::ConstantInt::get(
Int64Ty, 32);
9214 RtAndRt2 =
Builder.CreateShl(Rt, ShiftCast,
"shl",
true);
9215 RtAndRt2 =
Builder.CreateOr(RtAndRt2, Rt1);
9220 if (BuiltinID == clang::ARM::BI__builtin_arm_ldrexd ||
9221 ((BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
9222 BuiltinID == clang::ARM::BI__builtin_arm_ldaex) &&
9224 BuiltinID == clang::ARM::BI__ldrexd) {
9227 switch (BuiltinID) {
9228 default: llvm_unreachable(
"unexpected builtin");
9229 case clang::ARM::BI__builtin_arm_ldaex:
9232 case clang::ARM::BI__builtin_arm_ldrexd:
9233 case clang::ARM::BI__builtin_arm_ldrex:
9234 case clang::ARM::BI__ldrexd:
9248 Val =
Builder.CreateShl(Val0, ShiftCst,
"shl",
true );
9249 Val =
Builder.CreateOr(Val, Val1);
9253 if (BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
9254 BuiltinID == clang::ARM::BI__builtin_arm_ldaex) {
9263 BuiltinID == clang::ARM::BI__builtin_arm_ldaex ? Intrinsic::arm_ldaex
9264 : Intrinsic::arm_ldrex,
9266 CallInst *Val =
Builder.CreateCall(F, LoadAddr,
"ldrex");
9270 if (RealResTy->isPointerTy())
9271 return Builder.CreateIntToPtr(Val, RealResTy);
9273 llvm::Type *IntResTy = llvm::IntegerType::get(
9275 return Builder.CreateBitCast(
Builder.CreateTruncOrBitCast(Val, IntResTy),
9280 if (BuiltinID == clang::ARM::BI__builtin_arm_strexd ||
9281 ((BuiltinID == clang::ARM::BI__builtin_arm_stlex ||
9282 BuiltinID == clang::ARM::BI__builtin_arm_strex) &&
9285 BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlexd
9286 : Intrinsic::arm_strexd);
9299 return Builder.CreateCall(F, {Arg0, Arg1, StPtr},
"strexd");
9302 if (BuiltinID == clang::ARM::BI__builtin_arm_strex ||
9303 BuiltinID == clang::ARM::BI__builtin_arm_stlex) {
9308 llvm::Type *StoreTy =
9311 if (StoreVal->
getType()->isPointerTy())
9314 llvm::Type *
IntTy = llvm::IntegerType::get(
9322 BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlex
9323 : Intrinsic::arm_strex,
9326 CallInst *CI =
Builder.CreateCall(F, {StoreVal, StoreAddr},
"strex");
9328 1, Attribute::get(
getLLVMContext(), Attribute::ElementType, StoreTy));
9332 if (BuiltinID == clang::ARM::BI__builtin_arm_clrex) {
9338 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
9339 switch (BuiltinID) {
9340 case clang::ARM::BI__builtin_arm_crc32b:
9341 CRCIntrinsicID = Intrinsic::arm_crc32b;
break;
9342 case clang::ARM::BI__builtin_arm_crc32cb:
9343 CRCIntrinsicID = Intrinsic::arm_crc32cb;
break;
9344 case clang::ARM::BI__builtin_arm_crc32h:
9345 CRCIntrinsicID = Intrinsic::arm_crc32h;
break;
9346 case clang::ARM::BI__builtin_arm_crc32ch:
9347 CRCIntrinsicID = Intrinsic::arm_crc32ch;
break;
9348 case clang::ARM::BI__builtin_arm_crc32w:
9349 case clang::ARM::BI__builtin_arm_crc32d:
9350 CRCIntrinsicID = Intrinsic::arm_crc32w;
break;
9351 case clang::ARM::BI__builtin_arm_crc32cw:
9352 case clang::ARM::BI__builtin_arm_crc32cd:
9353 CRCIntrinsicID = Intrinsic::arm_crc32cw;
break;
9356 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
9362 if (BuiltinID == clang::ARM::BI__builtin_arm_crc32d ||
9363 BuiltinID == clang::ARM::BI__builtin_arm_crc32cd) {
9371 return Builder.CreateCall(F, {Res, Arg1b});
9376 return Builder.CreateCall(F, {Arg0, Arg1});
9380 if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
9381 BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
9382 BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
9383 BuiltinID == clang::ARM::BI__builtin_arm_wsr ||
9384 BuiltinID == clang::ARM::BI__builtin_arm_wsr64 ||
9385 BuiltinID == clang::ARM::BI__builtin_arm_wsrp) {
9388 if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
9389 BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
9390 BuiltinID == clang::ARM::BI__builtin_arm_rsrp)
9393 bool IsPointerBuiltin = BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
9394 BuiltinID == clang::ARM::BI__builtin_arm_wsrp;
9396 bool Is64Bit = BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
9397 BuiltinID == clang::ARM::BI__builtin_arm_wsr64;
9399 llvm::Type *ValueType;
9401 if (IsPointerBuiltin) {
9404 }
else if (Is64Bit) {
9414 if (BuiltinID == ARM::BI__builtin_sponentry) {
9433 return P.first == BuiltinID;
9436 BuiltinID = It->second;
9440 unsigned ICEArguments = 0;
9445 auto getAlignmentValue32 = [&](
Address addr) ->
Value* {
9446 return Builder.getInt32(addr.getAlignment().getQuantity());
9453 unsigned NumArgs =
E->getNumArgs() - (HasExtraArg ? 1 : 0);
9454 for (
unsigned i = 0, e = NumArgs; i != e; i++) {
9456 switch (BuiltinID) {
9457 case NEON::BI__builtin_neon_vld1_v:
9458 case NEON::BI__builtin_neon_vld1q_v:
9459 case NEON::BI__builtin_neon_vld1q_lane_v:
9460 case NEON::BI__builtin_neon_vld1_lane_v:
9461 case NEON::BI__builtin_neon_vld1_dup_v:
9462 case NEON::BI__builtin_neon_vld1q_dup_v:
9463 case NEON::BI__builtin_neon_vst1_v:
9464 case NEON::BI__builtin_neon_vst1q_v:
9465 case NEON::BI__builtin_neon_vst1q_lane_v:
9466 case NEON::BI__builtin_neon_vst1_lane_v:
9467 case NEON::BI__builtin_neon_vst2_v:
9468 case NEON::BI__builtin_neon_vst2q_v:
9469 case NEON::BI__builtin_neon_vst2_lane_v:
9470 case NEON::BI__builtin_neon_vst2q_lane_v:
9471 case NEON::BI__builtin_neon_vst3_v:
9472 case NEON::BI__builtin_neon_vst3q_v:
9473 case NEON::BI__builtin_neon_vst3_lane_v:
9474 case NEON::BI__builtin_neon_vst3q_lane_v:
9475 case NEON::BI__builtin_neon_vst4_v:
9476 case NEON::BI__builtin_neon_vst4q_v:
9477 case NEON::BI__builtin_neon_vst4_lane_v:
9478 case NEON::BI__builtin_neon_vst4q_lane_v:
9487 switch (BuiltinID) {
9488 case NEON::BI__builtin_neon_vld2_v:
9489 case NEON::BI__builtin_neon_vld2q_v:
9490 case NEON::BI__builtin_neon_vld3_v:
9491 case NEON::BI__builtin_neon_vld3q_v:
9492 case NEON::BI__builtin_neon_vld4_v:
9493 case NEON::BI__builtin_neon_vld4q_v:
9494 case NEON::BI__builtin_neon_vld2_lane_v:
9495 case NEON::BI__builtin_neon_vld2q_lane_v:
9496 case NEON::BI__builtin_neon_vld3_lane_v:
9497 case NEON::BI__builtin_neon_vld3q_lane_v:
9498 case NEON::BI__builtin_neon_vld4_lane_v:
9499 case NEON::BI__builtin_neon_vld4q_lane_v:
9500 case NEON::BI__builtin_neon_vld2_dup_v:
9501 case NEON::BI__builtin_neon_vld2q_dup_v:
9502 case NEON::BI__builtin_neon_vld3_dup_v:
9503 case NEON::BI__builtin_neon_vld3q_dup_v:
9504 case NEON::BI__builtin_neon_vld4_dup_v:
9505 case NEON::BI__builtin_neon_vld4q_dup_v:
9517 switch (BuiltinID) {
9520 case NEON::BI__builtin_neon_vget_lane_i8:
9521 case NEON::BI__builtin_neon_vget_lane_i16:
9522 case NEON::BI__builtin_neon_vget_lane_i32:
9523 case NEON::BI__builtin_neon_vget_lane_i64:
9524 case NEON::BI__builtin_neon_vget_lane_bf16:
9525 case NEON::BI__builtin_neon_vget_lane_f32:
9526 case NEON::BI__builtin_neon_vgetq_lane_i8:
9527 case NEON::BI__builtin_neon_vgetq_lane_i16:
9528 case NEON::BI__builtin_neon_vgetq_lane_i32:
9529 case NEON::BI__builtin_neon_vgetq_lane_i64:
9530 case NEON::BI__builtin_neon_vgetq_lane_bf16:
9531 case NEON::BI__builtin_neon_vgetq_lane_f32:
9532 case NEON::BI__builtin_neon_vduph_lane_bf16:
9533 case NEON::BI__builtin_neon_vduph_laneq_bf16:
9534 return Builder.CreateExtractElement(Ops[0], Ops[1],
"vget_lane");
9536 case NEON::BI__builtin_neon_vrndns_f32: {
9538 llvm::Type *Tys[] = {Arg->
getType()};
9540 return Builder.CreateCall(F, {Arg},
"vrndn"); }
9542 case NEON::BI__builtin_neon_vset_lane_i8:
9543 case NEON::BI__builtin_neon_vset_lane_i16:
9544 case NEON::BI__builtin_neon_vset_lane_i32:
9545 case NEON::BI__builtin_neon_vset_lane_i64:
9546 case NEON::BI__builtin_neon_vset_lane_bf16:
9547 case NEON::BI__builtin_neon_vset_lane_f32:
9548 case NEON::BI__builtin_neon_vsetq_lane_i8:
9549 case NEON::BI__builtin_neon_vsetq_lane_i16:
9550 case NEON::BI__builtin_neon_vsetq_lane_i32:
9551 case NEON::BI__builtin_neon_vsetq_lane_i64:
9552 case NEON::BI__builtin_neon_vsetq_lane_bf16:
9553 case NEON::BI__builtin_neon_vsetq_lane_f32:
9554 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
9556 case NEON::BI__builtin_neon_vsha1h_u32:
9559 case NEON::BI__builtin_neon_vsha1cq_u32:
9562 case NEON::BI__builtin_neon_vsha1pq_u32:
9565 case NEON::BI__builtin_neon_vsha1mq_u32:
9569 case NEON::BI__builtin_neon_vcvth_bf16_f32: {
9576 case clang::ARM::BI_MoveToCoprocessor:
9577 case clang::ARM::BI_MoveToCoprocessor2: {
9579 ? Intrinsic::arm_mcr
9580 : Intrinsic::arm_mcr2);
9581 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0],
9582 Ops[3], Ops[4], Ops[5]});
9587 assert(HasExtraArg);
9588 const Expr *Arg =
E->getArg(
E->getNumArgs()-1);
9589 std::optional<llvm::APSInt>
Result =
9594 if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f ||
9595 BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_d) {
9598 if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f)
9604 bool usgn =
Result->getZExtValue() == 1;
9605 unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr;
9609 return Builder.CreateCall(F, Ops,
"vcvtr");
9614 bool usgn =
Type.isUnsigned();
9615 bool rightShift =
false;
9617 llvm::FixedVectorType *VTy =
9620 llvm::Type *Ty = VTy;
9631 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
9632 Builtin->NameHint, Builtin->TypeModifier,
E, Ops, PtrOp0, PtrOp1, Arch);
9635 switch (BuiltinID) {
9636 default:
return nullptr;
9637 case NEON::BI__builtin_neon_vld1q_lane_v:
9640 if (VTy->getElementType()->isIntegerTy(64)) {
9642 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
9643 int Lane = cast<ConstantInt>(Ops[2])->getZExtValue();
9644 Value *SV = llvm::ConstantVector::get(ConstantInt::get(
Int32Ty, 1-Lane));
9645 Ops[1] =
Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
9647 Ty = llvm::FixedVectorType::get(VTy->getElementType(), 1);
9650 Value *Align = getAlignmentValue32(PtrOp0);
9653 int Indices[] = {1 - Lane, Lane};
9654 return Builder.CreateShuffleVector(Ops[1], Ld, Indices,
"vld1q_lane");
9657 case NEON::BI__builtin_neon_vld1_lane_v: {
9658 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
9661 return Builder.CreateInsertElement(Ops[1], Ld, Ops[2],
"vld1_lane");
9663 case NEON::BI__builtin_neon_vqrshrn_n_v:
9665 usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns;
9668 case NEON::BI__builtin_neon_vqrshrun_n_v:
9670 Ops,
"vqrshrun_n", 1,
true);
9671 case NEON::BI__builtin_neon_vqshrn_n_v:
9672 Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns;
9675 case NEON::BI__builtin_neon_vqshrun_n_v:
9677 Ops,
"vqshrun_n", 1,
true);
9678 case NEON::BI__builtin_neon_vrecpe_v:
9679 case NEON::BI__builtin_neon_vrecpeq_v:
9682 case NEON::BI__builtin_neon_vrshrn_n_v:
9684 Ops,
"vrshrn_n", 1,
true);
9685 case NEON::BI__builtin_neon_vrsra_n_v:
9686 case NEON::BI__builtin_neon_vrsraq_n_v:
9687 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
9688 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
9690 Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
9692 return Builder.CreateAdd(Ops[0], Ops[1],
"vrsra_n");
9693 case NEON::BI__builtin_neon_vsri_n_v:
9694 case NEON::BI__builtin_neon_vsriq_n_v:
9697 case NEON::BI__builtin_neon_vsli_n_v:
9698 case NEON::BI__builtin_neon_vsliq_n_v:
9702 case NEON::BI__builtin_neon_vsra_n_v:
9703 case NEON::BI__builtin_neon_vsraq_n_v:
9704 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
9706 return Builder.CreateAdd(Ops[0], Ops[1]);
9707 case NEON::BI__builtin_neon_vst1q_lane_v:
9710 if (VTy->getElementType()->isIntegerTy(64)) {
9711 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
9712 Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2]));
9713 Ops[1] =
Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
9714 Ops[2] = getAlignmentValue32(PtrOp0);
9715 llvm::Type *Tys[] = {
Int8PtrTy, Ops[1]->getType()};
9720 case NEON::BI__builtin_neon_vst1_lane_v: {
9721 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
9722 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2]);
9726 case NEON::BI__builtin_neon_vtbl1_v:
9729 case NEON::BI__builtin_neon_vtbl2_v:
9732 case NEON::BI__builtin_neon_vtbl3_v:
9735 case NEON::BI__builtin_neon_vtbl4_v:
9738 case NEON::BI__builtin_neon_vtbx1_v:
9741 case NEON::BI__builtin_neon_vtbx2_v:
9744 case NEON::BI__builtin_neon_vtbx3_v:
9747 case NEON::BI__builtin_neon_vtbx4_v:
9753template<
typename Integer>
9762 return Unsigned ? Builder.CreateZExt(
V,
T) : Builder.CreateSExt(
V,
T);
9772 unsigned LaneBits = cast<llvm::VectorType>(
V->getType())
9774 ->getPrimitiveSizeInBits();
9775 if (Shift == LaneBits) {
9780 return llvm::Constant::getNullValue(
V->getType());
9784 return Unsigned ? Builder.CreateLShr(
V, Shift) : Builder.CreateAShr(
V, Shift);
9791 unsigned Elements = 128 /
V->getType()->getPrimitiveSizeInBits();
9792 return Builder.CreateVectorSplat(Elements,
V);
9798 llvm::Type *DestType) {
9811 V->getType()->getScalarSizeInBits() != DestType->getScalarSizeInBits()) {
9812 return Builder.CreateCall(
9814 {DestType, V->getType()}),
9817 return Builder.CreateBitCast(
V, DestType);
9825 unsigned InputElements =
9826 cast<llvm::FixedVectorType>(
V->getType())->getNumElements();
9827 for (
unsigned i = 0; i < InputElements; i += 2)
9828 Indices.push_back(i + Odd);
9829 return Builder.CreateShuffleVector(
V, Indices);
9835 assert(V0->getType() == V1->getType() &&
"Can't zip different vector types");
9837 unsigned InputElements =
9838 cast<llvm::FixedVectorType>(V0->getType())->getNumElements();
9839 for (
unsigned i = 0; i < InputElements; i++) {
9840 Indices.push_back(i);
9841 Indices.push_back(i + InputElements);
9843 return Builder.CreateShuffleVector(V0, V1, Indices);
9846template<
unsigned HighBit,
unsigned OtherBits>
9850 llvm::Type *
T = cast<llvm::VectorType>(VT)->getElementType();
9851 unsigned LaneBits =
T->getPrimitiveSizeInBits();
9852 uint32_t
Value = HighBit << (LaneBits - 1);
9854 Value |= (1UL << (LaneBits - 1)) - 1;
9855 llvm::Value *Lane = llvm::ConstantInt::get(
T,
Value);
9861 unsigned ReverseWidth) {
9865 unsigned LaneSize =
V->getType()->getScalarSizeInBits();
9866 unsigned Elements = 128 / LaneSize;
9867 unsigned Mask = ReverseWidth / LaneSize - 1;
9868 for (
unsigned i = 0; i < Elements; i++)
9869 Indices.push_back(i ^ Mask);
9870 return Builder.CreateShuffleVector(
V, Indices);
9876 llvm::Triple::ArchType Arch) {
9877 enum class CustomCodeGen { VLD24, VST24 } CustomCodeGenType;
9878 Intrinsic::ID IRIntr;
9879 unsigned NumVectors;
9882 switch (BuiltinID) {
9883 #include "clang/Basic/arm_mve_builtin_cg.inc"
9894 switch (CustomCodeGenType) {
9896 case CustomCodeGen::VLD24: {
9902 assert(MvecLType->isStructTy() &&
9903 "Return type for vld[24]q should be a struct");
9904 assert(MvecLType->getStructNumElements() == 1 &&
9905 "Return-type struct for vld[24]q should have one element");
9906 auto MvecLTypeInner = MvecLType->getStructElementType(0);
9907 assert(MvecLTypeInner->isArrayTy() &&
9908 "Return-type struct for vld[24]q should contain an array");
9909 assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
9910 "Array member of return-type struct vld[24]q has wrong length");
9911 auto VecLType = MvecLTypeInner->getArrayElementType();
9913 Tys.push_back(VecLType);
9915 auto Addr =
E->getArg(0);
9921 Value *MvecOut = PoisonValue::get(MvecLType);
9922 for (
unsigned i = 0; i < NumVectors; ++i) {
9923 Value *Vec =
Builder.CreateExtractValue(LoadResult, i);
9924 MvecOut =
Builder.CreateInsertValue(MvecOut, Vec, {0, i});
9933 case CustomCodeGen::VST24: {
9937 auto Addr =
E->getArg(0);
9941 auto MvecCType =
E->getArg(1)->
getType();
9943 assert(MvecLType->isStructTy() &&
"Data type for vst2q should be a struct");
9944 assert(MvecLType->getStructNumElements() == 1 &&
9945 "Data-type struct for vst2q should have one element");
9946 auto MvecLTypeInner = MvecLType->getStructElementType(0);
9947 assert(MvecLTypeInner->isArrayTy() &&
9948 "Data-type struct for vst2q should contain an array");
9949 assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
9950 "Array member of return-type struct vld[24]q has wrong length");
9951 auto VecLType = MvecLTypeInner->getArrayElementType();
9953 Tys.push_back(VecLType);
9958 for (
unsigned i = 0; i < NumVectors; i++)
9959 Ops.push_back(
Builder.CreateExtractValue(Mvec, {0, i}));
9962 Value *ToReturn =
nullptr;
9963 for (
unsigned i = 0; i < NumVectors; i++) {
9964 Ops.push_back(llvm::ConstantInt::get(
Int32Ty, i));
9965 ToReturn =
Builder.CreateCall(F, Ops);
9971 llvm_unreachable(
"unknown custom codegen type.");
9977 llvm::Triple::ArchType Arch) {
9978 switch (BuiltinID) {
9981#include "clang/Basic/arm_cde_builtin_cg.inc"
9988 llvm::Triple::ArchType Arch) {
9989 unsigned int Int = 0;
9990 const char *
s =
nullptr;
9992 switch (BuiltinID) {
9995 case NEON::BI__builtin_neon_vtbl1_v:
9996 case NEON::BI__builtin_neon_vqtbl1_v:
9997 case NEON::BI__builtin_neon_vqtbl1q_v:
9998 case NEON::BI__builtin_neon_vtbl2_v:
9999 case NEON::BI__builtin_neon_vqtbl2_v:
10000 case NEON::BI__builtin_neon_vqtbl2q_v:
10001 case NEON::BI__builtin_neon_vtbl3_v:
10002 case NEON::BI__builtin_neon_vqtbl3_v:
10003 case NEON::BI__builtin_neon_vqtbl3q_v:
10004 case NEON::BI__builtin_neon_vtbl4_v:
10005 case NEON::BI__builtin_neon_vqtbl4_v:
10006 case NEON::BI__builtin_neon_vqtbl4q_v:
10008 case NEON::BI__builtin_neon_vtbx1_v:
10009 case NEON::BI__builtin_neon_vqtbx1_v:
10010 case NEON::BI__builtin_neon_vqtbx1q_v:
10011 case NEON::BI__builtin_neon_vtbx2_v:
10012 case NEON::BI__builtin_neon_vqtbx2_v:
10013 case NEON::BI__builtin_neon_vqtbx2q_v:
10014 case NEON::BI__builtin_neon_vtbx3_v:
10015 case NEON::BI__builtin_neon_vqtbx3_v:
10016 case NEON::BI__builtin_neon_vqtbx3q_v:
10017 case NEON::BI__builtin_neon_vtbx4_v:
10018 case NEON::BI__builtin_neon_vqtbx4_v:
10019 case NEON::BI__builtin_neon_vqtbx4q_v:
10023 assert(
E->getNumArgs() >= 3);
10026 const Expr *Arg =
E->getArg(
E->getNumArgs() - 1);
10027 std::optional<llvm::APSInt>
Result =
10042 switch (BuiltinID) {
10043 case NEON::BI__builtin_neon_vtbl1_v: {
10045 Ty, Intrinsic::aarch64_neon_tbl1,
"vtbl1");
10047 case NEON::BI__builtin_neon_vtbl2_v: {
10049 Ty, Intrinsic::aarch64_neon_tbl1,
"vtbl1");
10051 case NEON::BI__builtin_neon_vtbl3_v: {
10053 Ty, Intrinsic::aarch64_neon_tbl2,
"vtbl2");
10055 case NEON::BI__builtin_neon_vtbl4_v: {
10057 Ty, Intrinsic::aarch64_neon_tbl2,
"vtbl2");
10059 case NEON::BI__builtin_neon_vtbx1_v: {
10062 Intrinsic::aarch64_neon_tbl1,
"vtbl1");
10064 llvm::Constant *EightV = ConstantInt::get(Ty, 8);
10065 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV);
10066 CmpRes = Builder.CreateSExt(CmpRes, Ty);
10068 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
10069 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
10070 return Builder.CreateOr(EltsFromInput, EltsFromTbl,
"vtbx");
10072 case NEON::BI__builtin_neon_vtbx2_v: {
10074 Ty, Intrinsic::aarch64_neon_tbx1,
"vtbx1");
10076 case NEON::BI__builtin_neon_vtbx3_v: {
10079 Intrinsic::aarch64_neon_tbl2,
"vtbl2");
10081 llvm::Constant *TwentyFourV = ConstantInt::get(Ty, 24);
10082 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4],
10084 CmpRes = Builder.CreateSExt(CmpRes, Ty);
10086 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
10087 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
10088 return Builder.CreateOr(EltsFromInput, EltsFromTbl,
"vtbx");
10090 case NEON::BI__builtin_neon_vtbx4_v: {
10092 Ty, Intrinsic::aarch64_neon_tbx2,
"vtbx2");
10094 case NEON::BI__builtin_neon_vqtbl1_v:
10095 case NEON::BI__builtin_neon_vqtbl1q_v:
10096 Int = Intrinsic::aarch64_neon_tbl1;
s =
"vtbl1";
break;
10097 case NEON::BI__builtin_neon_vqtbl2_v:
10098 case NEON::BI__builtin_neon_vqtbl2q_v: {
10099 Int = Intrinsic::aarch64_neon_tbl2;
s =
"vtbl2";
break;
10100 case NEON::BI__builtin_neon_vqtbl3_v:
10101 case NEON::BI__builtin_neon_vqtbl3q_v:
10102 Int = Intrinsic::aarch64_neon_tbl3;
s =
"vtbl3";
break;
10103 case NEON::BI__builtin_neon_vqtbl4_v:
10104 case NEON::BI__builtin_neon_vqtbl4q_v:
10105 Int = Intrinsic::aarch64_neon_tbl4;
s =
"vtbl4";
break;
10106 case NEON::BI__builtin_neon_vqtbx1_v:
10107 case NEON::BI__builtin_neon_vqtbx1q_v:
10108 Int = Intrinsic::aarch64_neon_tbx1;
s =
"vtbx1";
break;
10109 case NEON::BI__builtin_neon_vqtbx2_v:
10110 case NEON::BI__builtin_neon_vqtbx2q_v:
10111 Int = Intrinsic::aarch64_neon_tbx2;
s =
"vtbx2";
break;
10112 case NEON::BI__builtin_neon_vqtbx3_v:
10113 case NEON::BI__builtin_neon_vqtbx3q_v:
10114 Int = Intrinsic::aarch64_neon_tbx3;
s =
"vtbx3";
break;
10115 case NEON::BI__builtin_neon_vqtbx4_v:
10116 case NEON::BI__builtin_neon_vqtbx4q_v:
10117 Int = Intrinsic::aarch64_neon_tbx4;
s =
"vtbx4";
break;
10129 auto *VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
10131 Value *
V = PoisonValue::get(VTy);
10132 llvm::Constant *CI = ConstantInt::get(
SizeTy, 0);
10133 Op =
Builder.CreateInsertElement(
V, Op, CI);
10142 case SVETypeFlags::MemEltTyDefault:
10144 case SVETypeFlags::MemEltTyInt8:
10146 case SVETypeFlags::MemEltTyInt16:
10148 case SVETypeFlags::MemEltTyInt32:
10150 case SVETypeFlags::MemEltTyInt64:
10153 llvm_unreachable(
"Unknown MemEltType");
10159 llvm_unreachable(
"Invalid SVETypeFlag!");
10161 case SVETypeFlags::EltTyInt8:
10163 case SVETypeFlags::EltTyInt16:
10165 case SVETypeFlags::EltTyInt32:
10167 case SVETypeFlags::EltTyInt64:
10169 case SVETypeFlags::EltTyInt128:
10170 return Builder.getInt128Ty();
10172 case SVETypeFlags::EltTyFloat16:
10174 case SVETypeFlags::EltTyFloat32:
10176 case SVETypeFlags::EltTyFloat64:
10177 return Builder.getDoubleTy();
10179 case SVETypeFlags::EltTyBFloat16:
10180 return Builder.getBFloatTy();
10182 case SVETypeFlags::EltTyBool8:
10183 case SVETypeFlags::EltTyBool16:
10184 case SVETypeFlags::EltTyBool32:
10185 case SVETypeFlags::EltTyBool64:
10192llvm::ScalableVectorType *
10195 default: llvm_unreachable(
"Unhandled SVETypeFlag!");
10197 case SVETypeFlags::EltTyInt8:
10198 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 16);
10199 case SVETypeFlags::EltTyInt16:
10200 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 8);
10201 case SVETypeFlags::EltTyInt32:
10202 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 4);
10203 case SVETypeFlags::EltTyInt64:
10204 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 2);
10206 case SVETypeFlags::EltTyBFloat16:
10207 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 8);
10208 case SVETypeFlags::EltTyFloat16:
10209 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 8);
10210 case SVETypeFlags::EltTyFloat32:
10211 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 4);
10212 case SVETypeFlags::EltTyFloat64:
10213 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 2);
10215 case SVETypeFlags::EltTyBool8:
10216 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 16);
10217 case SVETypeFlags::EltTyBool16:
10218 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 8);
10219 case SVETypeFlags::EltTyBool32:
10220 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 4);
10221 case SVETypeFlags::EltTyBool64:
10222 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 2);
10227llvm::ScalableVectorType *
10231 llvm_unreachable(
"Invalid SVETypeFlag!");
10233 case SVETypeFlags::EltTyInt8:
10234 return llvm::ScalableVectorType::get(
Builder.getInt8Ty(), 16);
10235 case SVETypeFlags::EltTyInt16:
10236 return llvm::ScalableVectorType::get(
Builder.getInt16Ty(), 8);
10237 case SVETypeFlags::EltTyInt32:
10238 return llvm::ScalableVectorType::get(
Builder.getInt32Ty(), 4);
10239 case SVETypeFlags::EltTyInt64:
10240 return llvm::ScalableVectorType::get(
Builder.getInt64Ty(), 2);
10242 case SVETypeFlags::EltTyMFloat8:
10243 return llvm::ScalableVectorType::get(
Builder.getInt8Ty(), 16);
10244 case SVETypeFlags::EltTyFloat16:
10245 return llvm::ScalableVectorType::get(
Builder.getHalfTy(), 8);
10246 case SVETypeFlags::EltTyBFloat16:
10247 return llvm::ScalableVectorType::get(
Builder.getBFloatTy(), 8);
10248 case SVETypeFlags::EltTyFloat32:
10249 return llvm::ScalableVectorType::get(
Builder.getFloatTy(), 4);
10250 case SVETypeFlags::EltTyFloat64:
10251 return llvm::ScalableVectorType::get(
Builder.getDoubleTy(), 2);
10253 case SVETypeFlags::EltTyBool8:
10254 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 16);
10255 case SVETypeFlags::EltTyBool16:
10256 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 8);
10257 case SVETypeFlags::EltTyBool32:
10258 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 4);
10259 case SVETypeFlags::EltTyBool64:
10260 return llvm::ScalableVectorType::get(
Builder.getInt1Ty(), 2);
10275 return llvm::ScalableVectorType::get(EltTy, NumElts);
10281 llvm::ScalableVectorType *VTy) {
10283 if (isa<TargetExtType>(Pred->
getType()) &&
10284 cast<TargetExtType>(Pred->
getType())->getName() ==
"aarch64.svcount")
10287 auto *RTy = llvm::VectorType::get(IntegerType::get(
getLLVMContext(), 1), VTy);
10292 llvm::Type *IntrinsicTy;
10293 switch (VTy->getMinNumElements()) {
10295 llvm_unreachable(
"unsupported element count!");
10300 IntID = Intrinsic::aarch64_sve_convert_from_svbool;
10304 IntID = Intrinsic::aarch64_sve_convert_to_svbool;
10305 IntrinsicTy = Pred->
getType();
10311 assert(
C->getType() == RTy &&
"Unexpected return type!");
10316 llvm::StructType *Ty) {
10317 if (PredTuple->
getType() == Ty)
10320 Value *
Ret = llvm::PoisonValue::get(Ty);
10321 for (
unsigned I = 0; I < Ty->getNumElements(); ++I) {
10322 Value *Pred =
Builder.CreateExtractValue(PredTuple, I);
10324 Pred, cast<llvm::ScalableVectorType>(Ty->getTypeAtIndex(I)));
10325 Ret =
Builder.CreateInsertValue(Ret, Pred, I);
10335 auto *OverloadedTy =
10339 if (Ops[1]->getType()->isVectorTy())
10359 Ops[0], cast<llvm::ScalableVectorType>(F->getArg(0)->getType()));
10364 if (Ops.size() == 2) {
10365 assert(Ops[1]->getType()->isVectorTy() &&
"Scalar base requires an offset");
10366 Ops.push_back(ConstantInt::get(
Int64Ty, 0));
10371 if (!TypeFlags.
isByteIndexed() && Ops[1]->getType()->isVectorTy()) {
10372 unsigned BytesPerElt =
10373 OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
10374 Ops[2] =
Builder.CreateShl(Ops[2], Log2_32(BytesPerElt));
10389 auto *OverloadedTy =
10394 Ops.insert(Ops.begin(), Ops.pop_back_val());
10397 if (Ops[2]->getType()->isVectorTy())
10412 if (Ops.size() == 3) {
10413 assert(Ops[1]->getType()->isVectorTy() &&
"Scalar base requires an offset");
10414 Ops.push_back(ConstantInt::get(
Int64Ty, 0));
10419 Ops[0] =
Builder.CreateTrunc(Ops[0], OverloadedTy);
10429 Ops[1], cast<llvm::ScalableVectorType>(F->getArg(1)->getType()));
10433 if (!TypeFlags.
isByteIndexed() && Ops[2]->getType()->isVectorTy()) {
10434 unsigned BytesPerElt =
10435 OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
10436 Ops[3] =
Builder.CreateShl(Ops[3], Log2_32(BytesPerElt));
10439 return Builder.CreateCall(F, Ops);
10447 auto *OverloadedTy = dyn_cast<llvm::ScalableVectorType>(Ops[1]->getType());
10449 OverloadedTy = cast<llvm::ScalableVectorType>(Ops[2]->getType());
10455 if (Ops[1]->getType()->isVectorTy()) {
10456 if (Ops.size() == 3) {
10458 Ops.push_back(ConstantInt::get(
Int64Ty, 0));
10461 std::swap(Ops[2], Ops[3]);
10465 unsigned BytesPerElt = MemEltTy->getPrimitiveSizeInBits() / 8;
10466 if (BytesPerElt > 1)
10467 Ops[2] =
Builder.CreateShl(Ops[2], Log2_32(BytesPerElt));
10472 return Builder.CreateCall(F, Ops);
10478 llvm::ScalableVectorType *VTy =
getSVEType(TypeFlags);
10480 Value *BasePtr = Ops[1];
10483 if (Ops.size() > 2)
10487 return Builder.CreateCall(F, {Predicate, BasePtr});
10493 llvm::ScalableVectorType *VTy =
getSVEType(TypeFlags);
10497 case Intrinsic::aarch64_sve_st2:
10498 case Intrinsic::aarch64_sve_st1_pn_x2:
10499 case Intrinsic::aarch64_sve_stnt1_pn_x2:
10500 case Intrinsic::aarch64_sve_st2q:
10503 case Intrinsic::aarch64_sve_st3:
10504 case Intrinsic::aarch64_sve_st3q:
10507 case Intrinsic::aarch64_sve_st4:
10508 case Intrinsic::aarch64_sve_st1_pn_x4:
10509 case Intrinsic::aarch64_sve_stnt1_pn_x4:
10510 case Intrinsic::aarch64_sve_st4q:
10514 llvm_unreachable(
"unknown intrinsic!");
10518 Value *BasePtr = Ops[1];
10521 if (Ops.size() > (2 + N))
10527 for (
unsigned I = Ops.size() - N; I < Ops.size(); ++I)
10528 Operands.push_back(Ops[I]);
10529 Operands.append({Predicate, BasePtr});
10532 return Builder.CreateCall(F, Operands);
10540 unsigned BuiltinID) {
10552 llvm::ScalableVectorType *Ty =
getSVEType(TypeFlags);
10558 llvm::Type *OverloadedTy =
getSVEType(TypeFlags);
10565 unsigned BuiltinID) {
10568 auto *MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
10571 Value *BasePtr = Ops[1];
10574 if (Ops.size() > 3)
10577 Value *PrfOp = Ops.back();
10580 return Builder.CreateCall(F, {Predicate, BasePtr, PrfOp});
10584 llvm::Type *ReturnTy,
10586 unsigned IntrinsicID,
10587 bool IsZExtReturn) {
10594 auto VectorTy = cast<llvm::ScalableVectorType>(ReturnTy);
10595 llvm::ScalableVectorType *MemoryTy =
nullptr;
10596 llvm::ScalableVectorType *PredTy =
nullptr;
10597 bool IsQuadLoad =
false;
10598 switch (IntrinsicID) {
10599 case Intrinsic::aarch64_sve_ld1uwq:
10600 case Intrinsic::aarch64_sve_ld1udq:
10601 MemoryTy = llvm::ScalableVectorType::get(MemEltTy, 1);
10602 PredTy = llvm::ScalableVectorType::get(
10607 MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
10613 Value *BasePtr = Ops[1];
10616 if (Ops.size() > 2)
10621 cast<llvm::Instruction>(
Builder.CreateCall(F, {Predicate, BasePtr}));
10628 return IsZExtReturn ?
Builder.CreateZExt(Load, VectorTy)
10629 :
Builder.CreateSExt(Load, VectorTy);
10634 unsigned IntrinsicID) {
10641 auto VectorTy = cast<llvm::ScalableVectorType>(Ops.back()->getType());
10642 auto MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
10644 auto PredTy = MemoryTy;
10645 auto AddrMemoryTy = MemoryTy;
10646 bool IsQuadStore =
false;
10648 switch (IntrinsicID) {
10649 case Intrinsic::aarch64_sve_st1wq:
10650 case Intrinsic::aarch64_sve_st1dq:
10651 AddrMemoryTy = llvm::ScalableVectorType::get(MemEltTy, 1);
10653 llvm::ScalableVectorType::get(IntegerType::get(
getLLVMContext(), 1), 1);
10654 IsQuadStore =
true;
10660 Value *BasePtr = Ops[1];
10663 if (Ops.size() == 4)
10668 IsQuadStore ? Ops.back() :
Builder.CreateTrunc(Ops.back(), MemoryTy);
10673 cast<llvm::Instruction>(
Builder.CreateCall(F, {Val, Predicate, BasePtr}));
10686 NewOps.push_back(Ops[2]);
10688 llvm::Value *BasePtr = Ops[3];
10689 llvm::Value *RealSlice = Ops[1];
10692 if (Ops.size() == 5) {
10695 llvm::Value *StreamingVectorLengthCall =
10696 Builder.CreateCall(StreamingVectorLength);
10697 llvm::Value *Mulvl =
10698 Builder.CreateMul(StreamingVectorLengthCall, Ops[4],
"mulvl");
10702 RealSlice =
Builder.CreateAdd(RealSlice, Ops[4]);
10705 NewOps.push_back(BasePtr);
10706 NewOps.push_back(Ops[0]);
10707 NewOps.push_back(RealSlice);
10709 return Builder.CreateCall(F, NewOps);
10721 return Builder.CreateCall(F, Ops);
10728 if (Ops.size() == 0)
10729 Ops.push_back(llvm::ConstantInt::get(
Int32Ty, 255));
10731 return Builder.CreateCall(F, Ops);
10737 if (Ops.size() == 2)
10738 Ops.push_back(
Builder.getInt32(0));
10742 return Builder.CreateCall(F, Ops);
10748 return Builder.CreateVectorSplat(
10749 cast<llvm::VectorType>(Ty)->getElementCount(), Scalar);
10753 if (
auto *Ty =
Scalar->getType(); Ty->isVectorTy()) {
10755 auto *VecTy = cast<llvm::VectorType>(Ty);
10756 ElementCount EC = VecTy->getElementCount();
10757 assert(EC.isScalar() && VecTy->getElementType() ==
Int8Ty &&
10758 "Only <1 x i8> expected");
10773 if (
auto *StructTy = dyn_cast<StructType>(Ty)) {
10774 Value *Tuple = llvm::PoisonValue::get(Ty);
10776 for (
unsigned I = 0; I < StructTy->getNumElements(); ++I) {
10778 Value *Out =
Builder.CreateBitCast(In, StructTy->getTypeAtIndex(I));
10779 Tuple =
Builder.CreateInsertValue(Tuple, Out, I);
10785 return Builder.CreateBitCast(Val, Ty);
10790 auto *SplatZero = Constant::getNullValue(Ty);
10791 Ops.insert(Ops.begin(), SplatZero);
10796 auto *SplatUndef = UndefValue::get(Ty);
10797 Ops.insert(Ops.begin(), SplatUndef);
10802 llvm::Type *ResultType,
10807 llvm::Type *DefaultType =
getSVEType(TypeFlags);
10810 return {DefaultType, Ops[1]->getType()};
10816 return {Ops[0]->getType(), Ops.back()->getType()};
10818 if (TypeFlags.
isReductionQV() && !ResultType->isScalableTy() &&
10819 ResultType->isVectorTy())
10820 return {ResultType, Ops[1]->getType()};
10823 return {DefaultType};
10829 "Expects TypleFlags.isTupleSet() or TypeFlags.isTupleGet()");
10830 unsigned Idx = cast<ConstantInt>(Ops[1])->getZExtValue();
10833 return Builder.CreateInsertValue(Ops[0], Ops[2], Idx);
10834 return Builder.CreateExtractValue(Ops[0], Idx);
10840 assert(TypeFlags.
isTupleCreate() &&
"Expects TypleFlag isTupleCreate");
10842 Value *Tuple = llvm::PoisonValue::get(Ty);
10843 for (
unsigned Idx = 0; Idx < Ops.size(); Idx++)
10844 Tuple =
Builder.CreateInsertValue(Tuple, Ops[Idx], Idx);
10853 unsigned ICEArguments = 0;
10862 for (
unsigned i = 0, e =
E->getNumArgs(); i != e; i++) {
10863 bool IsICE = ICEArguments & (1 << i);
10869 std::optional<llvm::APSInt>
Result =
10871 assert(
Result &&
"Expected argument to be a constant");
10881 if (isa<StructType>(Arg->getType()) && !IsTupleGetOrSet) {
10882 for (
unsigned I = 0; I < Arg->getType()->getStructNumElements(); ++I)
10883 Ops.push_back(
Builder.CreateExtractValue(Arg, I));
10888 Ops.push_back(Arg);
10895 if (BuiltinID >= SVE::BI__builtin_sve_reinterpret_s8_s8 &&
10896 BuiltinID <= SVE::BI__builtin_sve_reinterpret_f64_f64_x4) {
10911 else if (TypeFlags.
isStore())
10929 else if (TypeFlags.
isUndef())
10930 return UndefValue::get(Ty);
10931 else if (Builtin->LLVMIntrinsic != 0) {
10935 Ops.pop_back_val());
10936 if (TypeFlags.
getMergeType() == SVETypeFlags::MergeZeroExp)
10939 if (TypeFlags.
getMergeType() == SVETypeFlags::MergeAnyExp)
10945 Ops.push_back(
Builder.getInt32( 31));
10947 Ops.insert(&Ops[1],
Builder.getInt32( 31));
10950 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
10951 if (
auto PredTy = dyn_cast<llvm::VectorType>(Ops[i]->getType()))
10952 if (PredTy->getElementType()->isIntegerTy(1))
10962 std::swap(Ops[1], Ops[2]);
10964 std::swap(Ops[1], Ops[2]);
10967 std::swap(Ops[1], Ops[2]);
10970 std::swap(Ops[1], Ops[3]);
10973 if (TypeFlags.
getMergeType() == SVETypeFlags::MergeZero) {
10974 llvm::Type *OpndTy = Ops[1]->getType();
10975 auto *SplatZero = Constant::getNullValue(OpndTy);
10976 Ops[1] =
Builder.CreateSelect(Ops[0], Ops[1], SplatZero);
10983 if (
Call->getType() == Ty)
10987 if (
auto PredTy = dyn_cast<llvm::ScalableVectorType>(Ty))
10989 if (
auto PredTupleTy = dyn_cast<llvm::StructType>(Ty))
10992 llvm_unreachable(
"unsupported element count!");
10995 switch (BuiltinID) {
10999 case SVE::BI__builtin_sve_svreinterpret_b: {
11003 CGM.
getIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool, SVCountTy);
11004 return Builder.CreateCall(CastFromSVCountF, Ops[0]);
11006 case SVE::BI__builtin_sve_svreinterpret_c: {
11010 CGM.
getIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool, SVCountTy);
11011 return Builder.CreateCall(CastToSVCountF, Ops[0]);
11014 case SVE::BI__builtin_sve_svpsel_lane_b8:
11015 case SVE::BI__builtin_sve_svpsel_lane_b16:
11016 case SVE::BI__builtin_sve_svpsel_lane_b32:
11017 case SVE::BI__builtin_sve_svpsel_lane_b64:
11018 case SVE::BI__builtin_sve_svpsel_lane_c8:
11019 case SVE::BI__builtin_sve_svpsel_lane_c16:
11020 case SVE::BI__builtin_sve_svpsel_lane_c32:
11021 case SVE::BI__builtin_sve_svpsel_lane_c64: {
11022 bool IsSVCount = isa<TargetExtType>(Ops[0]->getType());
11023 assert(((!IsSVCount || cast<TargetExtType>(Ops[0]->getType())->
getName() ==
11024 "aarch64.svcount")) &&
11025 "Unexpected TargetExtType");
11029 CGM.
getIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool, SVCountTy);
11031 CGM.
getIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool, SVCountTy);
11035 llvm::Value *Ops0 =
11036 IsSVCount ?
Builder.CreateCall(CastFromSVCountF, Ops[0]) : Ops[0];
11038 llvm::Value *PSel =
Builder.CreateCall(F, {Ops0, Ops1, Ops[2]});
11039 return IsSVCount ?
Builder.CreateCall(CastToSVCountF, PSel) : PSel;
11041 case SVE::BI__builtin_sve_svmov_b_z: {
11044 llvm::Type* OverloadedTy =
getSVEType(TypeFlags);
11046 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[1]});
11049 case SVE::BI__builtin_sve_svnot_b_z: {
11052 llvm::Type* OverloadedTy =
getSVEType(TypeFlags);
11054 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[0]});
11057 case SVE::BI__builtin_sve_svmovlb_u16:
11058 case SVE::BI__builtin_sve_svmovlb_u32:
11059 case SVE::BI__builtin_sve_svmovlb_u64:
11060 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllb);
11062 case SVE::BI__builtin_sve_svmovlb_s16:
11063 case SVE::BI__builtin_sve_svmovlb_s32:
11064 case SVE::BI__builtin_sve_svmovlb_s64:
11065 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllb);
11067 case SVE::BI__builtin_sve_svmovlt_u16:
11068 case SVE::BI__builtin_sve_svmovlt_u32:
11069 case SVE::BI__builtin_sve_svmovlt_u64:
11070 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllt);
11072 case SVE::BI__builtin_sve_svmovlt_s16:
11073 case SVE::BI__builtin_sve_svmovlt_s32:
11074 case SVE::BI__builtin_sve_svmovlt_s64:
11075 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllt);
11077 case SVE::BI__builtin_sve_svpmullt_u16:
11078 case SVE::BI__builtin_sve_svpmullt_u64:
11079 case SVE::BI__builtin_sve_svpmullt_n_u16:
11080 case SVE::BI__builtin_sve_svpmullt_n_u64:
11081 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullt_pair);
11083 case SVE::BI__builtin_sve_svpmullb_u16:
11084 case SVE::BI__builtin_sve_svpmullb_u64:
11085 case SVE::BI__builtin_sve_svpmullb_n_u16:
11086 case SVE::BI__builtin_sve_svpmullb_n_u64:
11087 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullb_pair);
11089 case SVE::BI__builtin_sve_svdup_n_b8:
11090 case SVE::BI__builtin_sve_svdup_n_b16:
11091 case SVE::BI__builtin_sve_svdup_n_b32:
11092 case SVE::BI__builtin_sve_svdup_n_b64: {
11094 Builder.CreateICmpNE(Ops[0], Constant::getNullValue(Ops[0]->getType()));
11095 llvm::ScalableVectorType *OverloadedTy =
getSVEType(TypeFlags);
11100 case SVE::BI__builtin_sve_svdupq_n_b8:
11101 case SVE::BI__builtin_sve_svdupq_n_b16:
11102 case SVE::BI__builtin_sve_svdupq_n_b32:
11103 case SVE::BI__builtin_sve_svdupq_n_b64:
11104 case SVE::BI__builtin_sve_svdupq_n_u8:
11105 case SVE::BI__builtin_sve_svdupq_n_s8:
11106 case SVE::BI__builtin_sve_svdupq_n_u64:
11107 case SVE::BI__builtin_sve_svdupq_n_f64:
11108 case SVE::BI__builtin_sve_svdupq_n_s64:
11109 case SVE::BI__builtin_sve_svdupq_n_u16:
11110 case SVE::BI__builtin_sve_svdupq_n_f16:
11111 case SVE::BI__builtin_sve_svdupq_n_bf16:
11112 case SVE::BI__builtin_sve_svdupq_n_s16:
11113 case SVE::BI__builtin_sve_svdupq_n_u32:
11114 case SVE::BI__builtin_sve_svdupq_n_f32:
11115 case SVE::BI__builtin_sve_svdupq_n_s32: {
11118 unsigned NumOpnds = Ops.size();
11121 cast<llvm::VectorType>(Ty)->getElementType()->isIntegerTy(1);
11126 llvm::Type *EltTy = Ops[0]->getType();
11131 for (
unsigned I = 0; I < NumOpnds; ++I)
11132 VecOps.push_back(
Builder.CreateZExt(Ops[I], EltTy));
11137 OverloadedTy, PoisonValue::get(OverloadedTy), Vec,
Builder.getInt64(0));
11152 : Intrinsic::aarch64_sve_cmpne_wide,
11159 case SVE::BI__builtin_sve_svpfalse_b:
11160 return ConstantInt::getFalse(Ty);
11162 case SVE::BI__builtin_sve_svpfalse_c: {
11163 auto SVBoolTy = ScalableVectorType::get(
Builder.getInt1Ty(), 16);
11166 return Builder.CreateCall(CastToSVCountF, ConstantInt::getFalse(SVBoolTy));
11169 case SVE::BI__builtin_sve_svlen_bf16:
11170 case SVE::BI__builtin_sve_svlen_f16:
11171 case SVE::BI__builtin_sve_svlen_f32:
11172 case SVE::BI__builtin_sve_svlen_f64:
11173 case SVE::BI__builtin_sve_svlen_s8:
11174 case SVE::BI__builtin_sve_svlen_s16:
11175 case SVE::BI__builtin_sve_svlen_s32:
11176 case SVE::BI__builtin_sve_svlen_s64:
11177 case SVE::BI__builtin_sve_svlen_u8:
11178 case SVE::BI__builtin_sve_svlen_u16:
11179 case SVE::BI__builtin_sve_svlen_u32:
11180 case SVE::BI__builtin_sve_svlen_u64: {
11182 auto VTy = cast<llvm::VectorType>(
getSVEType(TF));
11184 llvm::ConstantInt::get(Ty, VTy->getElementCount().getKnownMinValue());
11190 case SVE::BI__builtin_sve_svtbl2_u8:
11191 case SVE::BI__builtin_sve_svtbl2_s8:
11192 case SVE::BI__builtin_sve_svtbl2_u16:
11193 case SVE::BI__builtin_sve_svtbl2_s16:
11194 case SVE::BI__builtin_sve_svtbl2_u32:
11195 case SVE::BI__builtin_sve_svtbl2_s32:
11196 case SVE::BI__builtin_sve_svtbl2_u64:
11197 case SVE::BI__builtin_sve_svtbl2_s64:
11198 case SVE::BI__builtin_sve_svtbl2_f16:
11199 case SVE::BI__builtin_sve_svtbl2_bf16:
11200 case SVE::BI__builtin_sve_svtbl2_f32:
11201 case SVE::BI__builtin_sve_svtbl2_f64: {
11203 auto VTy = cast<llvm::ScalableVectorType>(
getSVEType(TF));
11205 return Builder.CreateCall(F, Ops);
11208 case SVE::BI__builtin_sve_svset_neonq_s8:
11209 case SVE::BI__builtin_sve_svset_neonq_s16:
11210 case SVE::BI__builtin_sve_svset_neonq_s32:
11211 case SVE::BI__builtin_sve_svset_neonq_s64:
11212 case SVE::BI__builtin_sve_svset_neonq_u8:
11213 case SVE::BI__builtin_sve_svset_neonq_u16:
11214 case SVE::BI__builtin_sve_svset_neonq_u32:
11215 case SVE::BI__builtin_sve_svset_neonq_u64:
11216 case SVE::BI__builtin_sve_svset_neonq_f16:
11217 case SVE::BI__builtin_sve_svset_neonq_f32:
11218 case SVE::BI__builtin_sve_svset_neonq_f64:
11219 case SVE::BI__builtin_sve_svset_neonq_bf16: {
11220 return Builder.CreateInsertVector(Ty, Ops[0], Ops[1],
Builder.getInt64(0));
11223 case SVE::BI__builtin_sve_svget_neonq_s8:
11224 case SVE::BI__builtin_sve_svget_neonq_s16:
11225 case SVE::BI__builtin_sve_svget_neonq_s32:
11226 case SVE::BI__builtin_sve_svget_neonq_s64:
11227 case SVE::BI__builtin_sve_svget_neonq_u8:
11228 case SVE::BI__builtin_sve_svget_neonq_u16:
11229 case SVE::BI__builtin_sve_svget_neonq_u32:
11230 case SVE::BI__builtin_sve_svget_neonq_u64:
11231 case SVE::BI__builtin_sve_svget_neonq_f16:
11232 case SVE::BI__builtin_sve_svget_neonq_f32:
11233 case SVE::BI__builtin_sve_svget_neonq_f64:
11234 case SVE::BI__builtin_sve_svget_neonq_bf16: {
11235 return Builder.CreateExtractVector(Ty, Ops[0],
Builder.getInt64(0));
11238 case SVE::BI__builtin_sve_svdup_neonq_s8:
11239 case SVE::BI__builtin_sve_svdup_neonq_s16:
11240 case SVE::BI__builtin_sve_svdup_neonq_s32:
11241 case SVE::BI__builtin_sve_svdup_neonq_s64:
11242 case SVE::BI__builtin_sve_svdup_neonq_u8:
11243 case SVE::BI__builtin_sve_svdup_neonq_u16:
11244 case SVE::BI__builtin_sve_svdup_neonq_u32:
11245 case SVE::BI__builtin_sve_svdup_neonq_u64:
11246 case SVE::BI__builtin_sve_svdup_neonq_f16:
11247 case SVE::BI__builtin_sve_svdup_neonq_f32:
11248 case SVE::BI__builtin_sve_svdup_neonq_f64:
11249 case SVE::BI__builtin_sve_svdup_neonq_bf16: {
11252 return Builder.CreateIntrinsic(Intrinsic::aarch64_sve_dupq_lane, {Ty},
11264 switch (BuiltinID) {
11267 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x1:
11270 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x2:
11271 case SME::BI__builtin_sme_svsudot_za32_s8_vg1x2:
11274 case SME::BI__builtin_sme_svsudot_za32_s8_vg1x4:
11275 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x4:
11281 for (
unsigned I = 0; I < MultiVec; ++I)
11282 std::swap(Ops[I + 1], Ops[I + 1 + MultiVec]);
11295 return EmitSMELd1St1(TypeFlags, Ops, Builtin->LLVMIntrinsic);
11298 else if (BuiltinID == SME::BI__builtin_sme_svzero_mask_za ||
11299 BuiltinID == SME::BI__builtin_sme_svzero_za)
11300 return EmitSMEZero(TypeFlags, Ops, Builtin->LLVMIntrinsic);
11301 else if (BuiltinID == SME::BI__builtin_sme_svldr_vnum_za ||
11302 BuiltinID == SME::BI__builtin_sme_svstr_vnum_za ||
11303 BuiltinID == SME::BI__builtin_sme_svldr_za ||
11304 BuiltinID == SME::BI__builtin_sme_svstr_za)
11305 return EmitSMELdrStr(TypeFlags, Ops, Builtin->LLVMIntrinsic);
11310 Ops.pop_back_val());
11315 if (Builtin->LLVMIntrinsic == 0)
11318 if (BuiltinID == SME::BI__builtin_sme___arm_in_streaming_mode) {
11321 const auto *FD = cast<FunctionDecl>(
CurFuncDecl);
11323 unsigned SMEAttrs = FPT->getAArch64SMEAttributes();
11326 return ConstantInt::getBool(
Builder.getContext(), IsStreaming);
11332 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
11333 if (
auto PredTy = dyn_cast<llvm::VectorType>(Ops[i]->getType()))
11334 if (PredTy->getElementType()->isIntegerTy(1))
11342 return Builder.CreateCall(F, Ops);
11347 llvm::Triple::ArchType Arch) {
11356 if (BuiltinID == Builtin::BI__builtin_cpu_supports)
11357 return EmitAArch64CpuSupports(
E);
11359 unsigned HintID =
static_cast<unsigned>(-1);
11360 switch (BuiltinID) {
11362 case clang::AArch64::BI__builtin_arm_nop:
11365 case clang::AArch64::BI__builtin_arm_yield:
11366 case clang::AArch64::BI__yield:
11369 case clang::AArch64::BI__builtin_arm_wfe:
11370 case clang::AArch64::BI__wfe:
11373 case clang::AArch64::BI__builtin_arm_wfi:
11374 case clang::AArch64::BI__wfi:
11377 case clang::AArch64::BI__builtin_arm_sev:
11378 case clang::AArch64::BI__sev:
11381 case clang::AArch64::BI__builtin_arm_sevl:
11382 case clang::AArch64::BI__sevl:
11387 if (HintID !=
static_cast<unsigned>(-1)) {
11389 return Builder.CreateCall(F, llvm::ConstantInt::get(
Int32Ty, HintID));
11392 if (BuiltinID == clang::AArch64::BI__builtin_arm_trap) {
11398 if (BuiltinID == clang::AArch64::BI__builtin_arm_get_sme_state) {
11403 "__arm_sme_state"));
11405 "aarch64_pstate_sm_compatible");
11406 CI->setAttributes(Attrs);
11407 CI->setCallingConv(
11408 llvm::CallingConv::
11409 AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2);
11416 if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit) {
11418 "rbit of unusual size!");
11421 CGM.
getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
11423 if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit64) {
11425 "rbit of unusual size!");
11428 CGM.
getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
11431 if (BuiltinID == clang::AArch64::BI__builtin_arm_clz ||
11432 BuiltinID == clang::AArch64::BI__builtin_arm_clz64) {
11436 if (BuiltinID == clang::AArch64::BI__builtin_arm_clz64)
11441 if (BuiltinID == clang::AArch64::BI__builtin_arm_cls) {
11446 if (BuiltinID == clang::AArch64::BI__builtin_arm_cls64) {
11452 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint32zf ||
11453 BuiltinID == clang::AArch64::BI__builtin_arm_rint32z) {
11455 llvm::Type *Ty = Arg->getType();
11460 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint64zf ||
11461 BuiltinID == clang::AArch64::BI__builtin_arm_rint64z) {
11463 llvm::Type *Ty = Arg->getType();
11468 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint32xf ||
11469 BuiltinID == clang::AArch64::BI__builtin_arm_rint32x) {
11471 llvm::Type *Ty = Arg->getType();
11476 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint64xf ||
11477 BuiltinID == clang::AArch64::BI__builtin_arm_rint64x) {
11479 llvm::Type *Ty = Arg->getType();
11484 if (BuiltinID == clang::AArch64::BI__builtin_arm_jcvt) {
11486 "__jcvt of unusual size!");
11492 if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b ||
11493 BuiltinID == clang::AArch64::BI__builtin_arm_st64b ||
11494 BuiltinID == clang::AArch64::BI__builtin_arm_st64bv ||
11495 BuiltinID == clang::AArch64::BI__builtin_arm_st64bv0) {
11499 if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b) {
11503 llvm::Value *Val =
Builder.CreateCall(F, MemAddr);
11504 llvm::Value *ToRet;
11505 for (
size_t i = 0; i < 8; i++) {
11506 llvm::Value *ValOffsetPtr =
11517 Args.push_back(MemAddr);
11518 for (
size_t i = 0; i < 8; i++) {
11519 llvm::Value *ValOffsetPtr =
11526 auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_st64b
11527 ? Intrinsic::aarch64_st64b
11528 : BuiltinID == clang::AArch64::BI__builtin_arm_st64bv
11529 ? Intrinsic::aarch64_st64bv
11530 : Intrinsic::aarch64_st64bv0);
11532 return Builder.CreateCall(F, Args);
11536 if (BuiltinID == clang::AArch64::BI__builtin_arm_rndr ||
11537 BuiltinID == clang::AArch64::BI__builtin_arm_rndrrs) {
11539 auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_rndr
11540 ? Intrinsic::aarch64_rndr
11541 : Intrinsic::aarch64_rndrrs);
11543 llvm::Value *Val =
Builder.CreateCall(F);
11544 Value *RandomValue =
Builder.CreateExtractValue(Val, 0);
11553 if (BuiltinID == clang::AArch64::BI__clear_cache) {
11554 assert(
E->getNumArgs() == 2 &&
"__clear_cache takes 2 arguments");
11557 for (
unsigned i = 0; i < 2; i++)
11560 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
11561 StringRef Name = FD->
getName();
11565 if ((BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
11566 BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) &&
11570 ? Intrinsic::aarch64_ldaxp
11571 : Intrinsic::aarch64_ldxp);
11578 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
11579 Val0 =
Builder.CreateZExt(Val0, Int128Ty);
11580 Val1 =
Builder.CreateZExt(Val1, Int128Ty);
11582 Value *ShiftCst = llvm::ConstantInt::get(Int128Ty, 64);
11583 Val =
Builder.CreateShl(Val0, ShiftCst,
"shl",
true );
11584 Val =
Builder.CreateOr(Val, Val1);
11586 }
else if (BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
11587 BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) {
11592 llvm::Type *
IntTy =
11597 ? Intrinsic::aarch64_ldaxr
11598 : Intrinsic::aarch64_ldxr,
11600 CallInst *Val =
Builder.CreateCall(F, LoadAddr,
"ldxr");
11604 if (RealResTy->isPointerTy())
11605 return Builder.CreateIntToPtr(Val, RealResTy);
11607 llvm::Type *IntResTy = llvm::IntegerType::get(
11609 return Builder.CreateBitCast(
Builder.CreateTruncOrBitCast(Val, IntResTy),
11613 if ((BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
11614 BuiltinID == clang::AArch64::BI__builtin_arm_stlex) &&
11618 ? Intrinsic::aarch64_stlxp
11619 : Intrinsic::aarch64_stxp);
11631 return Builder.CreateCall(F, {Arg0, Arg1, StPtr},
"stxp");
11634 if (BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
11635 BuiltinID == clang::AArch64::BI__builtin_arm_stlex) {
11640 llvm::Type *StoreTy =
11643 if (StoreVal->
getType()->isPointerTy())
11646 llvm::Type *
IntTy = llvm::IntegerType::get(
11655 ? Intrinsic::aarch64_stlxr
11656 : Intrinsic::aarch64_stxr,
11658 CallInst *CI =
Builder.CreateCall(F, {StoreVal, StoreAddr},
"stxr");
11660 1, Attribute::get(
getLLVMContext(), Attribute::ElementType, StoreTy));
11664 if (BuiltinID == clang::AArch64::BI__getReg) {
11667 llvm_unreachable(
"Sema will ensure that the parameter is constant");
11673 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, Reg)};
11674 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
11675 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
11677 llvm::Function *F =
11679 return Builder.CreateCall(F, Metadata);
11682 if (BuiltinID == clang::AArch64::BI__break) {
11685 llvm_unreachable(
"Sema will ensure that the parameter is constant");
11687 llvm::Function *F =
CGM.
getIntrinsic(llvm::Intrinsic::aarch64_break);
11691 if (BuiltinID == clang::AArch64::BI__builtin_arm_clrex) {
11693 return Builder.CreateCall(F);
11696 if (BuiltinID == clang::AArch64::BI_ReadWriteBarrier)
11697 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
11698 llvm::SyncScope::SingleThread);
11701 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
11702 switch (BuiltinID) {
11703 case clang::AArch64::BI__builtin_arm_crc32b:
11704 CRCIntrinsicID = Intrinsic::aarch64_crc32b;
break;
11705 case clang::AArch64::BI__builtin_arm_crc32cb:
11706 CRCIntrinsicID = Intrinsic::aarch64_crc32cb;
break;
11707 case clang::AArch64::BI__builtin_arm_crc32h:
11708 CRCIntrinsicID = Intrinsic::aarch64_crc32h;
break;
11709 case clang::AArch64::BI__builtin_arm_crc32ch:
11710 CRCIntrinsicID = Intrinsic::aarch64_crc32ch;
break;
11711 case clang::AArch64::BI__builtin_arm_crc32w:
11712 CRCIntrinsicID = Intrinsic::aarch64_crc32w;
break;
11713 case clang::AArch64::BI__builtin_arm_crc32cw:
11714 CRCIntrinsicID = Intrinsic::aarch64_crc32cw;
break;
11715 case clang::AArch64::BI__builtin_arm_crc32d:
11716 CRCIntrinsicID = Intrinsic::aarch64_crc32x;
break;
11717 case clang::AArch64::BI__builtin_arm_crc32cd:
11718 CRCIntrinsicID = Intrinsic::aarch64_crc32cx;
break;
11721 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
11726 llvm::Type *DataTy = F->getFunctionType()->getParamType(1);
11727 Arg1 =
Builder.CreateZExtOrBitCast(Arg1, DataTy);
11729 return Builder.CreateCall(F, {Arg0, Arg1});
11733 if (BuiltinID == AArch64::BI__builtin_arm_mops_memset_tag) {
11740 CGM.
getIntrinsic(Intrinsic::aarch64_mops_memset_tag), {Dst, Val, Size});
11744 Intrinsic::ID MTEIntrinsicID = Intrinsic::not_intrinsic;
11745 switch (BuiltinID) {
11746 case clang::AArch64::BI__builtin_arm_irg:
11747 MTEIntrinsicID = Intrinsic::aarch64_irg;
break;
11748 case clang::AArch64::BI__builtin_arm_addg:
11749 MTEIntrinsicID = Intrinsic::aarch64_addg;
break;
11750 case clang::AArch64::BI__builtin_arm_gmi:
11751 MTEIntrinsicID = Intrinsic::aarch64_gmi;
break;
11752 case clang::AArch64::BI__builtin_arm_ldg:
11753 MTEIntrinsicID = Intrinsic::aarch64_ldg;
break;
11754 case clang::AArch64::BI__builtin_arm_stg:
11755 MTEIntrinsicID = Intrinsic::aarch64_stg;
break;
11756 case clang::AArch64::BI__builtin_arm_subp:
11757 MTEIntrinsicID = Intrinsic::aarch64_subp;
break;
11760 if (MTEIntrinsicID != Intrinsic::not_intrinsic) {
11761 if (MTEIntrinsicID == Intrinsic::aarch64_irg) {
11769 if (MTEIntrinsicID == Intrinsic::aarch64_addg) {
11775 {Pointer, TagOffset});
11777 if (MTEIntrinsicID == Intrinsic::aarch64_gmi) {
11788 if (MTEIntrinsicID == Intrinsic::aarch64_ldg) {
11791 {TagAddress, TagAddress});
11796 if (MTEIntrinsicID == Intrinsic::aarch64_stg) {
11799 {TagAddress, TagAddress});
11801 if (MTEIntrinsicID == Intrinsic::aarch64_subp) {
11809 if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
11810 BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
11811 BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
11812 BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
11813 BuiltinID == clang::AArch64::BI__builtin_arm_wsr ||
11814 BuiltinID == clang::AArch64::BI__builtin_arm_wsr64 ||
11815 BuiltinID == clang::AArch64::BI__builtin_arm_wsr128 ||
11816 BuiltinID == clang::AArch64::BI__builtin_arm_wsrp) {
11819 if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
11820 BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
11821 BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
11822 BuiltinID == clang::AArch64::BI__builtin_arm_rsrp)
11825 bool IsPointerBuiltin = BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
11826 BuiltinID == clang::AArch64::BI__builtin_arm_wsrp;
11828 bool Is32Bit = BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
11829 BuiltinID == clang::AArch64::BI__builtin_arm_wsr;
11831 bool Is128Bit = BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
11832 BuiltinID == clang::AArch64::BI__builtin_arm_wsr128;
11834 llvm::Type *ValueType;
11838 }
else if (Is128Bit) {
11839 llvm::Type *Int128Ty =
11841 ValueType = Int128Ty;
11843 }
else if (IsPointerBuiltin) {
11853 if (BuiltinID == clang::AArch64::BI_ReadStatusReg ||
11854 BuiltinID == clang::AArch64::BI_WriteStatusReg) {
11860 std::string SysRegStr;
11861 llvm::raw_string_ostream(SysRegStr) <<
11862 ((1 << 1) | ((SysReg >> 14) & 1)) <<
":" <<
11863 ((SysReg >> 11) & 7) <<
":" <<
11864 ((SysReg >> 7) & 15) <<
":" <<
11865 ((SysReg >> 3) & 15) <<
":" <<
11868 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysRegStr) };
11869 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
11870 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
11875 if (BuiltinID == clang::AArch64::BI_ReadStatusReg) {
11876 llvm::Function *F =
CGM.
getIntrinsic(llvm::Intrinsic::read_register, Types);
11878 return Builder.CreateCall(F, Metadata);
11881 llvm::Function *F =
CGM.
getIntrinsic(llvm::Intrinsic::write_register, Types);
11884 return Builder.CreateCall(F, { Metadata, ArgValue });
11887 if (BuiltinID == clang::AArch64::BI_AddressOfReturnAddress) {
11888 llvm::Function *F =
11890 return Builder.CreateCall(F);
11893 if (BuiltinID == clang::AArch64::BI__builtin_sponentry) {
11895 return Builder.CreateCall(F);
11898 if (BuiltinID == clang::AArch64::BI__mulh ||
11899 BuiltinID == clang::AArch64::BI__umulh) {
11901 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
11903 bool IsSigned = BuiltinID == clang::AArch64::BI__mulh;
11909 Value *MulResult, *HigherBits;
11911 MulResult =
Builder.CreateNSWMul(LHS, RHS);
11912 HigherBits =
Builder.CreateAShr(MulResult, 64);
11914 MulResult =
Builder.CreateNUWMul(LHS, RHS);
11915 HigherBits =
Builder.CreateLShr(MulResult, 64);
11917 HigherBits =
Builder.CreateIntCast(HigherBits, ResType, IsSigned);
11922 if (BuiltinID == AArch64::BI__writex18byte ||
11923 BuiltinID == AArch64::BI__writex18word ||
11924 BuiltinID == AArch64::BI__writex18dword ||
11925 BuiltinID == AArch64::BI__writex18qword) {
11941 if (BuiltinID == AArch64::BI__readx18byte ||
11942 BuiltinID == AArch64::BI__readx18word ||
11943 BuiltinID == AArch64::BI__readx18dword ||
11944 BuiltinID == AArch64::BI__readx18qword) {
11959 if (BuiltinID == AArch64::BI__addx18byte ||
11960 BuiltinID == AArch64::BI__addx18word ||
11961 BuiltinID == AArch64::BI__addx18dword ||
11962 BuiltinID == AArch64::BI__addx18qword ||
11963 BuiltinID == AArch64::BI__incx18byte ||
11964 BuiltinID == AArch64::BI__incx18word ||
11965 BuiltinID == AArch64::BI__incx18dword ||
11966 BuiltinID == AArch64::BI__incx18qword) {
11969 switch (BuiltinID) {
11970 case AArch64::BI__incx18byte:
11972 isIncrement =
true;
11974 case AArch64::BI__incx18word:
11976 isIncrement =
true;
11978 case AArch64::BI__incx18dword:
11980 isIncrement =
true;
11982 case AArch64::BI__incx18qword:
11984 isIncrement =
true;
11988 isIncrement =
false;
12013 if (BuiltinID == AArch64::BI_CopyDoubleFromInt64 ||
12014 BuiltinID == AArch64::BI_CopyFloatFromInt32 ||
12015 BuiltinID == AArch64::BI_CopyInt32FromFloat ||
12016 BuiltinID == AArch64::BI_CopyInt64FromDouble) {
12019 return Builder.CreateBitCast(Arg, RetTy);
12022 if (BuiltinID == AArch64::BI_CountLeadingOnes ||
12023 BuiltinID == AArch64::BI_CountLeadingOnes64 ||
12024 BuiltinID == AArch64::BI_CountLeadingZeros ||
12025 BuiltinID == AArch64::BI_CountLeadingZeros64) {
12027 llvm::Type *ArgType = Arg->
getType();
12029 if (BuiltinID == AArch64::BI_CountLeadingOnes ||
12030 BuiltinID == AArch64::BI_CountLeadingOnes64)
12031 Arg =
Builder.CreateXor(Arg, Constant::getAllOnesValue(ArgType));
12036 if (BuiltinID == AArch64::BI_CountLeadingOnes64 ||
12037 BuiltinID == AArch64::BI_CountLeadingZeros64)
12042 if (BuiltinID == AArch64::BI_CountLeadingSigns ||
12043 BuiltinID == AArch64::BI_CountLeadingSigns64) {
12046 Function *F = (BuiltinID == AArch64::BI_CountLeadingSigns)
12051 if (BuiltinID == AArch64::BI_CountLeadingSigns64)
12056 if (BuiltinID == AArch64::BI_CountOneBits ||
12057 BuiltinID == AArch64::BI_CountOneBits64) {
12059 llvm::Type *ArgType = ArgValue->
getType();
12063 if (BuiltinID == AArch64::BI_CountOneBits64)
12068 if (BuiltinID == AArch64::BI__prefetch) {
12077 if (BuiltinID == AArch64::BI__hlt) {
12083 return ConstantInt::get(
Builder.getInt32Ty(), 0);
12088 if (std::optional<MSVCIntrin> MsvcIntId =
12094 return P.first == BuiltinID;
12097 BuiltinID = It->second;
12101 unsigned ICEArguments = 0;
12108 for (
unsigned i = 0, e =
E->getNumArgs() - 1; i != e; i++) {
12110 switch (BuiltinID) {
12111 case NEON::BI__builtin_neon_vld1_v:
12112 case NEON::BI__builtin_neon_vld1q_v:
12113 case NEON::BI__builtin_neon_vld1_dup_v:
12114 case NEON::BI__builtin_neon_vld1q_dup_v:
12115 case NEON::BI__builtin_neon_vld1_lane_v:
12116 case NEON::BI__builtin_neon_vld1q_lane_v:
12117 case NEON::BI__builtin_neon_vst1_v:
12118 case NEON::BI__builtin_neon_vst1q_v:
12119 case NEON::BI__builtin_neon_vst1_lane_v:
12120 case NEON::BI__builtin_neon_vst1q_lane_v:
12121 case NEON::BI__builtin_neon_vldap1_lane_s64:
12122 case NEON::BI__builtin_neon_vldap1q_lane_s64:
12123 case NEON::BI__builtin_neon_vstl1_lane_s64:
12124 case NEON::BI__builtin_neon_vstl1q_lane_s64:
12142 assert(
Result &&
"SISD intrinsic should have been handled");
12146 const Expr *Arg =
E->getArg(
E->getNumArgs()-1);
12148 if (std::optional<llvm::APSInt>
Result =
12153 bool usgn =
Type.isUnsigned();
12154 bool quad =
Type.isQuad();
12157 switch (BuiltinID) {
12159 case NEON::BI__builtin_neon_vabsh_f16:
12162 case NEON::BI__builtin_neon_vaddq_p128: {
12165 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
12166 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
12167 Ops[0] =
Builder.CreateXor(Ops[0], Ops[1]);
12168 llvm::Type *Int128Ty = llvm::Type::getIntNTy(
getLLVMContext(), 128);
12169 return Builder.CreateBitCast(Ops[0], Int128Ty);
12171 case NEON::BI__builtin_neon_vldrq_p128: {
12172 llvm::Type *Int128Ty = llvm::Type::getIntNTy(
getLLVMContext(), 128);
12177 case NEON::BI__builtin_neon_vstrq_p128: {
12178 Value *Ptr = Ops[0];
12181 case NEON::BI__builtin_neon_vcvts_f32_u32:
12182 case NEON::BI__builtin_neon_vcvtd_f64_u64:
12185 case NEON::BI__builtin_neon_vcvts_f32_s32:
12186 case NEON::BI__builtin_neon_vcvtd_f64_s64: {
12188 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
12191 Ops[0] =
Builder.CreateBitCast(Ops[0], InTy);
12193 return Builder.CreateUIToFP(Ops[0], FTy);
12194 return Builder.CreateSIToFP(Ops[0], FTy);
12196 case NEON::BI__builtin_neon_vcvth_f16_u16:
12197 case NEON::BI__builtin_neon_vcvth_f16_u32:
12198 case NEON::BI__builtin_neon_vcvth_f16_u64:
12201 case NEON::BI__builtin_neon_vcvth_f16_s16:
12202 case NEON::BI__builtin_neon_vcvth_f16_s32:
12203 case NEON::BI__builtin_neon_vcvth_f16_s64: {
12205 llvm::Type *FTy =
HalfTy;
12207 if (Ops[0]->getType()->getPrimitiveSizeInBits() == 64)
12209 else if (Ops[0]->getType()->getPrimitiveSizeInBits() == 32)
12213 Ops[0] =
Builder.CreateBitCast(Ops[0], InTy);
12215 return Builder.CreateUIToFP(Ops[0], FTy);
12216 return Builder.CreateSIToFP(Ops[0], FTy);
12218 case NEON::BI__builtin_neon_vcvtah_u16_f16:
12219 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
12220 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
12221 case NEON::BI__builtin_neon_vcvtph_u16_f16:
12222 case NEON::BI__builtin_neon_vcvth_u16_f16:
12223 case NEON::BI__builtin_neon_vcvtah_s16_f16:
12224 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
12225 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
12226 case NEON::BI__builtin_neon_vcvtph_s16_f16:
12227 case NEON::BI__builtin_neon_vcvth_s16_f16: {
12230 llvm::Type* FTy =
HalfTy;
12231 llvm::Type *Tys[2] = {InTy, FTy};
12233 switch (BuiltinID) {
12234 default: llvm_unreachable(
"missing builtin ID in switch!");
12235 case NEON::BI__builtin_neon_vcvtah_u16_f16:
12236 Int = Intrinsic::aarch64_neon_fcvtau;
break;
12237 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
12238 Int = Intrinsic::aarch64_neon_fcvtmu;
break;
12239 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
12240 Int = Intrinsic::aarch64_neon_fcvtnu;
break;
12241 case NEON::BI__builtin_neon_vcvtph_u16_f16:
12242 Int = Intrinsic::aarch64_neon_fcvtpu;
break;
12243 case NEON::BI__builtin_neon_vcvth_u16_f16:
12244 Int = Intrinsic::aarch64_neon_fcvtzu;
break;
12245 case NEON::BI__builtin_neon_vcvtah_s16_f16:
12246 Int = Intrinsic::aarch64_neon_fcvtas;
break;
12247 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
12248 Int = Intrinsic::aarch64_neon_fcvtms;
break;
12249 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
12250 Int = Intrinsic::aarch64_neon_fcvtns;
break;
12251 case NEON::BI__builtin_neon_vcvtph_s16_f16:
12252 Int = Intrinsic::aarch64_neon_fcvtps;
break;
12253 case NEON::BI__builtin_neon_vcvth_s16_f16:
12254 Int = Intrinsic::aarch64_neon_fcvtzs;
break;
12259 case NEON::BI__builtin_neon_vcaleh_f16:
12260 case NEON::BI__builtin_neon_vcalth_f16:
12261 case NEON::BI__builtin_neon_vcageh_f16:
12262 case NEON::BI__builtin_neon_vcagth_f16: {
12265 llvm::Type* FTy =
HalfTy;
12266 llvm::Type *Tys[2] = {InTy, FTy};
12268 switch (BuiltinID) {
12269 default: llvm_unreachable(
"missing builtin ID in switch!");
12270 case NEON::BI__builtin_neon_vcageh_f16:
12271 Int = Intrinsic::aarch64_neon_facge;
break;
12272 case NEON::BI__builtin_neon_vcagth_f16:
12273 Int = Intrinsic::aarch64_neon_facgt;
break;
12274 case NEON::BI__builtin_neon_vcaleh_f16:
12275 Int = Intrinsic::aarch64_neon_facge; std::swap(Ops[0], Ops[1]);
break;
12276 case NEON::BI__builtin_neon_vcalth_f16:
12277 Int = Intrinsic::aarch64_neon_facgt; std::swap(Ops[0], Ops[1]);
break;
12282 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
12283 case NEON::BI__builtin_neon_vcvth_n_u16_f16: {
12286 llvm::Type* FTy =
HalfTy;
12287 llvm::Type *Tys[2] = {InTy, FTy};
12289 switch (BuiltinID) {
12290 default: llvm_unreachable(
"missing builtin ID in switch!");
12291 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
12292 Int = Intrinsic::aarch64_neon_vcvtfp2fxs;
break;
12293 case NEON::BI__builtin_neon_vcvth_n_u16_f16:
12294 Int = Intrinsic::aarch64_neon_vcvtfp2fxu;
break;
12299 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
12300 case NEON::BI__builtin_neon_vcvth_n_f16_u16: {
12302 llvm::Type* FTy =
HalfTy;
12304 llvm::Type *Tys[2] = {FTy, InTy};
12306 switch (BuiltinID) {
12307 default: llvm_unreachable(
"missing builtin ID in switch!");
12308 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
12309 Int = Intrinsic::aarch64_neon_vcvtfxs2fp;
12310 Ops[0] =
Builder.CreateSExt(Ops[0], InTy,
"sext");
12312 case NEON::BI__builtin_neon_vcvth_n_f16_u16:
12313 Int = Intrinsic::aarch64_neon_vcvtfxu2fp;
12314 Ops[0] =
Builder.CreateZExt(Ops[0], InTy);
12319 case NEON::BI__builtin_neon_vpaddd_s64: {
12320 auto *Ty = llvm::FixedVectorType::get(
Int64Ty, 2);
12323 Vec =
Builder.CreateBitCast(Vec, Ty,
"v2i64");
12324 llvm::Value *Idx0 = llvm::ConstantInt::get(
SizeTy, 0);
12325 llvm::Value *Idx1 = llvm::ConstantInt::get(
SizeTy, 1);
12326 Value *Op0 =
Builder.CreateExtractElement(Vec, Idx0,
"lane0");
12327 Value *Op1 =
Builder.CreateExtractElement(Vec, Idx1,
"lane1");
12329 return Builder.CreateAdd(Op0, Op1,
"vpaddd");
12331 case NEON::BI__builtin_neon_vpaddd_f64: {
12332 auto *Ty = llvm::FixedVectorType::get(
DoubleTy, 2);
12335 Vec =
Builder.CreateBitCast(Vec, Ty,
"v2f64");
12336 llvm::Value *Idx0 = llvm::ConstantInt::get(
SizeTy, 0);
12337 llvm::Value *Idx1 = llvm::ConstantInt::get(
SizeTy, 1);
12338 Value *Op0 =
Builder.CreateExtractElement(Vec, Idx0,
"lane0");
12339 Value *Op1 =
Builder.CreateExtractElement(Vec, Idx1,
"lane1");
12341 return Builder.CreateFAdd(Op0, Op1,
"vpaddd");
12343 case NEON::BI__builtin_neon_vpadds_f32: {
12344 auto *Ty = llvm::FixedVectorType::get(
FloatTy, 2);
12347 Vec =
Builder.CreateBitCast(Vec, Ty,
"v2f32");
12348 llvm::Value *Idx0 = llvm::ConstantInt::get(
SizeTy, 0);
12349 llvm::Value *Idx1 = llvm::ConstantInt::get(
SizeTy, 1);
12350 Value *Op0 =
Builder.CreateExtractElement(Vec, Idx0,
"lane0");
12351 Value *Op1 =
Builder.CreateExtractElement(Vec, Idx1,
"lane1");
12353 return Builder.CreateFAdd(Op0, Op1,
"vpaddd");
12355 case NEON::BI__builtin_neon_vceqzd_s64:
12356 case NEON::BI__builtin_neon_vceqzd_f64:
12357 case NEON::BI__builtin_neon_vceqzs_f32:
12358 case NEON::BI__builtin_neon_vceqzh_f16:
12362 ICmpInst::FCMP_OEQ, ICmpInst::ICMP_EQ,
"vceqz");
12363 case NEON::BI__builtin_neon_vcgezd_s64:
12364 case NEON::BI__builtin_neon_vcgezd_f64:
12365 case NEON::BI__builtin_neon_vcgezs_f32:
12366 case NEON::BI__builtin_neon_vcgezh_f16:
12370 ICmpInst::FCMP_OGE, ICmpInst::ICMP_SGE,
"vcgez");
12371 case NEON::BI__builtin_neon_vclezd_s64:
12372 case NEON::BI__builtin_neon_vclezd_f64:
12373 case NEON::BI__builtin_neon_vclezs_f32:
12374 case NEON::BI__builtin_neon_vclezh_f16:
12378 ICmpInst::FCMP_OLE, ICmpInst::ICMP_SLE,
"vclez");
12379 case NEON::BI__builtin_neon_vcgtzd_s64:
12380 case NEON::BI__builtin_neon_vcgtzd_f64:
12381 case NEON::BI__builtin_neon_vcgtzs_f32:
12382 case NEON::BI__builtin_neon_vcgtzh_f16:
12386 ICmpInst::FCMP_OGT, ICmpInst::ICMP_SGT,
"vcgtz");
12387 case NEON::BI__builtin_neon_vcltzd_s64:
12388 case NEON::BI__builtin_neon_vcltzd_f64:
12389 case NEON::BI__builtin_neon_vcltzs_f32:
12390 case NEON::BI__builtin_neon_vcltzh_f16:
12394 ICmpInst::FCMP_OLT, ICmpInst::ICMP_SLT,
"vcltz");
12396 case NEON::BI__builtin_neon_vceqzd_u64: {
12400 Builder.CreateICmpEQ(Ops[0], llvm::Constant::getNullValue(
Int64Ty));
12403 case NEON::BI__builtin_neon_vceqd_f64:
12404 case NEON::BI__builtin_neon_vcled_f64:
12405 case NEON::BI__builtin_neon_vcltd_f64:
12406 case NEON::BI__builtin_neon_vcged_f64:
12407 case NEON::BI__builtin_neon_vcgtd_f64: {
12408 llvm::CmpInst::Predicate
P;
12409 switch (BuiltinID) {
12410 default: llvm_unreachable(
"missing builtin ID in switch!");
12411 case NEON::BI__builtin_neon_vceqd_f64:
P = llvm::FCmpInst::FCMP_OEQ;
break;
12412 case NEON::BI__builtin_neon_vcled_f64:
P = llvm::FCmpInst::FCMP_OLE;
break;
12413 case NEON::BI__builtin_neon_vcltd_f64:
P = llvm::FCmpInst::FCMP_OLT;
break;
12414 case NEON::BI__builtin_neon_vcged_f64:
P = llvm::FCmpInst::FCMP_OGE;
break;
12415 case NEON::BI__builtin_neon_vcgtd_f64:
P = llvm::FCmpInst::FCMP_OGT;
break;
12420 if (
P == llvm::FCmpInst::FCMP_OEQ)
12421 Ops[0] =
Builder.CreateFCmp(
P, Ops[0], Ops[1]);
12423 Ops[0] =
Builder.CreateFCmpS(
P, Ops[0], Ops[1]);
12426 case NEON::BI__builtin_neon_vceqs_f32:
12427 case NEON::BI__builtin_neon_vcles_f32:
12428 case NEON::BI__builtin_neon_vclts_f32:
12429 case NEON::BI__builtin_neon_vcges_f32:
12430 case NEON::BI__builtin_neon_vcgts_f32: {
12431 llvm::CmpInst::Predicate
P;
12432 switch (BuiltinID) {
12433 default: llvm_unreachable(
"missing builtin ID in switch!");
12434 case NEON::BI__builtin_neon_vceqs_f32:
P = llvm::FCmpInst::FCMP_OEQ;
break;
12435 case NEON::BI__builtin_neon_vcles_f32:
P = llvm::FCmpInst::FCMP_OLE;
break;
12436 case NEON::BI__builtin_neon_vclts_f32:
P = llvm::FCmpInst::FCMP_OLT;
break;
12437 case NEON::BI__builtin_neon_vcges_f32:
P = llvm::FCmpInst::FCMP_OGE;
break;
12438 case NEON::BI__builtin_neon_vcgts_f32:
P = llvm::FCmpInst::FCMP_OGT;
break;
12443 if (
P == llvm::FCmpInst::FCMP_OEQ)
12444 Ops[0] =
Builder.CreateFCmp(
P, Ops[0], Ops[1]);
12446 Ops[0] =
Builder.CreateFCmpS(
P, Ops[0], Ops[1]);
12449 case NEON::BI__builtin_neon_vceqh_f16:
12450 case NEON::BI__builtin_neon_vcleh_f16:
12451 case NEON::BI__builtin_neon_vclth_f16:
12452 case NEON::BI__builtin_neon_vcgeh_f16:
12453 case NEON::BI__builtin_neon_vcgth_f16: {
12454 llvm::CmpInst::Predicate
P;
12455 switch (BuiltinID) {
12456 default: llvm_unreachable(
"missing builtin ID in switch!");
12457 case NEON::BI__builtin_neon_vceqh_f16:
P = llvm::FCmpInst::FCMP_OEQ;
break;
12458 case NEON::BI__builtin_neon_vcleh_f16:
P = llvm::FCmpInst::FCMP_OLE;
break;
12459 case NEON::BI__builtin_neon_vclth_f16:
P = llvm::FCmpInst::FCMP_OLT;
break;
12460 case NEON::BI__builtin_neon_vcgeh_f16:
P = llvm::FCmpInst::FCMP_OGE;
break;
12461 case NEON::BI__builtin_neon_vcgth_f16:
P = llvm::FCmpInst::FCMP_OGT;
break;
12466 if (
P == llvm::FCmpInst::FCMP_OEQ)
12467 Ops[0] =
Builder.CreateFCmp(
P, Ops[0], Ops[1]);
12469 Ops[0] =
Builder.CreateFCmpS(
P, Ops[0], Ops[1]);
12472 case NEON::BI__builtin_neon_vceqd_s64:
12473 case NEON::BI__builtin_neon_vceqd_u64:
12474 case NEON::BI__builtin_neon_vcgtd_s64:
12475 case NEON::BI__builtin_neon_vcgtd_u64:
12476 case NEON::BI__builtin_neon_vcltd_s64:
12477 case NEON::BI__builtin_neon_vcltd_u64:
12478 case NEON::BI__builtin_neon_vcged_u64:
12479 case NEON::BI__builtin_neon_vcged_s64:
12480 case NEON::BI__builtin_neon_vcled_u64:
12481 case NEON::BI__builtin_neon_vcled_s64: {
12482 llvm::CmpInst::Predicate
P;
12483 switch (BuiltinID) {
12484 default: llvm_unreachable(
"missing builtin ID in switch!");
12485 case NEON::BI__builtin_neon_vceqd_s64:
12486 case NEON::BI__builtin_neon_vceqd_u64:
P = llvm::ICmpInst::ICMP_EQ;
break;
12487 case NEON::BI__builtin_neon_vcgtd_s64:
P = llvm::ICmpInst::ICMP_SGT;
break;
12488 case NEON::BI__builtin_neon_vcgtd_u64:
P = llvm::ICmpInst::ICMP_UGT;
break;
12489 case NEON::BI__builtin_neon_vcltd_s64:
P = llvm::ICmpInst::ICMP_SLT;
break;
12490 case NEON::BI__builtin_neon_vcltd_u64:
P = llvm::ICmpInst::ICMP_ULT;
break;
12491 case NEON::BI__builtin_neon_vcged_u64:
P = llvm::ICmpInst::ICMP_UGE;
break;
12492 case NEON::BI__builtin_neon_vcged_s64:
P = llvm::ICmpInst::ICMP_SGE;
break;
12493 case NEON::BI__builtin_neon_vcled_u64:
P = llvm::ICmpInst::ICMP_ULE;
break;
12494 case NEON::BI__builtin_neon_vcled_s64:
P = llvm::ICmpInst::ICMP_SLE;
break;
12499 Ops[0] =
Builder.CreateICmp(
P, Ops[0], Ops[1]);
12502 case NEON::BI__builtin_neon_vtstd_s64:
12503 case NEON::BI__builtin_neon_vtstd_u64: {
12507 Ops[0] =
Builder.CreateAnd(Ops[0], Ops[1]);
12508 Ops[0] =
Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
12509 llvm::Constant::getNullValue(
Int64Ty));
12512 case NEON::BI__builtin_neon_vset_lane_i8:
12513 case NEON::BI__builtin_neon_vset_lane_i16:
12514 case NEON::BI__builtin_neon_vset_lane_i32:
12515 case NEON::BI__builtin_neon_vset_lane_i64:
12516 case NEON::BI__builtin_neon_vset_lane_bf16:
12517 case NEON::BI__builtin_neon_vset_lane_f32:
12518 case NEON::BI__builtin_neon_vsetq_lane_i8:
12519 case NEON::BI__builtin_neon_vsetq_lane_i16:
12520 case NEON::BI__builtin_neon_vsetq_lane_i32:
12521 case NEON::BI__builtin_neon_vsetq_lane_i64:
12522 case NEON::BI__builtin_neon_vsetq_lane_bf16:
12523 case NEON::BI__builtin_neon_vsetq_lane_f32:
12525 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
12526 case NEON::BI__builtin_neon_vset_lane_f64:
12529 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(
DoubleTy, 1));
12531 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
12532 case NEON::BI__builtin_neon_vsetq_lane_f64:
12535 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(
DoubleTy, 2));
12537 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
12539 case NEON::BI__builtin_neon_vget_lane_i8:
12540 case NEON::BI__builtin_neon_vdupb_lane_i8:
12542 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int8Ty, 8));
12545 case NEON::BI__builtin_neon_vgetq_lane_i8:
12546 case NEON::BI__builtin_neon_vdupb_laneq_i8:
12548 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int8Ty, 16));
12551 case NEON::BI__builtin_neon_vget_lane_i16:
12552 case NEON::BI__builtin_neon_vduph_lane_i16:
12554 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int16Ty, 4));
12557 case NEON::BI__builtin_neon_vgetq_lane_i16:
12558 case NEON::BI__builtin_neon_vduph_laneq_i16:
12560 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int16Ty, 8));
12563 case NEON::BI__builtin_neon_vget_lane_i32:
12564 case NEON::BI__builtin_neon_vdups_lane_i32:
12566 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int32Ty, 2));
12569 case NEON::BI__builtin_neon_vdups_lane_f32:
12571 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
FloatTy, 2));
12574 case NEON::BI__builtin_neon_vgetq_lane_i32:
12575 case NEON::BI__builtin_neon_vdups_laneq_i32:
12577 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int32Ty, 4));
12580 case NEON::BI__builtin_neon_vget_lane_i64:
12581 case NEON::BI__builtin_neon_vdupd_lane_i64:
12583 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int64Ty, 1));
12586 case NEON::BI__builtin_neon_vdupd_lane_f64:
12588 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
DoubleTy, 1));
12591 case NEON::BI__builtin_neon_vgetq_lane_i64:
12592 case NEON::BI__builtin_neon_vdupd_laneq_i64:
12594 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int64Ty, 2));
12597 case NEON::BI__builtin_neon_vget_lane_f32:
12599 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
FloatTy, 2));
12602 case NEON::BI__builtin_neon_vget_lane_f64:
12604 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
DoubleTy, 1));
12607 case NEON::BI__builtin_neon_vgetq_lane_f32:
12608 case NEON::BI__builtin_neon_vdups_laneq_f32:
12610 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
FloatTy, 4));
12613 case NEON::BI__builtin_neon_vgetq_lane_f64:
12614 case NEON::BI__builtin_neon_vdupd_laneq_f64:
12616 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
DoubleTy, 2));
12619 case NEON::BI__builtin_neon_vaddh_f16:
12621 return Builder.CreateFAdd(Ops[0], Ops[1],
"vaddh");
12622 case NEON::BI__builtin_neon_vsubh_f16:
12624 return Builder.CreateFSub(Ops[0], Ops[1],
"vsubh");
12625 case NEON::BI__builtin_neon_vmulh_f16:
12627 return Builder.CreateFMul(Ops[0], Ops[1],
"vmulh");
12628 case NEON::BI__builtin_neon_vdivh_f16:
12630 return Builder.CreateFDiv(Ops[0], Ops[1],
"vdivh");
12631 case NEON::BI__builtin_neon_vfmah_f16:
12634 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma,
HalfTy,
12636 case NEON::BI__builtin_neon_vfmsh_f16: {
12641 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma,
HalfTy,
12644 case NEON::BI__builtin_neon_vaddd_s64:
12645 case NEON::BI__builtin_neon_vaddd_u64:
12647 case NEON::BI__builtin_neon_vsubd_s64:
12648 case NEON::BI__builtin_neon_vsubd_u64:
12650 case NEON::BI__builtin_neon_vqdmlalh_s16:
12651 case NEON::BI__builtin_neon_vqdmlslh_s16: {
12655 auto *VTy = llvm::FixedVectorType::get(
Int32Ty, 4);
12657 ProductOps,
"vqdmlXl");
12658 Constant *CI = ConstantInt::get(
SizeTy, 0);
12659 Ops[1] =
Builder.CreateExtractElement(Ops[1], CI,
"lane0");
12661 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlalh_s16
12662 ? Intrinsic::aarch64_neon_sqadd
12663 : Intrinsic::aarch64_neon_sqsub;
12666 case NEON::BI__builtin_neon_vqshlud_n_s64: {
12672 case NEON::BI__builtin_neon_vqshld_n_u64:
12673 case NEON::BI__builtin_neon_vqshld_n_s64: {
12674 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vqshld_n_u64
12675 ? Intrinsic::aarch64_neon_uqshl
12676 : Intrinsic::aarch64_neon_sqshl;
12681 case NEON::BI__builtin_neon_vrshrd_n_u64:
12682 case NEON::BI__builtin_neon_vrshrd_n_s64: {
12683 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrshrd_n_u64
12684 ? Intrinsic::aarch64_neon_urshl
12685 : Intrinsic::aarch64_neon_srshl;
12687 int SV = cast<ConstantInt>(Ops[1])->getSExtValue();
12688 Ops[1] = ConstantInt::get(
Int64Ty, -SV);
12691 case NEON::BI__builtin_neon_vrsrad_n_u64:
12692 case NEON::BI__builtin_neon_vrsrad_n_s64: {
12693 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrsrad_n_u64
12694 ? Intrinsic::aarch64_neon_urshl
12695 : Intrinsic::aarch64_neon_srshl;
12699 {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)});
12702 case NEON::BI__builtin_neon_vshld_n_s64:
12703 case NEON::BI__builtin_neon_vshld_n_u64: {
12704 llvm::ConstantInt *Amt = cast<ConstantInt>(
EmitScalarExpr(
E->getArg(1)));
12706 Ops[0], ConstantInt::get(
Int64Ty, Amt->getZExtValue()),
"shld_n");
12708 case NEON::BI__builtin_neon_vshrd_n_s64: {
12709 llvm::ConstantInt *Amt = cast<ConstantInt>(
EmitScalarExpr(
E->getArg(1)));
12711 Ops[0], ConstantInt::get(
Int64Ty, std::min(
static_cast<uint64_t>(63),
12712 Amt->getZExtValue())),
12715 case NEON::BI__builtin_neon_vshrd_n_u64: {
12716 llvm::ConstantInt *Amt = cast<ConstantInt>(
EmitScalarExpr(
E->getArg(1)));
12717 uint64_t ShiftAmt = Amt->getZExtValue();
12719 if (ShiftAmt == 64)
12720 return ConstantInt::get(
Int64Ty, 0);
12721 return Builder.CreateLShr(Ops[0], ConstantInt::get(
Int64Ty, ShiftAmt),
12724 case NEON::BI__builtin_neon_vsrad_n_s64: {
12725 llvm::ConstantInt *Amt = cast<ConstantInt>(
EmitScalarExpr(
E->getArg(2)));
12727 Ops[1], ConstantInt::get(
Int64Ty, std::min(
static_cast<uint64_t>(63),
12728 Amt->getZExtValue())),
12730 return Builder.CreateAdd(Ops[0], Ops[1]);
12732 case NEON::BI__builtin_neon_vsrad_n_u64: {
12733 llvm::ConstantInt *Amt = cast<ConstantInt>(
EmitScalarExpr(
E->getArg(2)));
12734 uint64_t ShiftAmt = Amt->getZExtValue();
12737 if (ShiftAmt == 64)
12739 Ops[1] =
Builder.CreateLShr(Ops[1], ConstantInt::get(
Int64Ty, ShiftAmt),
12741 return Builder.CreateAdd(Ops[0], Ops[1]);
12743 case NEON::BI__builtin_neon_vqdmlalh_lane_s16:
12744 case NEON::BI__builtin_neon_vqdmlalh_laneq_s16:
12745 case NEON::BI__builtin_neon_vqdmlslh_lane_s16:
12746 case NEON::BI__builtin_neon_vqdmlslh_laneq_s16: {
12752 auto *VTy = llvm::FixedVectorType::get(
Int32Ty, 4);
12754 ProductOps,
"vqdmlXl");
12755 Constant *CI = ConstantInt::get(
SizeTy, 0);
12756 Ops[1] =
Builder.CreateExtractElement(Ops[1], CI,
"lane0");
12759 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlalh_lane_s16 ||
12760 BuiltinID == NEON::BI__builtin_neon_vqdmlalh_laneq_s16)
12761 ? Intrinsic::aarch64_neon_sqadd
12762 : Intrinsic::aarch64_neon_sqsub;
12765 case NEON::BI__builtin_neon_vqdmlals_s32:
12766 case NEON::BI__builtin_neon_vqdmlsls_s32: {
12768 ProductOps.push_back(Ops[1]);
12772 ProductOps,
"vqdmlXl");
12774 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlals_s32
12775 ? Intrinsic::aarch64_neon_sqadd
12776 : Intrinsic::aarch64_neon_sqsub;
12779 case NEON::BI__builtin_neon_vqdmlals_lane_s32:
12780 case NEON::BI__builtin_neon_vqdmlals_laneq_s32:
12781 case NEON::BI__builtin_neon_vqdmlsls_lane_s32:
12782 case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: {
12786 ProductOps.push_back(Ops[1]);
12787 ProductOps.push_back(Ops[2]);
12790 ProductOps,
"vqdmlXl");
12793 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlals_lane_s32 ||
12794 BuiltinID == NEON::BI__builtin_neon_vqdmlals_laneq_s32)
12795 ? Intrinsic::aarch64_neon_sqadd
12796 : Intrinsic::aarch64_neon_sqsub;
12799 case NEON::BI__builtin_neon_vget_lane_bf16:
12800 case NEON::BI__builtin_neon_vduph_lane_bf16:
12801 case NEON::BI__builtin_neon_vduph_lane_f16: {
12805 case NEON::BI__builtin_neon_vgetq_lane_bf16:
12806 case NEON::BI__builtin_neon_vduph_laneq_bf16:
12807 case NEON::BI__builtin_neon_vduph_laneq_f16: {
12812 case clang::AArch64::BI_InterlockedAdd:
12813 case clang::AArch64::BI_InterlockedAdd64: {
12816 AtomicRMWInst *RMWI =
12818 llvm::AtomicOrdering::SequentiallyConsistent);
12819 return Builder.CreateAdd(RMWI, Val);
12824 llvm::Type *Ty = VTy;
12835 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
12836 Builtin->NameHint, Builtin->TypeModifier,
E, Ops,
12843 switch (BuiltinID) {
12844 default:
return nullptr;
12845 case NEON::BI__builtin_neon_vbsl_v:
12846 case NEON::BI__builtin_neon_vbslq_v: {
12847 llvm::Type *BitTy = llvm::VectorType::getInteger(VTy);
12848 Ops[0] =
Builder.CreateBitCast(Ops[0], BitTy,
"vbsl");
12849 Ops[1] =
Builder.CreateBitCast(Ops[1], BitTy,
"vbsl");
12850 Ops[2] =
Builder.CreateBitCast(Ops[2], BitTy,
"vbsl");
12852 Ops[1] =
Builder.CreateAnd(Ops[0], Ops[1],
"vbsl");
12853 Ops[2] =
Builder.CreateAnd(
Builder.CreateNot(Ops[0]), Ops[2],
"vbsl");
12854 Ops[0] =
Builder.CreateOr(Ops[1], Ops[2],
"vbsl");
12855 return Builder.CreateBitCast(Ops[0], Ty);
12857 case NEON::BI__builtin_neon_vfma_lane_v:
12858 case NEON::BI__builtin_neon_vfmaq_lane_v: {
12861 Value *Addend = Ops[0];
12862 Value *Multiplicand = Ops[1];
12863 Value *LaneSource = Ops[2];
12864 Ops[0] = Multiplicand;
12865 Ops[1] = LaneSource;
12869 auto *SourceTy = BuiltinID == NEON::BI__builtin_neon_vfmaq_lane_v
12870 ? llvm::FixedVectorType::get(VTy->getElementType(),
12871 VTy->getNumElements() / 2)
12873 llvm::Constant *cst = cast<Constant>(Ops[3]);
12874 Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(), cst);
12875 Ops[1] =
Builder.CreateBitCast(Ops[1], SourceTy);
12876 Ops[1] =
Builder.CreateShuffleVector(Ops[1], Ops[1], SV,
"lane");
12879 Int =
Builder.getIsFPConstrained() ? Intrinsic::experimental_constrained_fma
12883 case NEON::BI__builtin_neon_vfma_laneq_v: {
12884 auto *VTy = cast<llvm::FixedVectorType>(Ty);
12886 if (VTy && VTy->getElementType() ==
DoubleTy) {
12889 llvm::FixedVectorType *VTy =
12891 Ops[2] =
Builder.CreateBitCast(Ops[2], VTy);
12892 Ops[2] =
Builder.CreateExtractElement(Ops[2], Ops[3],
"extract");
12895 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma,
12896 DoubleTy, {Ops[1], Ops[2], Ops[0]});
12899 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
12900 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
12902 auto *STy = llvm::FixedVectorType::get(VTy->getElementType(),
12903 VTy->getNumElements() * 2);
12904 Ops[2] =
Builder.CreateBitCast(Ops[2], STy);
12905 Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(),
12906 cast<ConstantInt>(Ops[3]));
12907 Ops[2] =
Builder.CreateShuffleVector(Ops[2], Ops[2], SV,
"lane");
12910 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
12911 {Ops[2], Ops[1], Ops[0]});
12913 case NEON::BI__builtin_neon_vfmaq_laneq_v: {
12914 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
12915 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
12917 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
12920 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
12921 {Ops[2], Ops[1], Ops[0]});
12923 case NEON::BI__builtin_neon_vfmah_lane_f16:
12924 case NEON::BI__builtin_neon_vfmas_lane_f32:
12925 case NEON::BI__builtin_neon_vfmah_laneq_f16:
12926 case NEON::BI__builtin_neon_vfmas_laneq_f32:
12927 case NEON::BI__builtin_neon_vfmad_lane_f64:
12928 case NEON::BI__builtin_neon_vfmad_laneq_f64: {
12931 Ops[2] =
Builder.CreateExtractElement(Ops[2], Ops[3],
"extract");
12933 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
12934 {Ops[1], Ops[2], Ops[0]});
12936 case NEON::BI__builtin_neon_vmull_v:
12938 Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull;
12939 if (
Type.isPoly())
Int = Intrinsic::aarch64_neon_pmull;
12941 case NEON::BI__builtin_neon_vmax_v:
12942 case NEON::BI__builtin_neon_vmaxq_v:
12944 Int = usgn ? Intrinsic::aarch64_neon_umax : Intrinsic::aarch64_neon_smax;
12945 if (Ty->isFPOrFPVectorTy())
Int = Intrinsic::aarch64_neon_fmax;
12947 case NEON::BI__builtin_neon_vmaxh_f16: {
12949 Int = Intrinsic::aarch64_neon_fmax;
12952 case NEON::BI__builtin_neon_vmin_v:
12953 case NEON::BI__builtin_neon_vminq_v:
12955 Int = usgn ? Intrinsic::aarch64_neon_umin : Intrinsic::aarch64_neon_smin;
12956 if (Ty->isFPOrFPVectorTy())
Int = Intrinsic::aarch64_neon_fmin;
12958 case NEON::BI__builtin_neon_vminh_f16: {
12960 Int = Intrinsic::aarch64_neon_fmin;
12963 case NEON::BI__builtin_neon_vabd_v:
12964 case NEON::BI__builtin_neon_vabdq_v:
12966 Int = usgn ? Intrinsic::aarch64_neon_uabd : Intrinsic::aarch64_neon_sabd;
12967 if (Ty->isFPOrFPVectorTy())
Int = Intrinsic::aarch64_neon_fabd;
12969 case NEON::BI__builtin_neon_vpadal_v:
12970 case NEON::BI__builtin_neon_vpadalq_v: {
12971 unsigned ArgElts = VTy->getNumElements();
12972 llvm::IntegerType *EltTy = cast<IntegerType>(VTy->getElementType());
12973 unsigned BitWidth = EltTy->getBitWidth();
12974 auto *ArgTy = llvm::FixedVectorType::get(
12975 llvm::IntegerType::get(
getLLVMContext(), BitWidth / 2), 2 * ArgElts);
12976 llvm::Type* Tys[2] = { VTy, ArgTy };
12977 Int = usgn ? Intrinsic::aarch64_neon_uaddlp : Intrinsic::aarch64_neon_saddlp;
12979 TmpOps.push_back(Ops[1]);
12982 llvm::Value *addend =
Builder.CreateBitCast(Ops[0], tmp->getType());
12983 return Builder.CreateAdd(tmp, addend);
12985 case NEON::BI__builtin_neon_vpmin_v:
12986 case NEON::BI__builtin_neon_vpminq_v:
12988 Int = usgn ? Intrinsic::aarch64_neon_uminp : Intrinsic::aarch64_neon_sminp;
12989 if (Ty->isFPOrFPVectorTy())
Int = Intrinsic::aarch64_neon_fminp;
12991 case NEON::BI__builtin_neon_vpmax_v:
12992 case NEON::BI__builtin_neon_vpmaxq_v:
12994 Int = usgn ? Intrinsic::aarch64_neon_umaxp : Intrinsic::aarch64_neon_smaxp;
12995 if (Ty->isFPOrFPVectorTy())
Int = Intrinsic::aarch64_neon_fmaxp;
12997 case NEON::BI__builtin_neon_vminnm_v:
12998 case NEON::BI__builtin_neon_vminnmq_v:
12999 Int = Intrinsic::aarch64_neon_fminnm;
13001 case NEON::BI__builtin_neon_vminnmh_f16:
13003 Int = Intrinsic::aarch64_neon_fminnm;
13005 case NEON::BI__builtin_neon_vmaxnm_v:
13006 case NEON::BI__builtin_neon_vmaxnmq_v:
13007 Int = Intrinsic::aarch64_neon_fmaxnm;
13009 case NEON::BI__builtin_neon_vmaxnmh_f16:
13011 Int = Intrinsic::aarch64_neon_fmaxnm;
13013 case NEON::BI__builtin_neon_vrecpss_f32: {
13018 case NEON::BI__builtin_neon_vrecpsd_f64:
13022 case NEON::BI__builtin_neon_vrecpsh_f16:
13026 case NEON::BI__builtin_neon_vqshrun_n_v:
13027 Int = Intrinsic::aarch64_neon_sqshrun;
13029 case NEON::BI__builtin_neon_vqrshrun_n_v:
13030 Int = Intrinsic::aarch64_neon_sqrshrun;
13032 case NEON::BI__builtin_neon_vqshrn_n_v:
13033 Int = usgn ? Intrinsic::aarch64_neon_uqshrn : Intrinsic::aarch64_neon_sqshrn;
13035 case NEON::BI__builtin_neon_vrshrn_n_v:
13036 Int = Intrinsic::aarch64_neon_rshrn;
13038 case NEON::BI__builtin_neon_vqrshrn_n_v:
13039 Int = usgn ? Intrinsic::aarch64_neon_uqrshrn : Intrinsic::aarch64_neon_sqrshrn;
13041 case NEON::BI__builtin_neon_vrndah_f16: {
13044 ? Intrinsic::experimental_constrained_round
13045 : Intrinsic::round;
13048 case NEON::BI__builtin_neon_vrnda_v:
13049 case NEON::BI__builtin_neon_vrndaq_v: {
13051 ? Intrinsic::experimental_constrained_round
13052 : Intrinsic::round;
13055 case NEON::BI__builtin_neon_vrndih_f16: {
13058 ? Intrinsic::experimental_constrained_nearbyint
13059 : Intrinsic::nearbyint;
13062 case NEON::BI__builtin_neon_vrndmh_f16: {
13065 ? Intrinsic::experimental_constrained_floor
13066 : Intrinsic::floor;
13069 case NEON::BI__builtin_neon_vrndm_v:
13070 case NEON::BI__builtin_neon_vrndmq_v: {
13072 ? Intrinsic::experimental_constrained_floor
13073 : Intrinsic::floor;
13076 case NEON::BI__builtin_neon_vrndnh_f16: {
13079 ? Intrinsic::experimental_constrained_roundeven
13080 : Intrinsic::roundeven;
13083 case NEON::BI__builtin_neon_vrndn_v:
13084 case NEON::BI__builtin_neon_vrndnq_v: {
13086 ? Intrinsic::experimental_constrained_roundeven
13087 : Intrinsic::roundeven;
13090 case NEON::BI__builtin_neon_vrndns_f32: {
13093 ? Intrinsic::experimental_constrained_roundeven
13094 : Intrinsic::roundeven;
13097 case NEON::BI__builtin_neon_vrndph_f16: {
13100 ? Intrinsic::experimental_constrained_ceil
13104 case NEON::BI__builtin_neon_vrndp_v:
13105 case NEON::BI__builtin_neon_vrndpq_v: {
13107 ? Intrinsic::experimental_constrained_ceil
13111 case NEON::BI__builtin_neon_vrndxh_f16: {
13114 ? Intrinsic::experimental_constrained_rint
13118 case NEON::BI__builtin_neon_vrndx_v:
13119 case NEON::BI__builtin_neon_vrndxq_v: {
13121 ? Intrinsic::experimental_constrained_rint
13125 case NEON::BI__builtin_neon_vrndh_f16: {
13128 ? Intrinsic::experimental_constrained_trunc
13129 : Intrinsic::trunc;
13132 case NEON::BI__builtin_neon_vrnd32x_f32:
13133 case NEON::BI__builtin_neon_vrnd32xq_f32:
13134 case NEON::BI__builtin_neon_vrnd32x_f64:
13135 case NEON::BI__builtin_neon_vrnd32xq_f64: {
13137 Int = Intrinsic::aarch64_neon_frint32x;
13140 case NEON::BI__builtin_neon_vrnd32z_f32:
13141 case NEON::BI__builtin_neon_vrnd32zq_f32:
13142 case NEON::BI__builtin_neon_vrnd32z_f64:
13143 case NEON::BI__builtin_neon_vrnd32zq_f64: {
13145 Int = Intrinsic::aarch64_neon_frint32z;
13148 case NEON::BI__builtin_neon_vrnd64x_f32:
13149 case NEON::BI__builtin_neon_vrnd64xq_f32:
13150 case NEON::BI__builtin_neon_vrnd64x_f64:
13151 case NEON::BI__builtin_neon_vrnd64xq_f64: {
13153 Int = Intrinsic::aarch64_neon_frint64x;
13156 case NEON::BI__builtin_neon_vrnd64z_f32:
13157 case NEON::BI__builtin_neon_vrnd64zq_f32:
13158 case NEON::BI__builtin_neon_vrnd64z_f64:
13159 case NEON::BI__builtin_neon_vrnd64zq_f64: {
13161 Int = Intrinsic::aarch64_neon_frint64z;
13164 case NEON::BI__builtin_neon_vrnd_v:
13165 case NEON::BI__builtin_neon_vrndq_v: {
13167 ? Intrinsic::experimental_constrained_trunc
13168 : Intrinsic::trunc;
13171 case NEON::BI__builtin_neon_vcvt_f64_v:
13172 case NEON::BI__builtin_neon_vcvtq_f64_v:
13173 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
13175 return usgn ?
Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
13176 :
Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
13177 case NEON::BI__builtin_neon_vcvt_f64_f32: {
13179 "unexpected vcvt_f64_f32 builtin");
13183 return Builder.CreateFPExt(Ops[0], Ty,
"vcvt");
13185 case NEON::BI__builtin_neon_vcvt_f32_f64: {
13187 "unexpected vcvt_f32_f64 builtin");
13191 return Builder.CreateFPTrunc(Ops[0], Ty,
"vcvt");
13193 case NEON::BI__builtin_neon_vcvt_s32_v:
13194 case NEON::BI__builtin_neon_vcvt_u32_v:
13195 case NEON::BI__builtin_neon_vcvt_s64_v:
13196 case NEON::BI__builtin_neon_vcvt_u64_v:
13197 case NEON::BI__builtin_neon_vcvt_s16_f16:
13198 case NEON::BI__builtin_neon_vcvt_u16_f16:
13199 case NEON::BI__builtin_neon_vcvtq_s32_v:
13200 case NEON::BI__builtin_neon_vcvtq_u32_v:
13201 case NEON::BI__builtin_neon_vcvtq_s64_v:
13202 case NEON::BI__builtin_neon_vcvtq_u64_v:
13203 case NEON::BI__builtin_neon_vcvtq_s16_f16:
13204 case NEON::BI__builtin_neon_vcvtq_u16_f16: {
13206 usgn ? Intrinsic::aarch64_neon_fcvtzu : Intrinsic::aarch64_neon_fcvtzs;
13210 case NEON::BI__builtin_neon_vcvta_s16_f16:
13211 case NEON::BI__builtin_neon_vcvta_u16_f16:
13212 case NEON::BI__builtin_neon_vcvta_s32_v:
13213 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
13214 case NEON::BI__builtin_neon_vcvtaq_s32_v:
13215 case NEON::BI__builtin_neon_vcvta_u32_v:
13216 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
13217 case NEON::BI__builtin_neon_vcvtaq_u32_v:
13218 case NEON::BI__builtin_neon_vcvta_s64_v:
13219 case NEON::BI__builtin_neon_vcvtaq_s64_v:
13220 case NEON::BI__builtin_neon_vcvta_u64_v:
13221 case NEON::BI__builtin_neon_vcvtaq_u64_v: {
13222 Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas;
13226 case NEON::BI__builtin_neon_vcvtm_s16_f16:
13227 case NEON::BI__builtin_neon_vcvtm_s32_v:
13228 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
13229 case NEON::BI__builtin_neon_vcvtmq_s32_v:
13230 case NEON::BI__builtin_neon_vcvtm_u16_f16:
13231 case NEON::BI__builtin_neon_vcvtm_u32_v:
13232 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
13233 case NEON::BI__builtin_neon_vcvtmq_u32_v:
13234 case NEON::BI__builtin_neon_vcvtm_s64_v:
13235 case NEON::BI__builtin_neon_vcvtmq_s64_v:
13236 case NEON::BI__builtin_neon_vcvtm_u64_v:
13237 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
13238 Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms;
13242 case NEON::BI__builtin_neon_vcvtn_s16_f16:
13243 case NEON::BI__builtin_neon_vcvtn_s32_v:
13244 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
13245 case NEON::BI__builtin_neon_vcvtnq_s32_v:
13246 case NEON::BI__builtin_neon_vcvtn_u16_f16:
13247 case NEON::BI__builtin_neon_vcvtn_u32_v:
13248 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
13249 case NEON::BI__builtin_neon_vcvtnq_u32_v:
13250 case NEON::BI__builtin_neon_vcvtn_s64_v:
13251 case NEON::BI__builtin_neon_vcvtnq_s64_v:
13252 case NEON::BI__builtin_neon_vcvtn_u64_v:
13253 case NEON::BI__builtin_neon_vcvtnq_u64_v: {
13254 Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns;
13258 case NEON::BI__builtin_neon_vcvtp_s16_f16:
13259 case NEON::BI__builtin_neon_vcvtp_s32_v:
13260 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
13261 case NEON::BI__builtin_neon_vcvtpq_s32_v:
13262 case NEON::BI__builtin_neon_vcvtp_u16_f16:
13263 case NEON::BI__builtin_neon_vcvtp_u32_v:
13264 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
13265 case NEON::BI__builtin_neon_vcvtpq_u32_v:
13266 case NEON::BI__builtin_neon_vcvtp_s64_v:
13267 case NEON::BI__builtin_neon_vcvtpq_s64_v:
13268 case NEON::BI__builtin_neon_vcvtp_u64_v:
13269 case NEON::BI__builtin_neon_vcvtpq_u64_v: {
13270 Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps;
13274 case NEON::BI__builtin_neon_vmulx_v:
13275 case NEON::BI__builtin_neon_vmulxq_v: {
13276 Int = Intrinsic::aarch64_neon_fmulx;
13279 case NEON::BI__builtin_neon_vmulxh_lane_f16:
13280 case NEON::BI__builtin_neon_vmulxh_laneq_f16: {
13284 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2],
"extract");
13286 Int = Intrinsic::aarch64_neon_fmulx;
13289 case NEON::BI__builtin_neon_vmul_lane_v:
13290 case NEON::BI__builtin_neon_vmul_laneq_v: {
13293 if (BuiltinID == NEON::BI__builtin_neon_vmul_laneq_v)
13296 llvm::FixedVectorType *VTy =
13298 Ops[1] =
Builder.CreateBitCast(Ops[1], VTy);
13299 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2],
"extract");
13303 case NEON::BI__builtin_neon_vnegd_s64:
13305 case NEON::BI__builtin_neon_vnegh_f16:
13307 case NEON::BI__builtin_neon_vpmaxnm_v:
13308 case NEON::BI__builtin_neon_vpmaxnmq_v: {
13309 Int = Intrinsic::aarch64_neon_fmaxnmp;
13312 case NEON::BI__builtin_neon_vpminnm_v:
13313 case NEON::BI__builtin_neon_vpminnmq_v: {
13314 Int = Intrinsic::aarch64_neon_fminnmp;
13317 case NEON::BI__builtin_neon_vsqrth_f16: {
13320 ? Intrinsic::experimental_constrained_sqrt
13324 case NEON::BI__builtin_neon_vsqrt_v:
13325 case NEON::BI__builtin_neon_vsqrtq_v: {
13327 ? Intrinsic::experimental_constrained_sqrt
13329 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
13332 case NEON::BI__builtin_neon_vrbit_v:
13333 case NEON::BI__builtin_neon_vrbitq_v: {
13334 Int = Intrinsic::bitreverse;
13337 case NEON::BI__builtin_neon_vaddv_u8:
13341 case NEON::BI__builtin_neon_vaddv_s8: {
13342 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
13344 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
13345 llvm::Type *Tys[2] = { Ty, VTy };
13350 case NEON::BI__builtin_neon_vaddv_u16:
13353 case NEON::BI__builtin_neon_vaddv_s16: {
13354 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
13356 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
13357 llvm::Type *Tys[2] = { Ty, VTy };
13362 case NEON::BI__builtin_neon_vaddvq_u8:
13365 case NEON::BI__builtin_neon_vaddvq_s8: {
13366 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
13368 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
13369 llvm::Type *Tys[2] = { Ty, VTy };
13374 case NEON::BI__builtin_neon_vaddvq_u16:
13377 case NEON::BI__builtin_neon_vaddvq_s16: {
13378 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
13380 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
13381 llvm::Type *Tys[2] = { Ty, VTy };
13386 case NEON::BI__builtin_neon_vmaxv_u8: {
13387 Int = Intrinsic::aarch64_neon_umaxv;
13389 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
13390 llvm::Type *Tys[2] = { Ty, VTy };
13395 case NEON::BI__builtin_neon_vmaxv_u16: {
13396 Int = Intrinsic::aarch64_neon_umaxv;
13398 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
13399 llvm::Type *Tys[2] = { Ty, VTy };
13404 case NEON::BI__builtin_neon_vmaxvq_u8: {
13405 Int = Intrinsic::aarch64_neon_umaxv;
13407 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
13408 llvm::Type *Tys[2] = { Ty, VTy };
13413 case NEON::BI__builtin_neon_vmaxvq_u16: {
13414 Int = Intrinsic::aarch64_neon_umaxv;
13416 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
13417 llvm::Type *Tys[2] = { Ty, VTy };
13422 case NEON::BI__builtin_neon_vmaxv_s8: {
13423 Int = Intrinsic::aarch64_neon_smaxv;
13425 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
13426 llvm::Type *Tys[2] = { Ty, VTy };
13431 case NEON::BI__builtin_neon_vmaxv_s16: {
13432 Int = Intrinsic::aarch64_neon_smaxv;
13434 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
13435 llvm::Type *Tys[2] = { Ty, VTy };
13440 case NEON::BI__builtin_neon_vmaxvq_s8: {
13441 Int = Intrinsic::aarch64_neon_smaxv;
13443 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
13444 llvm::Type *Tys[2] = { Ty, VTy };
13449 case NEON::BI__builtin_neon_vmaxvq_s16: {
13450 Int = Intrinsic::aarch64_neon_smaxv;
13452 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
13453 llvm::Type *Tys[2] = { Ty, VTy };
13458 case NEON::BI__builtin_neon_vmaxv_f16: {
13459 Int = Intrinsic::aarch64_neon_fmaxv;
13461 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
13462 llvm::Type *Tys[2] = { Ty, VTy };
13467 case NEON::BI__builtin_neon_vmaxvq_f16: {
13468 Int = Intrinsic::aarch64_neon_fmaxv;
13470 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
13471 llvm::Type *Tys[2] = { Ty, VTy };
13476 case NEON::BI__builtin_neon_vminv_u8: {
13477 Int = Intrinsic::aarch64_neon_uminv;
13479 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
13480 llvm::Type *Tys[2] = { Ty, VTy };
13485 case NEON::BI__builtin_neon_vminv_u16: {
13486 Int = Intrinsic::aarch64_neon_uminv;
13488 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
13489 llvm::Type *Tys[2] = { Ty, VTy };
13494 case NEON::BI__builtin_neon_vminvq_u8: {
13495 Int = Intrinsic::aarch64_neon_uminv;
13497 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
13498 llvm::Type *Tys[2] = { Ty, VTy };
13503 case NEON::BI__builtin_neon_vminvq_u16: {
13504 Int = Intrinsic::aarch64_neon_uminv;
13506 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
13507 llvm::Type *Tys[2] = { Ty, VTy };
13512 case NEON::BI__builtin_neon_vminv_s8: {
13513 Int = Intrinsic::aarch64_neon_sminv;
13515 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
13516 llvm::Type *Tys[2] = { Ty, VTy };
13521 case NEON::BI__builtin_neon_vminv_s16: {
13522 Int = Intrinsic::aarch64_neon_sminv;
13524 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
13525 llvm::Type *Tys[2] = { Ty, VTy };
13530 case NEON::BI__builtin_neon_vminvq_s8: {
13531 Int = Intrinsic::aarch64_neon_sminv;
13533 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
13534 llvm::Type *Tys[2] = { Ty, VTy };
13539 case NEON::BI__builtin_neon_vminvq_s16: {
13540 Int = Intrinsic::aarch64_neon_sminv;
13542 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
13543 llvm::Type *Tys[2] = { Ty, VTy };
13548 case NEON::BI__builtin_neon_vminv_f16: {
13549 Int = Intrinsic::aarch64_neon_fminv;
13551 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
13552 llvm::Type *Tys[2] = { Ty, VTy };
13557 case NEON::BI__builtin_neon_vminvq_f16: {
13558 Int = Intrinsic::aarch64_neon_fminv;
13560 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
13561 llvm::Type *Tys[2] = { Ty, VTy };
13566 case NEON::BI__builtin_neon_vmaxnmv_f16: {
13567 Int = Intrinsic::aarch64_neon_fmaxnmv;
13569 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
13570 llvm::Type *Tys[2] = { Ty, VTy };
13575 case NEON::BI__builtin_neon_vmaxnmvq_f16: {
13576 Int = Intrinsic::aarch64_neon_fmaxnmv;
13578 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
13579 llvm::Type *Tys[2] = { Ty, VTy };
13584 case NEON::BI__builtin_neon_vminnmv_f16: {
13585 Int = Intrinsic::aarch64_neon_fminnmv;
13587 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
13588 llvm::Type *Tys[2] = { Ty, VTy };
13593 case NEON::BI__builtin_neon_vminnmvq_f16: {
13594 Int = Intrinsic::aarch64_neon_fminnmv;
13596 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
13597 llvm::Type *Tys[2] = { Ty, VTy };
13602 case NEON::BI__builtin_neon_vmul_n_f64: {
13605 return Builder.CreateFMul(Ops[0], RHS);
13607 case NEON::BI__builtin_neon_vaddlv_u8: {
13608 Int = Intrinsic::aarch64_neon_uaddlv;
13610 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
13611 llvm::Type *Tys[2] = { Ty, VTy };
13616 case NEON::BI__builtin_neon_vaddlv_u16: {
13617 Int = Intrinsic::aarch64_neon_uaddlv;
13619 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
13620 llvm::Type *Tys[2] = { Ty, VTy };
13624 case NEON::BI__builtin_neon_vaddlvq_u8: {
13625 Int = Intrinsic::aarch64_neon_uaddlv;
13627 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
13628 llvm::Type *Tys[2] = { Ty, VTy };
13633 case NEON::BI__builtin_neon_vaddlvq_u16: {
13634 Int = Intrinsic::aarch64_neon_uaddlv;
13636 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
13637 llvm::Type *Tys[2] = { Ty, VTy };
13641 case NEON::BI__builtin_neon_vaddlv_s8: {
13642 Int = Intrinsic::aarch64_neon_saddlv;
13644 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
13645 llvm::Type *Tys[2] = { Ty, VTy };
13650 case NEON::BI__builtin_neon_vaddlv_s16: {
13651 Int = Intrinsic::aarch64_neon_saddlv;
13653 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
13654 llvm::Type *Tys[2] = { Ty, VTy };
13658 case NEON::BI__builtin_neon_vaddlvq_s8: {
13659 Int = Intrinsic::aarch64_neon_saddlv;
13661 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
13662 llvm::Type *Tys[2] = { Ty, VTy };
13667 case NEON::BI__builtin_neon_vaddlvq_s16: {
13668 Int = Intrinsic::aarch64_neon_saddlv;
13670 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
13671 llvm::Type *Tys[2] = { Ty, VTy };
13675 case NEON::BI__builtin_neon_vsri_n_v:
13676 case NEON::BI__builtin_neon_vsriq_n_v: {
13677 Int = Intrinsic::aarch64_neon_vsri;
13681 case NEON::BI__builtin_neon_vsli_n_v:
13682 case NEON::BI__builtin_neon_vsliq_n_v: {
13683 Int = Intrinsic::aarch64_neon_vsli;
13687 case NEON::BI__builtin_neon_vsra_n_v:
13688 case NEON::BI__builtin_neon_vsraq_n_v:
13689 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
13691 return Builder.CreateAdd(Ops[0], Ops[1]);
13692 case NEON::BI__builtin_neon_vrsra_n_v:
13693 case NEON::BI__builtin_neon_vrsraq_n_v: {
13694 Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl;
13696 TmpOps.push_back(Ops[1]);
13697 TmpOps.push_back(Ops[2]);
13699 llvm::Value *tmp =
EmitNeonCall(F, TmpOps,
"vrshr_n", 1,
true);
13700 Ops[0] =
Builder.CreateBitCast(Ops[0], VTy);
13701 return Builder.CreateAdd(Ops[0], tmp);
13703 case NEON::BI__builtin_neon_vld1_v:
13704 case NEON::BI__builtin_neon_vld1q_v: {
13707 case NEON::BI__builtin_neon_vst1_v:
13708 case NEON::BI__builtin_neon_vst1q_v:
13709 Ops[1] =
Builder.CreateBitCast(Ops[1], VTy);
13711 case NEON::BI__builtin_neon_vld1_lane_v:
13712 case NEON::BI__builtin_neon_vld1q_lane_v: {
13713 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13716 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vld1_lane");
13718 case NEON::BI__builtin_neon_vldap1_lane_s64:
13719 case NEON::BI__builtin_neon_vldap1q_lane_s64: {
13720 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13722 VTy->getElementType(), Ops[0], PtrOp0.
getAlignment());
13723 LI->setAtomic(llvm::AtomicOrdering::Acquire);
13725 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vldap1_lane");
13727 case NEON::BI__builtin_neon_vld1_dup_v:
13728 case NEON::BI__builtin_neon_vld1q_dup_v: {
13729 Value *
V = PoisonValue::get(Ty);
13732 llvm::Constant *CI = ConstantInt::get(
Int32Ty, 0);
13733 Ops[0] =
Builder.CreateInsertElement(
V, Ops[0], CI);
13736 case NEON::BI__builtin_neon_vst1_lane_v:
13737 case NEON::BI__builtin_neon_vst1q_lane_v:
13738 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13739 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2]);
13741 case NEON::BI__builtin_neon_vstl1_lane_s64:
13742 case NEON::BI__builtin_neon_vstl1q_lane_s64: {
13743 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13744 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2]);
13745 llvm::StoreInst *SI =
13747 SI->setAtomic(llvm::AtomicOrdering::Release);
13750 case NEON::BI__builtin_neon_vld2_v:
13751 case NEON::BI__builtin_neon_vld2q_v: {
13754 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld2");
13757 case NEON::BI__builtin_neon_vld3_v:
13758 case NEON::BI__builtin_neon_vld3q_v: {
13761 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld3");
13764 case NEON::BI__builtin_neon_vld4_v:
13765 case NEON::BI__builtin_neon_vld4q_v: {
13768 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld4");
13771 case NEON::BI__builtin_neon_vld2_dup_v:
13772 case NEON::BI__builtin_neon_vld2q_dup_v: {
13775 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld2");
13778 case NEON::BI__builtin_neon_vld3_dup_v:
13779 case NEON::BI__builtin_neon_vld3q_dup_v: {
13782 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld3");
13785 case NEON::BI__builtin_neon_vld4_dup_v:
13786 case NEON::BI__builtin_neon_vld4q_dup_v: {
13789 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld4");
13792 case NEON::BI__builtin_neon_vld2_lane_v:
13793 case NEON::BI__builtin_neon_vld2q_lane_v: {
13794 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
13796 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
13797 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13798 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13803 case NEON::BI__builtin_neon_vld3_lane_v:
13804 case NEON::BI__builtin_neon_vld3q_lane_v: {
13805 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
13807 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
13808 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13809 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13810 Ops[3] =
Builder.CreateBitCast(Ops[3], Ty);
13815 case NEON::BI__builtin_neon_vld4_lane_v:
13816 case NEON::BI__builtin_neon_vld4q_lane_v: {
13817 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
13819 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
13820 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13821 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13822 Ops[3] =
Builder.CreateBitCast(Ops[3], Ty);
13823 Ops[4] =
Builder.CreateBitCast(Ops[4], Ty);
13828 case NEON::BI__builtin_neon_vst2_v:
13829 case NEON::BI__builtin_neon_vst2q_v: {
13830 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13831 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() };
13835 case NEON::BI__builtin_neon_vst2_lane_v:
13836 case NEON::BI__builtin_neon_vst2q_lane_v: {
13837 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13839 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
13843 case NEON::BI__builtin_neon_vst3_v:
13844 case NEON::BI__builtin_neon_vst3q_v: {
13845 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13846 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
13850 case NEON::BI__builtin_neon_vst3_lane_v:
13851 case NEON::BI__builtin_neon_vst3q_lane_v: {
13852 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13854 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
13858 case NEON::BI__builtin_neon_vst4_v:
13859 case NEON::BI__builtin_neon_vst4q_v: {
13860 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13861 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
13865 case NEON::BI__builtin_neon_vst4_lane_v:
13866 case NEON::BI__builtin_neon_vst4q_lane_v: {
13867 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
13869 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() };
13873 case NEON::BI__builtin_neon_vtrn_v:
13874 case NEON::BI__builtin_neon_vtrnq_v: {
13875 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13876 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13877 Value *SV =
nullptr;
13879 for (
unsigned vi = 0; vi != 2; ++vi) {
13881 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
13882 Indices.push_back(i+vi);
13883 Indices.push_back(i+e+vi);
13885 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
13886 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vtrn");
13891 case NEON::BI__builtin_neon_vuzp_v:
13892 case NEON::BI__builtin_neon_vuzpq_v: {
13893 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13894 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13895 Value *SV =
nullptr;
13897 for (
unsigned vi = 0; vi != 2; ++vi) {
13899 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
13900 Indices.push_back(2*i+vi);
13902 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
13903 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vuzp");
13908 case NEON::BI__builtin_neon_vzip_v:
13909 case NEON::BI__builtin_neon_vzipq_v: {
13910 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
13911 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
13912 Value *SV =
nullptr;
13914 for (
unsigned vi = 0; vi != 2; ++vi) {
13916 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
13917 Indices.push_back((i + vi*e) >> 1);
13918 Indices.push_back(((i + vi*e) >> 1)+e);
13920 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
13921 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vzip");
13926 case NEON::BI__builtin_neon_vqtbl1q_v: {
13930 case NEON::BI__builtin_neon_vqtbl2q_v: {
13934 case NEON::BI__builtin_neon_vqtbl3q_v: {
13938 case NEON::BI__builtin_neon_vqtbl4q_v: {
13942 case NEON::BI__builtin_neon_vqtbx1q_v: {
13946 case NEON::BI__builtin_neon_vqtbx2q_v: {
13950 case NEON::BI__builtin_neon_vqtbx3q_v: {
13954 case NEON::BI__builtin_neon_vqtbx4q_v: {
13958 case NEON::BI__builtin_neon_vsqadd_v:
13959 case NEON::BI__builtin_neon_vsqaddq_v: {
13960 Int = Intrinsic::aarch64_neon_usqadd;
13963 case NEON::BI__builtin_neon_vuqadd_v:
13964 case NEON::BI__builtin_neon_vuqaddq_v: {
13965 Int = Intrinsic::aarch64_neon_suqadd;
13969 case NEON::BI__builtin_neon_vluti2_laneq_bf16:
13970 case NEON::BI__builtin_neon_vluti2_laneq_f16:
13971 case NEON::BI__builtin_neon_vluti2_laneq_p16:
13972 case NEON::BI__builtin_neon_vluti2_laneq_p8:
13973 case NEON::BI__builtin_neon_vluti2_laneq_s16:
13974 case NEON::BI__builtin_neon_vluti2_laneq_s8:
13975 case NEON::BI__builtin_neon_vluti2_laneq_u16:
13976 case NEON::BI__builtin_neon_vluti2_laneq_u8: {
13977 Int = Intrinsic::aarch64_neon_vluti2_laneq;
13978 llvm::Type *Tys[2];
13984 case NEON::BI__builtin_neon_vluti2q_laneq_bf16:
13985 case NEON::BI__builtin_neon_vluti2q_laneq_f16:
13986 case NEON::BI__builtin_neon_vluti2q_laneq_p16:
13987 case NEON::BI__builtin_neon_vluti2q_laneq_p8:
13988 case NEON::BI__builtin_neon_vluti2q_laneq_s16:
13989 case NEON::BI__builtin_neon_vluti2q_laneq_s8:
13990 case NEON::BI__builtin_neon_vluti2q_laneq_u16:
13991 case NEON::BI__builtin_neon_vluti2q_laneq_u8: {
13992 Int = Intrinsic::aarch64_neon_vluti2_laneq;
13993 llvm::Type *Tys[2];
13999 case NEON::BI__builtin_neon_vluti2_lane_bf16:
14000 case NEON::BI__builtin_neon_vluti2_lane_f16:
14001 case NEON::BI__builtin_neon_vluti2_lane_p16:
14002 case NEON::BI__builtin_neon_vluti2_lane_p8:
14003 case NEON::BI__builtin_neon_vluti2_lane_s16:
14004 case NEON::BI__builtin_neon_vluti2_lane_s8:
14005 case NEON::BI__builtin_neon_vluti2_lane_u16:
14006 case NEON::BI__builtin_neon_vluti2_lane_u8: {
14007 Int = Intrinsic::aarch64_neon_vluti2_lane;
14008 llvm::Type *Tys[2];
14014 case NEON::BI__builtin_neon_vluti2q_lane_bf16:
14015 case NEON::BI__builtin_neon_vluti2q_lane_f16:
14016 case NEON::BI__builtin_neon_vluti2q_lane_p16:
14017 case NEON::BI__builtin_neon_vluti2q_lane_p8:
14018 case NEON::BI__builtin_neon_vluti2q_lane_s16:
14019 case NEON::BI__builtin_neon_vluti2q_lane_s8:
14020 case NEON::BI__builtin_neon_vluti2q_lane_u16:
14021 case NEON::BI__builtin_neon_vluti2q_lane_u8: {
14022 Int = Intrinsic::aarch64_neon_vluti2_lane;
14023 llvm::Type *Tys[2];
14029 case NEON::BI__builtin_neon_vluti4q_lane_p8:
14030 case NEON::BI__builtin_neon_vluti4q_lane_s8:
14031 case NEON::BI__builtin_neon_vluti4q_lane_u8: {
14032 Int = Intrinsic::aarch64_neon_vluti4q_lane;
14035 case NEON::BI__builtin_neon_vluti4q_laneq_p8:
14036 case NEON::BI__builtin_neon_vluti4q_laneq_s8:
14037 case NEON::BI__builtin_neon_vluti4q_laneq_u8: {
14038 Int = Intrinsic::aarch64_neon_vluti4q_laneq;
14041 case NEON::BI__builtin_neon_vluti4q_lane_bf16_x2:
14042 case NEON::BI__builtin_neon_vluti4q_lane_f16_x2:
14043 case NEON::BI__builtin_neon_vluti4q_lane_p16_x2:
14044 case NEON::BI__builtin_neon_vluti4q_lane_s16_x2:
14045 case NEON::BI__builtin_neon_vluti4q_lane_u16_x2: {
14046 Int = Intrinsic::aarch64_neon_vluti4q_lane_x2;
14049 case NEON::BI__builtin_neon_vluti4q_laneq_bf16_x2:
14050 case NEON::BI__builtin_neon_vluti4q_laneq_f16_x2:
14051 case NEON::BI__builtin_neon_vluti4q_laneq_p16_x2:
14052 case NEON::BI__builtin_neon_vluti4q_laneq_s16_x2:
14053 case NEON::BI__builtin_neon_vluti4q_laneq_u16_x2: {
14054 Int = Intrinsic::aarch64_neon_vluti4q_laneq_x2;
14058 case NEON::BI__builtin_neon_vamin_f16:
14059 case NEON::BI__builtin_neon_vaminq_f16:
14060 case NEON::BI__builtin_neon_vamin_f32:
14061 case NEON::BI__builtin_neon_vaminq_f32:
14062 case NEON::BI__builtin_neon_vaminq_f64: {
14063 Int = Intrinsic::aarch64_neon_famin;
14066 case NEON::BI__builtin_neon_vamax_f16:
14067 case NEON::BI__builtin_neon_vamaxq_f16:
14068 case NEON::BI__builtin_neon_vamax_f32:
14069 case NEON::BI__builtin_neon_vamaxq_f32:
14070 case NEON::BI__builtin_neon_vamaxq_f64: {
14071 Int = Intrinsic::aarch64_neon_famax;
14074 case NEON::BI__builtin_neon_vscale_f16:
14075 case NEON::BI__builtin_neon_vscaleq_f16:
14076 case NEON::BI__builtin_neon_vscale_f32:
14077 case NEON::BI__builtin_neon_vscaleq_f32:
14078 case NEON::BI__builtin_neon_vscaleq_f64: {
14079 Int = Intrinsic::aarch64_neon_fp8_fscale;
14087 assert((BuiltinID == BPF::BI__builtin_preserve_field_info ||
14088 BuiltinID == BPF::BI__builtin_btf_type_id ||
14089 BuiltinID == BPF::BI__builtin_preserve_type_info ||
14090 BuiltinID == BPF::BI__builtin_preserve_enum_value) &&
14091 "unexpected BPF builtin");
14098 switch (BuiltinID) {
14100 llvm_unreachable(
"Unexpected BPF builtin");
14101 case BPF::BI__builtin_preserve_field_info: {
14102 const Expr *Arg =
E->getArg(0);
14107 "using __builtin_preserve_field_info() without -g");
14120 Value *InfoKind = ConstantInt::get(
Int64Ty,
C->getSExtValue());
14123 llvm::Function *FnGetFieldInfo = llvm::Intrinsic::getOrInsertDeclaration(
14124 &
CGM.
getModule(), llvm::Intrinsic::bpf_preserve_field_info,
14125 {FieldAddr->getType()});
14126 return Builder.CreateCall(FnGetFieldInfo, {FieldAddr, InfoKind});
14128 case BPF::BI__builtin_btf_type_id:
14129 case BPF::BI__builtin_preserve_type_info: {
14135 const Expr *Arg0 =
E->getArg(0);
14140 Value *FlagValue = ConstantInt::get(
Int64Ty, Flag->getSExtValue());
14141 Value *SeqNumVal = ConstantInt::get(
Int32Ty, BuiltinSeqNum++);
14143 llvm::Function *FnDecl;
14144 if (BuiltinID == BPF::BI__builtin_btf_type_id)
14145 FnDecl = llvm::Intrinsic::getOrInsertDeclaration(
14146 &
CGM.
getModule(), llvm::Intrinsic::bpf_btf_type_id, {});
14148 FnDecl = llvm::Intrinsic::getOrInsertDeclaration(
14149 &
CGM.
getModule(), llvm::Intrinsic::bpf_preserve_type_info, {});
14150 CallInst *
Fn =
Builder.CreateCall(FnDecl, {SeqNumVal, FlagValue});
14151 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
14154 case BPF::BI__builtin_preserve_enum_value: {
14160 const Expr *Arg0 =
E->getArg(0);
14165 const auto *UO = cast<UnaryOperator>(Arg0->
IgnoreParens());
14166 const auto *CE = cast<CStyleCastExpr>(UO->getSubExpr());
14167 const auto *DR = cast<DeclRefExpr>(CE->getSubExpr());
14168 const auto *Enumerator = cast<EnumConstantDecl>(DR->getDecl());
14170 auto InitVal = Enumerator->getInitVal();
14171 std::string InitValStr;
14172 if (InitVal.isNegative() || InitVal >
uint64_t(INT64_MAX))
14173 InitValStr = std::to_string(InitVal.getSExtValue());
14175 InitValStr = std::to_string(InitVal.getZExtValue());
14176 std::string EnumStr = Enumerator->getNameAsString() +
":" + InitValStr;
14177 Value *EnumStrVal =
Builder.CreateGlobalString(EnumStr);
14180 Value *FlagValue = ConstantInt::get(
Int64Ty, Flag->getSExtValue());
14181 Value *SeqNumVal = ConstantInt::get(
Int32Ty, BuiltinSeqNum++);
14183 llvm::Function *IntrinsicFn = llvm::Intrinsic::getOrInsertDeclaration(
14184 &
CGM.
getModule(), llvm::Intrinsic::bpf_preserve_enum_value, {});
14186 Builder.CreateCall(IntrinsicFn, {SeqNumVal, EnumStrVal, FlagValue});
14187 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
14195 assert((Ops.size() & (Ops.size() - 1)) == 0 &&
14196 "Not a power-of-two sized vector!");
14197 bool AllConstants =
true;
14198 for (
unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i)
14199 AllConstants &= isa<Constant>(Ops[i]);
14202 if (AllConstants) {
14204 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
14205 CstOps.push_back(cast<Constant>(Ops[i]));
14206 return llvm::ConstantVector::get(CstOps);
14211 llvm::FixedVectorType::get(Ops[0]->getType(), Ops.size()));
14213 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
14221 unsigned NumElts) {
14223 auto *MaskTy = llvm::FixedVectorType::get(
14225 cast<IntegerType>(Mask->
getType())->getBitWidth());
14226 Value *MaskVec = CGF.
Builder.CreateBitCast(Mask, MaskTy);
14232 for (
unsigned i = 0; i != NumElts; ++i)
14234 MaskVec = CGF.
Builder.CreateShuffleVector(
14235 MaskVec, MaskVec,
ArrayRef(Indices, NumElts),
"extract");
14242 Value *Ptr = Ops[0];
14246 cast<llvm::FixedVectorType>(Ops[1]->getType())->getNumElements());
14248 return CGF.
Builder.CreateMaskedStore(Ops[1], Ptr, Alignment, MaskVec);
14253 llvm::Type *Ty = Ops[1]->getType();
14254 Value *Ptr = Ops[0];
14257 CGF, Ops[2], cast<llvm::FixedVectorType>(Ty)->getNumElements());
14259 return CGF.
Builder.CreateMaskedLoad(Ty, Ptr, Alignment, MaskVec, Ops[1]);
14264 auto *ResultTy = cast<llvm::VectorType>(Ops[1]->getType());
14265 Value *Ptr = Ops[0];
14268 CGF, Ops[2], cast<FixedVectorType>(ResultTy)->getNumElements());
14270 llvm::Function *F = CGF.
CGM.
getIntrinsic(Intrinsic::masked_expandload,
14272 return CGF.
Builder.CreateCall(F, { Ptr, MaskVec, Ops[1] });
14278 auto *ResultTy = cast<llvm::FixedVectorType>(Ops[1]->getType());
14282 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
14283 : Intrinsic::x86_avx512_mask_expand;
14285 return CGF.
Builder.CreateCall(F, { Ops[0], Ops[1], MaskVec });
14290 auto *ResultTy = cast<llvm::FixedVectorType>(Ops[1]->getType());
14291 Value *Ptr = Ops[0];
14295 llvm::Function *F = CGF.
CGM.
getIntrinsic(Intrinsic::masked_compressstore,
14297 return CGF.
Builder.CreateCall(F, { Ops[1], Ptr, MaskVec });
14302 bool InvertLHS =
false) {
14303 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
14308 LHS = CGF.
Builder.CreateNot(LHS);
14310 return CGF.
Builder.CreateBitCast(CGF.
Builder.CreateBinOp(Opc, LHS, RHS),
14311 Ops[0]->getType());
14315 Value *Amt,
bool IsRight) {
14316 llvm::Type *Ty = Op0->
getType();
14322 unsigned NumElts = cast<llvm::FixedVectorType>(Ty)->getNumElements();
14323 Amt = CGF.
Builder.CreateIntCast(Amt, Ty->getScalarType(),
false);
14324 Amt = CGF.
Builder.CreateVectorSplat(NumElts, Amt);
14327 unsigned IID = IsRight ? Intrinsic::fshr : Intrinsic::fshl;
14329 return CGF.
Builder.CreateCall(F, {Op0, Op1, Amt});
14334 Value *Op0 = Ops[0];
14335 Value *Op1 = Ops[1];
14336 llvm::Type *Ty = Op0->
getType();
14337 uint64_t Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
14339 CmpInst::Predicate Pred;
14342 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
14345 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
14348 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
14351 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
14354 Pred = ICmpInst::ICMP_EQ;
14357 Pred = ICmpInst::ICMP_NE;
14360 return llvm::Constant::getNullValue(Ty);
14362 return llvm::Constant::getAllOnesValue(Ty);
14364 llvm_unreachable(
"Unexpected XOP vpcom/vpcomu predicate");
14376 if (
const auto *
C = dyn_cast<Constant>(Mask))
14377 if (
C->isAllOnesValue())
14381 CGF, Mask, cast<llvm::FixedVectorType>(Op0->
getType())->getNumElements());
14383 return CGF.
Builder.CreateSelect(Mask, Op0, Op1);
14389 if (
const auto *
C = dyn_cast<Constant>(Mask))
14390 if (
C->isAllOnesValue())
14393 auto *MaskTy = llvm::FixedVectorType::get(
14394 CGF.
Builder.getInt1Ty(), Mask->
getType()->getIntegerBitWidth());
14395 Mask = CGF.
Builder.CreateBitCast(Mask, MaskTy);
14396 Mask = CGF.
Builder.CreateExtractElement(Mask, (uint64_t)0);
14397 return CGF.
Builder.CreateSelect(Mask, Op0, Op1);
14401 unsigned NumElts,
Value *MaskIn) {
14403 const auto *
C = dyn_cast<Constant>(MaskIn);
14404 if (!
C || !
C->isAllOnesValue())
14410 for (
unsigned i = 0; i != NumElts; ++i)
14412 for (
unsigned i = NumElts; i != 8; ++i)
14413 Indices[i] = i % NumElts + NumElts;
14414 Cmp = CGF.
Builder.CreateShuffleVector(
14415 Cmp, llvm::Constant::getNullValue(Cmp->getType()), Indices);
14418 return CGF.
Builder.CreateBitCast(Cmp,
14420 std::max(NumElts, 8U)));
14425 assert((Ops.size() == 2 || Ops.size() == 4) &&
14426 "Unexpected number of arguments");
14428 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
14432 Cmp = Constant::getNullValue(
14433 llvm::FixedVectorType::get(CGF.
Builder.getInt1Ty(), NumElts));
14434 }
else if (CC == 7) {
14435 Cmp = Constant::getAllOnesValue(
14436 llvm::FixedVectorType::get(CGF.
Builder.getInt1Ty(), NumElts));
14438 ICmpInst::Predicate Pred;
14440 default: llvm_unreachable(
"Unknown condition code");
14441 case 0: Pred = ICmpInst::ICMP_EQ;
break;
14442 case 1: Pred =
Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
break;
14443 case 2: Pred =
Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
break;
14444 case 4: Pred = ICmpInst::ICMP_NE;
break;
14445 case 5: Pred =
Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
break;
14446 case 6: Pred =
Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
break;
14448 Cmp = CGF.
Builder.CreateICmp(Pred, Ops[0], Ops[1]);
14451 Value *MaskIn =
nullptr;
14452 if (Ops.size() == 4)
14459 Value *Zero = Constant::getNullValue(In->getType());
14465 unsigned Rnd = cast<llvm::ConstantInt>(Ops[3])->getZExtValue();
14466 llvm::Type *Ty = Ops[1]->getType();
14470 Intrinsic::ID IID = IsSigned ? Intrinsic::x86_avx512_sitofp_round
14471 : Intrinsic::x86_avx512_uitofp_round;
14473 Res = CGF.
Builder.CreateCall(F, { Ops[0], Ops[3] });
14475 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
14476 Res = IsSigned ? CGF.
Builder.CreateSIToFP(Ops[0], Ty)
14477 : CGF.
Builder.CreateUIToFP(Ops[0], Ty);
14488 bool Subtract =
false;
14489 Intrinsic::ID IID = Intrinsic::not_intrinsic;
14490 switch (BuiltinID) {
14492 case clang::X86::BI__builtin_ia32_vfmsubph512_mask3:
14495 case clang::X86::BI__builtin_ia32_vfmaddph512_mask:
14496 case clang::X86::BI__builtin_ia32_vfmaddph512_maskz:
14497 case clang::X86::BI__builtin_ia32_vfmaddph512_mask3:
14498 IID = llvm::Intrinsic::x86_avx512fp16_vfmadd_ph_512;
14500 case clang::X86::BI__builtin_ia32_vfmsubaddph512_mask3:
14503 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask:
14504 case clang::X86::BI__builtin_ia32_vfmaddsubph512_maskz:
14505 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask3:
14506 IID = llvm::Intrinsic::x86_avx512fp16_vfmaddsub_ph_512;
14508 case clang::X86::BI__builtin_ia32_vfmsubps512_mask3:
14511 case clang::X86::BI__builtin_ia32_vfmaddps512_mask:
14512 case clang::X86::BI__builtin_ia32_vfmaddps512_maskz:
14513 case clang::X86::BI__builtin_ia32_vfmaddps512_mask3:
14514 IID = llvm::Intrinsic::x86_avx512_vfmadd_ps_512;
break;
14515 case clang::X86::BI__builtin_ia32_vfmsubpd512_mask3:
14518 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask:
14519 case clang::X86::BI__builtin_ia32_vfmaddpd512_maskz:
14520 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask3:
14521 IID = llvm::Intrinsic::x86_avx512_vfmadd_pd_512;
break;
14522 case clang::X86::BI__builtin_ia32_vfmsubaddps512_mask3:
14525 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask:
14526 case clang::X86::BI__builtin_ia32_vfmaddsubps512_maskz:
14527 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask3:
14528 IID = llvm::Intrinsic::x86_avx512_vfmaddsub_ps_512;
14530 case clang::X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
14533 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask:
14534 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
14535 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
14536 IID = llvm::Intrinsic::x86_avx512_vfmaddsub_pd_512;
14538 case clang::X86::BI__builtin_ia32_vfmsubph256_round_mask3:
14541 case clang::X86::BI__builtin_ia32_vfmaddph256_round_mask:
14542 case clang::X86::BI__builtin_ia32_vfmaddph256_round_maskz:
14543 case clang::X86::BI__builtin_ia32_vfmaddph256_round_mask3:
14544 IID = llvm::Intrinsic::x86_avx10_vfmaddph256;
14546 case clang::X86::BI__builtin_ia32_vfmsubaddph256_round_mask3:
14549 case clang::X86::BI__builtin_ia32_vfmaddsubph256_round_mask:
14550 case clang::X86::BI__builtin_ia32_vfmaddsubph256_round_maskz:
14551 case clang::X86::BI__builtin_ia32_vfmaddsubph256_round_mask3:
14552 IID = llvm::Intrinsic::x86_avx10_vfmaddsubph256;
14554 case clang::X86::BI__builtin_ia32_vfmsubps256_round_mask3:
14557 case clang::X86::BI__builtin_ia32_vfmaddps256_round_mask:
14558 case clang::X86::BI__builtin_ia32_vfmaddps256_round_maskz:
14559 case clang::X86::BI__builtin_ia32_vfmaddps256_round_mask3:
14560 IID = llvm::Intrinsic::x86_avx10_vfmaddps256;
14562 case clang::X86::BI__builtin_ia32_vfmsubpd256_round_mask3:
14565 case clang::X86::BI__builtin_ia32_vfmaddpd256_round_mask:
14566 case clang::X86::BI__builtin_ia32_vfmaddpd256_round_maskz:
14567 case clang::X86::BI__builtin_ia32_vfmaddpd256_round_mask3:
14568 IID = llvm::Intrinsic::x86_avx10_vfmaddpd256;
14570 case clang::X86::BI__builtin_ia32_vfmsubaddps256_round_mask3:
14573 case clang::X86::BI__builtin_ia32_vfmaddsubps256_round_mask:
14574 case clang::X86::BI__builtin_ia32_vfmaddsubps256_round_maskz:
14575 case clang::X86::BI__builtin_ia32_vfmaddsubps256_round_mask3:
14576 IID = llvm::Intrinsic::x86_avx10_vfmaddsubps256;
14578 case clang::X86::BI__builtin_ia32_vfmsubaddpd256_round_mask3:
14581 case clang::X86::BI__builtin_ia32_vfmaddsubpd256_round_mask:
14582 case clang::X86::BI__builtin_ia32_vfmaddsubpd256_round_maskz:
14583 case clang::X86::BI__builtin_ia32_vfmaddsubpd256_round_mask3:
14584 IID = llvm::Intrinsic::x86_avx10_vfmaddsubpd256;
14598 if (IID != Intrinsic::not_intrinsic &&
14599 (cast<llvm::ConstantInt>(Ops.back())->getZExtValue() != (uint64_t)4 ||
14602 Res = CGF.
Builder.CreateCall(Intr, {A, B,
C, Ops.back() });
14604 llvm::Type *Ty = A->
getType();
14606 if (CGF.
Builder.getIsFPConstrained()) {
14607 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
14608 FMA = CGF.
CGM.
getIntrinsic(Intrinsic::experimental_constrained_fma, Ty);
14609 Res = CGF.
Builder.CreateConstrainedFPCall(FMA, {A, B,
C});
14612 Res = CGF.
Builder.CreateCall(FMA, {A, B,
C});
14617 Value *MaskFalseVal =
nullptr;
14618 switch (BuiltinID) {
14619 case clang::X86::BI__builtin_ia32_vfmaddph512_mask:
14620 case clang::X86::BI__builtin_ia32_vfmaddps512_mask:
14621 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask:
14622 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask:
14623 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask:
14624 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask:
14625 case clang::X86::BI__builtin_ia32_vfmaddph256_round_mask:
14626 case clang::X86::BI__builtin_ia32_vfmaddps256_round_mask:
14627 case clang::X86::BI__builtin_ia32_vfmaddpd256_round_mask:
14628 case clang::X86::BI__builtin_ia32_vfmaddsubph256_round_mask:
14629 case clang::X86::BI__builtin_ia32_vfmaddsubps256_round_mask:
14630 case clang::X86::BI__builtin_ia32_vfmaddsubpd256_round_mask:
14631 MaskFalseVal = Ops[0];
14633 case clang::X86::BI__builtin_ia32_vfmaddph512_maskz:
14634 case clang::X86::BI__builtin_ia32_vfmaddps512_maskz:
14635 case clang::X86::BI__builtin_ia32_vfmaddpd512_maskz:
14636 case clang::X86::BI__builtin_ia32_vfmaddsubph512_maskz:
14637 case clang::X86::BI__builtin_ia32_vfmaddsubps512_maskz:
14638 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
14639 case clang::X86::BI__builtin_ia32_vfmaddph256_round_maskz:
14640 case clang::X86::BI__builtin_ia32_vfmaddps256_round_maskz:
14641 case clang::X86::BI__builtin_ia32_vfmaddpd256_round_maskz:
14642 case clang::X86::BI__builtin_ia32_vfmaddsubph256_round_maskz:
14643 case clang::X86::BI__builtin_ia32_vfmaddsubps256_round_maskz:
14644 case clang::X86::BI__builtin_ia32_vfmaddsubpd256_round_maskz:
14645 MaskFalseVal = Constant::getNullValue(Ops[0]->getType());
14647 case clang::X86::BI__builtin_ia32_vfmsubph512_mask3:
14648 case clang::X86::BI__builtin_ia32_vfmaddph512_mask3:
14649 case clang::X86::BI__builtin_ia32_vfmsubps512_mask3:
14650 case clang::X86::BI__builtin_ia32_vfmaddps512_mask3:
14651 case clang::X86::BI__builtin_ia32_vfmsubpd512_mask3:
14652 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask3:
14653 case clang::X86::BI__builtin_ia32_vfmsubaddph512_mask3:
14654 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask3:
14655 case clang::X86::BI__builtin_ia32_vfmsubaddps512_mask3:
14656 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask3:
14657 case clang::X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
14658 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
14659 case clang::X86::BI__builtin_ia32_vfmsubph256_round_mask3:
14660 case clang::X86::BI__builtin_ia32_vfmaddph256_round_mask3:
14661 case clang::X86::BI__builtin_ia32_vfmsubps256_round_mask3:
14662 case clang::X86::BI__builtin_ia32_vfmaddps256_round_mask3:
14663 case clang::X86::BI__builtin_ia32_vfmsubpd256_round_mask3:
14664 case clang::X86::BI__builtin_ia32_vfmaddpd256_round_mask3:
14665 case clang::X86::BI__builtin_ia32_vfmsubaddph256_round_mask3:
14666 case clang::X86::BI__builtin_ia32_vfmaddsubph256_round_mask3:
14667 case clang::X86::BI__builtin_ia32_vfmsubaddps256_round_mask3:
14668 case clang::X86::BI__builtin_ia32_vfmaddsubps256_round_mask3:
14669 case clang::X86::BI__builtin_ia32_vfmsubaddpd256_round_mask3:
14670 case clang::X86::BI__builtin_ia32_vfmaddsubpd256_round_mask3:
14671 MaskFalseVal = Ops[2];
14683 bool ZeroMask =
false,
unsigned PTIdx = 0,
14684 bool NegAcc =
false) {
14686 if (Ops.size() > 4)
14687 Rnd = cast<llvm::ConstantInt>(Ops[4])->getZExtValue();
14690 Ops[2] = CGF.
Builder.CreateFNeg(Ops[2]);
14692 Ops[0] = CGF.
Builder.CreateExtractElement(Ops[0], (uint64_t)0);
14693 Ops[1] = CGF.
Builder.CreateExtractElement(Ops[1], (uint64_t)0);
14694 Ops[2] = CGF.
Builder.CreateExtractElement(Ops[2], (uint64_t)0);
14699 switch (Ops[0]->getType()->getPrimitiveSizeInBits()) {
14701 IID = Intrinsic::x86_avx512fp16_vfmadd_f16;
14704 IID = Intrinsic::x86_avx512_vfmadd_f32;
14707 IID = Intrinsic::x86_avx512_vfmadd_f64;
14710 llvm_unreachable(
"Unexpected size");
14713 {Ops[0], Ops[1], Ops[2], Ops[4]});
14714 }
else if (CGF.
Builder.getIsFPConstrained()) {
14715 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF,
E);
14717 Intrinsic::experimental_constrained_fma, Ops[0]->getType());
14718 Res = CGF.
Builder.CreateConstrainedFPCall(FMA, Ops.slice(0, 3));
14721 Res = CGF.
Builder.CreateCall(FMA, Ops.slice(0, 3));
14724 if (Ops.size() > 3) {
14725 Value *PassThru = ZeroMask ? Constant::getNullValue(Res->
getType())
14731 if (NegAcc && PTIdx == 2)
14732 PassThru = CGF.
Builder.CreateExtractElement(Upper, (uint64_t)0);
14736 return CGF.
Builder.CreateInsertElement(Upper, Res, (uint64_t)0);
14741 llvm::Type *Ty = Ops[0]->getType();
14743 Ty = llvm::FixedVectorType::get(CGF.
Int64Ty,
14744 Ty->getPrimitiveSizeInBits() / 64);
14750 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
14751 LHS = CGF.
Builder.CreateShl(LHS, ShiftAmt);
14752 LHS = CGF.
Builder.CreateAShr(LHS, ShiftAmt);
14753 RHS = CGF.
Builder.CreateShl(RHS, ShiftAmt);
14754 RHS = CGF.
Builder.CreateAShr(RHS, ShiftAmt);
14757 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
14758 LHS = CGF.
Builder.CreateAnd(LHS, Mask);
14759 RHS = CGF.
Builder.CreateAnd(RHS, Mask);
14762 return CGF.
Builder.CreateMul(LHS, RHS);
14770 llvm::Type *Ty = Ops[0]->getType();
14772 unsigned VecWidth = Ty->getPrimitiveSizeInBits();
14773 unsigned EltWidth = Ty->getScalarSizeInBits();
14775 if (VecWidth == 128 && EltWidth == 32)
14776 IID = Intrinsic::x86_avx512_pternlog_d_128;
14777 else if (VecWidth == 256 && EltWidth == 32)
14778 IID = Intrinsic::x86_avx512_pternlog_d_256;
14779 else if (VecWidth == 512 && EltWidth == 32)
14780 IID = Intrinsic::x86_avx512_pternlog_d_512;
14781 else if (VecWidth == 128 && EltWidth == 64)
14782 IID = Intrinsic::x86_avx512_pternlog_q_128;
14783 else if (VecWidth == 256 && EltWidth == 64)
14784 IID = Intrinsic::x86_avx512_pternlog_q_256;
14785 else if (VecWidth == 512 && EltWidth == 64)
14786 IID = Intrinsic::x86_avx512_pternlog_q_512;
14788 llvm_unreachable(
"Unexpected intrinsic");
14792 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty) : Ops[0];
14797 llvm::Type *DstTy) {
14798 unsigned NumberOfElements =
14799 cast<llvm::FixedVectorType>(DstTy)->getNumElements();
14801 return CGF.
Builder.CreateSExt(Mask, DstTy,
"vpmovm2");
14806 StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
14807 return EmitX86CpuIs(CPUStr);
14813 llvm::Type *DstTy) {
14814 assert((Ops.size() == 1 || Ops.size() == 3 || Ops.size() == 4) &&
14815 "Unknown cvtph2ps intrinsic");
14818 if (Ops.size() == 4 && cast<llvm::ConstantInt>(Ops[3])->getZExtValue() != 4) {
14821 return CGF.
Builder.CreateCall(F, {Ops[0], Ops[1], Ops[2], Ops[3]});
14824 unsigned NumDstElts = cast<llvm::FixedVectorType>(DstTy)->getNumElements();
14825 Value *Src = Ops[0];
14829 cast<llvm::FixedVectorType>(Src->
getType())->getNumElements()) {
14830 assert(NumDstElts == 4 &&
"Unexpected vector size");
14835 auto *HalfTy = llvm::FixedVectorType::get(
14837 Src = CGF.
Builder.CreateBitCast(Src, HalfTy);
14840 Value *Res = CGF.
Builder.CreateFPExt(Src, DstTy,
"cvtph2ps");
14842 if (Ops.size() >= 3)
14847Value *CodeGenFunction::EmitX86CpuIs(StringRef CPUStr) {
14858 llvm::ArrayType::get(
Int32Ty, 1));
14862 cast<llvm::GlobalValue>(CpuModel)->setDSOLocal(
true);
14868 std::tie(Index,
Value) = StringSwitch<std::pair<unsigned, unsigned>>(CPUStr)
14870 .Case(STRING, {0u,
static_cast<unsigned>(llvm::X86::ENUM)})
14872 .Case(
ALIAS, {1u,
static_cast<unsigned>(llvm::X86::ENUM)})
14874 .Case(STR, {1u,
static_cast<unsigned>(llvm::X86::ENUM)})
14876 .Case(
ALIAS, {2u,
static_cast<unsigned>(llvm::X86::ENUM)})
14878 .Case(STR, {2u,
static_cast<unsigned>(llvm::X86::ENUM)})
14879#include
"llvm/TargetParser/X86TargetParser.def"
14881 assert(
Value != 0 &&
"Invalid CPUStr passed to CpuIs");
14884 llvm::Value *Idxs[] = {ConstantInt::get(
Int32Ty, 0),
14885 ConstantInt::get(
Int32Ty, Index)};
14891 return Builder.CreateICmpEQ(CpuValue,
14897 StringRef FeatureStr = cast<StringLiteral>(FeatureExpr)->getString();
14898 if (!
getContext().getTargetInfo().validateCpuSupports(FeatureStr))
14900 return EmitX86CpuSupports(FeatureStr);
14904 return EmitX86CpuSupports(llvm::X86::getCpuSupportsMask(FeatureStrs));
14908CodeGenFunction::EmitX86CpuSupports(std::array<uint32_t, 4> FeatureMask) {
14910 if (FeatureMask[0] != 0) {
14918 llvm::ArrayType::get(
Int32Ty, 1));
14922 cast<llvm::GlobalValue>(CpuModel)->setDSOLocal(
true);
14939 llvm::Type *ATy = llvm::ArrayType::get(
Int32Ty, 3);
14940 llvm::Constant *CpuFeatures2 =
14942 cast<llvm::GlobalValue>(CpuFeatures2)->setDSOLocal(
true);
14943 for (
int i = 1; i != 4; ++i) {
14944 const uint32_t M = FeatureMask[i];
14961Value *CodeGenFunction::EmitAArch64CpuInit() {
14962 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy,
false);
14963 llvm::FunctionCallee
Func =
14965 cast<llvm::GlobalValue>(
Func.getCallee())->setDSOLocal(
true);
14966 cast<llvm::GlobalValue>(
Func.getCallee())
14967 ->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
14972 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy, {
VoidPtrTy},
false);
14973 llvm::FunctionCallee
Func =
14975 auto *CalleeGV = cast<llvm::GlobalValue>(
Func.getCallee());
14976 CalleeGV->setDSOLocal(
true);
14977 CalleeGV->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
14981Value *CodeGenFunction::EmitX86CpuInit() {
14982 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy,
14984 llvm::FunctionCallee
Func =
14986 cast<llvm::GlobalValue>(
Func.getCallee())->setDSOLocal(
true);
14987 cast<llvm::GlobalValue>(
Func.getCallee())
14988 ->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
14992Value *CodeGenFunction::EmitAArch64CpuSupports(
const CallExpr *
E) {
14994 StringRef ArgStr = cast<StringLiteral>(ArgExpr)->getString();
14996 ArgStr.split(Features,
"+");
14997 for (
auto &Feature : Features) {
14998 Feature = Feature.trim();
14999 if (!llvm::AArch64::parseFMVExtension(Feature))
15001 if (Feature !=
"default")
15002 Features.push_back(Feature);
15004 return EmitAArch64CpuSupports(Features);
15009 uint64_t FeaturesMask = llvm::AArch64::getCpuSupportsMask(FeaturesStrs);
15011 if (FeaturesMask != 0) {
15016 llvm::Type *STy = llvm::StructType::get(
Int64Ty);
15017 llvm::Constant *AArch64CPUFeatures =
15019 cast<llvm::GlobalValue>(AArch64CPUFeatures)->setDSOLocal(
true);
15021 STy, AArch64CPUFeatures,
15036 StringRef FeatureStr = cast<StringLiteral>(FeatureExpr)->getString();
15037 if (!
getContext().getTargetInfo().validateCpuSupports(FeatureStr))
15045 llvm::Type *Int32Ty = Builder.getInt32Ty();
15046 llvm::Type *Int64Ty = Builder.getInt64Ty();
15047 llvm::ArrayType *ArrayOfInt64Ty =
15048 llvm::ArrayType::get(Int64Ty, llvm::RISCVISAInfo::FeatureBitSize);
15049 llvm::Type *StructTy = llvm::StructType::get(Int32Ty, ArrayOfInt64Ty);
15050 llvm::Constant *RISCVFeaturesBits =
15052 cast<llvm::GlobalValue>(RISCVFeaturesBits)->setDSOLocal(
true);
15053 Value *IndexVal = llvm::ConstantInt::get(Int32Ty, Index);
15054 llvm::Value *GEPIndices[] = {Builder.getInt32(0), Builder.getInt32(1),
15057 Builder.CreateInBoundsGEP(StructTy, RISCVFeaturesBits, GEPIndices);
15058 Value *FeaturesBit =
15060 return FeaturesBit;
15064 const unsigned RISCVFeatureLength = llvm::RISCVISAInfo::FeatureBitSize;
15065 uint64_t RequireBitMasks[RISCVFeatureLength] = {0};
15067 for (
auto Feat : FeaturesStrs) {
15068 auto [GroupID, BitPos] = RISCVISAInfo::getRISCVFeaturesBitsInfo(Feat);
15075 RequireBitMasks[GroupID] |= (1ULL << BitPos);
15079 for (
unsigned Idx = 0; Idx < RISCVFeatureLength; Idx++) {
15080 if (RequireBitMasks[Idx] == 0)
15090 assert(
Result &&
"Should have value here.");
15097 if (BuiltinID == Builtin::BI__builtin_cpu_is)
15098 return EmitX86CpuIs(
E);
15099 if (BuiltinID == Builtin::BI__builtin_cpu_supports)
15100 return EmitX86CpuSupports(
E);
15101 if (BuiltinID == Builtin::BI__builtin_cpu_init)
15102 return EmitX86CpuInit();
15110 bool IsMaskFCmp =
false;
15111 bool IsConjFMA =
false;
15114 unsigned ICEArguments = 0;
15119 for (
unsigned i = 0, e =
E->getNumArgs(); i != e; i++) {
15129 auto getCmpIntrinsicCall = [
this, &Ops](Intrinsic::ID
ID,
unsigned Imm) {
15130 Ops.push_back(llvm::ConstantInt::get(
Int8Ty, Imm));
15132 return Builder.CreateCall(F, Ops);
15140 auto getVectorFCmpIR = [
this, &Ops,
E](CmpInst::Predicate Pred,
15141 bool IsSignaling) {
15142 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
15145 Cmp =
Builder.CreateFCmpS(Pred, Ops[0], Ops[1]);
15147 Cmp =
Builder.CreateFCmp(Pred, Ops[0], Ops[1]);
15148 llvm::VectorType *FPVecTy = cast<llvm::VectorType>(Ops[0]->getType());
15149 llvm::VectorType *IntVecTy = llvm::VectorType::getInteger(FPVecTy);
15151 return Builder.CreateBitCast(Sext, FPVecTy);
15154 switch (BuiltinID) {
15155 default:
return nullptr;
15156 case X86::BI_mm_prefetch: {
15158 ConstantInt *
C = cast<ConstantInt>(Ops[1]);
15159 Value *RW = ConstantInt::get(
Int32Ty, (
C->getZExtValue() >> 2) & 0x1);
15160 Value *Locality = ConstantInt::get(
Int32Ty,
C->getZExtValue() & 0x3);
15165 case X86::BI_mm_clflush: {
15169 case X86::BI_mm_lfence: {
15172 case X86::BI_mm_mfence: {
15175 case X86::BI_mm_sfence: {
15178 case X86::BI_mm_pause: {
15181 case X86::BI__rdtsc: {
15184 case X86::BI__builtin_ia32_rdtscp: {
15190 case X86::BI__builtin_ia32_lzcnt_u16:
15191 case X86::BI__builtin_ia32_lzcnt_u32:
15192 case X86::BI__builtin_ia32_lzcnt_u64: {
15196 case X86::BI__builtin_ia32_tzcnt_u16:
15197 case X86::BI__builtin_ia32_tzcnt_u32:
15198 case X86::BI__builtin_ia32_tzcnt_u64: {
15202 case X86::BI__builtin_ia32_undef128:
15203 case X86::BI__builtin_ia32_undef256:
15204 case X86::BI__builtin_ia32_undef512:
15211 case X86::BI__builtin_ia32_vec_ext_v4hi:
15212 case X86::BI__builtin_ia32_vec_ext_v16qi:
15213 case X86::BI__builtin_ia32_vec_ext_v8hi:
15214 case X86::BI__builtin_ia32_vec_ext_v4si:
15215 case X86::BI__builtin_ia32_vec_ext_v4sf:
15216 case X86::BI__builtin_ia32_vec_ext_v2di:
15217 case X86::BI__builtin_ia32_vec_ext_v32qi:
15218 case X86::BI__builtin_ia32_vec_ext_v16hi:
15219 case X86::BI__builtin_ia32_vec_ext_v8si:
15220 case X86::BI__builtin_ia32_vec_ext_v4di: {
15222 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15223 uint64_t Index = cast<ConstantInt>(Ops[1])->getZExtValue();
15224 Index &= NumElts - 1;
15227 return Builder.CreateExtractElement(Ops[0], Index);
15229 case X86::BI__builtin_ia32_vec_set_v4hi:
15230 case X86::BI__builtin_ia32_vec_set_v16qi:
15231 case X86::BI__builtin_ia32_vec_set_v8hi:
15232 case X86::BI__builtin_ia32_vec_set_v4si:
15233 case X86::BI__builtin_ia32_vec_set_v2di:
15234 case X86::BI__builtin_ia32_vec_set_v32qi:
15235 case X86::BI__builtin_ia32_vec_set_v16hi:
15236 case X86::BI__builtin_ia32_vec_set_v8si:
15237 case X86::BI__builtin_ia32_vec_set_v4di: {
15239 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15240 unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue();
15241 Index &= NumElts - 1;
15244 return Builder.CreateInsertElement(Ops[0], Ops[1], Index);
15246 case X86::BI_mm_setcsr:
15247 case X86::BI__builtin_ia32_ldmxcsr: {
15253 case X86::BI_mm_getcsr:
15254 case X86::BI__builtin_ia32_stmxcsr: {
15260 case X86::BI__builtin_ia32_xsave:
15261 case X86::BI__builtin_ia32_xsave64:
15262 case X86::BI__builtin_ia32_xrstor:
15263 case X86::BI__builtin_ia32_xrstor64:
15264 case X86::BI__builtin_ia32_xsaveopt:
15265 case X86::BI__builtin_ia32_xsaveopt64:
15266 case X86::BI__builtin_ia32_xrstors:
15267 case X86::BI__builtin_ia32_xrstors64:
15268 case X86::BI__builtin_ia32_xsavec:
15269 case X86::BI__builtin_ia32_xsavec64:
15270 case X86::BI__builtin_ia32_xsaves:
15271 case X86::BI__builtin_ia32_xsaves64:
15272 case X86::BI__builtin_ia32_xsetbv:
15273 case X86::BI_xsetbv: {
15275#define INTRINSIC_X86_XSAVE_ID(NAME) \
15276 case X86::BI__builtin_ia32_##NAME: \
15277 ID = Intrinsic::x86_##NAME; \
15279 switch (BuiltinID) {
15280 default: llvm_unreachable(
"Unsupported intrinsic!");
15294 case X86::BI_xsetbv:
15295 ID = Intrinsic::x86_xsetbv;
15298#undef INTRINSIC_X86_XSAVE_ID
15303 Ops.push_back(Mlo);
15306 case X86::BI__builtin_ia32_xgetbv:
15307 case X86::BI_xgetbv:
15309 case X86::BI__builtin_ia32_storedqudi128_mask:
15310 case X86::BI__builtin_ia32_storedqusi128_mask:
15311 case X86::BI__builtin_ia32_storedquhi128_mask:
15312 case X86::BI__builtin_ia32_storedquqi128_mask:
15313 case X86::BI__builtin_ia32_storeupd128_mask:
15314 case X86::BI__builtin_ia32_storeups128_mask:
15315 case X86::BI__builtin_ia32_storedqudi256_mask:
15316 case X86::BI__builtin_ia32_storedqusi256_mask:
15317 case X86::BI__builtin_ia32_storedquhi256_mask:
15318 case X86::BI__builtin_ia32_storedquqi256_mask:
15319 case X86::BI__builtin_ia32_storeupd256_mask:
15320 case X86::BI__builtin_ia32_storeups256_mask:
15321 case X86::BI__builtin_ia32_storedqudi512_mask:
15322 case X86::BI__builtin_ia32_storedqusi512_mask:
15323 case X86::BI__builtin_ia32_storedquhi512_mask:
15324 case X86::BI__builtin_ia32_storedquqi512_mask:
15325 case X86::BI__builtin_ia32_storeupd512_mask:
15326 case X86::BI__builtin_ia32_storeups512_mask:
15329 case X86::BI__builtin_ia32_storesbf16128_mask:
15330 case X86::BI__builtin_ia32_storesh128_mask:
15331 case X86::BI__builtin_ia32_storess128_mask:
15332 case X86::BI__builtin_ia32_storesd128_mask:
15335 case X86::BI__builtin_ia32_cvtmask2b128:
15336 case X86::BI__builtin_ia32_cvtmask2b256:
15337 case X86::BI__builtin_ia32_cvtmask2b512:
15338 case X86::BI__builtin_ia32_cvtmask2w128:
15339 case X86::BI__builtin_ia32_cvtmask2w256:
15340 case X86::BI__builtin_ia32_cvtmask2w512:
15341 case X86::BI__builtin_ia32_cvtmask2d128:
15342 case X86::BI__builtin_ia32_cvtmask2d256:
15343 case X86::BI__builtin_ia32_cvtmask2d512:
15344 case X86::BI__builtin_ia32_cvtmask2q128:
15345 case X86::BI__builtin_ia32_cvtmask2q256:
15346 case X86::BI__builtin_ia32_cvtmask2q512:
15349 case X86::BI__builtin_ia32_cvtb2mask128:
15350 case X86::BI__builtin_ia32_cvtb2mask256:
15351 case X86::BI__builtin_ia32_cvtb2mask512:
15352 case X86::BI__builtin_ia32_cvtw2mask128:
15353 case X86::BI__builtin_ia32_cvtw2mask256:
15354 case X86::BI__builtin_ia32_cvtw2mask512:
15355 case X86::BI__builtin_ia32_cvtd2mask128:
15356 case X86::BI__builtin_ia32_cvtd2mask256:
15357 case X86::BI__builtin_ia32_cvtd2mask512:
15358 case X86::BI__builtin_ia32_cvtq2mask128:
15359 case X86::BI__builtin_ia32_cvtq2mask256:
15360 case X86::BI__builtin_ia32_cvtq2mask512:
15363 case X86::BI__builtin_ia32_cvtdq2ps512_mask:
15364 case X86::BI__builtin_ia32_cvtqq2ps512_mask:
15365 case X86::BI__builtin_ia32_cvtqq2pd512_mask:
15366 case X86::BI__builtin_ia32_vcvtw2ph512_mask:
15367 case X86::BI__builtin_ia32_vcvtdq2ph512_mask:
15368 case X86::BI__builtin_ia32_vcvtqq2ph512_mask:
15369 case X86::BI__builtin_ia32_vcvtdq2ph256_round_mask:
15370 case X86::BI__builtin_ia32_vcvtdq2ps256_round_mask:
15371 case X86::BI__builtin_ia32_vcvtqq2pd256_round_mask:
15372 case X86::BI__builtin_ia32_vcvtqq2ph256_round_mask:
15373 case X86::BI__builtin_ia32_vcvtqq2ps256_round_mask:
15374 case X86::BI__builtin_ia32_vcvtw2ph256_round_mask:
15376 case X86::BI__builtin_ia32_cvtudq2ps512_mask:
15377 case X86::BI__builtin_ia32_cvtuqq2ps512_mask:
15378 case X86::BI__builtin_ia32_cvtuqq2pd512_mask:
15379 case X86::BI__builtin_ia32_vcvtuw2ph512_mask:
15380 case X86::BI__builtin_ia32_vcvtudq2ph512_mask:
15381 case X86::BI__builtin_ia32_vcvtuqq2ph512_mask:
15382 case X86::BI__builtin_ia32_vcvtudq2ph256_round_mask:
15383 case X86::BI__builtin_ia32_vcvtudq2ps256_round_mask:
15384 case X86::BI__builtin_ia32_vcvtuqq2pd256_round_mask:
15385 case X86::BI__builtin_ia32_vcvtuqq2ph256_round_mask:
15386 case X86::BI__builtin_ia32_vcvtuqq2ps256_round_mask:
15387 case X86::BI__builtin_ia32_vcvtuw2ph256_round_mask:
15390 case X86::BI__builtin_ia32_vfmaddss3:
15391 case X86::BI__builtin_ia32_vfmaddsd3:
15392 case X86::BI__builtin_ia32_vfmaddsh3_mask:
15393 case X86::BI__builtin_ia32_vfmaddss3_mask:
15394 case X86::BI__builtin_ia32_vfmaddsd3_mask:
15396 case X86::BI__builtin_ia32_vfmaddss:
15397 case X86::BI__builtin_ia32_vfmaddsd:
15399 Constant::getNullValue(Ops[0]->getType()));
15400 case X86::BI__builtin_ia32_vfmaddsh3_maskz:
15401 case X86::BI__builtin_ia32_vfmaddss3_maskz:
15402 case X86::BI__builtin_ia32_vfmaddsd3_maskz:
15404 case X86::BI__builtin_ia32_vfmaddsh3_mask3:
15405 case X86::BI__builtin_ia32_vfmaddss3_mask3:
15406 case X86::BI__builtin_ia32_vfmaddsd3_mask3:
15408 case X86::BI__builtin_ia32_vfmsubsh3_mask3:
15409 case X86::BI__builtin_ia32_vfmsubss3_mask3:
15410 case X86::BI__builtin_ia32_vfmsubsd3_mask3:
15413 case X86::BI__builtin_ia32_vfmaddph:
15414 case X86::BI__builtin_ia32_vfmaddps:
15415 case X86::BI__builtin_ia32_vfmaddpd:
15416 case X86::BI__builtin_ia32_vfmaddph256:
15417 case X86::BI__builtin_ia32_vfmaddps256:
15418 case X86::BI__builtin_ia32_vfmaddpd256:
15419 case X86::BI__builtin_ia32_vfmaddph512_mask:
15420 case X86::BI__builtin_ia32_vfmaddph512_maskz:
15421 case X86::BI__builtin_ia32_vfmaddph512_mask3:
15422 case X86::BI__builtin_ia32_vfmaddnepbh128:
15423 case X86::BI__builtin_ia32_vfmaddnepbh256:
15424 case X86::BI__builtin_ia32_vfmaddnepbh512:
15425 case X86::BI__builtin_ia32_vfmaddps512_mask:
15426 case X86::BI__builtin_ia32_vfmaddps512_maskz:
15427 case X86::BI__builtin_ia32_vfmaddps512_mask3:
15428 case X86::BI__builtin_ia32_vfmsubps512_mask3:
15429 case X86::BI__builtin_ia32_vfmaddpd512_mask:
15430 case X86::BI__builtin_ia32_vfmaddpd512_maskz:
15431 case X86::BI__builtin_ia32_vfmaddpd512_mask3:
15432 case X86::BI__builtin_ia32_vfmsubpd512_mask3:
15433 case X86::BI__builtin_ia32_vfmsubph512_mask3:
15434 case X86::BI__builtin_ia32_vfmaddph256_round_mask:
15435 case X86::BI__builtin_ia32_vfmaddph256_round_maskz:
15436 case X86::BI__builtin_ia32_vfmaddph256_round_mask3:
15437 case X86::BI__builtin_ia32_vfmaddps256_round_mask:
15438 case X86::BI__builtin_ia32_vfmaddps256_round_maskz:
15439 case X86::BI__builtin_ia32_vfmaddps256_round_mask3:
15440 case X86::BI__builtin_ia32_vfmsubps256_round_mask3:
15441 case X86::BI__builtin_ia32_vfmaddpd256_round_mask:
15442 case X86::BI__builtin_ia32_vfmaddpd256_round_maskz:
15443 case X86::BI__builtin_ia32_vfmaddpd256_round_mask3:
15444 case X86::BI__builtin_ia32_vfmsubpd256_round_mask3:
15445 case X86::BI__builtin_ia32_vfmsubph256_round_mask3:
15447 case X86::BI__builtin_ia32_vfmaddsubph512_mask:
15448 case X86::BI__builtin_ia32_vfmaddsubph512_maskz:
15449 case X86::BI__builtin_ia32_vfmaddsubph512_mask3:
15450 case X86::BI__builtin_ia32_vfmsubaddph512_mask3:
15451 case X86::BI__builtin_ia32_vfmaddsubps512_mask:
15452 case X86::BI__builtin_ia32_vfmaddsubps512_maskz:
15453 case X86::BI__builtin_ia32_vfmaddsubps512_mask3:
15454 case X86::BI__builtin_ia32_vfmsubaddps512_mask3:
15455 case X86::BI__builtin_ia32_vfmaddsubpd512_mask:
15456 case X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
15457 case X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
15458 case X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
15459 case X86::BI__builtin_ia32_vfmaddsubph256_round_mask:
15460 case X86::BI__builtin_ia32_vfmaddsubph256_round_maskz:
15461 case X86::BI__builtin_ia32_vfmaddsubph256_round_mask3:
15462 case X86::BI__builtin_ia32_vfmsubaddph256_round_mask3:
15463 case X86::BI__builtin_ia32_vfmaddsubps256_round_mask:
15464 case X86::BI__builtin_ia32_vfmaddsubps256_round_maskz:
15465 case X86::BI__builtin_ia32_vfmaddsubps256_round_mask3:
15466 case X86::BI__builtin_ia32_vfmsubaddps256_round_mask3:
15467 case X86::BI__builtin_ia32_vfmaddsubpd256_round_mask:
15468 case X86::BI__builtin_ia32_vfmaddsubpd256_round_maskz:
15469 case X86::BI__builtin_ia32_vfmaddsubpd256_round_mask3:
15470 case X86::BI__builtin_ia32_vfmsubaddpd256_round_mask3:
15473 case X86::BI__builtin_ia32_movdqa32store128_mask:
15474 case X86::BI__builtin_ia32_movdqa64store128_mask:
15475 case X86::BI__builtin_ia32_storeaps128_mask:
15476 case X86::BI__builtin_ia32_storeapd128_mask:
15477 case X86::BI__builtin_ia32_movdqa32store256_mask:
15478 case X86::BI__builtin_ia32_movdqa64store256_mask:
15479 case X86::BI__builtin_ia32_storeaps256_mask:
15480 case X86::BI__builtin_ia32_storeapd256_mask:
15481 case X86::BI__builtin_ia32_movdqa32store512_mask:
15482 case X86::BI__builtin_ia32_movdqa64store512_mask:
15483 case X86::BI__builtin_ia32_storeaps512_mask:
15484 case X86::BI__builtin_ia32_storeapd512_mask:
15489 case X86::BI__builtin_ia32_loadups128_mask:
15490 case X86::BI__builtin_ia32_loadups256_mask:
15491 case X86::BI__builtin_ia32_loadups512_mask:
15492 case X86::BI__builtin_ia32_loadupd128_mask:
15493 case X86::BI__builtin_ia32_loadupd256_mask:
15494 case X86::BI__builtin_ia32_loadupd512_mask:
15495 case X86::BI__builtin_ia32_loaddquqi128_mask:
15496 case X86::BI__builtin_ia32_loaddquqi256_mask:
15497 case X86::BI__builtin_ia32_loaddquqi512_mask:
15498 case X86::BI__builtin_ia32_loaddquhi128_mask:
15499 case X86::BI__builtin_ia32_loaddquhi256_mask:
15500 case X86::BI__builtin_ia32_loaddquhi512_mask:
15501 case X86::BI__builtin_ia32_loaddqusi128_mask:
15502 case X86::BI__builtin_ia32_loaddqusi256_mask:
15503 case X86::BI__builtin_ia32_loaddqusi512_mask:
15504 case X86::BI__builtin_ia32_loaddqudi128_mask:
15505 case X86::BI__builtin_ia32_loaddqudi256_mask:
15506 case X86::BI__builtin_ia32_loaddqudi512_mask:
15509 case X86::BI__builtin_ia32_loadsbf16128_mask:
15510 case X86::BI__builtin_ia32_loadsh128_mask:
15511 case X86::BI__builtin_ia32_loadss128_mask:
15512 case X86::BI__builtin_ia32_loadsd128_mask:
15515 case X86::BI__builtin_ia32_loadaps128_mask:
15516 case X86::BI__builtin_ia32_loadaps256_mask:
15517 case X86::BI__builtin_ia32_loadaps512_mask:
15518 case X86::BI__builtin_ia32_loadapd128_mask:
15519 case X86::BI__builtin_ia32_loadapd256_mask:
15520 case X86::BI__builtin_ia32_loadapd512_mask:
15521 case X86::BI__builtin_ia32_movdqa32load128_mask:
15522 case X86::BI__builtin_ia32_movdqa32load256_mask:
15523 case X86::BI__builtin_ia32_movdqa32load512_mask:
15524 case X86::BI__builtin_ia32_movdqa64load128_mask:
15525 case X86::BI__builtin_ia32_movdqa64load256_mask:
15526 case X86::BI__builtin_ia32_movdqa64load512_mask:
15531 case X86::BI__builtin_ia32_expandloaddf128_mask:
15532 case X86::BI__builtin_ia32_expandloaddf256_mask:
15533 case X86::BI__builtin_ia32_expandloaddf512_mask:
15534 case X86::BI__builtin_ia32_expandloadsf128_mask:
15535 case X86::BI__builtin_ia32_expandloadsf256_mask:
15536 case X86::BI__builtin_ia32_expandloadsf512_mask:
15537 case X86::BI__builtin_ia32_expandloaddi128_mask:
15538 case X86::BI__builtin_ia32_expandloaddi256_mask:
15539 case X86::BI__builtin_ia32_expandloaddi512_mask:
15540 case X86::BI__builtin_ia32_expandloadsi128_mask:
15541 case X86::BI__builtin_ia32_expandloadsi256_mask:
15542 case X86::BI__builtin_ia32_expandloadsi512_mask:
15543 case X86::BI__builtin_ia32_expandloadhi128_mask:
15544 case X86::BI__builtin_ia32_expandloadhi256_mask:
15545 case X86::BI__builtin_ia32_expandloadhi512_mask:
15546 case X86::BI__builtin_ia32_expandloadqi128_mask:
15547 case X86::BI__builtin_ia32_expandloadqi256_mask:
15548 case X86::BI__builtin_ia32_expandloadqi512_mask:
15551 case X86::BI__builtin_ia32_compressstoredf128_mask:
15552 case X86::BI__builtin_ia32_compressstoredf256_mask:
15553 case X86::BI__builtin_ia32_compressstoredf512_mask:
15554 case X86::BI__builtin_ia32_compressstoresf128_mask:
15555 case X86::BI__builtin_ia32_compressstoresf256_mask:
15556 case X86::BI__builtin_ia32_compressstoresf512_mask:
15557 case X86::BI__builtin_ia32_compressstoredi128_mask:
15558 case X86::BI__builtin_ia32_compressstoredi256_mask:
15559 case X86::BI__builtin_ia32_compressstoredi512_mask:
15560 case X86::BI__builtin_ia32_compressstoresi128_mask:
15561 case X86::BI__builtin_ia32_compressstoresi256_mask:
15562 case X86::BI__builtin_ia32_compressstoresi512_mask:
15563 case X86::BI__builtin_ia32_compressstorehi128_mask:
15564 case X86::BI__builtin_ia32_compressstorehi256_mask:
15565 case X86::BI__builtin_ia32_compressstorehi512_mask:
15566 case X86::BI__builtin_ia32_compressstoreqi128_mask:
15567 case X86::BI__builtin_ia32_compressstoreqi256_mask:
15568 case X86::BI__builtin_ia32_compressstoreqi512_mask:
15571 case X86::BI__builtin_ia32_expanddf128_mask:
15572 case X86::BI__builtin_ia32_expanddf256_mask:
15573 case X86::BI__builtin_ia32_expanddf512_mask:
15574 case X86::BI__builtin_ia32_expandsf128_mask:
15575 case X86::BI__builtin_ia32_expandsf256_mask:
15576 case X86::BI__builtin_ia32_expandsf512_mask:
15577 case X86::BI__builtin_ia32_expanddi128_mask:
15578 case X86::BI__builtin_ia32_expanddi256_mask:
15579 case X86::BI__builtin_ia32_expanddi512_mask:
15580 case X86::BI__builtin_ia32_expandsi128_mask:
15581 case X86::BI__builtin_ia32_expandsi256_mask:
15582 case X86::BI__builtin_ia32_expandsi512_mask:
15583 case X86::BI__builtin_ia32_expandhi128_mask:
15584 case X86::BI__builtin_ia32_expandhi256_mask:
15585 case X86::BI__builtin_ia32_expandhi512_mask:
15586 case X86::BI__builtin_ia32_expandqi128_mask:
15587 case X86::BI__builtin_ia32_expandqi256_mask:
15588 case X86::BI__builtin_ia32_expandqi512_mask:
15591 case X86::BI__builtin_ia32_compressdf128_mask:
15592 case X86::BI__builtin_ia32_compressdf256_mask:
15593 case X86::BI__builtin_ia32_compressdf512_mask:
15594 case X86::BI__builtin_ia32_compresssf128_mask:
15595 case X86::BI__builtin_ia32_compresssf256_mask:
15596 case X86::BI__builtin_ia32_compresssf512_mask:
15597 case X86::BI__builtin_ia32_compressdi128_mask:
15598 case X86::BI__builtin_ia32_compressdi256_mask:
15599 case X86::BI__builtin_ia32_compressdi512_mask:
15600 case X86::BI__builtin_ia32_compresssi128_mask:
15601 case X86::BI__builtin_ia32_compresssi256_mask:
15602 case X86::BI__builtin_ia32_compresssi512_mask:
15603 case X86::BI__builtin_ia32_compresshi128_mask:
15604 case X86::BI__builtin_ia32_compresshi256_mask:
15605 case X86::BI__builtin_ia32_compresshi512_mask:
15606 case X86::BI__builtin_ia32_compressqi128_mask:
15607 case X86::BI__builtin_ia32_compressqi256_mask:
15608 case X86::BI__builtin_ia32_compressqi512_mask:
15611 case X86::BI__builtin_ia32_gather3div2df:
15612 case X86::BI__builtin_ia32_gather3div2di:
15613 case X86::BI__builtin_ia32_gather3div4df:
15614 case X86::BI__builtin_ia32_gather3div4di:
15615 case X86::BI__builtin_ia32_gather3div4sf:
15616 case X86::BI__builtin_ia32_gather3div4si:
15617 case X86::BI__builtin_ia32_gather3div8sf:
15618 case X86::BI__builtin_ia32_gather3div8si:
15619 case X86::BI__builtin_ia32_gather3siv2df:
15620 case X86::BI__builtin_ia32_gather3siv2di:
15621 case X86::BI__builtin_ia32_gather3siv4df:
15622 case X86::BI__builtin_ia32_gather3siv4di:
15623 case X86::BI__builtin_ia32_gather3siv4sf:
15624 case X86::BI__builtin_ia32_gather3siv4si:
15625 case X86::BI__builtin_ia32_gather3siv8sf:
15626 case X86::BI__builtin_ia32_gather3siv8si:
15627 case X86::BI__builtin_ia32_gathersiv8df:
15628 case X86::BI__builtin_ia32_gathersiv16sf:
15629 case X86::BI__builtin_ia32_gatherdiv8df:
15630 case X86::BI__builtin_ia32_gatherdiv16sf:
15631 case X86::BI__builtin_ia32_gathersiv8di:
15632 case X86::BI__builtin_ia32_gathersiv16si:
15633 case X86::BI__builtin_ia32_gatherdiv8di:
15634 case X86::BI__builtin_ia32_gatherdiv16si: {
15636 switch (BuiltinID) {
15637 default: llvm_unreachable(
"Unexpected builtin");
15638 case X86::BI__builtin_ia32_gather3div2df:
15639 IID = Intrinsic::x86_avx512_mask_gather3div2_df;
15641 case X86::BI__builtin_ia32_gather3div2di:
15642 IID = Intrinsic::x86_avx512_mask_gather3div2_di;
15644 case X86::BI__builtin_ia32_gather3div4df:
15645 IID = Intrinsic::x86_avx512_mask_gather3div4_df;
15647 case X86::BI__builtin_ia32_gather3div4di:
15648 IID = Intrinsic::x86_avx512_mask_gather3div4_di;
15650 case X86::BI__builtin_ia32_gather3div4sf:
15651 IID = Intrinsic::x86_avx512_mask_gather3div4_sf;
15653 case X86::BI__builtin_ia32_gather3div4si:
15654 IID = Intrinsic::x86_avx512_mask_gather3div4_si;
15656 case X86::BI__builtin_ia32_gather3div8sf:
15657 IID = Intrinsic::x86_avx512_mask_gather3div8_sf;
15659 case X86::BI__builtin_ia32_gather3div8si:
15660 IID = Intrinsic::x86_avx512_mask_gather3div8_si;
15662 case X86::BI__builtin_ia32_gather3siv2df:
15663 IID = Intrinsic::x86_avx512_mask_gather3siv2_df;
15665 case X86::BI__builtin_ia32_gather3siv2di:
15666 IID = Intrinsic::x86_avx512_mask_gather3siv2_di;
15668 case X86::BI__builtin_ia32_gather3siv4df:
15669 IID = Intrinsic::x86_avx512_mask_gather3siv4_df;
15671 case X86::BI__builtin_ia32_gather3siv4di:
15672 IID = Intrinsic::x86_avx512_mask_gather3siv4_di;
15674 case X86::BI__builtin_ia32_gather3siv4sf:
15675 IID = Intrinsic::x86_avx512_mask_gather3siv4_sf;
15677 case X86::BI__builtin_ia32_gather3siv4si:
15678 IID = Intrinsic::x86_avx512_mask_gather3siv4_si;
15680 case X86::BI__builtin_ia32_gather3siv8sf:
15681 IID = Intrinsic::x86_avx512_mask_gather3siv8_sf;
15683 case X86::BI__builtin_ia32_gather3siv8si:
15684 IID = Intrinsic::x86_avx512_mask_gather3siv8_si;
15686 case X86::BI__builtin_ia32_gathersiv8df:
15687 IID = Intrinsic::x86_avx512_mask_gather_dpd_512;
15689 case X86::BI__builtin_ia32_gathersiv16sf:
15690 IID = Intrinsic::x86_avx512_mask_gather_dps_512;
15692 case X86::BI__builtin_ia32_gatherdiv8df:
15693 IID = Intrinsic::x86_avx512_mask_gather_qpd_512;
15695 case X86::BI__builtin_ia32_gatherdiv16sf:
15696 IID = Intrinsic::x86_avx512_mask_gather_qps_512;
15698 case X86::BI__builtin_ia32_gathersiv8di:
15699 IID = Intrinsic::x86_avx512_mask_gather_dpq_512;
15701 case X86::BI__builtin_ia32_gathersiv16si:
15702 IID = Intrinsic::x86_avx512_mask_gather_dpi_512;
15704 case X86::BI__builtin_ia32_gatherdiv8di:
15705 IID = Intrinsic::x86_avx512_mask_gather_qpq_512;
15707 case X86::BI__builtin_ia32_gatherdiv16si:
15708 IID = Intrinsic::x86_avx512_mask_gather_qpi_512;
15712 unsigned MinElts = std::min(
15713 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(),
15714 cast<llvm::FixedVectorType>(Ops[2]->getType())->getNumElements());
15717 return Builder.CreateCall(Intr, Ops);
15720 case X86::BI__builtin_ia32_scattersiv8df:
15721 case X86::BI__builtin_ia32_scattersiv16sf:
15722 case X86::BI__builtin_ia32_scatterdiv8df:
15723 case X86::BI__builtin_ia32_scatterdiv16sf:
15724 case X86::BI__builtin_ia32_scattersiv8di:
15725 case X86::BI__builtin_ia32_scattersiv16si:
15726 case X86::BI__builtin_ia32_scatterdiv8di:
15727 case X86::BI__builtin_ia32_scatterdiv16si:
15728 case X86::BI__builtin_ia32_scatterdiv2df:
15729 case X86::BI__builtin_ia32_scatterdiv2di:
15730 case X86::BI__builtin_ia32_scatterdiv4df:
15731 case X86::BI__builtin_ia32_scatterdiv4di:
15732 case X86::BI__builtin_ia32_scatterdiv4sf:
15733 case X86::BI__builtin_ia32_scatterdiv4si:
15734 case X86::BI__builtin_ia32_scatterdiv8sf:
15735 case X86::BI__builtin_ia32_scatterdiv8si:
15736 case X86::BI__builtin_ia32_scattersiv2df:
15737 case X86::BI__builtin_ia32_scattersiv2di:
15738 case X86::BI__builtin_ia32_scattersiv4df:
15739 case X86::BI__builtin_ia32_scattersiv4di:
15740 case X86::BI__builtin_ia32_scattersiv4sf:
15741 case X86::BI__builtin_ia32_scattersiv4si:
15742 case X86::BI__builtin_ia32_scattersiv8sf:
15743 case X86::BI__builtin_ia32_scattersiv8si: {
15745 switch (BuiltinID) {
15746 default: llvm_unreachable(
"Unexpected builtin");
15747 case X86::BI__builtin_ia32_scattersiv8df:
15748 IID = Intrinsic::x86_avx512_mask_scatter_dpd_512;
15750 case X86::BI__builtin_ia32_scattersiv16sf:
15751 IID = Intrinsic::x86_avx512_mask_scatter_dps_512;
15753 case X86::BI__builtin_ia32_scatterdiv8df:
15754 IID = Intrinsic::x86_avx512_mask_scatter_qpd_512;
15756 case X86::BI__builtin_ia32_scatterdiv16sf:
15757 IID = Intrinsic::x86_avx512_mask_scatter_qps_512;
15759 case X86::BI__builtin_ia32_scattersiv8di:
15760 IID = Intrinsic::x86_avx512_mask_scatter_dpq_512;
15762 case X86::BI__builtin_ia32_scattersiv16si:
15763 IID = Intrinsic::x86_avx512_mask_scatter_dpi_512;
15765 case X86::BI__builtin_ia32_scatterdiv8di:
15766 IID = Intrinsic::x86_avx512_mask_scatter_qpq_512;
15768 case X86::BI__builtin_ia32_scatterdiv16si:
15769 IID = Intrinsic::x86_avx512_mask_scatter_qpi_512;
15771 case X86::BI__builtin_ia32_scatterdiv2df:
15772 IID = Intrinsic::x86_avx512_mask_scatterdiv2_df;
15774 case X86::BI__builtin_ia32_scatterdiv2di:
15775 IID = Intrinsic::x86_avx512_mask_scatterdiv2_di;
15777 case X86::BI__builtin_ia32_scatterdiv4df:
15778 IID = Intrinsic::x86_avx512_mask_scatterdiv4_df;
15780 case X86::BI__builtin_ia32_scatterdiv4di:
15781 IID = Intrinsic::x86_avx512_mask_scatterdiv4_di;
15783 case X86::BI__builtin_ia32_scatterdiv4sf:
15784 IID = Intrinsic::x86_avx512_mask_scatterdiv4_sf;
15786 case X86::BI__builtin_ia32_scatterdiv4si:
15787 IID = Intrinsic::x86_avx512_mask_scatterdiv4_si;
15789 case X86::BI__builtin_ia32_scatterdiv8sf:
15790 IID = Intrinsic::x86_avx512_mask_scatterdiv8_sf;
15792 case X86::BI__builtin_ia32_scatterdiv8si:
15793 IID = Intrinsic::x86_avx512_mask_scatterdiv8_si;
15795 case X86::BI__builtin_ia32_scattersiv2df:
15796 IID = Intrinsic::x86_avx512_mask_scattersiv2_df;
15798 case X86::BI__builtin_ia32_scattersiv2di:
15799 IID = Intrinsic::x86_avx512_mask_scattersiv2_di;
15801 case X86::BI__builtin_ia32_scattersiv4df:
15802 IID = Intrinsic::x86_avx512_mask_scattersiv4_df;
15804 case X86::BI__builtin_ia32_scattersiv4di:
15805 IID = Intrinsic::x86_avx512_mask_scattersiv4_di;
15807 case X86::BI__builtin_ia32_scattersiv4sf:
15808 IID = Intrinsic::x86_avx512_mask_scattersiv4_sf;
15810 case X86::BI__builtin_ia32_scattersiv4si:
15811 IID = Intrinsic::x86_avx512_mask_scattersiv4_si;
15813 case X86::BI__builtin_ia32_scattersiv8sf:
15814 IID = Intrinsic::x86_avx512_mask_scattersiv8_sf;
15816 case X86::BI__builtin_ia32_scattersiv8si:
15817 IID = Intrinsic::x86_avx512_mask_scattersiv8_si;
15821 unsigned MinElts = std::min(
15822 cast<llvm::FixedVectorType>(Ops[2]->getType())->getNumElements(),
15823 cast<llvm::FixedVectorType>(Ops[3]->getType())->getNumElements());
15826 return Builder.CreateCall(Intr, Ops);
15829 case X86::BI__builtin_ia32_vextractf128_pd256:
15830 case X86::BI__builtin_ia32_vextractf128_ps256:
15831 case X86::BI__builtin_ia32_vextractf128_si256:
15832 case X86::BI__builtin_ia32_extract128i256:
15833 case X86::BI__builtin_ia32_extractf64x4_mask:
15834 case X86::BI__builtin_ia32_extractf32x4_mask:
15835 case X86::BI__builtin_ia32_extracti64x4_mask:
15836 case X86::BI__builtin_ia32_extracti32x4_mask:
15837 case X86::BI__builtin_ia32_extractf32x8_mask:
15838 case X86::BI__builtin_ia32_extracti32x8_mask:
15839 case X86::BI__builtin_ia32_extractf32x4_256_mask:
15840 case X86::BI__builtin_ia32_extracti32x4_256_mask:
15841 case X86::BI__builtin_ia32_extractf64x2_256_mask:
15842 case X86::BI__builtin_ia32_extracti64x2_256_mask:
15843 case X86::BI__builtin_ia32_extractf64x2_512_mask:
15844 case X86::BI__builtin_ia32_extracti64x2_512_mask: {
15846 unsigned NumElts = DstTy->getNumElements();
15847 unsigned SrcNumElts =
15848 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15849 unsigned SubVectors = SrcNumElts / NumElts;
15850 unsigned Index = cast<ConstantInt>(Ops[1])->getZExtValue();
15851 assert(llvm::isPowerOf2_32(SubVectors) &&
"Expected power of 2 subvectors");
15852 Index &= SubVectors - 1;
15856 for (
unsigned i = 0; i != NumElts; ++i)
15857 Indices[i] = i + Index;
15862 if (Ops.size() == 4)
15867 case X86::BI__builtin_ia32_vinsertf128_pd256:
15868 case X86::BI__builtin_ia32_vinsertf128_ps256:
15869 case X86::BI__builtin_ia32_vinsertf128_si256:
15870 case X86::BI__builtin_ia32_insert128i256:
15871 case X86::BI__builtin_ia32_insertf64x4:
15872 case X86::BI__builtin_ia32_insertf32x4:
15873 case X86::BI__builtin_ia32_inserti64x4:
15874 case X86::BI__builtin_ia32_inserti32x4:
15875 case X86::BI__builtin_ia32_insertf32x8:
15876 case X86::BI__builtin_ia32_inserti32x8:
15877 case X86::BI__builtin_ia32_insertf32x4_256:
15878 case X86::BI__builtin_ia32_inserti32x4_256:
15879 case X86::BI__builtin_ia32_insertf64x2_256:
15880 case X86::BI__builtin_ia32_inserti64x2_256:
15881 case X86::BI__builtin_ia32_insertf64x2_512:
15882 case X86::BI__builtin_ia32_inserti64x2_512: {
15883 unsigned DstNumElts =
15884 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15885 unsigned SrcNumElts =
15886 cast<llvm::FixedVectorType>(Ops[1]->getType())->getNumElements();
15887 unsigned SubVectors = DstNumElts / SrcNumElts;
15888 unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue();
15889 assert(llvm::isPowerOf2_32(SubVectors) &&
"Expected power of 2 subvectors");
15890 Index &= SubVectors - 1;
15891 Index *= SrcNumElts;
15894 for (
unsigned i = 0; i != DstNumElts; ++i)
15895 Indices[i] = (i >= SrcNumElts) ? SrcNumElts + (i % SrcNumElts) : i;
15898 Ops[1],
ArrayRef(Indices, DstNumElts),
"widen");
15900 for (
unsigned i = 0; i != DstNumElts; ++i) {
15901 if (i >= Index && i < (Index + SrcNumElts))
15902 Indices[i] = (i - Index) + DstNumElts;
15907 return Builder.CreateShuffleVector(Ops[0], Op1,
15908 ArrayRef(Indices, DstNumElts),
"insert");
15910 case X86::BI__builtin_ia32_pmovqd512_mask:
15911 case X86::BI__builtin_ia32_pmovwb512_mask: {
15912 Value *Res =
Builder.CreateTrunc(Ops[0], Ops[1]->getType());
15915 case X86::BI__builtin_ia32_pmovdb512_mask:
15916 case X86::BI__builtin_ia32_pmovdw512_mask:
15917 case X86::BI__builtin_ia32_pmovqw512_mask: {
15918 if (
const auto *
C = dyn_cast<Constant>(Ops[2]))
15919 if (
C->isAllOnesValue())
15920 return Builder.CreateTrunc(Ops[0], Ops[1]->getType());
15923 switch (BuiltinID) {
15924 default: llvm_unreachable(
"Unsupported intrinsic!");
15925 case X86::BI__builtin_ia32_pmovdb512_mask:
15926 IID = Intrinsic::x86_avx512_mask_pmov_db_512;
15928 case X86::BI__builtin_ia32_pmovdw512_mask:
15929 IID = Intrinsic::x86_avx512_mask_pmov_dw_512;
15931 case X86::BI__builtin_ia32_pmovqw512_mask:
15932 IID = Intrinsic::x86_avx512_mask_pmov_qw_512;
15937 return Builder.CreateCall(Intr, Ops);
15939 case X86::BI__builtin_ia32_pblendw128:
15940 case X86::BI__builtin_ia32_blendpd:
15941 case X86::BI__builtin_ia32_blendps:
15942 case X86::BI__builtin_ia32_blendpd256:
15943 case X86::BI__builtin_ia32_blendps256:
15944 case X86::BI__builtin_ia32_pblendw256:
15945 case X86::BI__builtin_ia32_pblendd128:
15946 case X86::BI__builtin_ia32_pblendd256: {
15948 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15949 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
15954 for (
unsigned i = 0; i != NumElts; ++i)
15955 Indices[i] = ((Imm >> (i % 8)) & 0x1) ? NumElts + i : i;
15957 return Builder.CreateShuffleVector(Ops[0], Ops[1],
15958 ArrayRef(Indices, NumElts),
"blend");
15960 case X86::BI__builtin_ia32_pshuflw:
15961 case X86::BI__builtin_ia32_pshuflw256:
15962 case X86::BI__builtin_ia32_pshuflw512: {
15963 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
15964 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15965 unsigned NumElts = Ty->getNumElements();
15968 Imm = (Imm & 0xff) * 0x01010101;
15971 for (
unsigned l = 0; l != NumElts; l += 8) {
15972 for (
unsigned i = 0; i != 4; ++i) {
15973 Indices[l + i] = l + (Imm & 3);
15976 for (
unsigned i = 4; i != 8; ++i)
15977 Indices[l + i] = l + i;
15980 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
15983 case X86::BI__builtin_ia32_pshufhw:
15984 case X86::BI__builtin_ia32_pshufhw256:
15985 case X86::BI__builtin_ia32_pshufhw512: {
15986 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
15987 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15988 unsigned NumElts = Ty->getNumElements();
15991 Imm = (Imm & 0xff) * 0x01010101;
15994 for (
unsigned l = 0; l != NumElts; l += 8) {
15995 for (
unsigned i = 0; i != 4; ++i)
15996 Indices[l + i] = l + i;
15997 for (
unsigned i = 4; i != 8; ++i) {
15998 Indices[l + i] = l + 4 + (Imm & 3);
16003 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
16006 case X86::BI__builtin_ia32_pshufd:
16007 case X86::BI__builtin_ia32_pshufd256:
16008 case X86::BI__builtin_ia32_pshufd512:
16009 case X86::BI__builtin_ia32_vpermilpd:
16010 case X86::BI__builtin_ia32_vpermilps:
16011 case X86::BI__builtin_ia32_vpermilpd256:
16012 case X86::BI__builtin_ia32_vpermilps256:
16013 case X86::BI__builtin_ia32_vpermilpd512:
16014 case X86::BI__builtin_ia32_vpermilps512: {
16015 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
16016 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
16017 unsigned NumElts = Ty->getNumElements();
16018 unsigned NumLanes = Ty->getPrimitiveSizeInBits() / 128;
16019 unsigned NumLaneElts = NumElts / NumLanes;
16022 Imm = (Imm & 0xff) * 0x01010101;
16025 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
16026 for (
unsigned i = 0; i != NumLaneElts; ++i) {
16027 Indices[i + l] = (Imm % NumLaneElts) + l;
16028 Imm /= NumLaneElts;
16032 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
16035 case X86::BI__builtin_ia32_shufpd:
16036 case X86::BI__builtin_ia32_shufpd256:
16037 case X86::BI__builtin_ia32_shufpd512:
16038 case X86::BI__builtin_ia32_shufps:
16039 case X86::BI__builtin_ia32_shufps256:
16040 case X86::BI__builtin_ia32_shufps512: {
16041 uint32_t Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
16042 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
16043 unsigned NumElts = Ty->getNumElements();
16044 unsigned NumLanes = Ty->getPrimitiveSizeInBits() / 128;
16045 unsigned NumLaneElts = NumElts / NumLanes;
16048 Imm = (Imm & 0xff) * 0x01010101;
16051 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
16052 for (
unsigned i = 0; i != NumLaneElts; ++i) {
16053 unsigned Index = Imm % NumLaneElts;
16054 Imm /= NumLaneElts;
16055 if (i >= (NumLaneElts / 2))
16057 Indices[l + i] = l + Index;
16061 return Builder.CreateShuffleVector(Ops[0], Ops[1],
16062 ArrayRef(Indices, NumElts),
"shufp");
16064 case X86::BI__builtin_ia32_permdi256:
16065 case X86::BI__builtin_ia32_permdf256:
16066 case X86::BI__builtin_ia32_permdi512:
16067 case X86::BI__builtin_ia32_permdf512: {
16068 unsigned Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
16069 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
16070 unsigned NumElts = Ty->getNumElements();
16074 for (
unsigned l = 0; l != NumElts; l += 4)
16075 for (
unsigned i = 0; i != 4; ++i)
16076 Indices[l + i] = l + ((Imm >> (2 * i)) & 0x3);
16078 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
16081 case X86::BI__builtin_ia32_palignr128:
16082 case X86::BI__builtin_ia32_palignr256:
16083 case X86::BI__builtin_ia32_palignr512: {
16084 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff;
16087 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16088 assert(NumElts % 16 == 0);
16092 if (ShiftVal >= 32)
16097 if (ShiftVal > 16) {
16100 Ops[0] = llvm::Constant::getNullValue(Ops[0]->getType());
16105 for (
unsigned l = 0; l != NumElts; l += 16) {
16106 for (
unsigned i = 0; i != 16; ++i) {
16107 unsigned Idx = ShiftVal + i;
16109 Idx += NumElts - 16;
16110 Indices[l + i] = Idx + l;
16114 return Builder.CreateShuffleVector(Ops[1], Ops[0],
16115 ArrayRef(Indices, NumElts),
"palignr");
16117 case X86::BI__builtin_ia32_alignd128:
16118 case X86::BI__builtin_ia32_alignd256:
16119 case X86::BI__builtin_ia32_alignd512:
16120 case X86::BI__builtin_ia32_alignq128:
16121 case X86::BI__builtin_ia32_alignq256:
16122 case X86::BI__builtin_ia32_alignq512: {
16124 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16125 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff;
16128 ShiftVal &= NumElts - 1;
16131 for (
unsigned i = 0; i != NumElts; ++i)
16132 Indices[i] = i + ShiftVal;
16134 return Builder.CreateShuffleVector(Ops[1], Ops[0],
16135 ArrayRef(Indices, NumElts),
"valign");
16137 case X86::BI__builtin_ia32_shuf_f32x4_256:
16138 case X86::BI__builtin_ia32_shuf_f64x2_256:
16139 case X86::BI__builtin_ia32_shuf_i32x4_256:
16140 case X86::BI__builtin_ia32_shuf_i64x2_256:
16141 case X86::BI__builtin_ia32_shuf_f32x4:
16142 case X86::BI__builtin_ia32_shuf_f64x2:
16143 case X86::BI__builtin_ia32_shuf_i32x4:
16144 case X86::BI__builtin_ia32_shuf_i64x2: {
16145 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
16146 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
16147 unsigned NumElts = Ty->getNumElements();
16148 unsigned NumLanes = Ty->getPrimitiveSizeInBits() == 512 ? 4 : 2;
16149 unsigned NumLaneElts = NumElts / NumLanes;
16152 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
16153 unsigned Index = (Imm % NumLanes) * NumLaneElts;
16155 if (l >= (NumElts / 2))
16157 for (
unsigned i = 0; i != NumLaneElts; ++i) {
16158 Indices[l + i] = Index + i;
16162 return Builder.CreateShuffleVector(Ops[0], Ops[1],
16163 ArrayRef(Indices, NumElts),
"shuf");
16166 case X86::BI__builtin_ia32_vperm2f128_pd256:
16167 case X86::BI__builtin_ia32_vperm2f128_ps256:
16168 case X86::BI__builtin_ia32_vperm2f128_si256:
16169 case X86::BI__builtin_ia32_permti256: {
16170 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
16172 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16181 for (
unsigned l = 0; l != 2; ++l) {
16183 if (Imm & (1 << ((l * 4) + 3)))
16184 OutOps[l] = llvm::ConstantAggregateZero::get(Ops[0]->getType());
16185 else if (Imm & (1 << ((l * 4) + 1)))
16186 OutOps[l] = Ops[1];
16188 OutOps[l] = Ops[0];
16190 for (
unsigned i = 0; i != NumElts/2; ++i) {
16192 unsigned Idx = (l * NumElts) + i;
16195 if (Imm & (1 << (l * 4)))
16197 Indices[(l * (NumElts/2)) + i] = Idx;
16201 return Builder.CreateShuffleVector(OutOps[0], OutOps[1],
16202 ArrayRef(Indices, NumElts),
"vperm");
16205 case X86::BI__builtin_ia32_pslldqi128_byteshift:
16206 case X86::BI__builtin_ia32_pslldqi256_byteshift:
16207 case X86::BI__builtin_ia32_pslldqi512_byteshift: {
16208 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
16209 auto *ResultType = cast<llvm::FixedVectorType>(Ops[0]->getType());
16211 unsigned NumElts = ResultType->getNumElements() * 8;
16214 if (ShiftVal >= 16)
16215 return llvm::Constant::getNullValue(ResultType);
16219 for (
unsigned l = 0; l != NumElts; l += 16) {
16220 for (
unsigned i = 0; i != 16; ++i) {
16221 unsigned Idx = NumElts + i - ShiftVal;
16222 if (Idx < NumElts) Idx -= NumElts - 16;
16223 Indices[l + i] = Idx + l;
16227 auto *VecTy = llvm::FixedVectorType::get(
Int8Ty, NumElts);
16229 Value *
Zero = llvm::Constant::getNullValue(VecTy);
16231 Zero, Cast,
ArrayRef(Indices, NumElts),
"pslldq");
16232 return Builder.CreateBitCast(SV, Ops[0]->getType(),
"cast");
16234 case X86::BI__builtin_ia32_psrldqi128_byteshift:
16235 case X86::BI__builtin_ia32_psrldqi256_byteshift:
16236 case X86::BI__builtin_ia32_psrldqi512_byteshift: {
16237 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
16238 auto *ResultType = cast<llvm::FixedVectorType>(Ops[0]->getType());
16240 unsigned NumElts = ResultType->getNumElements() * 8;
16243 if (ShiftVal >= 16)
16244 return llvm::Constant::getNullValue(ResultType);
16248 for (
unsigned l = 0; l != NumElts; l += 16) {
16249 for (
unsigned i = 0; i != 16; ++i) {
16250 unsigned Idx = i + ShiftVal;
16251 if (Idx >= 16) Idx += NumElts - 16;
16252 Indices[l + i] = Idx + l;
16256 auto *VecTy = llvm::FixedVectorType::get(
Int8Ty, NumElts);
16258 Value *
Zero = llvm::Constant::getNullValue(VecTy);
16260 Cast, Zero,
ArrayRef(Indices, NumElts),
"psrldq");
16261 return Builder.CreateBitCast(SV, ResultType,
"cast");
16263 case X86::BI__builtin_ia32_kshiftliqi:
16264 case X86::BI__builtin_ia32_kshiftlihi:
16265 case X86::BI__builtin_ia32_kshiftlisi:
16266 case X86::BI__builtin_ia32_kshiftlidi: {
16267 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
16268 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
16270 if (ShiftVal >= NumElts)
16271 return llvm::Constant::getNullValue(Ops[0]->getType());
16276 for (
unsigned i = 0; i != NumElts; ++i)
16277 Indices[i] = NumElts + i - ShiftVal;
16279 Value *
Zero = llvm::Constant::getNullValue(
In->getType());
16281 Zero, In,
ArrayRef(Indices, NumElts),
"kshiftl");
16282 return Builder.CreateBitCast(SV, Ops[0]->getType());
16284 case X86::BI__builtin_ia32_kshiftriqi:
16285 case X86::BI__builtin_ia32_kshiftrihi:
16286 case X86::BI__builtin_ia32_kshiftrisi:
16287 case X86::BI__builtin_ia32_kshiftridi: {
16288 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
16289 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
16291 if (ShiftVal >= NumElts)
16292 return llvm::Constant::getNullValue(Ops[0]->getType());
16297 for (
unsigned i = 0; i != NumElts; ++i)
16298 Indices[i] = i + ShiftVal;
16300 Value *
Zero = llvm::Constant::getNullValue(
In->getType());
16302 In, Zero,
ArrayRef(Indices, NumElts),
"kshiftr");
16303 return Builder.CreateBitCast(SV, Ops[0]->getType());
16305 case X86::BI__builtin_ia32_movnti:
16306 case X86::BI__builtin_ia32_movnti64:
16307 case X86::BI__builtin_ia32_movntsd:
16308 case X86::BI__builtin_ia32_movntss: {
16309 llvm::MDNode *
Node = llvm::MDNode::get(
16312 Value *Ptr = Ops[0];
16313 Value *Src = Ops[1];
16316 if (BuiltinID == X86::BI__builtin_ia32_movntsd ||
16317 BuiltinID == X86::BI__builtin_ia32_movntss)
16318 Src =
Builder.CreateExtractElement(Src, (uint64_t)0,
"extract");
16322 SI->setMetadata(llvm::LLVMContext::MD_nontemporal,
Node);
16323 SI->setAlignment(llvm::Align(1));
16327 case X86::BI__builtin_ia32_vprotb:
16328 case X86::BI__builtin_ia32_vprotw:
16329 case X86::BI__builtin_ia32_vprotd:
16330 case X86::BI__builtin_ia32_vprotq:
16331 case X86::BI__builtin_ia32_vprotbi:
16332 case X86::BI__builtin_ia32_vprotwi:
16333 case X86::BI__builtin_ia32_vprotdi:
16334 case X86::BI__builtin_ia32_vprotqi:
16335 case X86::BI__builtin_ia32_prold128:
16336 case X86::BI__builtin_ia32_prold256:
16337 case X86::BI__builtin_ia32_prold512:
16338 case X86::BI__builtin_ia32_prolq128:
16339 case X86::BI__builtin_ia32_prolq256:
16340 case X86::BI__builtin_ia32_prolq512:
16341 case X86::BI__builtin_ia32_prolvd128:
16342 case X86::BI__builtin_ia32_prolvd256:
16343 case X86::BI__builtin_ia32_prolvd512:
16344 case X86::BI__builtin_ia32_prolvq128:
16345 case X86::BI__builtin_ia32_prolvq256:
16346 case X86::BI__builtin_ia32_prolvq512:
16348 case X86::BI__builtin_ia32_prord128:
16349 case X86::BI__builtin_ia32_prord256:
16350 case X86::BI__builtin_ia32_prord512:
16351 case X86::BI__builtin_ia32_prorq128:
16352 case X86::BI__builtin_ia32_prorq256:
16353 case X86::BI__builtin_ia32_prorq512:
16354 case X86::BI__builtin_ia32_prorvd128:
16355 case X86::BI__builtin_ia32_prorvd256:
16356 case X86::BI__builtin_ia32_prorvd512:
16357 case X86::BI__builtin_ia32_prorvq128:
16358 case X86::BI__builtin_ia32_prorvq256:
16359 case X86::BI__builtin_ia32_prorvq512:
16361 case X86::BI__builtin_ia32_selectb_128:
16362 case X86::BI__builtin_ia32_selectb_256:
16363 case X86::BI__builtin_ia32_selectb_512:
16364 case X86::BI__builtin_ia32_selectw_128:
16365 case X86::BI__builtin_ia32_selectw_256:
16366 case X86::BI__builtin_ia32_selectw_512:
16367 case X86::BI__builtin_ia32_selectd_128:
16368 case X86::BI__builtin_ia32_selectd_256:
16369 case X86::BI__builtin_ia32_selectd_512:
16370 case X86::BI__builtin_ia32_selectq_128:
16371 case X86::BI__builtin_ia32_selectq_256:
16372 case X86::BI__builtin_ia32_selectq_512:
16373 case X86::BI__builtin_ia32_selectph_128:
16374 case X86::BI__builtin_ia32_selectph_256:
16375 case X86::BI__builtin_ia32_selectph_512:
16376 case X86::BI__builtin_ia32_selectpbf_128:
16377 case X86::BI__builtin_ia32_selectpbf_256:
16378 case X86::BI__builtin_ia32_selectpbf_512:
16379 case X86::BI__builtin_ia32_selectps_128:
16380 case X86::BI__builtin_ia32_selectps_256:
16381 case X86::BI__builtin_ia32_selectps_512:
16382 case X86::BI__builtin_ia32_selectpd_128:
16383 case X86::BI__builtin_ia32_selectpd_256:
16384 case X86::BI__builtin_ia32_selectpd_512:
16386 case X86::BI__builtin_ia32_selectsh_128:
16387 case X86::BI__builtin_ia32_selectsbf_128:
16388 case X86::BI__builtin_ia32_selectss_128:
16389 case X86::BI__builtin_ia32_selectsd_128: {
16390 Value *A =
Builder.CreateExtractElement(Ops[1], (uint64_t)0);
16391 Value *B =
Builder.CreateExtractElement(Ops[2], (uint64_t)0);
16393 return Builder.CreateInsertElement(Ops[1], A, (uint64_t)0);
16395 case X86::BI__builtin_ia32_cmpb128_mask:
16396 case X86::BI__builtin_ia32_cmpb256_mask:
16397 case X86::BI__builtin_ia32_cmpb512_mask:
16398 case X86::BI__builtin_ia32_cmpw128_mask:
16399 case X86::BI__builtin_ia32_cmpw256_mask:
16400 case X86::BI__builtin_ia32_cmpw512_mask:
16401 case X86::BI__builtin_ia32_cmpd128_mask:
16402 case X86::BI__builtin_ia32_cmpd256_mask:
16403 case X86::BI__builtin_ia32_cmpd512_mask:
16404 case X86::BI__builtin_ia32_cmpq128_mask:
16405 case X86::BI__builtin_ia32_cmpq256_mask:
16406 case X86::BI__builtin_ia32_cmpq512_mask: {
16407 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
16410 case X86::BI__builtin_ia32_ucmpb128_mask:
16411 case X86::BI__builtin_ia32_ucmpb256_mask:
16412 case X86::BI__builtin_ia32_ucmpb512_mask:
16413 case X86::BI__builtin_ia32_ucmpw128_mask:
16414 case X86::BI__builtin_ia32_ucmpw256_mask:
16415 case X86::BI__builtin_ia32_ucmpw512_mask:
16416 case X86::BI__builtin_ia32_ucmpd128_mask:
16417 case X86::BI__builtin_ia32_ucmpd256_mask:
16418 case X86::BI__builtin_ia32_ucmpd512_mask:
16419 case X86::BI__builtin_ia32_ucmpq128_mask:
16420 case X86::BI__builtin_ia32_ucmpq256_mask:
16421 case X86::BI__builtin_ia32_ucmpq512_mask: {
16422 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
16425 case X86::BI__builtin_ia32_vpcomb:
16426 case X86::BI__builtin_ia32_vpcomw:
16427 case X86::BI__builtin_ia32_vpcomd:
16428 case X86::BI__builtin_ia32_vpcomq:
16430 case X86::BI__builtin_ia32_vpcomub:
16431 case X86::BI__builtin_ia32_vpcomuw:
16432 case X86::BI__builtin_ia32_vpcomud:
16433 case X86::BI__builtin_ia32_vpcomuq:
16436 case X86::BI__builtin_ia32_kortestcqi:
16437 case X86::BI__builtin_ia32_kortestchi:
16438 case X86::BI__builtin_ia32_kortestcsi:
16439 case X86::BI__builtin_ia32_kortestcdi: {
16441 Value *
C = llvm::Constant::getAllOnesValue(Ops[0]->getType());
16445 case X86::BI__builtin_ia32_kortestzqi:
16446 case X86::BI__builtin_ia32_kortestzhi:
16447 case X86::BI__builtin_ia32_kortestzsi:
16448 case X86::BI__builtin_ia32_kortestzdi: {
16450 Value *
C = llvm::Constant::getNullValue(Ops[0]->getType());
16455 case X86::BI__builtin_ia32_ktestcqi:
16456 case X86::BI__builtin_ia32_ktestzqi:
16457 case X86::BI__builtin_ia32_ktestchi:
16458 case X86::BI__builtin_ia32_ktestzhi:
16459 case X86::BI__builtin_ia32_ktestcsi:
16460 case X86::BI__builtin_ia32_ktestzsi:
16461 case X86::BI__builtin_ia32_ktestcdi:
16462 case X86::BI__builtin_ia32_ktestzdi: {
16464 switch (BuiltinID) {
16465 default: llvm_unreachable(
"Unsupported intrinsic!");
16466 case X86::BI__builtin_ia32_ktestcqi:
16467 IID = Intrinsic::x86_avx512_ktestc_b;
16469 case X86::BI__builtin_ia32_ktestzqi:
16470 IID = Intrinsic::x86_avx512_ktestz_b;
16472 case X86::BI__builtin_ia32_ktestchi:
16473 IID = Intrinsic::x86_avx512_ktestc_w;
16475 case X86::BI__builtin_ia32_ktestzhi:
16476 IID = Intrinsic::x86_avx512_ktestz_w;
16478 case X86::BI__builtin_ia32_ktestcsi:
16479 IID = Intrinsic::x86_avx512_ktestc_d;
16481 case X86::BI__builtin_ia32_ktestzsi:
16482 IID = Intrinsic::x86_avx512_ktestz_d;
16484 case X86::BI__builtin_ia32_ktestcdi:
16485 IID = Intrinsic::x86_avx512_ktestc_q;
16487 case X86::BI__builtin_ia32_ktestzdi:
16488 IID = Intrinsic::x86_avx512_ktestz_q;
16492 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
16496 return Builder.CreateCall(Intr, {LHS, RHS});
16499 case X86::BI__builtin_ia32_kaddqi:
16500 case X86::BI__builtin_ia32_kaddhi:
16501 case X86::BI__builtin_ia32_kaddsi:
16502 case X86::BI__builtin_ia32_kadddi: {
16504 switch (BuiltinID) {
16505 default: llvm_unreachable(
"Unsupported intrinsic!");
16506 case X86::BI__builtin_ia32_kaddqi:
16507 IID = Intrinsic::x86_avx512_kadd_b;
16509 case X86::BI__builtin_ia32_kaddhi:
16510 IID = Intrinsic::x86_avx512_kadd_w;
16512 case X86::BI__builtin_ia32_kaddsi:
16513 IID = Intrinsic::x86_avx512_kadd_d;
16515 case X86::BI__builtin_ia32_kadddi:
16516 IID = Intrinsic::x86_avx512_kadd_q;
16520 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
16525 return Builder.CreateBitCast(Res, Ops[0]->getType());
16527 case X86::BI__builtin_ia32_kandqi:
16528 case X86::BI__builtin_ia32_kandhi:
16529 case X86::BI__builtin_ia32_kandsi:
16530 case X86::BI__builtin_ia32_kanddi:
16532 case X86::BI__builtin_ia32_kandnqi:
16533 case X86::BI__builtin_ia32_kandnhi:
16534 case X86::BI__builtin_ia32_kandnsi:
16535 case X86::BI__builtin_ia32_kandndi:
16537 case X86::BI__builtin_ia32_korqi:
16538 case X86::BI__builtin_ia32_korhi:
16539 case X86::BI__builtin_ia32_korsi:
16540 case X86::BI__builtin_ia32_kordi:
16542 case X86::BI__builtin_ia32_kxnorqi:
16543 case X86::BI__builtin_ia32_kxnorhi:
16544 case X86::BI__builtin_ia32_kxnorsi:
16545 case X86::BI__builtin_ia32_kxnordi:
16547 case X86::BI__builtin_ia32_kxorqi:
16548 case X86::BI__builtin_ia32_kxorhi:
16549 case X86::BI__builtin_ia32_kxorsi:
16550 case X86::BI__builtin_ia32_kxordi:
16552 case X86::BI__builtin_ia32_knotqi:
16553 case X86::BI__builtin_ia32_knothi:
16554 case X86::BI__builtin_ia32_knotsi:
16555 case X86::BI__builtin_ia32_knotdi: {
16556 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
16559 Ops[0]->getType());
16561 case X86::BI__builtin_ia32_kmovb:
16562 case X86::BI__builtin_ia32_kmovw:
16563 case X86::BI__builtin_ia32_kmovd:
16564 case X86::BI__builtin_ia32_kmovq: {
16568 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
16570 return Builder.CreateBitCast(Res, Ops[0]->getType());
16573 case X86::BI__builtin_ia32_kunpckdi:
16574 case X86::BI__builtin_ia32_kunpcksi:
16575 case X86::BI__builtin_ia32_kunpckhi: {
16576 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
16580 for (
unsigned i = 0; i != NumElts; ++i)
16585 LHS =
Builder.CreateShuffleVector(LHS, LHS,
ArrayRef(Indices, NumElts / 2));
16586 RHS =
Builder.CreateShuffleVector(RHS, RHS,
ArrayRef(Indices, NumElts / 2));
16591 return Builder.CreateBitCast(Res, Ops[0]->getType());
16594 case X86::BI__builtin_ia32_vplzcntd_128:
16595 case X86::BI__builtin_ia32_vplzcntd_256:
16596 case X86::BI__builtin_ia32_vplzcntd_512:
16597 case X86::BI__builtin_ia32_vplzcntq_128:
16598 case X86::BI__builtin_ia32_vplzcntq_256:
16599 case X86::BI__builtin_ia32_vplzcntq_512: {
16603 case X86::BI__builtin_ia32_sqrtss:
16604 case X86::BI__builtin_ia32_sqrtsd: {
16605 Value *A =
Builder.CreateExtractElement(Ops[0], (uint64_t)0);
16607 if (
Builder.getIsFPConstrained()) {
16608 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
16611 A =
Builder.CreateConstrainedFPCall(F, {A});
16614 A =
Builder.CreateCall(F, {A});
16616 return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0);
16618 case X86::BI__builtin_ia32_sqrtsh_round_mask:
16619 case X86::BI__builtin_ia32_sqrtsd_round_mask:
16620 case X86::BI__builtin_ia32_sqrtss_round_mask: {
16621 unsigned CC = cast<llvm::ConstantInt>(Ops[4])->getZExtValue();
16627 switch (BuiltinID) {
16629 llvm_unreachable(
"Unsupported intrinsic!");
16630 case X86::BI__builtin_ia32_sqrtsh_round_mask:
16631 IID = Intrinsic::x86_avx512fp16_mask_sqrt_sh;
16633 case X86::BI__builtin_ia32_sqrtsd_round_mask:
16634 IID = Intrinsic::x86_avx512_mask_sqrt_sd;
16636 case X86::BI__builtin_ia32_sqrtss_round_mask:
16637 IID = Intrinsic::x86_avx512_mask_sqrt_ss;
16642 Value *A =
Builder.CreateExtractElement(Ops[1], (uint64_t)0);
16644 if (
Builder.getIsFPConstrained()) {
16645 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
16648 A =
Builder.CreateConstrainedFPCall(F, A);
16651 A =
Builder.CreateCall(F, A);
16653 Value *Src =
Builder.CreateExtractElement(Ops[2], (uint64_t)0);
16655 return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0);
16657 case X86::BI__builtin_ia32_sqrtpd256:
16658 case X86::BI__builtin_ia32_sqrtpd:
16659 case X86::BI__builtin_ia32_sqrtps256:
16660 case X86::BI__builtin_ia32_sqrtps:
16661 case X86::BI__builtin_ia32_sqrtph256:
16662 case X86::BI__builtin_ia32_sqrtph:
16663 case X86::BI__builtin_ia32_sqrtph512:
16664 case X86::BI__builtin_ia32_vsqrtnepbf16256:
16665 case X86::BI__builtin_ia32_vsqrtnepbf16:
16666 case X86::BI__builtin_ia32_vsqrtnepbf16512:
16667 case X86::BI__builtin_ia32_sqrtps512:
16668 case X86::BI__builtin_ia32_sqrtpd512: {
16669 if (Ops.size() == 2) {
16670 unsigned CC = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
16676 switch (BuiltinID) {
16678 llvm_unreachable(
"Unsupported intrinsic!");
16679 case X86::BI__builtin_ia32_sqrtph512:
16680 IID = Intrinsic::x86_avx512fp16_sqrt_ph_512;
16682 case X86::BI__builtin_ia32_sqrtps512:
16683 IID = Intrinsic::x86_avx512_sqrt_ps_512;
16685 case X86::BI__builtin_ia32_sqrtpd512:
16686 IID = Intrinsic::x86_avx512_sqrt_pd_512;
16692 if (
Builder.getIsFPConstrained()) {
16693 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
16695 Ops[0]->getType());
16696 return Builder.CreateConstrainedFPCall(F, Ops[0]);
16699 return Builder.CreateCall(F, Ops[0]);
16703 case X86::BI__builtin_ia32_pmuludq128:
16704 case X86::BI__builtin_ia32_pmuludq256:
16705 case X86::BI__builtin_ia32_pmuludq512:
16708 case X86::BI__builtin_ia32_pmuldq128:
16709 case X86::BI__builtin_ia32_pmuldq256:
16710 case X86::BI__builtin_ia32_pmuldq512:
16713 case X86::BI__builtin_ia32_pternlogd512_mask:
16714 case X86::BI__builtin_ia32_pternlogq512_mask:
16715 case X86::BI__builtin_ia32_pternlogd128_mask:
16716 case X86::BI__builtin_ia32_pternlogd256_mask:
16717 case X86::BI__builtin_ia32_pternlogq128_mask:
16718 case X86::BI__builtin_ia32_pternlogq256_mask:
16721 case X86::BI__builtin_ia32_pternlogd512_maskz:
16722 case X86::BI__builtin_ia32_pternlogq512_maskz:
16723 case X86::BI__builtin_ia32_pternlogd128_maskz:
16724 case X86::BI__builtin_ia32_pternlogd256_maskz:
16725 case X86::BI__builtin_ia32_pternlogq128_maskz:
16726 case X86::BI__builtin_ia32_pternlogq256_maskz:
16729 case X86::BI__builtin_ia32_vpshldd128:
16730 case X86::BI__builtin_ia32_vpshldd256:
16731 case X86::BI__builtin_ia32_vpshldd512:
16732 case X86::BI__builtin_ia32_vpshldq128:
16733 case X86::BI__builtin_ia32_vpshldq256:
16734 case X86::BI__builtin_ia32_vpshldq512:
16735 case X86::BI__builtin_ia32_vpshldw128:
16736 case X86::BI__builtin_ia32_vpshldw256:
16737 case X86::BI__builtin_ia32_vpshldw512:
16740 case X86::BI__builtin_ia32_vpshrdd128:
16741 case X86::BI__builtin_ia32_vpshrdd256:
16742 case X86::BI__builtin_ia32_vpshrdd512:
16743 case X86::BI__builtin_ia32_vpshrdq128:
16744 case X86::BI__builtin_ia32_vpshrdq256:
16745 case X86::BI__builtin_ia32_vpshrdq512:
16746 case X86::BI__builtin_ia32_vpshrdw128:
16747 case X86::BI__builtin_ia32_vpshrdw256:
16748 case X86::BI__builtin_ia32_vpshrdw512:
16752 case X86::BI__builtin_ia32_vpshldvd128:
16753 case X86::BI__builtin_ia32_vpshldvd256:
16754 case X86::BI__builtin_ia32_vpshldvd512:
16755 case X86::BI__builtin_ia32_vpshldvq128:
16756 case X86::BI__builtin_ia32_vpshldvq256:
16757 case X86::BI__builtin_ia32_vpshldvq512:
16758 case X86::BI__builtin_ia32_vpshldvw128:
16759 case X86::BI__builtin_ia32_vpshldvw256:
16760 case X86::BI__builtin_ia32_vpshldvw512:
16763 case X86::BI__builtin_ia32_vpshrdvd128:
16764 case X86::BI__builtin_ia32_vpshrdvd256:
16765 case X86::BI__builtin_ia32_vpshrdvd512:
16766 case X86::BI__builtin_ia32_vpshrdvq128:
16767 case X86::BI__builtin_ia32_vpshrdvq256:
16768 case X86::BI__builtin_ia32_vpshrdvq512:
16769 case X86::BI__builtin_ia32_vpshrdvw128:
16770 case X86::BI__builtin_ia32_vpshrdvw256:
16771 case X86::BI__builtin_ia32_vpshrdvw512:
16776 case X86::BI__builtin_ia32_reduce_fadd_pd512:
16777 case X86::BI__builtin_ia32_reduce_fadd_ps512:
16778 case X86::BI__builtin_ia32_reduce_fadd_ph512:
16779 case X86::BI__builtin_ia32_reduce_fadd_ph256:
16780 case X86::BI__builtin_ia32_reduce_fadd_ph128: {
16783 IRBuilder<>::FastMathFlagGuard FMFGuard(
Builder);
16784 Builder.getFastMathFlags().setAllowReassoc();
16785 return Builder.CreateCall(F, {Ops[0], Ops[1]});
16787 case X86::BI__builtin_ia32_reduce_fmul_pd512:
16788 case X86::BI__builtin_ia32_reduce_fmul_ps512:
16789 case X86::BI__builtin_ia32_reduce_fmul_ph512:
16790 case X86::BI__builtin_ia32_reduce_fmul_ph256:
16791 case X86::BI__builtin_ia32_reduce_fmul_ph128: {
16794 IRBuilder<>::FastMathFlagGuard FMFGuard(
Builder);
16795 Builder.getFastMathFlags().setAllowReassoc();
16796 return Builder.CreateCall(F, {Ops[0], Ops[1]});
16798 case X86::BI__builtin_ia32_reduce_fmax_pd512:
16799 case X86::BI__builtin_ia32_reduce_fmax_ps512:
16800 case X86::BI__builtin_ia32_reduce_fmax_ph512:
16801 case X86::BI__builtin_ia32_reduce_fmax_ph256:
16802 case X86::BI__builtin_ia32_reduce_fmax_ph128: {
16805 IRBuilder<>::FastMathFlagGuard FMFGuard(
Builder);
16806 Builder.getFastMathFlags().setNoNaNs();
16807 return Builder.CreateCall(F, {Ops[0]});
16809 case X86::BI__builtin_ia32_reduce_fmin_pd512:
16810 case X86::BI__builtin_ia32_reduce_fmin_ps512:
16811 case X86::BI__builtin_ia32_reduce_fmin_ph512:
16812 case X86::BI__builtin_ia32_reduce_fmin_ph256:
16813 case X86::BI__builtin_ia32_reduce_fmin_ph128: {
16816 IRBuilder<>::FastMathFlagGuard FMFGuard(
Builder);
16817 Builder.getFastMathFlags().setNoNaNs();
16818 return Builder.CreateCall(F, {Ops[0]});
16821 case X86::BI__builtin_ia32_rdrand16_step:
16822 case X86::BI__builtin_ia32_rdrand32_step:
16823 case X86::BI__builtin_ia32_rdrand64_step:
16824 case X86::BI__builtin_ia32_rdseed16_step:
16825 case X86::BI__builtin_ia32_rdseed32_step:
16826 case X86::BI__builtin_ia32_rdseed64_step: {
16828 switch (BuiltinID) {
16829 default: llvm_unreachable(
"Unsupported intrinsic!");
16830 case X86::BI__builtin_ia32_rdrand16_step:
16831 ID = Intrinsic::x86_rdrand_16;
16833 case X86::BI__builtin_ia32_rdrand32_step:
16834 ID = Intrinsic::x86_rdrand_32;
16836 case X86::BI__builtin_ia32_rdrand64_step:
16837 ID = Intrinsic::x86_rdrand_64;
16839 case X86::BI__builtin_ia32_rdseed16_step:
16840 ID = Intrinsic::x86_rdseed_16;
16842 case X86::BI__builtin_ia32_rdseed32_step:
16843 ID = Intrinsic::x86_rdseed_32;
16845 case X86::BI__builtin_ia32_rdseed64_step:
16846 ID = Intrinsic::x86_rdseed_64;
16855 case X86::BI__builtin_ia32_addcarryx_u32:
16856 case X86::BI__builtin_ia32_addcarryx_u64:
16857 case X86::BI__builtin_ia32_subborrow_u32:
16858 case X86::BI__builtin_ia32_subborrow_u64: {
16860 switch (BuiltinID) {
16861 default: llvm_unreachable(
"Unsupported intrinsic!");
16862 case X86::BI__builtin_ia32_addcarryx_u32:
16863 IID = Intrinsic::x86_addcarry_32;
16865 case X86::BI__builtin_ia32_addcarryx_u64:
16866 IID = Intrinsic::x86_addcarry_64;
16868 case X86::BI__builtin_ia32_subborrow_u32:
16869 IID = Intrinsic::x86_subborrow_32;
16871 case X86::BI__builtin_ia32_subborrow_u64:
16872 IID = Intrinsic::x86_subborrow_64;
16877 { Ops[0], Ops[1], Ops[2] });
16883 case X86::BI__builtin_ia32_fpclassps128_mask:
16884 case X86::BI__builtin_ia32_fpclassps256_mask:
16885 case X86::BI__builtin_ia32_fpclassps512_mask:
16886 case X86::BI__builtin_ia32_vfpclasspbf16128_mask:
16887 case X86::BI__builtin_ia32_vfpclasspbf16256_mask:
16888 case X86::BI__builtin_ia32_vfpclasspbf16512_mask:
16889 case X86::BI__builtin_ia32_fpclassph128_mask:
16890 case X86::BI__builtin_ia32_fpclassph256_mask:
16891 case X86::BI__builtin_ia32_fpclassph512_mask:
16892 case X86::BI__builtin_ia32_fpclasspd128_mask:
16893 case X86::BI__builtin_ia32_fpclasspd256_mask:
16894 case X86::BI__builtin_ia32_fpclasspd512_mask: {
16896 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16897 Value *MaskIn = Ops[2];
16898 Ops.erase(&Ops[2]);
16901 switch (BuiltinID) {
16902 default: llvm_unreachable(
"Unsupported intrinsic!");
16903 case X86::BI__builtin_ia32_vfpclasspbf16128_mask:
16904 ID = Intrinsic::x86_avx10_fpclass_nepbf16_128;
16906 case X86::BI__builtin_ia32_vfpclasspbf16256_mask:
16907 ID = Intrinsic::x86_avx10_fpclass_nepbf16_256;
16909 case X86::BI__builtin_ia32_vfpclasspbf16512_mask:
16910 ID = Intrinsic::x86_avx10_fpclass_nepbf16_512;
16912 case X86::BI__builtin_ia32_fpclassph128_mask:
16913 ID = Intrinsic::x86_avx512fp16_fpclass_ph_128;
16915 case X86::BI__builtin_ia32_fpclassph256_mask:
16916 ID = Intrinsic::x86_avx512fp16_fpclass_ph_256;
16918 case X86::BI__builtin_ia32_fpclassph512_mask:
16919 ID = Intrinsic::x86_avx512fp16_fpclass_ph_512;
16921 case X86::BI__builtin_ia32_fpclassps128_mask:
16922 ID = Intrinsic::x86_avx512_fpclass_ps_128;
16924 case X86::BI__builtin_ia32_fpclassps256_mask:
16925 ID = Intrinsic::x86_avx512_fpclass_ps_256;
16927 case X86::BI__builtin_ia32_fpclassps512_mask:
16928 ID = Intrinsic::x86_avx512_fpclass_ps_512;
16930 case X86::BI__builtin_ia32_fpclasspd128_mask:
16931 ID = Intrinsic::x86_avx512_fpclass_pd_128;
16933 case X86::BI__builtin_ia32_fpclasspd256_mask:
16934 ID = Intrinsic::x86_avx512_fpclass_pd_256;
16936 case X86::BI__builtin_ia32_fpclasspd512_mask:
16937 ID = Intrinsic::x86_avx512_fpclass_pd_512;
16945 case X86::BI__builtin_ia32_vp2intersect_q_512:
16946 case X86::BI__builtin_ia32_vp2intersect_q_256:
16947 case X86::BI__builtin_ia32_vp2intersect_q_128:
16948 case X86::BI__builtin_ia32_vp2intersect_d_512:
16949 case X86::BI__builtin_ia32_vp2intersect_d_256:
16950 case X86::BI__builtin_ia32_vp2intersect_d_128: {
16952 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16955 switch (BuiltinID) {
16956 default: llvm_unreachable(
"Unsupported intrinsic!");
16957 case X86::BI__builtin_ia32_vp2intersect_q_512:
16958 ID = Intrinsic::x86_avx512_vp2intersect_q_512;
16960 case X86::BI__builtin_ia32_vp2intersect_q_256:
16961 ID = Intrinsic::x86_avx512_vp2intersect_q_256;
16963 case X86::BI__builtin_ia32_vp2intersect_q_128:
16964 ID = Intrinsic::x86_avx512_vp2intersect_q_128;
16966 case X86::BI__builtin_ia32_vp2intersect_d_512:
16967 ID = Intrinsic::x86_avx512_vp2intersect_d_512;
16969 case X86::BI__builtin_ia32_vp2intersect_d_256:
16970 ID = Intrinsic::x86_avx512_vp2intersect_d_256;
16972 case X86::BI__builtin_ia32_vp2intersect_d_128:
16973 ID = Intrinsic::x86_avx512_vp2intersect_d_128;
16987 case X86::BI__builtin_ia32_vpmultishiftqb128:
16988 case X86::BI__builtin_ia32_vpmultishiftqb256:
16989 case X86::BI__builtin_ia32_vpmultishiftqb512: {
16991 switch (BuiltinID) {
16992 default: llvm_unreachable(
"Unsupported intrinsic!");
16993 case X86::BI__builtin_ia32_vpmultishiftqb128:
16994 ID = Intrinsic::x86_avx512_pmultishift_qb_128;
16996 case X86::BI__builtin_ia32_vpmultishiftqb256:
16997 ID = Intrinsic::x86_avx512_pmultishift_qb_256;
16999 case X86::BI__builtin_ia32_vpmultishiftqb512:
17000 ID = Intrinsic::x86_avx512_pmultishift_qb_512;
17007 case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
17008 case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
17009 case X86::BI__builtin_ia32_vpshufbitqmb512_mask: {
17011 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
17012 Value *MaskIn = Ops[2];
17013 Ops.erase(&Ops[2]);
17016 switch (BuiltinID) {
17017 default: llvm_unreachable(
"Unsupported intrinsic!");
17018 case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
17019 ID = Intrinsic::x86_avx512_vpshufbitqmb_128;
17021 case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
17022 ID = Intrinsic::x86_avx512_vpshufbitqmb_256;
17024 case X86::BI__builtin_ia32_vpshufbitqmb512_mask:
17025 ID = Intrinsic::x86_avx512_vpshufbitqmb_512;
17034 case X86::BI__builtin_ia32_cmpeqps:
17035 case X86::BI__builtin_ia32_cmpeqpd:
17036 return getVectorFCmpIR(CmpInst::FCMP_OEQ,
false);
17037 case X86::BI__builtin_ia32_cmpltps:
17038 case X86::BI__builtin_ia32_cmpltpd:
17039 return getVectorFCmpIR(CmpInst::FCMP_OLT,
true);
17040 case X86::BI__builtin_ia32_cmpleps:
17041 case X86::BI__builtin_ia32_cmplepd:
17042 return getVectorFCmpIR(CmpInst::FCMP_OLE,
true);
17043 case X86::BI__builtin_ia32_cmpunordps:
17044 case X86::BI__builtin_ia32_cmpunordpd:
17045 return getVectorFCmpIR(CmpInst::FCMP_UNO,
false);
17046 case X86::BI__builtin_ia32_cmpneqps:
17047 case X86::BI__builtin_ia32_cmpneqpd:
17048 return getVectorFCmpIR(CmpInst::FCMP_UNE,
false);
17049 case X86::BI__builtin_ia32_cmpnltps:
17050 case X86::BI__builtin_ia32_cmpnltpd:
17051 return getVectorFCmpIR(CmpInst::FCMP_UGE,
true);
17052 case X86::BI__builtin_ia32_cmpnleps:
17053 case X86::BI__builtin_ia32_cmpnlepd:
17054 return getVectorFCmpIR(CmpInst::FCMP_UGT,
true);
17055 case X86::BI__builtin_ia32_cmpordps:
17056 case X86::BI__builtin_ia32_cmpordpd:
17057 return getVectorFCmpIR(CmpInst::FCMP_ORD,
false);
17058 case X86::BI__builtin_ia32_cmpph128_mask:
17059 case X86::BI__builtin_ia32_cmpph256_mask:
17060 case X86::BI__builtin_ia32_cmpph512_mask:
17061 case X86::BI__builtin_ia32_cmpps128_mask:
17062 case X86::BI__builtin_ia32_cmpps256_mask:
17063 case X86::BI__builtin_ia32_cmpps512_mask:
17064 case X86::BI__builtin_ia32_cmppd128_mask:
17065 case X86::BI__builtin_ia32_cmppd256_mask:
17066 case X86::BI__builtin_ia32_cmppd512_mask:
17067 case X86::BI__builtin_ia32_vcmppd256_round_mask:
17068 case X86::BI__builtin_ia32_vcmpps256_round_mask:
17069 case X86::BI__builtin_ia32_vcmpph256_round_mask:
17070 case X86::BI__builtin_ia32_vcmppbf16512_mask:
17071 case X86::BI__builtin_ia32_vcmppbf16256_mask:
17072 case X86::BI__builtin_ia32_vcmppbf16128_mask:
17075 case X86::BI__builtin_ia32_cmpps:
17076 case X86::BI__builtin_ia32_cmpps256:
17077 case X86::BI__builtin_ia32_cmppd:
17078 case X86::BI__builtin_ia32_cmppd256: {
17086 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x1f;
17091 FCmpInst::Predicate Pred;
17095 switch (CC & 0xf) {
17096 case 0x00: Pred = FCmpInst::FCMP_OEQ; IsSignaling =
false;
break;
17097 case 0x01: Pred = FCmpInst::FCMP_OLT; IsSignaling =
true;
break;
17098 case 0x02: Pred = FCmpInst::FCMP_OLE; IsSignaling =
true;
break;
17099 case 0x03: Pred = FCmpInst::FCMP_UNO; IsSignaling =
false;
break;
17100 case 0x04: Pred = FCmpInst::FCMP_UNE; IsSignaling =
false;
break;
17101 case 0x05: Pred = FCmpInst::FCMP_UGE; IsSignaling =
true;
break;
17102 case 0x06: Pred = FCmpInst::FCMP_UGT; IsSignaling =
true;
break;
17103 case 0x07: Pred = FCmpInst::FCMP_ORD; IsSignaling =
false;
break;
17104 case 0x08: Pred = FCmpInst::FCMP_UEQ; IsSignaling =
false;
break;
17105 case 0x09: Pred = FCmpInst::FCMP_ULT; IsSignaling =
true;
break;
17106 case 0x0a: Pred = FCmpInst::FCMP_ULE; IsSignaling =
true;
break;
17107 case 0x0b: Pred = FCmpInst::FCMP_FALSE; IsSignaling =
false;
break;
17108 case 0x0c: Pred = FCmpInst::FCMP_ONE; IsSignaling =
false;
break;
17109 case 0x0d: Pred = FCmpInst::FCMP_OGE; IsSignaling =
true;
break;
17110 case 0x0e: Pred = FCmpInst::FCMP_OGT; IsSignaling =
true;
break;
17111 case 0x0f: Pred = FCmpInst::FCMP_TRUE; IsSignaling =
false;
break;
17112 default: llvm_unreachable(
"Unhandled CC");
17117 IsSignaling = !IsSignaling;
17124 if (
Builder.getIsFPConstrained() &&
17125 (Pred == FCmpInst::FCMP_TRUE || Pred == FCmpInst::FCMP_FALSE ||
17129 switch (BuiltinID) {
17130 default: llvm_unreachable(
"Unexpected builtin");
17131 case X86::BI__builtin_ia32_cmpps:
17132 IID = Intrinsic::x86_sse_cmp_ps;
17134 case X86::BI__builtin_ia32_cmpps256:
17135 IID = Intrinsic::x86_avx_cmp_ps_256;
17137 case X86::BI__builtin_ia32_cmppd:
17138 IID = Intrinsic::x86_sse2_cmp_pd;
17140 case X86::BI__builtin_ia32_cmppd256:
17141 IID = Intrinsic::x86_avx_cmp_pd_256;
17143 case X86::BI__builtin_ia32_cmpph128_mask:
17144 IID = Intrinsic::x86_avx512fp16_mask_cmp_ph_128;
17146 case X86::BI__builtin_ia32_cmpph256_mask:
17147 IID = Intrinsic::x86_avx512fp16_mask_cmp_ph_256;
17149 case X86::BI__builtin_ia32_cmpph512_mask:
17150 IID = Intrinsic::x86_avx512fp16_mask_cmp_ph_512;
17152 case X86::BI__builtin_ia32_cmpps512_mask:
17153 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
17155 case X86::BI__builtin_ia32_cmppd512_mask:
17156 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
17158 case X86::BI__builtin_ia32_cmpps128_mask:
17159 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
17161 case X86::BI__builtin_ia32_cmpps256_mask:
17162 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
17164 case X86::BI__builtin_ia32_cmppd128_mask:
17165 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
17167 case X86::BI__builtin_ia32_cmppd256_mask:
17168 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
17175 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
17181 return Builder.CreateCall(Intr, Ops);
17192 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
17195 Cmp =
Builder.CreateFCmpS(Pred, Ops[0], Ops[1]);
17197 Cmp =
Builder.CreateFCmp(Pred, Ops[0], Ops[1]);
17201 return getVectorFCmpIR(Pred, IsSignaling);
17205 case X86::BI__builtin_ia32_cmpeqss:
17206 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 0);
17207 case X86::BI__builtin_ia32_cmpltss:
17208 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 1);
17209 case X86::BI__builtin_ia32_cmpless:
17210 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 2);
17211 case X86::BI__builtin_ia32_cmpunordss:
17212 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 3);
17213 case X86::BI__builtin_ia32_cmpneqss:
17214 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 4);
17215 case X86::BI__builtin_ia32_cmpnltss:
17216 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 5);
17217 case X86::BI__builtin_ia32_cmpnless:
17218 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 6);
17219 case X86::BI__builtin_ia32_cmpordss:
17220 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 7);
17221 case X86::BI__builtin_ia32_cmpeqsd:
17222 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 0);
17223 case X86::BI__builtin_ia32_cmpltsd:
17224 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 1);
17225 case X86::BI__builtin_ia32_cmplesd:
17226 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 2);
17227 case X86::BI__builtin_ia32_cmpunordsd:
17228 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 3);
17229 case X86::BI__builtin_ia32_cmpneqsd:
17230 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 4);
17231 case X86::BI__builtin_ia32_cmpnltsd:
17232 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 5);
17233 case X86::BI__builtin_ia32_cmpnlesd:
17234 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 6);
17235 case X86::BI__builtin_ia32_cmpordsd:
17236 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 7);
17239 case X86::BI__builtin_ia32_vcvtph2ps:
17240 case X86::BI__builtin_ia32_vcvtph2ps256:
17241 case X86::BI__builtin_ia32_vcvtph2ps_mask:
17242 case X86::BI__builtin_ia32_vcvtph2ps256_mask:
17243 case X86::BI__builtin_ia32_vcvtph2ps512_mask: {
17244 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this,
E);
17249 case X86::BI__builtin_ia32_cvtneps2bf16_128_mask: {
17252 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements());
17253 Intrinsic::ID IID = Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128;
17256 case X86::BI__builtin_ia32_cvtsbf162ss_32:
17259 case X86::BI__builtin_ia32_cvtneps2bf16_256_mask:
17260 case X86::BI__builtin_ia32_cvtneps2bf16_512_mask: {
17262 switch (BuiltinID) {
17263 default: llvm_unreachable(
"Unsupported intrinsic!");
17264 case X86::BI__builtin_ia32_cvtneps2bf16_256_mask:
17265 IID = Intrinsic::x86_avx512bf16_cvtneps2bf16_256;
17267 case X86::BI__builtin_ia32_cvtneps2bf16_512_mask:
17268 IID = Intrinsic::x86_avx512bf16_cvtneps2bf16_512;
17275 case X86::BI__cpuid:
17276 case X86::BI__cpuidex: {
17278 Value *SubFuncId = BuiltinID == X86::BI__cpuidex
17282 llvm::StructType *CpuidRetTy =
17284 llvm::FunctionType *FTy =
17287 StringRef
Asm, Constraints;
17288 if (
getTarget().getTriple().getArch() == llvm::Triple::x86) {
17290 Constraints =
"={ax},={bx},={cx},={dx},{ax},{cx}";
17293 Asm =
"xchgq %rbx, ${1:q}\n"
17295 "xchgq %rbx, ${1:q}";
17296 Constraints =
"={ax},=r,={cx},={dx},0,2";
17299 llvm::InlineAsm *IA = llvm::InlineAsm::get(FTy,
Asm, Constraints,
17301 Value *IACall =
Builder.CreateCall(IA, {FuncId, SubFuncId});
17304 for (
unsigned i = 0; i < 4; i++) {
17305 Value *Extracted =
Builder.CreateExtractValue(IACall, i);
17315 case X86::BI__emul:
17316 case X86::BI__emulu: {
17318 bool isSigned = (BuiltinID == X86::BI__emul);
17321 return Builder.CreateMul(LHS, RHS,
"", !isSigned, isSigned);
17323 case X86::BI__mulh:
17324 case X86::BI__umulh:
17325 case X86::BI_mul128:
17326 case X86::BI_umul128: {
17328 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
17330 bool IsSigned = (BuiltinID == X86::BI__mulh || BuiltinID == X86::BI_mul128);
17331 Value *LHS =
Builder.CreateIntCast(Ops[0], Int128Ty, IsSigned);
17332 Value *RHS =
Builder.CreateIntCast(Ops[1], Int128Ty, IsSigned);
17334 Value *MulResult, *HigherBits;
17336 MulResult =
Builder.CreateNSWMul(LHS, RHS);
17337 HigherBits =
Builder.CreateAShr(MulResult, 64);
17339 MulResult =
Builder.CreateNUWMul(LHS, RHS);
17340 HigherBits =
Builder.CreateLShr(MulResult, 64);
17342 HigherBits =
Builder.CreateIntCast(HigherBits, ResType, IsSigned);
17344 if (BuiltinID == X86::BI__mulh || BuiltinID == X86::BI__umulh)
17349 return Builder.CreateIntCast(MulResult, ResType, IsSigned);
17352 case X86::BI__faststorefence: {
17353 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
17354 llvm::SyncScope::System);
17356 case X86::BI__shiftleft128:
17357 case X86::BI__shiftright128: {
17359 BuiltinID == X86::BI__shiftleft128 ? Intrinsic::fshl : Intrinsic::fshr,
17364 std::swap(Ops[0], Ops[1]);
17366 return Builder.CreateCall(F, Ops);
17368 case X86::BI_ReadWriteBarrier:
17369 case X86::BI_ReadBarrier:
17370 case X86::BI_WriteBarrier: {
17371 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
17372 llvm::SyncScope::SingleThread);
17375 case X86::BI_AddressOfReturnAddress: {
17378 return Builder.CreateCall(F);
17380 case X86::BI__stosb: {
17386 case X86::BI__builtin_ia32_t2rpntlvwz0_internal:
17387 case X86::BI__builtin_ia32_t2rpntlvwz0rs_internal:
17388 case X86::BI__builtin_ia32_t2rpntlvwz0t1_internal:
17389 case X86::BI__builtin_ia32_t2rpntlvwz0rst1_internal:
17390 case X86::BI__builtin_ia32_t2rpntlvwz1_internal:
17391 case X86::BI__builtin_ia32_t2rpntlvwz1rs_internal:
17392 case X86::BI__builtin_ia32_t2rpntlvwz1t1_internal:
17393 case X86::BI__builtin_ia32_t2rpntlvwz1rst1_internal: {
17395 switch (BuiltinID) {
17397 llvm_unreachable(
"Unsupported intrinsic!");
17398 case X86::BI__builtin_ia32_t2rpntlvwz0_internal:
17399 IID = Intrinsic::x86_t2rpntlvwz0_internal;
17401 case X86::BI__builtin_ia32_t2rpntlvwz0rs_internal:
17402 IID = Intrinsic::x86_t2rpntlvwz0rs_internal;
17404 case X86::BI__builtin_ia32_t2rpntlvwz0t1_internal:
17405 IID = Intrinsic::x86_t2rpntlvwz0t1_internal;
17407 case X86::BI__builtin_ia32_t2rpntlvwz0rst1_internal:
17408 IID = Intrinsic::x86_t2rpntlvwz0rst1_internal;
17410 case X86::BI__builtin_ia32_t2rpntlvwz1_internal:
17411 IID = Intrinsic::x86_t2rpntlvwz1_internal;
17413 case X86::BI__builtin_ia32_t2rpntlvwz1rs_internal:
17414 IID = Intrinsic::x86_t2rpntlvwz1rs_internal;
17416 case X86::BI__builtin_ia32_t2rpntlvwz1t1_internal:
17417 IID = Intrinsic::x86_t2rpntlvwz1t1_internal;
17419 case X86::BI__builtin_ia32_t2rpntlvwz1rst1_internal:
17420 IID = Intrinsic::x86_t2rpntlvwz1rst1_internal;
17426 {Ops[0], Ops[1], Ops[2], Ops[5], Ops[6]});
17429 assert(PtrTy &&
"arg3 must be of pointer type");
17436 Value *VecT0 =
Builder.CreateIntrinsic(Intrinsic::x86_cast_tile_to_vector,
17442 Value *VecT1 =
Builder.CreateIntrinsic(Intrinsic::x86_cast_tile_to_vector,
17456 case X86::BI__int2c: {
17458 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy,
false);
17459 llvm::InlineAsm *IA =
17460 llvm::InlineAsm::get(FTy,
"int $$0x2c",
"",
true);
17461 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
17463 llvm::Attribute::NoReturn);
17464 llvm::CallInst *CI =
Builder.CreateCall(IA);
17465 CI->setAttributes(NoReturnAttr);
17468 case X86::BI__readfsbyte:
17469 case X86::BI__readfsword:
17470 case X86::BI__readfsdword:
17471 case X86::BI__readfsqword: {
17477 Load->setVolatile(
true);
17480 case X86::BI__readgsbyte:
17481 case X86::BI__readgsword:
17482 case X86::BI__readgsdword:
17483 case X86::BI__readgsqword: {
17489 Load->setVolatile(
true);
17492 case X86::BI__builtin_ia32_encodekey128_u32: {
17493 Intrinsic::ID IID = Intrinsic::x86_encodekey128;
17497 for (
int i = 0; i < 3; ++i) {
17505 case X86::BI__builtin_ia32_encodekey256_u32: {
17506 Intrinsic::ID IID = Intrinsic::x86_encodekey256;
17511 for (
int i = 0; i < 4; ++i) {
17519 case X86::BI__builtin_ia32_aesenc128kl_u8:
17520 case X86::BI__builtin_ia32_aesdec128kl_u8:
17521 case X86::BI__builtin_ia32_aesenc256kl_u8:
17522 case X86::BI__builtin_ia32_aesdec256kl_u8: {
17524 StringRef BlockName;
17525 switch (BuiltinID) {
17527 llvm_unreachable(
"Unexpected builtin");
17528 case X86::BI__builtin_ia32_aesenc128kl_u8:
17529 IID = Intrinsic::x86_aesenc128kl;
17530 BlockName =
"aesenc128kl";
17532 case X86::BI__builtin_ia32_aesdec128kl_u8:
17533 IID = Intrinsic::x86_aesdec128kl;
17534 BlockName =
"aesdec128kl";
17536 case X86::BI__builtin_ia32_aesenc256kl_u8:
17537 IID = Intrinsic::x86_aesenc256kl;
17538 BlockName =
"aesenc256kl";
17540 case X86::BI__builtin_ia32_aesdec256kl_u8:
17541 IID = Intrinsic::x86_aesdec256kl;
17542 BlockName =
"aesdec256kl";
17548 BasicBlock *NoError =
17556 Builder.CreateCondBr(Succ, NoError, Error);
17558 Builder.SetInsertPoint(NoError);
17562 Builder.SetInsertPoint(Error);
17563 Constant *
Zero = llvm::Constant::getNullValue(Out->getType());
17570 case X86::BI__builtin_ia32_aesencwide128kl_u8:
17571 case X86::BI__builtin_ia32_aesdecwide128kl_u8:
17572 case X86::BI__builtin_ia32_aesencwide256kl_u8:
17573 case X86::BI__builtin_ia32_aesdecwide256kl_u8: {
17575 StringRef BlockName;
17576 switch (BuiltinID) {
17577 case X86::BI__builtin_ia32_aesencwide128kl_u8:
17578 IID = Intrinsic::x86_aesencwide128kl;
17579 BlockName =
"aesencwide128kl";
17581 case X86::BI__builtin_ia32_aesdecwide128kl_u8:
17582 IID = Intrinsic::x86_aesdecwide128kl;
17583 BlockName =
"aesdecwide128kl";
17585 case X86::BI__builtin_ia32_aesencwide256kl_u8:
17586 IID = Intrinsic::x86_aesencwide256kl;
17587 BlockName =
"aesencwide256kl";
17589 case X86::BI__builtin_ia32_aesdecwide256kl_u8:
17590 IID = Intrinsic::x86_aesdecwide256kl;
17591 BlockName =
"aesdecwide256kl";
17595 llvm::Type *Ty = FixedVectorType::get(
Builder.getInt64Ty(), 2);
17598 for (
int i = 0; i != 8; ++i) {
17599 Value *Ptr =
Builder.CreateConstGEP1_32(Ty, Ops[1], i);
17605 BasicBlock *NoError =
17612 Builder.CreateCondBr(Succ, NoError, Error);
17614 Builder.SetInsertPoint(NoError);
17615 for (
int i = 0; i != 8; ++i) {
17622 Builder.SetInsertPoint(Error);
17623 for (
int i = 0; i != 8; ++i) {
17625 Constant *
Zero = llvm::Constant::getNullValue(Out->getType());
17626 Value *Ptr =
Builder.CreateConstGEP1_32(Out->getType(), Ops[0], i);
17634 case X86::BI__builtin_ia32_vfcmaddcph512_mask:
17637 case X86::BI__builtin_ia32_vfmaddcph512_mask: {
17638 Intrinsic::ID IID = IsConjFMA
17639 ? Intrinsic::x86_avx512fp16_mask_vfcmadd_cph_512
17640 : Intrinsic::x86_avx512fp16_mask_vfmadd_cph_512;
17644 case X86::BI__builtin_ia32_vfcmaddcph256_round_mask:
17647 case X86::BI__builtin_ia32_vfmaddcph256_round_mask: {
17648 Intrinsic::ID IID = IsConjFMA ? Intrinsic::x86_avx10_mask_vfcmaddcph256
17649 : Intrinsic::x86_avx10_mask_vfmaddcph256;
17653 case X86::BI__builtin_ia32_vfcmaddcsh_round_mask:
17656 case X86::BI__builtin_ia32_vfmaddcsh_round_mask: {
17657 Intrinsic::ID IID = IsConjFMA ? Intrinsic::x86_avx512fp16_mask_vfcmadd_csh
17658 : Intrinsic::x86_avx512fp16_mask_vfmadd_csh;
17663 case X86::BI__builtin_ia32_vfcmaddcsh_round_mask3:
17666 case X86::BI__builtin_ia32_vfmaddcsh_round_mask3: {
17667 Intrinsic::ID IID = IsConjFMA ? Intrinsic::x86_avx512fp16_mask_vfcmadd_csh
17668 : Intrinsic::x86_avx512fp16_mask_vfmadd_csh;
17670 static constexpr int Mask[] = {0, 5, 6, 7};
17671 return Builder.CreateShuffleVector(
Call, Ops[2], Mask);
17673 case X86::BI__builtin_ia32_prefetchi:
17676 {Ops[0], llvm::ConstantInt::get(Int32Ty, 0), Ops[1],
17677 llvm::ConstantInt::get(Int32Ty, 0)});
17695 Intrinsic::ID
ID = Intrinsic::not_intrinsic;
17697#include "llvm/TargetParser/PPCTargetParser.def"
17698 auto GenAIXPPCBuiltinCpuExpr = [&](
unsigned SupportMethod,
unsigned FieldIdx,
17699 unsigned Mask, CmpInst::Predicate CompOp,
17700 unsigned OpValue) ->
Value * {
17701 if (SupportMethod == BUILTIN_PPC_FALSE)
17704 if (SupportMethod == BUILTIN_PPC_TRUE)
17707 assert(SupportMethod <= SYS_CALL &&
"Invalid value for SupportMethod.");
17709 llvm::Value *FieldValue =
nullptr;
17710 if (SupportMethod == USE_SYS_CONF) {
17711 llvm::Type *STy = llvm::StructType::get(PPC_SYSTEMCONFIG_TYPE);
17712 llvm::Constant *SysConf =
17716 llvm::Value *Idxs[] = {ConstantInt::get(
Int32Ty, 0),
17717 ConstantInt::get(
Int32Ty, FieldIdx)};
17722 }
else if (SupportMethod == SYS_CALL) {
17723 llvm::FunctionType *FTy =
17725 llvm::FunctionCallee
Func =
17731 assert(FieldValue &&
17732 "SupportMethod value is not defined in PPCTargetParser.def.");
17735 FieldValue =
Builder.CreateAnd(FieldValue, Mask);
17737 llvm::Type *ValueType = FieldValue->getType();
17738 bool IsValueType64Bit = ValueType->isIntegerTy(64);
17740 (IsValueType64Bit || ValueType->isIntegerTy(32)) &&
17741 "Only 32/64-bit integers are supported in GenAIXPPCBuiltinCpuExpr().");
17744 CompOp, FieldValue,
17745 ConstantInt::get(IsValueType64Bit ?
Int64Ty :
Int32Ty, OpValue));
17748 switch (BuiltinID) {
17749 default:
return nullptr;
17751 case Builtin::BI__builtin_cpu_is: {
17753 StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
17756 unsigned LinuxSupportMethod, LinuxIDValue, AIXSupportMethod, AIXIDValue;
17757 typedef std::tuple<unsigned, unsigned, unsigned, unsigned>
CPUInfo;
17759 std::tie(LinuxSupportMethod, LinuxIDValue, AIXSupportMethod, AIXIDValue) =
17760 static_cast<CPUInfo>(StringSwitch<CPUInfo>(CPUStr)
17761#define PPC_CPU(NAME, Linux_SUPPORT_METHOD, LinuxID, AIX_SUPPORT_METHOD, \
17763 .Case(NAME, {Linux_SUPPORT_METHOD, LinuxID, AIX_SUPPORT_METHOD, AIXID})
17764#include "llvm/TargetParser/PPCTargetParser.def"
17765 .Default({BUILTIN_PPC_UNSUPPORTED, 0,
17766 BUILTIN_PPC_UNSUPPORTED, 0}));
17768 if (Triple.isOSAIX()) {
17769 assert((AIXSupportMethod != BUILTIN_PPC_UNSUPPORTED) &&
17770 "Invalid CPU name. Missed by SemaChecking?");
17771 return GenAIXPPCBuiltinCpuExpr(AIXSupportMethod, AIX_SYSCON_IMPL_IDX, 0,
17772 ICmpInst::ICMP_EQ, AIXIDValue);
17775 assert(Triple.isOSLinux() &&
17776 "__builtin_cpu_is() is only supported for AIX and Linux.");
17778 assert((LinuxSupportMethod != BUILTIN_PPC_UNSUPPORTED) &&
17779 "Invalid CPU name. Missed by SemaChecking?");
17781 if (LinuxSupportMethod == BUILTIN_PPC_FALSE)
17784 Value *Op0 = llvm::ConstantInt::get(
Int32Ty, PPC_FAWORD_CPUID);
17786 Value *TheCall =
Builder.CreateCall(F, {Op0},
"cpu_is");
17787 return Builder.CreateICmpEQ(TheCall,
17788 llvm::ConstantInt::get(
Int32Ty, LinuxIDValue));
17790 case Builtin::BI__builtin_cpu_supports: {
17793 StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
17794 if (Triple.isOSAIX()) {
17795 unsigned SupportMethod, FieldIdx, Mask,
Value;
17796 CmpInst::Predicate CompOp;
17800 std::tie(SupportMethod, FieldIdx, Mask, CompOp,
Value) =
17801 static_cast<CPUSupportType
>(StringSwitch<CPUSupportType>(CPUStr)
17802#define PPC_AIX_FEATURE(NAME, DESC, SUPPORT_METHOD, INDEX, MASK, COMP_OP, \
17804 .Case(NAME, {SUPPORT_METHOD, INDEX, MASK, COMP_OP, VALUE})
17805#include "llvm/TargetParser/PPCTargetParser.def"
17806 .Default({BUILTIN_PPC_FALSE, 0, 0,
17807 CmpInst::Predicate(), 0}));
17808 return GenAIXPPCBuiltinCpuExpr(SupportMethod, FieldIdx, Mask, CompOp,
17812 assert(Triple.isOSLinux() &&
17813 "__builtin_cpu_supports() is only supported for AIX and Linux.");
17814 unsigned FeatureWord;
17816 std::tie(FeatureWord, BitMask) =
17817 StringSwitch<std::pair<unsigned, unsigned>>(CPUStr)
17818#define
PPC_LNX_FEATURE(Name, Description, EnumName, Bitmask, FA_WORD) \
17819 .Case(Name, {FA_WORD, Bitmask})
17820#include
"llvm/TargetParser/PPCTargetParser.def"
17824 Value *Op0 = llvm::ConstantInt::get(
Int32Ty, FeatureWord);
17826 Value *TheCall =
Builder.CreateCall(F, {Op0},
"cpu_supports");
17828 Builder.CreateAnd(TheCall, llvm::ConstantInt::get(
Int32Ty, BitMask));
17829 return Builder.CreateICmpNE(Mask, llvm::Constant::getNullValue(
Int32Ty));
17830#undef PPC_FAWORD_HWCAP
17831#undef PPC_FAWORD_HWCAP2
17832#undef PPC_FAWORD_CPUID
17837 case PPC::BI__builtin_ppc_get_timebase:
17841 case PPC::BI__builtin_altivec_lvx:
17842 case PPC::BI__builtin_altivec_lvxl:
17843 case PPC::BI__builtin_altivec_lvebx:
17844 case PPC::BI__builtin_altivec_lvehx:
17845 case PPC::BI__builtin_altivec_lvewx:
17846 case PPC::BI__builtin_altivec_lvsl:
17847 case PPC::BI__builtin_altivec_lvsr:
17848 case PPC::BI__builtin_vsx_lxvd2x:
17849 case PPC::BI__builtin_vsx_lxvw4x:
17850 case PPC::BI__builtin_vsx_lxvd2x_be:
17851 case PPC::BI__builtin_vsx_lxvw4x_be:
17852 case PPC::BI__builtin_vsx_lxvl:
17853 case PPC::BI__builtin_vsx_lxvll:
17858 if (!(BuiltinID == PPC::BI__builtin_vsx_lxvl ||
17859 BuiltinID == PPC::BI__builtin_vsx_lxvll)) {
17864 switch (BuiltinID) {
17865 default: llvm_unreachable(
"Unsupported ld/lvsl/lvsr intrinsic!");
17866 case PPC::BI__builtin_altivec_lvx:
17867 ID = Intrinsic::ppc_altivec_lvx;
17869 case PPC::BI__builtin_altivec_lvxl:
17870 ID = Intrinsic::ppc_altivec_lvxl;
17872 case PPC::BI__builtin_altivec_lvebx:
17873 ID = Intrinsic::ppc_altivec_lvebx;
17875 case PPC::BI__builtin_altivec_lvehx:
17876 ID = Intrinsic::ppc_altivec_lvehx;
17878 case PPC::BI__builtin_altivec_lvewx:
17879 ID = Intrinsic::ppc_altivec_lvewx;
17881 case PPC::BI__builtin_altivec_lvsl:
17882 ID = Intrinsic::ppc_altivec_lvsl;
17884 case PPC::BI__builtin_altivec_lvsr:
17885 ID = Intrinsic::ppc_altivec_lvsr;
17887 case PPC::BI__builtin_vsx_lxvd2x:
17888 ID = Intrinsic::ppc_vsx_lxvd2x;
17890 case PPC::BI__builtin_vsx_lxvw4x:
17891 ID = Intrinsic::ppc_vsx_lxvw4x;
17893 case PPC::BI__builtin_vsx_lxvd2x_be:
17894 ID = Intrinsic::ppc_vsx_lxvd2x_be;
17896 case PPC::BI__builtin_vsx_lxvw4x_be:
17897 ID = Intrinsic::ppc_vsx_lxvw4x_be;
17899 case PPC::BI__builtin_vsx_lxvl:
17900 ID = Intrinsic::ppc_vsx_lxvl;
17902 case PPC::BI__builtin_vsx_lxvll:
17903 ID = Intrinsic::ppc_vsx_lxvll;
17907 return Builder.CreateCall(F, Ops,
"");
17911 case PPC::BI__builtin_altivec_stvx:
17912 case PPC::BI__builtin_altivec_stvxl:
17913 case PPC::BI__builtin_altivec_stvebx:
17914 case PPC::BI__builtin_altivec_stvehx:
17915 case PPC::BI__builtin_altivec_stvewx:
17916 case PPC::BI__builtin_vsx_stxvd2x:
17917 case PPC::BI__builtin_vsx_stxvw4x:
17918 case PPC::BI__builtin_vsx_stxvd2x_be:
17919 case PPC::BI__builtin_vsx_stxvw4x_be:
17920 case PPC::BI__builtin_vsx_stxvl:
17921 case PPC::BI__builtin_vsx_stxvll:
17927 if (!(BuiltinID == PPC::BI__builtin_vsx_stxvl ||
17928 BuiltinID == PPC::BI__builtin_vsx_stxvll)) {
17933 switch (BuiltinID) {
17934 default: llvm_unreachable(
"Unsupported st intrinsic!");
17935 case PPC::BI__builtin_altivec_stvx:
17936 ID = Intrinsic::ppc_altivec_stvx;
17938 case PPC::BI__builtin_altivec_stvxl:
17939 ID = Intrinsic::ppc_altivec_stvxl;
17941 case PPC::BI__builtin_altivec_stvebx:
17942 ID = Intrinsic::ppc_altivec_stvebx;
17944 case PPC::BI__builtin_altivec_stvehx:
17945 ID = Intrinsic::ppc_altivec_stvehx;
17947 case PPC::BI__builtin_altivec_stvewx:
17948 ID = Intrinsic::ppc_altivec_stvewx;
17950 case PPC::BI__builtin_vsx_stxvd2x:
17951 ID = Intrinsic::ppc_vsx_stxvd2x;
17953 case PPC::BI__builtin_vsx_stxvw4x:
17954 ID = Intrinsic::ppc_vsx_stxvw4x;
17956 case PPC::BI__builtin_vsx_stxvd2x_be:
17957 ID = Intrinsic::ppc_vsx_stxvd2x_be;
17959 case PPC::BI__builtin_vsx_stxvw4x_be:
17960 ID = Intrinsic::ppc_vsx_stxvw4x_be;
17962 case PPC::BI__builtin_vsx_stxvl:
17963 ID = Intrinsic::ppc_vsx_stxvl;
17965 case PPC::BI__builtin_vsx_stxvll:
17966 ID = Intrinsic::ppc_vsx_stxvll;
17970 return Builder.CreateCall(F, Ops,
"");
17972 case PPC::BI__builtin_vsx_ldrmb: {
17978 int64_t NumBytes = cast<ConstantInt>(Op1)->getZExtValue();
17983 if (NumBytes == 16) {
17991 for (
int Idx = 0; Idx < 16; Idx++)
17992 RevMask.push_back(15 - Idx);
17993 return Builder.CreateShuffleVector(LD, LD, RevMask);
17997 llvm::Function *Lvs =
CGM.
getIntrinsic(IsLE ? Intrinsic::ppc_altivec_lvsr
17998 : Intrinsic::ppc_altivec_lvsl);
17999 llvm::Function *Vperm =
CGM.
getIntrinsic(Intrinsic::ppc_altivec_vperm);
18001 Int8Ty, Op0, ConstantInt::get(Op1->
getType(), NumBytes - 1));
18003 Value *HiLd =
Builder.CreateCall(Lvx, HiMem,
"ld.hi");
18006 Op0 = IsLE ? HiLd : LoLd;
18007 Op1 = IsLE ? LoLd : HiLd;
18008 Value *AllElts =
Builder.CreateCall(Vperm, {Op0, Op1, Mask1},
"shuffle1");
18009 Constant *
Zero = llvm::Constant::getNullValue(IsLE ? ResTy : AllElts->
getType());
18013 for (
int Idx = 0; Idx < 16; Idx++) {
18014 int Val = (NumBytes - Idx - 1 >= 0) ? (NumBytes - Idx - 1)
18015 : 16 - (NumBytes - Idx);
18016 Consts.push_back(Val);
18018 return Builder.CreateShuffleVector(
Builder.CreateBitCast(AllElts, ResTy),
18022 for (
int Idx = 0; Idx < 16; Idx++)
18023 Consts.push_back(
Builder.getInt8(NumBytes + Idx));
18024 Value *Mask2 = ConstantVector::get(Consts);
18025 return Builder.CreateBitCast(
18026 Builder.CreateCall(Vperm, {Zero, AllElts, Mask2},
"shuffle2"), ResTy);
18028 case PPC::BI__builtin_vsx_strmb: {
18032 int64_t NumBytes = cast<ConstantInt>(Op1)->getZExtValue();
18034 auto StoreSubVec = [&](
unsigned Width,
unsigned Offset,
unsigned EltNo) {
18038 Value *StVec = Op2;
18041 for (
int Idx = 0; Idx < 16; Idx++)
18042 RevMask.push_back(15 - Idx);
18043 StVec =
Builder.CreateShuffleVector(Op2, Op2, RevMask);
18049 unsigned NumElts = 0;
18052 llvm_unreachable(
"width for stores must be a power of 2");
18071 Op2, llvm::FixedVectorType::get(ConvTy, NumElts));
18074 Value *Elt =
Builder.CreateExtractElement(Vec, EltNo);
18075 if (IsLE && Width > 1) {
18077 Elt =
Builder.CreateCall(F, Elt);
18082 unsigned Stored = 0;
18083 unsigned RemainingBytes = NumBytes;
18085 if (NumBytes == 16)
18086 return StoreSubVec(16, 0, 0);
18087 if (NumBytes >= 8) {
18088 Result = StoreSubVec(8, NumBytes - 8, IsLE ? 0 : 1);
18089 RemainingBytes -= 8;
18092 if (RemainingBytes >= 4) {
18093 Result = StoreSubVec(4, NumBytes - Stored - 4,
18094 IsLE ? (Stored >> 2) : 3 - (Stored >> 2));
18095 RemainingBytes -= 4;
18098 if (RemainingBytes >= 2) {
18099 Result = StoreSubVec(2, NumBytes - Stored - 2,
18100 IsLE ? (Stored >> 1) : 7 - (Stored >> 1));
18101 RemainingBytes -= 2;
18104 if (RemainingBytes)
18106 StoreSubVec(1, NumBytes - Stored - 1, IsLE ? Stored : 15 - Stored);
18110 case PPC::BI__builtin_vsx_xvsqrtsp:
18111 case PPC::BI__builtin_vsx_xvsqrtdp: {
18114 if (
Builder.getIsFPConstrained()) {
18116 Intrinsic::experimental_constrained_sqrt, ResultType);
18117 return Builder.CreateConstrainedFPCall(F,
X);
18124 case PPC::BI__builtin_altivec_vclzb:
18125 case PPC::BI__builtin_altivec_vclzh:
18126 case PPC::BI__builtin_altivec_vclzw:
18127 case PPC::BI__builtin_altivec_vclzd: {
18130 Value *Undef = ConstantInt::get(
Builder.getInt1Ty(),
false);
18132 return Builder.CreateCall(F, {
X, Undef});
18134 case PPC::BI__builtin_altivec_vctzb:
18135 case PPC::BI__builtin_altivec_vctzh:
18136 case PPC::BI__builtin_altivec_vctzw:
18137 case PPC::BI__builtin_altivec_vctzd: {
18140 Value *Undef = ConstantInt::get(
Builder.getInt1Ty(),
false);
18142 return Builder.CreateCall(F, {
X, Undef});
18144 case PPC::BI__builtin_altivec_vinsd:
18145 case PPC::BI__builtin_altivec_vinsw:
18146 case PPC::BI__builtin_altivec_vinsd_elt:
18147 case PPC::BI__builtin_altivec_vinsw_elt: {
18153 bool IsUnaligned = (BuiltinID == PPC::BI__builtin_altivec_vinsw ||
18154 BuiltinID == PPC::BI__builtin_altivec_vinsd);
18156 bool Is32bit = (BuiltinID == PPC::BI__builtin_altivec_vinsw ||
18157 BuiltinID == PPC::BI__builtin_altivec_vinsw_elt);
18160 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
18162 "Third Arg to vinsw/vinsd intrinsic must be a constant integer!");
18166 int ValidMaxValue = 0;
18168 ValidMaxValue = (Is32bit) ? 12 : 8;
18170 ValidMaxValue = (Is32bit) ? 3 : 1;
18173 int64_t ConstArg = ArgCI->getSExtValue();
18176 std::string RangeErrMsg = IsUnaligned ?
"byte" :
"element";
18177 RangeErrMsg +=
" number " + llvm::to_string(ConstArg);
18178 RangeErrMsg +=
" is outside of the valid range [0, ";
18179 RangeErrMsg += llvm::to_string(ValidMaxValue) +
"]";
18182 if (ConstArg < 0 || ConstArg > ValidMaxValue)
18186 if (!IsUnaligned) {
18187 ConstArg *= Is32bit ? 4 : 8;
18190 ConstArg = (Is32bit ? 12 : 8) - ConstArg;
18193 ID = Is32bit ? Intrinsic::ppc_altivec_vinsw : Intrinsic::ppc_altivec_vinsd;
18194 Op2 = ConstantInt::getSigned(
Int32Ty, ConstArg);
18198 ?
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int32Ty, 4))
18200 llvm::FixedVectorType::get(
Int64Ty, 2));
18201 return Builder.CreateBitCast(
18204 case PPC::BI__builtin_altivec_vadduqm:
18205 case PPC::BI__builtin_altivec_vsubuqm: {
18208 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
18209 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(Int128Ty, 1));
18210 Op1 =
Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(Int128Ty, 1));
18211 if (BuiltinID == PPC::BI__builtin_altivec_vadduqm)
18212 return Builder.CreateAdd(Op0, Op1,
"vadduqm");
18214 return Builder.CreateSub(Op0, Op1,
"vsubuqm");
18216 case PPC::BI__builtin_altivec_vaddcuq_c:
18217 case PPC::BI__builtin_altivec_vsubcuq_c: {
18221 llvm::Type *V1I128Ty = llvm::FixedVectorType::get(
18223 Ops.push_back(
Builder.CreateBitCast(Op0, V1I128Ty));
18224 Ops.push_back(
Builder.CreateBitCast(Op1, V1I128Ty));
18225 ID = (BuiltinID == PPC::BI__builtin_altivec_vaddcuq_c)
18226 ? Intrinsic::ppc_altivec_vaddcuq
18227 : Intrinsic::ppc_altivec_vsubcuq;
18230 case PPC::BI__builtin_altivec_vaddeuqm_c:
18231 case PPC::BI__builtin_altivec_vaddecuq_c:
18232 case PPC::BI__builtin_altivec_vsubeuqm_c:
18233 case PPC::BI__builtin_altivec_vsubecuq_c: {
18238 llvm::Type *V1I128Ty = llvm::FixedVectorType::get(
18240 Ops.push_back(
Builder.CreateBitCast(Op0, V1I128Ty));
18241 Ops.push_back(
Builder.CreateBitCast(Op1, V1I128Ty));
18242 Ops.push_back(
Builder.CreateBitCast(Op2, V1I128Ty));
18243 switch (BuiltinID) {
18245 llvm_unreachable(
"Unsupported intrinsic!");
18246 case PPC::BI__builtin_altivec_vaddeuqm_c:
18247 ID = Intrinsic::ppc_altivec_vaddeuqm;
18249 case PPC::BI__builtin_altivec_vaddecuq_c:
18250 ID = Intrinsic::ppc_altivec_vaddecuq;
18252 case PPC::BI__builtin_altivec_vsubeuqm_c:
18253 ID = Intrinsic::ppc_altivec_vsubeuqm;
18255 case PPC::BI__builtin_altivec_vsubecuq_c:
18256 ID = Intrinsic::ppc_altivec_vsubecuq;
18261 case PPC::BI__builtin_ppc_rldimi:
18262 case PPC::BI__builtin_ppc_rlwimi: {
18269 if (BuiltinID == PPC::BI__builtin_ppc_rldimi &&
18279 ? Intrinsic::ppc_rldimi
18280 : Intrinsic::ppc_rlwimi),
18281 {Op0, Op1, Op2, Op3});
18283 case PPC::BI__builtin_ppc_rlwnm: {
18290 case PPC::BI__builtin_ppc_poppar4:
18291 case PPC::BI__builtin_ppc_poppar8: {
18293 llvm::Type *ArgType = Op0->
getType();
18299 if (
Result->getType() != ResultType)
18304 case PPC::BI__builtin_ppc_cmpb: {
18307 if (
getTarget().getTriple().isPPC64()) {
18310 return Builder.CreateCall(F, {Op0, Op1},
"cmpb");
18330 Constant *ShiftAmt = ConstantInt::get(
Int64Ty, 32);
18339 Value *ResHi =
Builder.CreateShl(ResHiShift, ShiftAmt);
18340 return Builder.CreateOr(ResLo, ResHi);
18343 case PPC::BI__builtin_vsx_xvcpsgnsp:
18344 case PPC::BI__builtin_vsx_xvcpsgndp: {
18348 ID = Intrinsic::copysign;
18350 return Builder.CreateCall(F, {
X, Y});
18353 case PPC::BI__builtin_vsx_xvrspip:
18354 case PPC::BI__builtin_vsx_xvrdpip:
18355 case PPC::BI__builtin_vsx_xvrdpim:
18356 case PPC::BI__builtin_vsx_xvrspim:
18357 case PPC::BI__builtin_vsx_xvrdpi:
18358 case PPC::BI__builtin_vsx_xvrspi:
18359 case PPC::BI__builtin_vsx_xvrdpic:
18360 case PPC::BI__builtin_vsx_xvrspic:
18361 case PPC::BI__builtin_vsx_xvrdpiz:
18362 case PPC::BI__builtin_vsx_xvrspiz: {
18365 if (BuiltinID == PPC::BI__builtin_vsx_xvrdpim ||
18366 BuiltinID == PPC::BI__builtin_vsx_xvrspim)
18368 ? Intrinsic::experimental_constrained_floor
18369 : Intrinsic::floor;
18370 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpi ||
18371 BuiltinID == PPC::BI__builtin_vsx_xvrspi)
18373 ? Intrinsic::experimental_constrained_round
18374 : Intrinsic::round;
18375 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpic ||
18376 BuiltinID == PPC::BI__builtin_vsx_xvrspic)
18378 ? Intrinsic::experimental_constrained_rint
18380 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpip ||
18381 BuiltinID == PPC::BI__builtin_vsx_xvrspip)
18383 ? Intrinsic::experimental_constrained_ceil
18385 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpiz ||
18386 BuiltinID == PPC::BI__builtin_vsx_xvrspiz)
18388 ? Intrinsic::experimental_constrained_trunc
18389 : Intrinsic::trunc;
18391 return Builder.getIsFPConstrained() ?
Builder.CreateConstrainedFPCall(F,
X)
18396 case PPC::BI__builtin_vsx_xvabsdp:
18397 case PPC::BI__builtin_vsx_xvabssp: {
18405 case PPC::BI__builtin_ppc_recipdivf:
18406 case PPC::BI__builtin_ppc_recipdivd:
18407 case PPC::BI__builtin_ppc_rsqrtf:
18408 case PPC::BI__builtin_ppc_rsqrtd: {
18409 FastMathFlags FMF =
Builder.getFastMathFlags();
18410 Builder.getFastMathFlags().setFast();
18414 if (BuiltinID == PPC::BI__builtin_ppc_recipdivf ||
18415 BuiltinID == PPC::BI__builtin_ppc_recipdivd) {
18418 Builder.getFastMathFlags() &= (FMF);
18421 auto *One = ConstantFP::get(ResultType, 1.0);
18424 Builder.getFastMathFlags() &= (FMF);
18427 case PPC::BI__builtin_ppc_alignx: {
18430 ConstantInt *AlignmentCI = cast<ConstantInt>(Op0);
18431 if (AlignmentCI->getValue().ugt(llvm::Value::MaximumAlignment))
18432 AlignmentCI = ConstantInt::get(AlignmentCI->getIntegerType(),
18433 llvm::Value::MaximumAlignment);
18437 AlignmentCI,
nullptr);
18440 case PPC::BI__builtin_ppc_rdlam: {
18444 llvm::Type *Ty = Op0->
getType();
18445 Value *ShiftAmt =
Builder.CreateIntCast(Op1, Ty,
false);
18447 Value *Rotate =
Builder.CreateCall(F, {Op0, Op0, ShiftAmt});
18448 return Builder.CreateAnd(Rotate, Op2);
18450 case PPC::BI__builtin_ppc_load2r: {
18457 case PPC::BI__builtin_ppc_fnmsub:
18458 case PPC::BI__builtin_ppc_fnmsubs:
18459 case PPC::BI__builtin_vsx_xvmaddadp:
18460 case PPC::BI__builtin_vsx_xvmaddasp:
18461 case PPC::BI__builtin_vsx_xvnmaddadp:
18462 case PPC::BI__builtin_vsx_xvnmaddasp:
18463 case PPC::BI__builtin_vsx_xvmsubadp:
18464 case PPC::BI__builtin_vsx_xvmsubasp:
18465 case PPC::BI__builtin_vsx_xvnmsubadp:
18466 case PPC::BI__builtin_vsx_xvnmsubasp: {
18472 if (
Builder.getIsFPConstrained())
18473 F =
CGM.
getIntrinsic(Intrinsic::experimental_constrained_fma, ResultType);
18476 switch (BuiltinID) {
18477 case PPC::BI__builtin_vsx_xvmaddadp:
18478 case PPC::BI__builtin_vsx_xvmaddasp:
18479 if (
Builder.getIsFPConstrained())
18480 return Builder.CreateConstrainedFPCall(F, {
X, Y, Z});
18482 return Builder.CreateCall(F, {
X, Y, Z});
18483 case PPC::BI__builtin_vsx_xvnmaddadp:
18484 case PPC::BI__builtin_vsx_xvnmaddasp:
18485 if (
Builder.getIsFPConstrained())
18487 Builder.CreateConstrainedFPCall(F, {X, Y, Z}),
"neg");
18489 return Builder.CreateFNeg(
Builder.CreateCall(F, {X, Y, Z}),
"neg");
18490 case PPC::BI__builtin_vsx_xvmsubadp:
18491 case PPC::BI__builtin_vsx_xvmsubasp:
18492 if (
Builder.getIsFPConstrained())
18493 return Builder.CreateConstrainedFPCall(
18494 F, {
X, Y,
Builder.CreateFNeg(Z,
"neg")});
18497 case PPC::BI__builtin_ppc_fnmsub:
18498 case PPC::BI__builtin_ppc_fnmsubs:
18499 case PPC::BI__builtin_vsx_xvnmsubadp:
18500 case PPC::BI__builtin_vsx_xvnmsubasp:
18501 if (
Builder.getIsFPConstrained())
18503 Builder.CreateConstrainedFPCall(
18504 F, {X, Y, Builder.CreateFNeg(Z,
"neg")}),
18510 llvm_unreachable(
"Unknown FMA operation");
18514 case PPC::BI__builtin_vsx_insertword: {
18522 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
18524 "Third arg to xxinsertw intrinsic must be constant integer");
18526 int64_t Index = std::clamp(ArgCI->getSExtValue(), (int64_t)0, MaxIndex);
18533 std::swap(Op0, Op1);
18537 Op1 =
Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(
Int64Ty, 2));
18541 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int64Ty, 2));
18545 Index = MaxIndex - Index;
18549 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int32Ty, 4));
18550 Op2 = ConstantInt::getSigned(
Int32Ty, Index);
18551 return Builder.CreateCall(F, {Op0, Op1, Op2});
18554 case PPC::BI__builtin_vsx_extractuword: {
18557 llvm::Function *F =
CGM.
getIntrinsic(Intrinsic::ppc_vsx_xxextractuw);
18560 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int64Ty, 2));
18564 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op1);
18566 "Second Arg to xxextractuw intrinsic must be a constant integer!");
18568 int64_t Index = std::clamp(ArgCI->getSExtValue(), (int64_t)0, MaxIndex);
18572 Index = MaxIndex - Index;
18573 Op1 = ConstantInt::getSigned(
Int32Ty, Index);
18578 Value *ShuffleCall =
18580 return ShuffleCall;
18582 Op1 = ConstantInt::getSigned(
Int32Ty, Index);
18583 return Builder.CreateCall(F, {Op0, Op1});
18587 case PPC::BI__builtin_vsx_xxpermdi: {
18591 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
18592 assert(ArgCI &&
"Third arg must be constant integer!");
18594 unsigned Index = ArgCI->getZExtValue();
18595 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int64Ty, 2));
18596 Op1 =
Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(
Int64Ty, 2));
18601 int ElemIdx0 = (Index & 2) >> 1;
18602 int ElemIdx1 = 2 + (Index & 1);
18604 int ShuffleElts[2] = {ElemIdx0, ElemIdx1};
18605 Value *ShuffleCall =
Builder.CreateShuffleVector(Op0, Op1, ShuffleElts);
18608 return Builder.CreateBitCast(ShuffleCall, RetTy);
18611 case PPC::BI__builtin_vsx_xxsldwi: {
18615 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
18616 assert(ArgCI &&
"Third argument must be a compile time constant");
18617 unsigned Index = ArgCI->getZExtValue() & 0x3;
18618 Op0 =
Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int32Ty, 4));
18619 Op1 =
Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(
Int32Ty, 4));
18630 ElemIdx0 = (8 - Index) % 8;
18631 ElemIdx1 = (9 - Index) % 8;
18632 ElemIdx2 = (10 - Index) % 8;
18633 ElemIdx3 = (11 - Index) % 8;
18637 ElemIdx1 = Index + 1;
18638 ElemIdx2 = Index + 2;
18639 ElemIdx3 = Index + 3;
18642 int ShuffleElts[4] = {ElemIdx0, ElemIdx1, ElemIdx2, ElemIdx3};
18643 Value *ShuffleCall =
Builder.CreateShuffleVector(Op0, Op1, ShuffleElts);
18646 return Builder.CreateBitCast(ShuffleCall, RetTy);
18649 case PPC::BI__builtin_pack_vector_int128: {
18653 Value *PoisonValue =
18654 llvm::PoisonValue::get(llvm::FixedVectorType::get(Op0->
getType(), 2));
18656 PoisonValue, Op0, (uint64_t)(isLittleEndian ? 1 : 0));
18657 Res =
Builder.CreateInsertElement(Res, Op1,
18658 (uint64_t)(isLittleEndian ? 0 : 1));
18662 case PPC::BI__builtin_unpack_vector_int128: {
18665 ConstantInt *Index = cast<ConstantInt>(Op1);
18671 ConstantInt::get(Index->getIntegerType(), 1 - Index->getZExtValue());
18673 return Builder.CreateExtractElement(Unpacked, Index);
18676 case PPC::BI__builtin_ppc_sthcx: {
18680 return Builder.CreateCall(F, {Op0, Op1});
18689#define CUSTOM_BUILTIN(Name, Intr, Types, Accumulate, Feature) \
18690 case PPC::BI__builtin_##Name:
18691#include "clang/Basic/BuiltinsPPC.def"
18694 for (
unsigned i = 0, e =
E->getNumArgs(); i != e; i++)
18704 if (BuiltinID == PPC::BI__builtin_mma_disassemble_acc ||
18705 BuiltinID == PPC::BI__builtin_vsx_disassemble_pair ||
18706 BuiltinID == PPC::BI__builtin_mma_disassemble_pair) {
18707 unsigned NumVecs = 2;
18708 auto Intrinsic = Intrinsic::ppc_vsx_disassemble_pair;
18709 if (BuiltinID == PPC::BI__builtin_mma_disassemble_acc) {
18711 Intrinsic = Intrinsic::ppc_mma_disassemble_acc;
18717 llvm::Type *VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
18718 Value *Ptr = Ops[0];
18719 for (
unsigned i=0; i<NumVecs; i++) {
18721 llvm::ConstantInt* Index = llvm::ConstantInt::get(
IntTy, i);
18727 if (BuiltinID == PPC::BI__builtin_vsx_build_pair ||
18728 BuiltinID == PPC::BI__builtin_mma_build_acc) {
18736 std::reverse(Ops.begin() + 1, Ops.end());
18739 switch (BuiltinID) {
18740 #define CUSTOM_BUILTIN(Name, Intr, Types, Acc, Feature) \
18741 case PPC::BI__builtin_##Name: \
18742 ID = Intrinsic::ppc_##Intr; \
18743 Accumulate = Acc; \
18745 #include "clang/Basic/BuiltinsPPC.def"
18747 if (BuiltinID == PPC::BI__builtin_vsx_lxvp ||
18748 BuiltinID == PPC::BI__builtin_vsx_stxvp ||
18749 BuiltinID == PPC::BI__builtin_mma_lxvp ||
18750 BuiltinID == PPC::BI__builtin_mma_stxvp) {
18751 if (BuiltinID == PPC::BI__builtin_vsx_lxvp ||
18752 BuiltinID == PPC::BI__builtin_mma_lxvp) {
18759 return Builder.CreateCall(F, Ops,
"");
18765 CallOps.push_back(Acc);
18767 for (
unsigned i=1; i<Ops.size(); i++)
18768 CallOps.push_back(Ops[i]);
18774 case PPC::BI__builtin_ppc_compare_and_swap:
18775 case PPC::BI__builtin_ppc_compare_and_swaplp: {
18784 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Monotonic,
true);
18792 Value *LoadedVal = Pair.first.getScalarVal();
18796 case PPC::BI__builtin_ppc_fetch_and_add:
18797 case PPC::BI__builtin_ppc_fetch_and_addlp: {
18799 llvm::AtomicOrdering::Monotonic);
18801 case PPC::BI__builtin_ppc_fetch_and_and:
18802 case PPC::BI__builtin_ppc_fetch_and_andlp: {
18804 llvm::AtomicOrdering::Monotonic);
18807 case PPC::BI__builtin_ppc_fetch_and_or:
18808 case PPC::BI__builtin_ppc_fetch_and_orlp: {
18810 llvm::AtomicOrdering::Monotonic);
18812 case PPC::BI__builtin_ppc_fetch_and_swap:
18813 case PPC::BI__builtin_ppc_fetch_and_swaplp: {
18815 llvm::AtomicOrdering::Monotonic);
18817 case PPC::BI__builtin_ppc_ldarx:
18818 case PPC::BI__builtin_ppc_lwarx:
18819 case PPC::BI__builtin_ppc_lharx:
18820 case PPC::BI__builtin_ppc_lbarx:
18822 case PPC::BI__builtin_ppc_mfspr: {
18828 return Builder.CreateCall(F, {Op0});
18830 case PPC::BI__builtin_ppc_mtspr: {
18837 return Builder.CreateCall(F, {Op0, Op1});
18839 case PPC::BI__builtin_ppc_popcntb: {
18841 llvm::Type *ArgType = ArgValue->
getType();
18843 return Builder.CreateCall(F, {ArgValue},
"popcntb");
18845 case PPC::BI__builtin_ppc_mtfsf: {
18855 case PPC::BI__builtin_ppc_swdiv_nochk:
18856 case PPC::BI__builtin_ppc_swdivs_nochk: {
18859 FastMathFlags FMF =
Builder.getFastMathFlags();
18860 Builder.getFastMathFlags().setFast();
18861 Value *FDiv =
Builder.CreateFDiv(Op0, Op1,
"swdiv_nochk");
18862 Builder.getFastMathFlags() &= (FMF);
18865 case PPC::BI__builtin_ppc_fric:
18867 *
this,
E, Intrinsic::rint,
18868 Intrinsic::experimental_constrained_rint))
18870 case PPC::BI__builtin_ppc_frim:
18871 case PPC::BI__builtin_ppc_frims:
18873 *
this,
E, Intrinsic::floor,
18874 Intrinsic::experimental_constrained_floor))
18876 case PPC::BI__builtin_ppc_frin:
18877 case PPC::BI__builtin_ppc_frins:
18879 *
this,
E, Intrinsic::round,
18880 Intrinsic::experimental_constrained_round))
18882 case PPC::BI__builtin_ppc_frip:
18883 case PPC::BI__builtin_ppc_frips:
18885 *
this,
E, Intrinsic::ceil,
18886 Intrinsic::experimental_constrained_ceil))
18888 case PPC::BI__builtin_ppc_friz:
18889 case PPC::BI__builtin_ppc_frizs:
18891 *
this,
E, Intrinsic::trunc,
18892 Intrinsic::experimental_constrained_trunc))
18894 case PPC::BI__builtin_ppc_fsqrt:
18895 case PPC::BI__builtin_ppc_fsqrts:
18897 *
this,
E, Intrinsic::sqrt,
18898 Intrinsic::experimental_constrained_sqrt))
18900 case PPC::BI__builtin_ppc_test_data_class: {
18905 {Op0, Op1},
"test_data_class");
18907 case PPC::BI__builtin_ppc_maxfe: {
18913 {Op0, Op1, Op2, Op3});
18915 case PPC::BI__builtin_ppc_maxfl: {
18921 {Op0, Op1, Op2, Op3});
18923 case PPC::BI__builtin_ppc_maxfs: {
18929 {Op0, Op1, Op2, Op3});
18931 case PPC::BI__builtin_ppc_minfe: {
18937 {Op0, Op1, Op2, Op3});
18939 case PPC::BI__builtin_ppc_minfl: {
18945 {Op0, Op1, Op2, Op3});
18947 case PPC::BI__builtin_ppc_minfs: {
18953 {Op0, Op1, Op2, Op3});
18955 case PPC::BI__builtin_ppc_swdiv:
18956 case PPC::BI__builtin_ppc_swdivs: {
18959 return Builder.CreateFDiv(Op0, Op1,
"swdiv");
18961 case PPC::BI__builtin_ppc_set_fpscr_rn:
18963 {EmitScalarExpr(E->getArg(0))});
18964 case PPC::BI__builtin_ppc_mffs:
18977 Attribute::getWithDereferenceableBytes(
Call->getContext(), 64));
18978 Call->addRetAttr(Attribute::getWithAlignment(
Call->getContext(), Align(4)));
18982 auto *RetTy = cast<llvm::PointerType>(CGF.
ConvertType(BuiltinRetType));
18983 if (RetTy ==
Call->getType())
18992 Attribute::getWithDereferenceableBytes(
Call->getContext(), 256));
18993 Call->addRetAttr(Attribute::getWithAlignment(
Call->getContext(), Align(8)));
19008 llvm::LoadInst *LD;
19012 if (Cov == CodeObjectVersionKind::COV_None) {
19013 StringRef Name =
"__oclc_ABI_version";
19014 auto *ABIVersionC = CGF.
CGM.
getModule().getNamedGlobal(Name);
19016 ABIVersionC =
new llvm::GlobalVariable(
19018 llvm::GlobalValue::ExternalLinkage,
nullptr, Name,
nullptr,
19019 llvm::GlobalVariable::NotThreadLocal,
19030 llvm::ConstantInt::get(CGF.
Int32Ty, CodeObjectVersionKind::COV_5));
19034 CGF.
Int8Ty, EmitAMDGPUImplicitArgPtr(CGF), 12 + Index * 2);
19038 CGF.
Int8Ty, EmitAMDGPUDispatchPtr(CGF), 4 + Index * 2);
19040 auto Result = CGF.
Builder.CreateSelect(IsCOV5, ImplicitGEP, DispatchGEP);
19044 Value *GEP =
nullptr;
19045 if (Cov >= CodeObjectVersionKind::COV_5) {
19047 GEP = CGF.
Builder.CreateConstGEP1_32(
19048 CGF.
Int8Ty, EmitAMDGPUImplicitArgPtr(CGF), 12 + Index * 2);
19051 GEP = CGF.
Builder.CreateConstGEP1_32(
19052 CGF.
Int8Ty, EmitAMDGPUDispatchPtr(CGF), 4 + Index * 2);
19059 llvm::MDNode *RNode = MDHelper.createRange(
APInt(16, 1),
19061 LD->setMetadata(llvm::LLVMContext::MD_range, RNode);
19062 LD->setMetadata(llvm::LLVMContext::MD_noundef,
19064 LD->setMetadata(llvm::LLVMContext::MD_invariant_load,
19071 const unsigned XOffset = 12;
19072 auto *DP = EmitAMDGPUDispatchPtr(CGF);
19074 auto *Offset = llvm::ConstantInt::get(CGF.
Int32Ty, XOffset + Index * 4);
19082 LD->setMetadata(llvm::LLVMContext::MD_range,
19083 MDB.createRange(
APInt(32, 1), APInt::getZero(32)));
19084 LD->setMetadata(llvm::LLVMContext::MD_invariant_load,
19097 llvm::AtomicOrdering &AO,
19098 llvm::SyncScope::ID &SSID) {
19099 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
19102 assert(llvm::isValidAtomicOrderingCABI(ord));
19103 switch (
static_cast<llvm::AtomicOrderingCABI
>(ord)) {
19104 case llvm::AtomicOrderingCABI::acquire:
19105 case llvm::AtomicOrderingCABI::consume:
19106 AO = llvm::AtomicOrdering::Acquire;
19108 case llvm::AtomicOrderingCABI::release:
19109 AO = llvm::AtomicOrdering::Release;
19111 case llvm::AtomicOrderingCABI::acq_rel:
19112 AO = llvm::AtomicOrdering::AcquireRelease;
19114 case llvm::AtomicOrderingCABI::seq_cst:
19115 AO = llvm::AtomicOrdering::SequentiallyConsistent;
19117 case llvm::AtomicOrderingCABI::relaxed:
19118 AO = llvm::AtomicOrdering::Monotonic;
19124 if (llvm::getConstantStringInfo(
Scope, scp)) {
19130 int scope = cast<llvm::ConstantInt>(
Scope)->getZExtValue();
19133 SSID = llvm::SyncScope::System;
19145 SSID = llvm::SyncScope::SingleThread;
19148 SSID = llvm::SyncScope::System;
19156 llvm::Value *Arg =
nullptr;
19157 if ((ICEArguments & (1 << Idx)) == 0) {
19162 std::optional<llvm::APSInt>
Result =
19164 assert(
Result &&
"Expected argument to be a constant");
19173 return RT.getFDotIntrinsic();
19175 return RT.getSDotIntrinsic();
19177 return RT.getUDotIntrinsic();
19182 return RT.getFirstBitSHighIntrinsic();
19186 return RT.getFirstBitUHighIntrinsic();
19195 switch (BuiltinID) {
19196 case Builtin::BI__builtin_hlsl_resource_getpointer: {
19201 llvm::Type *RetTy = llvm::PointerType::getUnqual(
getLLVMContext());
19203 return Builder.CreateIntrinsic(
19207 case Builtin::BI__builtin_hlsl_all: {
19209 return Builder.CreateIntrinsic(
19214 case Builtin::BI__builtin_hlsl_any: {
19216 return Builder.CreateIntrinsic(
19221 case Builtin::BI__builtin_hlsl_asdouble:
19223 case Builtin::BI__builtin_hlsl_elementwise_clamp: {
19230 Ty = VecTy->getElementType();
19232 Intrinsic::ID Intr;
19241 return Builder.CreateIntrinsic(
19245 case Builtin::BI__builtin_hlsl_cross: {
19250 "cross operands must have a float representation");
19255 "input vectors must have 3 elements each");
19256 return Builder.CreateIntrinsic(
19260 case Builtin::BI__builtin_hlsl_dot: {
19263 llvm::Type *T0 = Op0->
getType();
19264 llvm::Type *T1 = Op1->
getType();
19267 if (!T0->isVectorTy() && !T1->isVectorTy()) {
19268 if (T0->isFloatingPointTy())
19269 return Builder.CreateFMul(Op0, Op1,
"hlsl.dot");
19271 if (T0->isIntegerTy())
19272 return Builder.CreateMul(Op0, Op1,
"hlsl.dot");
19275 "Scalar dot product is only supported on ints and floats.");
19280 assert(T0->isVectorTy() && T1->isVectorTy() &&
19281 "Dot product of vector and scalar is not supported.");
19284 [[maybe_unused]]
auto *VecTy1 =
19288 "Dot product of vectors need the same element types.");
19291 "Dot product requires vectors to be of the same size.");
19293 return Builder.CreateIntrinsic(
19294 T0->getScalarType(),
19298 case Builtin::BI__builtin_hlsl_dot4add_i8packed: {
19304 return Builder.CreateIntrinsic(
19306 "hlsl.dot4add.i8packed");
19308 case Builtin::BI__builtin_hlsl_dot4add_u8packed: {
19314 return Builder.CreateIntrinsic(
19316 "hlsl.dot4add.u8packed");
19318 case Builtin::BI__builtin_hlsl_elementwise_firstbithigh: {
19322 return Builder.CreateIntrinsic(
19327 case Builtin::BI__builtin_hlsl_lerp: {
19332 llvm_unreachable(
"lerp operand must have a float representation");
19333 return Builder.CreateIntrinsic(
19337 case Builtin::BI__builtin_hlsl_length: {
19341 "length operand must have a float representation");
19346 return Builder.CreateIntrinsic(
19347 X->getType()->getScalarType(),
19349 nullptr,
"hlsl.length");
19351 case Builtin::BI__builtin_hlsl_normalize: {
19355 "normalize operand must have a float representation");
19357 return Builder.CreateIntrinsic(
19360 nullptr,
"hlsl.normalize");
19362 case Builtin::BI__builtin_hlsl_elementwise_degrees: {
19366 "degree operand must have a float representation");
19368 return Builder.CreateIntrinsic(
19372 case Builtin::BI__builtin_hlsl_elementwise_frac: {
19375 llvm_unreachable(
"frac operand must have a float representation");
19376 return Builder.CreateIntrinsic(
19380case Builtin::BI__builtin_hlsl_elementwise_isinf: {
19382 llvm::Type *Xty = Op0->
getType();
19383 llvm::Type *retType = llvm::Type::getInt1Ty(this->
getLLVMContext());
19384 if (Xty->isVectorTy()) {
19386 retType = llvm::VectorType::get(
19387 retType, ElementCount::getFixed(XVecTy->getNumElements()));
19390 llvm_unreachable(
"isinf operand must have a float representation");
19391 return Builder.CreateIntrinsic(retType, Intrinsic::dx_isinf,
19394 case Builtin::BI__builtin_hlsl_mad: {
19399 return Builder.CreateIntrinsic(
19400 M->
getType(), Intrinsic::fmuladd,
19405 return Builder.CreateIntrinsic(
19406 M->
getType(), Intrinsic::dx_imad,
19410 return Builder.CreateNSWAdd(Mul, B);
19414 return Builder.CreateIntrinsic(
19415 M->
getType(), Intrinsic::dx_umad,
19419 return Builder.CreateNUWAdd(Mul, B);
19421 case Builtin::BI__builtin_hlsl_elementwise_rcp: {
19424 llvm_unreachable(
"rcp operand must have a float representation");
19425 llvm::Type *Ty = Op0->
getType();
19426 llvm::Type *EltTy = Ty->getScalarType();
19427 Constant *One = Ty->isVectorTy()
19428 ? ConstantVector::getSplat(
19429 ElementCount::getFixed(
19430 cast<FixedVectorType>(Ty)->getNumElements()),
19431 ConstantFP::get(EltTy, 1.0))
19432 : ConstantFP::get(EltTy, 1.0);
19433 return Builder.CreateFDiv(One, Op0,
"hlsl.rcp");
19435 case Builtin::BI__builtin_hlsl_elementwise_rsqrt: {
19438 llvm_unreachable(
"rsqrt operand must have a float representation");
19439 return Builder.CreateIntrinsic(
19443 case Builtin::BI__builtin_hlsl_elementwise_saturate: {
19446 "saturate operand must have a float representation");
19447 return Builder.CreateIntrinsic(
19450 nullptr,
"hlsl.saturate");
19452 case Builtin::BI__builtin_hlsl_select: {
19466 Builder.CreateSelect(OpCond, OpTrue, OpFalse,
"hlsl.select");
19473 case Builtin::BI__builtin_hlsl_step: {
19478 "step operands must have a float representation");
19479 return Builder.CreateIntrinsic(
19483 case Builtin::BI__builtin_hlsl_wave_active_all_true: {
19485 assert(Op->
getType()->isIntegerTy(1) &&
19486 "Intrinsic WaveActiveAllTrue operand must be a bool");
19490 Intrinsic::getOrInsertDeclaration(&
CGM.
getModule(), ID), {Op});
19492 case Builtin::BI__builtin_hlsl_wave_active_any_true: {
19494 assert(Op->
getType()->isIntegerTy(1) &&
19495 "Intrinsic WaveActiveAnyTrue operand must be a bool");
19499 Intrinsic::getOrInsertDeclaration(&
CGM.
getModule(), ID), {Op});
19501 case Builtin::BI__builtin_hlsl_wave_active_count_bits: {
19505 Intrinsic::getOrInsertDeclaration(&
CGM.
getModule(), ID),
19508 case Builtin::BI__builtin_hlsl_wave_get_lane_index: {
19513 case llvm::Triple::dxil:
19516 case llvm::Triple::spirv:
19518 llvm::FunctionType::get(
IntTy, {},
false),
19519 "__hlsl_wave_get_lane_index", {},
false,
true));
19522 "Intrinsic WaveGetLaneIndex not supported by target architecture");
19525 case Builtin::BI__builtin_hlsl_wave_is_first_lane: {
19528 Intrinsic::getOrInsertDeclaration(&
CGM.
getModule(), ID));
19530 case Builtin::BI__builtin_hlsl_wave_read_lane_at: {
19535 llvm::FunctionType *FT = llvm::FunctionType::get(
19546 ArrayRef{OpExpr, OpIndex},
"hlsl.wave.readlane");
19548 case Builtin::BI__builtin_hlsl_elementwise_sign: {
19549 auto *Arg0 =
E->getArg(0);
19551 llvm::Type *Xty = Op0->
getType();
19552 llvm::Type *retType = llvm::Type::getInt32Ty(this->
getLLVMContext());
19553 if (Xty->isVectorTy()) {
19555 retType = llvm::VectorType::get(
19556 retType, ElementCount::getFixed(XVecTy->getNumElements()));
19560 "sign operand must have a float or int representation");
19563 Value *Cmp =
Builder.CreateICmpEQ(Op0, ConstantInt::get(Xty, 0));
19564 return Builder.CreateSelect(Cmp, ConstantInt::get(retType, 0),
19565 ConstantInt::get(retType, 1),
"hlsl.sign");
19568 return Builder.CreateIntrinsic(
19572 case Builtin::BI__builtin_hlsl_elementwise_radians: {
19575 "radians operand must have a float representation");
19576 return Builder.CreateIntrinsic(
19579 nullptr,
"hlsl.radians");
19581 case Builtin::BI__builtin_hlsl_buffer_update_counter: {
19585 return Builder.CreateIntrinsic(
19590 case Builtin::BI__builtin_hlsl_elementwise_splitdouble: {
19595 "asuint operands types mismatch");
19598 case Builtin::BI__builtin_hlsl_elementwise_clip:
19600 "clip operands types mismatch");
19602 case Builtin::BI__builtin_hlsl_group_memory_barrier_with_group_sync: {
19606 Intrinsic::getOrInsertDeclaration(&
CGM.
getModule(), ID));
19614 constexpr const char *
Tag =
"amdgpu-as";
19616 LLVMContext &Ctx = Inst->getContext();
19618 for (
unsigned K = 2; K <
E->getNumArgs(); ++K) {
19621 if (llvm::getConstantStringInfo(
V, AS)) {
19622 MMRAs.push_back({
Tag, AS});
19627 "expected an address space name as a string literal");
19631 MMRAs.erase(llvm::unique(MMRAs), MMRAs.end());
19632 Inst->setMetadata(LLVMContext::MD_mmra, MMRAMetadata::getMD(Ctx, MMRAs));
19637 llvm::AtomicOrdering AO = llvm::AtomicOrdering::SequentiallyConsistent;
19638 llvm::SyncScope::ID SSID;
19639 switch (BuiltinID) {
19640 case AMDGPU::BI__builtin_amdgcn_div_scale:
19641 case AMDGPU::BI__builtin_amdgcn_div_scalef: {
19654 llvm::Value *Tmp =
Builder.CreateCall(Callee, {
X, Y, Z});
19657 llvm::Value *Flag =
Builder.CreateExtractValue(Tmp, 1);
19661 llvm::Value *FlagExt =
Builder.CreateZExt(Flag, RealFlagType);
19665 case AMDGPU::BI__builtin_amdgcn_div_fmas:
19666 case AMDGPU::BI__builtin_amdgcn_div_fmasf: {
19674 llvm::Value *Src3ToBool =
Builder.CreateIsNotNull(Src3);
19675 return Builder.CreateCall(F, {Src0, Src1, Src2, Src3ToBool});
19678 case AMDGPU::BI__builtin_amdgcn_ds_swizzle:
19679 return emitBuiltinWithOneOverloadedType<2>(*
this,
E,
19680 Intrinsic::amdgcn_ds_swizzle);
19681 case AMDGPU::BI__builtin_amdgcn_mov_dpp8:
19682 case AMDGPU::BI__builtin_amdgcn_mov_dpp:
19683 case AMDGPU::BI__builtin_amdgcn_update_dpp: {
19687 unsigned ICEArguments = 0;
19692 unsigned Size = DataTy->getPrimitiveSizeInBits();
19693 llvm::Type *
IntTy =
19694 llvm::IntegerType::get(
Builder.getContext(), std::max(Size, 32u));
19697 ? Intrinsic::amdgcn_mov_dpp8
19698 : Intrinsic::amdgcn_update_dpp,
19700 assert(
E->getNumArgs() == 5 ||
E->getNumArgs() == 6 ||
19701 E->getNumArgs() == 2);
19702 bool InsertOld = BuiltinID == AMDGPU::BI__builtin_amdgcn_mov_dpp;
19704 Args.push_back(llvm::PoisonValue::get(
IntTy));
19705 for (
unsigned I = 0; I !=
E->getNumArgs(); ++I) {
19707 if (I < (BuiltinID == AMDGPU::BI__builtin_amdgcn_update_dpp ? 2u : 1u) &&
19709 if (!DataTy->isIntegerTy())
19711 V, llvm::IntegerType::get(
Builder.getContext(), Size));
19714 llvm::Type *ExpTy =
19715 F->getFunctionType()->getFunctionParamType(I + InsertOld);
19716 Args.push_back(
Builder.CreateTruncOrBitCast(
V, ExpTy));
19719 if (Size < 32 && !DataTy->isIntegerTy())
19721 V, llvm::IntegerType::get(
Builder.getContext(), Size));
19722 return Builder.CreateTruncOrBitCast(
V, DataTy);
19724 case AMDGPU::BI__builtin_amdgcn_permlane16:
19725 case AMDGPU::BI__builtin_amdgcn_permlanex16:
19726 return emitBuiltinWithOneOverloadedType<6>(
19728 BuiltinID == AMDGPU::BI__builtin_amdgcn_permlane16
19729 ? Intrinsic::amdgcn_permlane16
19730 : Intrinsic::amdgcn_permlanex16);
19731 case AMDGPU::BI__builtin_amdgcn_permlane64:
19732 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
19733 Intrinsic::amdgcn_permlane64);
19734 case AMDGPU::BI__builtin_amdgcn_readlane:
19735 return emitBuiltinWithOneOverloadedType<2>(*
this,
E,
19736 Intrinsic::amdgcn_readlane);
19737 case AMDGPU::BI__builtin_amdgcn_readfirstlane:
19738 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
19739 Intrinsic::amdgcn_readfirstlane);
19740 case AMDGPU::BI__builtin_amdgcn_div_fixup:
19741 case AMDGPU::BI__builtin_amdgcn_div_fixupf:
19742 case AMDGPU::BI__builtin_amdgcn_div_fixuph:
19743 return emitBuiltinWithOneOverloadedType<3>(*
this,
E,
19744 Intrinsic::amdgcn_div_fixup);
19745 case AMDGPU::BI__builtin_amdgcn_trig_preop:
19746 case AMDGPU::BI__builtin_amdgcn_trig_preopf:
19748 case AMDGPU::BI__builtin_amdgcn_rcp:
19749 case AMDGPU::BI__builtin_amdgcn_rcpf:
19750 case AMDGPU::BI__builtin_amdgcn_rcph:
19751 return emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::amdgcn_rcp);
19752 case AMDGPU::BI__builtin_amdgcn_sqrt:
19753 case AMDGPU::BI__builtin_amdgcn_sqrtf:
19754 case AMDGPU::BI__builtin_amdgcn_sqrth:
19755 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
19756 Intrinsic::amdgcn_sqrt);
19757 case AMDGPU::BI__builtin_amdgcn_rsq:
19758 case AMDGPU::BI__builtin_amdgcn_rsqf:
19759 case AMDGPU::BI__builtin_amdgcn_rsqh:
19760 return emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::amdgcn_rsq);
19761 case AMDGPU::BI__builtin_amdgcn_rsq_clamp:
19762 case AMDGPU::BI__builtin_amdgcn_rsq_clampf:
19763 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
19764 Intrinsic::amdgcn_rsq_clamp);
19765 case AMDGPU::BI__builtin_amdgcn_sinf:
19766 case AMDGPU::BI__builtin_amdgcn_sinh:
19767 return emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::amdgcn_sin);
19768 case AMDGPU::BI__builtin_amdgcn_cosf:
19769 case AMDGPU::BI__builtin_amdgcn_cosh:
19770 return emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::amdgcn_cos);
19771 case AMDGPU::BI__builtin_amdgcn_dispatch_ptr:
19772 return EmitAMDGPUDispatchPtr(*
this,
E);
19773 case AMDGPU::BI__builtin_amdgcn_logf:
19774 return emitBuiltinWithOneOverloadedType<1>(*
this,
E, Intrinsic::amdgcn_log);
19775 case AMDGPU::BI__builtin_amdgcn_exp2f:
19776 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
19777 Intrinsic::amdgcn_exp2);
19778 case AMDGPU::BI__builtin_amdgcn_log_clampf:
19779 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
19780 Intrinsic::amdgcn_log_clamp);
19781 case AMDGPU::BI__builtin_amdgcn_ldexp:
19782 case AMDGPU::BI__builtin_amdgcn_ldexpf: {
19785 llvm::Function *F =
19786 CGM.
getIntrinsic(Intrinsic::ldexp, {Src0->getType(), Src1->getType()});
19787 return Builder.CreateCall(F, {Src0, Src1});
19789 case AMDGPU::BI__builtin_amdgcn_ldexph: {
19794 llvm::Function *F =
19798 case AMDGPU::BI__builtin_amdgcn_frexp_mant:
19799 case AMDGPU::BI__builtin_amdgcn_frexp_mantf:
19800 case AMDGPU::BI__builtin_amdgcn_frexp_manth:
19801 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
19802 Intrinsic::amdgcn_frexp_mant);
19803 case AMDGPU::BI__builtin_amdgcn_frexp_exp:
19804 case AMDGPU::BI__builtin_amdgcn_frexp_expf: {
19808 return Builder.CreateCall(F, Src0);
19810 case AMDGPU::BI__builtin_amdgcn_frexp_exph: {
19814 return Builder.CreateCall(F, Src0);
19816 case AMDGPU::BI__builtin_amdgcn_fract:
19817 case AMDGPU::BI__builtin_amdgcn_fractf:
19818 case AMDGPU::BI__builtin_amdgcn_fracth:
19819 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
19820 Intrinsic::amdgcn_fract);
19821 case AMDGPU::BI__builtin_amdgcn_lerp:
19822 return emitBuiltinWithOneOverloadedType<3>(*
this,
E,
19823 Intrinsic::amdgcn_lerp);
19824 case AMDGPU::BI__builtin_amdgcn_ubfe:
19825 return emitBuiltinWithOneOverloadedType<3>(*
this,
E,
19826 Intrinsic::amdgcn_ubfe);
19827 case AMDGPU::BI__builtin_amdgcn_sbfe:
19828 return emitBuiltinWithOneOverloadedType<3>(*
this,
E,
19829 Intrinsic::amdgcn_sbfe);
19830 case AMDGPU::BI__builtin_amdgcn_ballot_w32:
19831 case AMDGPU::BI__builtin_amdgcn_ballot_w64: {
19835 return Builder.CreateCall(F, { Src });
19837 case AMDGPU::BI__builtin_amdgcn_uicmp:
19838 case AMDGPU::BI__builtin_amdgcn_uicmpl:
19839 case AMDGPU::BI__builtin_amdgcn_sicmp:
19840 case AMDGPU::BI__builtin_amdgcn_sicmpl: {
19847 {
Builder.getInt64Ty(), Src0->getType() });
19848 return Builder.CreateCall(F, { Src0, Src1, Src2 });
19850 case AMDGPU::BI__builtin_amdgcn_fcmp:
19851 case AMDGPU::BI__builtin_amdgcn_fcmpf: {
19858 {
Builder.getInt64Ty(), Src0->getType() });
19859 return Builder.CreateCall(F, { Src0, Src1, Src2 });
19861 case AMDGPU::BI__builtin_amdgcn_class:
19862 case AMDGPU::BI__builtin_amdgcn_classf:
19863 case AMDGPU::BI__builtin_amdgcn_classh:
19865 case AMDGPU::BI__builtin_amdgcn_fmed3f:
19866 case AMDGPU::BI__builtin_amdgcn_fmed3h:
19867 return emitBuiltinWithOneOverloadedType<3>(*
this,
E,
19868 Intrinsic::amdgcn_fmed3);
19869 case AMDGPU::BI__builtin_amdgcn_ds_append:
19870 case AMDGPU::BI__builtin_amdgcn_ds_consume: {
19871 Intrinsic::ID Intrin = BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_append ?
19872 Intrinsic::amdgcn_ds_append : Intrinsic::amdgcn_ds_consume;
19877 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_i32:
19878 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_v2i32:
19879 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4i16:
19880 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4f16:
19881 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4bf16:
19882 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16:
19883 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8f16:
19884 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8bf16:
19885 case AMDGPU::BI__builtin_amdgcn_ds_read_tr4_b64_v2i32:
19886 case AMDGPU::BI__builtin_amdgcn_ds_read_tr8_b64_v2i32:
19887 case AMDGPU::BI__builtin_amdgcn_ds_read_tr6_b96_v3i32:
19888 case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4f16:
19889 case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4bf16:
19890 case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4i16: {
19892 switch (BuiltinID) {
19893 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_i32:
19894 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_v2i32:
19895 IID = Intrinsic::amdgcn_global_load_tr_b64;
19897 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4i16:
19898 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4f16:
19899 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4bf16:
19900 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16:
19901 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8f16:
19902 case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8bf16:
19903 IID = Intrinsic::amdgcn_global_load_tr_b128;
19905 case AMDGPU::BI__builtin_amdgcn_ds_read_tr4_b64_v2i32:
19906 IID = Intrinsic::amdgcn_ds_read_tr4_b64;
19908 case AMDGPU::BI__builtin_amdgcn_ds_read_tr8_b64_v2i32:
19909 IID = Intrinsic::amdgcn_ds_read_tr8_b64;
19911 case AMDGPU::BI__builtin_amdgcn_ds_read_tr6_b96_v3i32:
19912 IID = Intrinsic::amdgcn_ds_read_tr6_b96;
19914 case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4i16:
19915 case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4f16:
19916 case AMDGPU::BI__builtin_amdgcn_ds_read_tr16_b64_v4bf16:
19917 IID = Intrinsic::amdgcn_ds_read_tr16_b64;
19923 return Builder.CreateCall(F, {Addr});
19925 case AMDGPU::BI__builtin_amdgcn_get_fpenv: {
19928 return Builder.CreateCall(F);
19930 case AMDGPU::BI__builtin_amdgcn_set_fpenv: {
19936 case AMDGPU::BI__builtin_amdgcn_read_exec:
19938 case AMDGPU::BI__builtin_amdgcn_read_exec_lo:
19940 case AMDGPU::BI__builtin_amdgcn_read_exec_hi:
19942 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray:
19943 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_h:
19944 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_l:
19945 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_lh: {
19955 RayOrigin =
Builder.CreateShuffleVector(RayOrigin, RayOrigin,
19959 RayInverseDir =
Builder.CreateShuffleVector(RayInverseDir, RayInverseDir,
19963 {NodePtr->getType(), RayDir->getType()});
19964 return Builder.CreateCall(F, {NodePtr, RayExtent, RayOrigin, RayDir,
19965 RayInverseDir, TextureDescr});
19968 case AMDGPU::BI__builtin_amdgcn_ds_bvh_stack_rtn: {
19970 for (
int i = 0, e =
E->getNumArgs(); i != e; ++i)
19978 Value *I0 =
Builder.CreateInsertElement(PoisonValue::get(RetTy), Rtn,
19980 return Builder.CreateInsertElement(I0, A, 1);
19982 case AMDGPU::BI__builtin_amdgcn_mfma_scale_f32_16x16x128_f8f6f4:
19983 case AMDGPU::BI__builtin_amdgcn_mfma_scale_f32_32x32x64_f8f6f4: {
19984 llvm::FixedVectorType *VT = FixedVectorType::get(
Builder.getInt32Ty(), 8);
19986 BuiltinID == AMDGPU::BI__builtin_amdgcn_mfma_scale_f32_32x32x64_f8f6f4
19987 ? Intrinsic::amdgcn_mfma_scale_f32_32x32x64_f8f6f4
19988 : Intrinsic::amdgcn_mfma_scale_f32_16x16x128_f8f6f4,
19992 for (
unsigned I = 0, N =
E->getNumArgs(); I != N; ++I)
19994 return Builder.CreateCall(F, Args);
19996 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32:
19997 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w32:
19998 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64:
19999 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w64:
20000 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32:
20001 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w32:
20002 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64:
20003 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w64:
20004 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32:
20005 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64:
20006 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32:
20007 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64:
20008 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32:
20009 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64:
20010 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32:
20011 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64:
20012 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32_gfx12:
20013 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64_gfx12:
20014 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32_gfx12:
20015 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64_gfx12:
20016 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32_gfx12:
20017 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64_gfx12:
20018 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12:
20019 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64_gfx12:
20020 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32_gfx12:
20021 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64_gfx12:
20022 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12:
20023 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64_gfx12:
20024 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w32_gfx12:
20025 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w64_gfx12:
20026 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w32_gfx12:
20027 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w64_gfx12:
20028 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w32_gfx12:
20029 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w64_gfx12:
20030 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w32_gfx12:
20031 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w64_gfx12:
20032 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w32_gfx12:
20033 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w64_gfx12:
20034 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w32:
20035 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w64:
20036 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w32:
20037 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w64:
20038 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w32:
20039 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w64:
20040 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w32:
20041 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w64:
20042 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w32:
20043 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w64:
20044 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w32:
20045 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w64:
20046 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w32:
20047 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w64:
20048 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w32:
20049 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w64:
20050 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w32:
20051 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w64:
20052 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w32:
20053 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w64:
20054 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w32:
20055 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w64: {
20068 bool AppendFalseForOpselArg =
false;
20069 unsigned BuiltinWMMAOp;
20071 switch (BuiltinID) {
20072 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32:
20073 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64:
20074 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12:
20075 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64_gfx12:
20076 ArgsForMatchingMatrixTypes = {2, 0};
20077 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_f16;
20079 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32:
20080 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64:
20081 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32_gfx12:
20082 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64_gfx12:
20083 ArgsForMatchingMatrixTypes = {2, 0};
20084 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_bf16;
20086 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32_gfx12:
20087 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64_gfx12:
20088 AppendFalseForOpselArg =
true;
20090 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32:
20091 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64:
20092 ArgsForMatchingMatrixTypes = {2, 0};
20093 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f16_16x16x16_f16;
20095 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32_gfx12:
20096 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64_gfx12:
20097 AppendFalseForOpselArg =
true;
20099 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32:
20100 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64:
20101 ArgsForMatchingMatrixTypes = {2, 0};
20102 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_bf16_16x16x16_bf16;
20104 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w32:
20105 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_tied_w64:
20106 ArgsForMatchingMatrixTypes = {2, 0};
20107 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f16_16x16x16_f16_tied;
20109 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w32:
20110 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_tied_w64:
20111 ArgsForMatchingMatrixTypes = {2, 0};
20112 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_bf16_16x16x16_bf16_tied;
20114 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32:
20115 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64:
20116 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12:
20117 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64_gfx12:
20118 ArgsForMatchingMatrixTypes = {4, 1};
20119 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x16_iu8;
20121 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32:
20122 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64:
20123 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32_gfx12:
20124 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64_gfx12:
20125 ArgsForMatchingMatrixTypes = {4, 1};
20126 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x16_iu4;
20128 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w32_gfx12:
20129 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w64_gfx12:
20130 ArgsForMatchingMatrixTypes = {2, 0};
20131 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_fp8_fp8;
20133 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w32_gfx12:
20134 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_fp8_bf8_w64_gfx12:
20135 ArgsForMatchingMatrixTypes = {2, 0};
20136 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_fp8_bf8;
20138 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w32_gfx12:
20139 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_fp8_w64_gfx12:
20140 ArgsForMatchingMatrixTypes = {2, 0};
20141 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_bf8_fp8;
20143 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w32_gfx12:
20144 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf8_bf8_w64_gfx12:
20145 ArgsForMatchingMatrixTypes = {2, 0};
20146 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_bf8_bf8;
20148 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w32_gfx12:
20149 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x32_iu4_w64_gfx12:
20150 ArgsForMatchingMatrixTypes = {4, 1};
20151 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x32_iu4;
20153 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w32:
20154 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_f16_w64:
20155 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
20156 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_f16;
20158 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w32:
20159 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf16_w64:
20160 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
20161 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_bf16;
20163 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w32:
20164 case AMDGPU::BI__builtin_amdgcn_swmmac_f16_16x16x32_f16_w64:
20165 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
20166 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f16_16x16x32_f16;
20168 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w32:
20169 case AMDGPU::BI__builtin_amdgcn_swmmac_bf16_16x16x32_bf16_w64:
20170 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
20171 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_bf16_16x16x32_bf16;
20173 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w32:
20174 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu8_w64:
20175 ArgsForMatchingMatrixTypes = {4, 1, 3, 5};
20176 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_i32_16x16x32_iu8;
20178 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w32:
20179 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x32_iu4_w64:
20180 ArgsForMatchingMatrixTypes = {4, 1, 3, 5};
20181 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_i32_16x16x32_iu4;
20183 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w32:
20184 case AMDGPU::BI__builtin_amdgcn_swmmac_i32_16x16x64_iu4_w64:
20185 ArgsForMatchingMatrixTypes = {4, 1, 3, 5};
20186 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_i32_16x16x64_iu4;
20188 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w32:
20189 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_fp8_w64:
20190 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
20191 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_fp8;
20193 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w32:
20194 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_fp8_bf8_w64:
20195 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
20196 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_bf8;
20198 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w32:
20199 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_fp8_w64:
20200 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
20201 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_fp8;
20203 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w32:
20204 case AMDGPU::BI__builtin_amdgcn_swmmac_f32_16x16x32_bf8_bf8_w64:
20205 ArgsForMatchingMatrixTypes = {2, 0, 1, 3};
20206 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_bf8;
20211 for (
int i = 0, e =
E->getNumArgs(); i != e; ++i)
20213 if (AppendFalseForOpselArg)
20214 Args.push_back(
Builder.getFalse());
20217 for (
auto ArgIdx : ArgsForMatchingMatrixTypes)
20218 ArgTypes.push_back(Args[ArgIdx]->getType());
20221 return Builder.CreateCall(F, Args);
20225 case AMDGPU::BI__builtin_amdgcn_workitem_id_x:
20227 case AMDGPU::BI__builtin_amdgcn_workitem_id_y:
20229 case AMDGPU::BI__builtin_amdgcn_workitem_id_z:
20233 case AMDGPU::BI__builtin_amdgcn_workgroup_size_x:
20234 return EmitAMDGPUWorkGroupSize(*
this, 0);
20235 case AMDGPU::BI__builtin_amdgcn_workgroup_size_y:
20236 return EmitAMDGPUWorkGroupSize(*
this, 1);
20237 case AMDGPU::BI__builtin_amdgcn_workgroup_size_z:
20238 return EmitAMDGPUWorkGroupSize(*
this, 2);
20241 case AMDGPU::BI__builtin_amdgcn_grid_size_x:
20242 return EmitAMDGPUGridSize(*
this, 0);
20243 case AMDGPU::BI__builtin_amdgcn_grid_size_y:
20244 return EmitAMDGPUGridSize(*
this, 1);
20245 case AMDGPU::BI__builtin_amdgcn_grid_size_z:
20246 return EmitAMDGPUGridSize(*
this, 2);
20249 case AMDGPU::BI__builtin_r600_recipsqrt_ieee:
20250 case AMDGPU::BI__builtin_r600_recipsqrt_ieeef:
20251 return emitBuiltinWithOneOverloadedType<1>(*
this,
E,
20252 Intrinsic::r600_recipsqrt_ieee);
20253 case AMDGPU::BI__builtin_r600_read_tidig_x:
20255 case AMDGPU::BI__builtin_r600_read_tidig_y:
20257 case AMDGPU::BI__builtin_r600_read_tidig_z:
20259 case AMDGPU::BI__builtin_amdgcn_alignbit: {
20264 return Builder.CreateCall(F, { Src0, Src1, Src2 });
20266 case AMDGPU::BI__builtin_amdgcn_fence: {
20269 FenceInst *Fence =
Builder.CreateFence(AO, SSID);
20270 if (
E->getNumArgs() > 2)
20274 case AMDGPU::BI__builtin_amdgcn_atomic_inc32:
20275 case AMDGPU::BI__builtin_amdgcn_atomic_inc64:
20276 case AMDGPU::BI__builtin_amdgcn_atomic_dec32:
20277 case AMDGPU::BI__builtin_amdgcn_atomic_dec64:
20278 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f64:
20279 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f32:
20280 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2f16:
20281 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2bf16:
20282 case AMDGPU::BI__builtin_amdgcn_ds_faddf:
20283 case AMDGPU::BI__builtin_amdgcn_ds_fminf:
20284 case AMDGPU::BI__builtin_amdgcn_ds_fmaxf:
20285 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f32:
20286 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f64:
20287 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2f16:
20288 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2f16:
20289 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f32:
20290 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f64:
20291 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2bf16:
20292 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2bf16:
20293 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmin_f64:
20294 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmax_f64:
20295 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmin_f64:
20296 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmax_f64: {
20297 llvm::AtomicRMWInst::BinOp BinOp;
20298 switch (BuiltinID) {
20299 case AMDGPU::BI__builtin_amdgcn_atomic_inc32:
20300 case AMDGPU::BI__builtin_amdgcn_atomic_inc64:
20301 BinOp = llvm::AtomicRMWInst::UIncWrap;
20303 case AMDGPU::BI__builtin_amdgcn_atomic_dec32:
20304 case AMDGPU::BI__builtin_amdgcn_atomic_dec64:
20305 BinOp = llvm::AtomicRMWInst::UDecWrap;
20307 case AMDGPU::BI__builtin_amdgcn_ds_faddf:
20308 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f64:
20309 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f32:
20310 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2f16:
20311 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2bf16:
20312 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f32:
20313 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f64:
20314 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2f16:
20315 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2f16:
20316 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f32:
20317 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f64:
20318 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2bf16:
20319 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2bf16:
20320 BinOp = llvm::AtomicRMWInst::FAdd;
20322 case AMDGPU::BI__builtin_amdgcn_ds_fminf:
20323 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmin_f64:
20324 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmin_f64:
20325 BinOp = llvm::AtomicRMWInst::FMin;
20327 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmax_f64:
20328 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmax_f64:
20329 case AMDGPU::BI__builtin_amdgcn_ds_fmaxf:
20330 BinOp = llvm::AtomicRMWInst::FMax;
20336 llvm::Type *OrigTy = Val->
getType();
20341 if (BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_faddf ||
20342 BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_fminf ||
20343 BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_fmaxf) {
20353 if (
E->getNumArgs() >= 4) {
20365 AO = AtomicOrdering::Monotonic;
20368 if (BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2bf16 ||
20369 BuiltinID == AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2bf16 ||
20370 BuiltinID == AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2bf16) {
20371 llvm::Type *V2BF16Ty = FixedVectorType::get(
20372 llvm::Type::getBFloatTy(
Builder.getContext()), 2);
20373 Val =
Builder.CreateBitCast(Val, V2BF16Ty);
20377 llvm::AtomicRMWInst *RMW =
20380 RMW->setVolatile(
true);
20382 unsigned AddrSpace = Ptr.
getType()->getAddressSpace();
20383 if (AddrSpace != llvm::AMDGPUAS::LOCAL_ADDRESS) {
20387 RMW->setMetadata(
"amdgpu.no.fine.grained.memory", EmptyMD);
20391 if (BinOp == llvm::AtomicRMWInst::FAdd && Val->
getType()->isFloatTy())
20392 RMW->setMetadata(
"amdgpu.ignore.denormal.mode", EmptyMD);
20395 return Builder.CreateBitCast(RMW, OrigTy);
20397 case AMDGPU::BI__builtin_amdgcn_s_sendmsg_rtn:
20398 case AMDGPU::BI__builtin_amdgcn_s_sendmsg_rtnl: {
20404 return Builder.CreateCall(F, {Arg});
20406 case AMDGPU::BI__builtin_amdgcn_permlane16_swap:
20407 case AMDGPU::BI__builtin_amdgcn_permlane32_swap: {
20415 CGM.
getIntrinsic(BuiltinID == AMDGPU::BI__builtin_amdgcn_permlane16_swap
20416 ? Intrinsic::amdgcn_permlane16_swap
20417 : Intrinsic::amdgcn_permlane32_swap);
20418 llvm::CallInst *
Call =
20419 Builder.CreateCall(F, {VDstOld, VSrcOld, FI, BoundCtrl});
20421 llvm::Value *Elt0 =
Builder.CreateExtractValue(
Call, 0);
20422 llvm::Value *Elt1 =
Builder.CreateExtractValue(
Call, 1);
20426 llvm::Value *Insert0 =
Builder.CreateInsertElement(
20427 llvm::PoisonValue::get(ResultType), Elt0, UINT64_C(0));
20428 llvm::Value *AsVector =
20429 Builder.CreateInsertElement(Insert0, Elt1, UINT64_C(1));
20432 case AMDGPU::BI__builtin_amdgcn_bitop3_b32:
20433 case AMDGPU::BI__builtin_amdgcn_bitop3_b16:
20434 return emitBuiltinWithOneOverloadedType<4>(*
this,
E,
20435 Intrinsic::amdgcn_bitop3);
20436 case AMDGPU::BI__builtin_amdgcn_make_buffer_rsrc:
20437 return emitBuiltinWithOneOverloadedType<4>(
20438 *
this,
E, Intrinsic::amdgcn_make_buffer_rsrc);
20439 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b8:
20440 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b16:
20441 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b32:
20442 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b64:
20443 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b96:
20444 case AMDGPU::BI__builtin_amdgcn_raw_buffer_store_b128:
20445 return emitBuiltinWithOneOverloadedType<5>(
20446 *
this,
E, Intrinsic::amdgcn_raw_ptr_buffer_store);
20447 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b8:
20448 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b16:
20449 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b32:
20450 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b64:
20451 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b96:
20452 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b128: {
20453 llvm::Type *RetTy =
nullptr;
20454 switch (BuiltinID) {
20455 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b8:
20458 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b16:
20461 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b32:
20464 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b64:
20465 RetTy = llvm::FixedVectorType::get(
Int32Ty, 2);
20467 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b96:
20468 RetTy = llvm::FixedVectorType::get(
Int32Ty, 3);
20470 case AMDGPU::BI__builtin_amdgcn_raw_buffer_load_b128:
20471 RetTy = llvm::FixedVectorType::get(
Int32Ty, 4);
20480 case AMDGPU::BI__builtin_amdgcn_s_prefetch_data:
20481 return emitBuiltinWithOneOverloadedType<2>(
20482 *
this,
E, Intrinsic::amdgcn_s_prefetch_data);
20490 switch (BuiltinID) {
20491 case SPIRV::BI__builtin_spirv_distance: {
20496 "Distance operands must have a float representation");
20499 "Distance operands must be a vector");
20500 return Builder.CreateIntrinsic(
20501 X->getType()->getScalarType(), Intrinsic::spv_distance,
20512 unsigned IntrinsicID,
20514 unsigned NumArgs =
E->getNumArgs() - 1;
20516 for (
unsigned I = 0; I < NumArgs; ++I)
20528 switch (BuiltinID) {
20529 case SystemZ::BI__builtin_tbegin: {
20531 Value *Control = llvm::ConstantInt::get(
Int32Ty, 0xff0c);
20533 return Builder.CreateCall(F, {TDB, Control});
20535 case SystemZ::BI__builtin_tbegin_nofloat: {
20537 Value *Control = llvm::ConstantInt::get(
Int32Ty, 0xff0c);
20539 return Builder.CreateCall(F, {TDB, Control});
20541 case SystemZ::BI__builtin_tbeginc: {
20543 Value *Control = llvm::ConstantInt::get(
Int32Ty, 0xff08);
20545 return Builder.CreateCall(F, {TDB, Control});
20547 case SystemZ::BI__builtin_tabort: {
20552 case SystemZ::BI__builtin_non_tx_store: {
20564 case SystemZ::BI__builtin_s390_vclzb:
20565 case SystemZ::BI__builtin_s390_vclzh:
20566 case SystemZ::BI__builtin_s390_vclzf:
20567 case SystemZ::BI__builtin_s390_vclzg: {
20570 Value *Undef = ConstantInt::get(
Builder.getInt1Ty(),
false);
20572 return Builder.CreateCall(F, {
X, Undef});
20575 case SystemZ::BI__builtin_s390_vctzb:
20576 case SystemZ::BI__builtin_s390_vctzh:
20577 case SystemZ::BI__builtin_s390_vctzf:
20578 case SystemZ::BI__builtin_s390_vctzg: {
20581 Value *Undef = ConstantInt::get(
Builder.getInt1Ty(),
false);
20583 return Builder.CreateCall(F, {
X, Undef});
20586 case SystemZ::BI__builtin_s390_verllb:
20587 case SystemZ::BI__builtin_s390_verllh:
20588 case SystemZ::BI__builtin_s390_verllf:
20589 case SystemZ::BI__builtin_s390_verllg: {
20594 unsigned NumElts = cast<llvm::FixedVectorType>(ResultType)->getNumElements();
20595 Amt =
Builder.CreateIntCast(Amt, ResultType->getScalarType(),
false);
20596 Amt =
Builder.CreateVectorSplat(NumElts, Amt);
20598 return Builder.CreateCall(F, { Src, Src, Amt });
20601 case SystemZ::BI__builtin_s390_verllvb:
20602 case SystemZ::BI__builtin_s390_verllvh:
20603 case SystemZ::BI__builtin_s390_verllvf:
20604 case SystemZ::BI__builtin_s390_verllvg: {
20609 return Builder.CreateCall(F, { Src, Src, Amt });
20612 case SystemZ::BI__builtin_s390_vfsqsb:
20613 case SystemZ::BI__builtin_s390_vfsqdb: {
20616 if (
Builder.getIsFPConstrained()) {
20618 return Builder.CreateConstrainedFPCall(F, {
X });
20624 case SystemZ::BI__builtin_s390_vfmasb:
20625 case SystemZ::BI__builtin_s390_vfmadb: {
20630 if (
Builder.getIsFPConstrained()) {
20632 return Builder.CreateConstrainedFPCall(F, {
X, Y, Z});
20635 return Builder.CreateCall(F, {
X, Y, Z});
20638 case SystemZ::BI__builtin_s390_vfmssb:
20639 case SystemZ::BI__builtin_s390_vfmsdb: {
20644 if (
Builder.getIsFPConstrained()) {
20646 return Builder.CreateConstrainedFPCall(F, {
X, Y,
Builder.CreateFNeg(Z,
"neg")});
20652 case SystemZ::BI__builtin_s390_vfnmasb:
20653 case SystemZ::BI__builtin_s390_vfnmadb: {
20658 if (
Builder.getIsFPConstrained()) {
20660 return Builder.CreateFNeg(
Builder.CreateConstrainedFPCall(F, {X, Y, Z}),
"neg");
20663 return Builder.CreateFNeg(
Builder.CreateCall(F, {X, Y, Z}),
"neg");
20666 case SystemZ::BI__builtin_s390_vfnmssb:
20667 case SystemZ::BI__builtin_s390_vfnmsdb: {
20672 if (
Builder.getIsFPConstrained()) {
20675 return Builder.CreateFNeg(
Builder.CreateConstrainedFPCall(F, {X, Y, NegZ}));
20682 case SystemZ::BI__builtin_s390_vflpsb:
20683 case SystemZ::BI__builtin_s390_vflpdb: {
20689 case SystemZ::BI__builtin_s390_vflnsb:
20690 case SystemZ::BI__builtin_s390_vflndb: {
20696 case SystemZ::BI__builtin_s390_vfisb:
20697 case SystemZ::BI__builtin_s390_vfidb: {
20705 Intrinsic::ID
ID = Intrinsic::not_intrinsic;
20707 switch (M4.getZExtValue()) {
20710 switch (M5.getZExtValue()) {
20712 case 0:
ID = Intrinsic::rint;
20713 CI = Intrinsic::experimental_constrained_rint;
break;
20717 switch (M5.getZExtValue()) {
20719 case 0:
ID = Intrinsic::nearbyint;
20720 CI = Intrinsic::experimental_constrained_nearbyint;
break;
20721 case 1:
ID = Intrinsic::round;
20722 CI = Intrinsic::experimental_constrained_round;
break;
20723 case 5:
ID = Intrinsic::trunc;
20724 CI = Intrinsic::experimental_constrained_trunc;
break;
20725 case 6:
ID = Intrinsic::ceil;
20726 CI = Intrinsic::experimental_constrained_ceil;
break;
20727 case 7:
ID = Intrinsic::floor;
20728 CI = Intrinsic::experimental_constrained_floor;
break;
20732 if (ID != Intrinsic::not_intrinsic) {
20733 if (
Builder.getIsFPConstrained()) {
20735 return Builder.CreateConstrainedFPCall(F,
X);
20741 switch (BuiltinID) {
20742 case SystemZ::BI__builtin_s390_vfisb:
ID = Intrinsic::s390_vfisb;
break;
20743 case SystemZ::BI__builtin_s390_vfidb:
ID = Intrinsic::s390_vfidb;
break;
20744 default: llvm_unreachable(
"Unknown BuiltinID");
20749 return Builder.CreateCall(F, {
X, M4Value, M5Value});
20751 case SystemZ::BI__builtin_s390_vfmaxsb:
20752 case SystemZ::BI__builtin_s390_vfmaxdb: {
20760 Intrinsic::ID
ID = Intrinsic::not_intrinsic;
20762 switch (M4.getZExtValue()) {
20764 case 4:
ID = Intrinsic::maxnum;
20765 CI = Intrinsic::experimental_constrained_maxnum;
break;
20767 if (ID != Intrinsic::not_intrinsic) {
20768 if (
Builder.getIsFPConstrained()) {
20770 return Builder.CreateConstrainedFPCall(F, {
X, Y});
20773 return Builder.CreateCall(F, {
X, Y});
20776 switch (BuiltinID) {
20777 case SystemZ::BI__builtin_s390_vfmaxsb:
ID = Intrinsic::s390_vfmaxsb;
break;
20778 case SystemZ::BI__builtin_s390_vfmaxdb:
ID = Intrinsic::s390_vfmaxdb;
break;
20779 default: llvm_unreachable(
"Unknown BuiltinID");
20783 return Builder.CreateCall(F, {
X, Y, M4Value});
20785 case SystemZ::BI__builtin_s390_vfminsb:
20786 case SystemZ::BI__builtin_s390_vfmindb: {
20794 Intrinsic::ID
ID = Intrinsic::not_intrinsic;
20796 switch (M4.getZExtValue()) {
20798 case 4:
ID = Intrinsic::minnum;
20799 CI = Intrinsic::experimental_constrained_minnum;
break;
20801 if (ID != Intrinsic::not_intrinsic) {
20802 if (
Builder.getIsFPConstrained()) {
20804 return Builder.CreateConstrainedFPCall(F, {
X, Y});
20807 return Builder.CreateCall(F, {
X, Y});
20810 switch (BuiltinID) {
20811 case SystemZ::BI__builtin_s390_vfminsb:
ID = Intrinsic::s390_vfminsb;
break;
20812 case SystemZ::BI__builtin_s390_vfmindb:
ID = Intrinsic::s390_vfmindb;
break;
20813 default: llvm_unreachable(
"Unknown BuiltinID");
20817 return Builder.CreateCall(F, {
X, Y, M4Value});
20820 case SystemZ::BI__builtin_s390_vlbrh:
20821 case SystemZ::BI__builtin_s390_vlbrf:
20822 case SystemZ::BI__builtin_s390_vlbrg: {
20831#define INTRINSIC_WITH_CC(NAME) \
20832 case SystemZ::BI__builtin_##NAME: \
20833 return EmitSystemZIntrinsicWithCC(*this, Intrinsic::NAME, E)
20912#undef INTRINSIC_WITH_CC
20921struct NVPTXMmaLdstInfo {
20922 unsigned NumResults;
20928#define MMA_INTR(geom_op_type, layout) \
20929 Intrinsic::nvvm_wmma_##geom_op_type##_##layout##_stride
20930#define MMA_LDST(n, geom_op_type) \
20931 { n, MMA_INTR(geom_op_type, col), MMA_INTR(geom_op_type, row) }
20933static NVPTXMmaLdstInfo getNVPTXMmaLdstInfo(
unsigned BuiltinID) {
20934 switch (BuiltinID) {
20936 case NVPTX::BI__hmma_m16n16k16_ld_a:
20937 return MMA_LDST(8, m16n16k16_load_a_f16);
20938 case NVPTX::BI__hmma_m16n16k16_ld_b:
20939 return MMA_LDST(8, m16n16k16_load_b_f16);
20940 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
20941 return MMA_LDST(4, m16n16k16_load_c_f16);
20942 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
20943 return MMA_LDST(8, m16n16k16_load_c_f32);
20944 case NVPTX::BI__hmma_m32n8k16_ld_a:
20945 return MMA_LDST(8, m32n8k16_load_a_f16);
20946 case NVPTX::BI__hmma_m32n8k16_ld_b:
20947 return MMA_LDST(8, m32n8k16_load_b_f16);
20948 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
20949 return MMA_LDST(4, m32n8k16_load_c_f16);
20950 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
20951 return MMA_LDST(8, m32n8k16_load_c_f32);
20952 case NVPTX::BI__hmma_m8n32k16_ld_a:
20953 return MMA_LDST(8, m8n32k16_load_a_f16);
20954 case NVPTX::BI__hmma_m8n32k16_ld_b:
20955 return MMA_LDST(8, m8n32k16_load_b_f16);
20956 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
20957 return MMA_LDST(4, m8n32k16_load_c_f16);
20958 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
20959 return MMA_LDST(8, m8n32k16_load_c_f32);
20962 case NVPTX::BI__imma_m16n16k16_ld_a_s8:
20963 return MMA_LDST(2, m16n16k16_load_a_s8);
20964 case NVPTX::BI__imma_m16n16k16_ld_a_u8:
20965 return MMA_LDST(2, m16n16k16_load_a_u8);
20966 case NVPTX::BI__imma_m16n16k16_ld_b_s8:
20967 return MMA_LDST(2, m16n16k16_load_b_s8);
20968 case NVPTX::BI__imma_m16n16k16_ld_b_u8:
20969 return MMA_LDST(2, m16n16k16_load_b_u8);
20970 case NVPTX::BI__imma_m16n16k16_ld_c:
20971 return MMA_LDST(8, m16n16k16_load_c_s32);
20972 case NVPTX::BI__imma_m32n8k16_ld_a_s8:
20973 return MMA_LDST(4, m32n8k16_load_a_s8);
20974 case NVPTX::BI__imma_m32n8k16_ld_a_u8:
20975 return MMA_LDST(4, m32n8k16_load_a_u8);
20976 case NVPTX::BI__imma_m32n8k16_ld_b_s8:
20977 return MMA_LDST(1, m32n8k16_load_b_s8);
20978 case NVPTX::BI__imma_m32n8k16_ld_b_u8:
20979 return MMA_LDST(1, m32n8k16_load_b_u8);
20980 case NVPTX::BI__imma_m32n8k16_ld_c:
20981 return MMA_LDST(8, m32n8k16_load_c_s32);
20982 case NVPTX::BI__imma_m8n32k16_ld_a_s8:
20983 return MMA_LDST(1, m8n32k16_load_a_s8);
20984 case NVPTX::BI__imma_m8n32k16_ld_a_u8:
20985 return MMA_LDST(1, m8n32k16_load_a_u8);
20986 case NVPTX::BI__imma_m8n32k16_ld_b_s8:
20987 return MMA_LDST(4, m8n32k16_load_b_s8);
20988 case NVPTX::BI__imma_m8n32k16_ld_b_u8:
20989 return MMA_LDST(4, m8n32k16_load_b_u8);
20990 case NVPTX::BI__imma_m8n32k16_ld_c:
20991 return MMA_LDST(8, m8n32k16_load_c_s32);
20995 case NVPTX::BI__imma_m8n8k32_ld_a_s4:
20996 return {1, 0,
MMA_INTR(m8n8k32_load_a_s4, row)};
20997 case NVPTX::BI__imma_m8n8k32_ld_a_u4:
20998 return {1, 0,
MMA_INTR(m8n8k32_load_a_u4, row)};
20999 case NVPTX::BI__imma_m8n8k32_ld_b_s4:
21000 return {1,
MMA_INTR(m8n8k32_load_b_s4, col), 0};
21001 case NVPTX::BI__imma_m8n8k32_ld_b_u4:
21002 return {1,
MMA_INTR(m8n8k32_load_b_u4, col), 0};
21003 case NVPTX::BI__imma_m8n8k32_ld_c:
21004 return MMA_LDST(2, m8n8k32_load_c_s32);
21005 case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
21006 return {1, 0,
MMA_INTR(m8n8k128_load_a_b1, row)};
21007 case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
21008 return {1,
MMA_INTR(m8n8k128_load_b_b1, col), 0};
21009 case NVPTX::BI__bmma_m8n8k128_ld_c:
21010 return MMA_LDST(2, m8n8k128_load_c_s32);
21013 case NVPTX::BI__dmma_m8n8k4_ld_a:
21014 return MMA_LDST(1, m8n8k4_load_a_f64);
21015 case NVPTX::BI__dmma_m8n8k4_ld_b:
21016 return MMA_LDST(1, m8n8k4_load_b_f64);
21017 case NVPTX::BI__dmma_m8n8k4_ld_c:
21018 return MMA_LDST(2, m8n8k4_load_c_f64);
21021 case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
21022 return MMA_LDST(4, m16n16k16_load_a_bf16);
21023 case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
21024 return MMA_LDST(4, m16n16k16_load_b_bf16);
21025 case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
21026 return MMA_LDST(2, m8n32k16_load_a_bf16);
21027 case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
21028 return MMA_LDST(8, m8n32k16_load_b_bf16);
21029 case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
21030 return MMA_LDST(8, m32n8k16_load_a_bf16);
21031 case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
21032 return MMA_LDST(2, m32n8k16_load_b_bf16);
21033 case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
21034 return MMA_LDST(4, m16n16k8_load_a_tf32);
21035 case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
21036 return MMA_LDST(4, m16n16k8_load_b_tf32);
21037 case NVPTX::BI__mma_tf32_m16n16k8_ld_c:
21038 return MMA_LDST(8, m16n16k8_load_c_f32);
21044 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
21045 return MMA_LDST(4, m16n16k16_store_d_f16);
21046 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
21047 return MMA_LDST(8, m16n16k16_store_d_f32);
21048 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
21049 return MMA_LDST(4, m32n8k16_store_d_f16);
21050 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
21051 return MMA_LDST(8, m32n8k16_store_d_f32);
21052 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
21053 return MMA_LDST(4, m8n32k16_store_d_f16);
21054 case NVPTX::BI__hmma_m8n32k16_st_c_f32:
21055 return MMA_LDST(8, m8n32k16_store_d_f32);
21060 case NVPTX::BI__imma_m16n16k16_st_c_i32:
21061 return MMA_LDST(8, m16n16k16_store_d_s32);
21062 case NVPTX::BI__imma_m32n8k16_st_c_i32:
21063 return MMA_LDST(8, m32n8k16_store_d_s32);
21064 case NVPTX::BI__imma_m8n32k16_st_c_i32:
21065 return MMA_LDST(8, m8n32k16_store_d_s32);
21066 case NVPTX::BI__imma_m8n8k32_st_c_i32:
21067 return MMA_LDST(2, m8n8k32_store_d_s32);
21068 case NVPTX::BI__bmma_m8n8k128_st_c_i32:
21069 return MMA_LDST(2, m8n8k128_store_d_s32);
21072 case NVPTX::BI__dmma_m8n8k4_st_c_f64:
21073 return MMA_LDST(2, m8n8k4_store_d_f64);
21076 case NVPTX::BI__mma_m16n16k8_st_c_f32:
21077 return MMA_LDST(8, m16n16k8_store_d_f32);
21080 llvm_unreachable(
"Unknown MMA builtin");
21087struct NVPTXMmaInfo {
21096 std::array<unsigned, 8> Variants;
21098 unsigned getMMAIntrinsic(
int Layout,
bool Satf) {
21099 unsigned Index = Layout + 4 * Satf;
21100 if (Index >= Variants.size())
21102 return Variants[Index];
21108static NVPTXMmaInfo getNVPTXMmaInfo(
unsigned BuiltinID) {
21110#define MMA_VARIANTS(geom, type) \
21111 Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type, \
21112 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type, \
21113 Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type, \
21114 Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type
21115#define MMA_SATF_VARIANTS(geom, type) \
21116 MMA_VARIANTS(geom, type), \
21117 Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type##_satfinite, \
21118 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type##_satfinite, \
21119 Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type##_satfinite, \
21120 Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type##_satfinite
21122#define MMA_VARIANTS_I4(geom, type) \
21124 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type, \
21128 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type##_satfinite, \
21132#define MMA_VARIANTS_B1_XOR(geom, type) \
21134 Intrinsic::nvvm_wmma_##geom##_mma_xor_popc_row_col_##type, \
21141#define MMA_VARIANTS_B1_AND(geom, type) \
21143 Intrinsic::nvvm_wmma_##geom##_mma_and_popc_row_col_##type, \
21151 switch (BuiltinID) {
21155 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
21157 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
21159 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
21161 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
21163 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
21165 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
21167 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
21169 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
21171 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
21173 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
21175 case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
21177 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
21181 case NVPTX::BI__imma_m16n16k16_mma_s8:
21183 case NVPTX::BI__imma_m16n16k16_mma_u8:
21185 case NVPTX::BI__imma_m32n8k16_mma_s8:
21187 case NVPTX::BI__imma_m32n8k16_mma_u8:
21189 case NVPTX::BI__imma_m8n32k16_mma_s8:
21191 case NVPTX::BI__imma_m8n32k16_mma_u8:
21195 case NVPTX::BI__imma_m8n8k32_mma_s4:
21197 case NVPTX::BI__imma_m8n8k32_mma_u4:
21199 case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1:
21201 case NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1:
21205 case NVPTX::BI__dmma_m8n8k4_mma_f64:
21209 case NVPTX::BI__mma_bf16_m16n16k16_mma_f32:
21210 return {4, 4, 8, 8, {{
MMA_VARIANTS(m16n16k16, bf16)}}};
21211 case NVPTX::BI__mma_bf16_m8n32k16_mma_f32:
21212 return {2, 8, 8, 8, {{
MMA_VARIANTS(m8n32k16, bf16)}}};
21213 case NVPTX::BI__mma_bf16_m32n8k16_mma_f32:
21214 return {8, 2, 8, 8, {{
MMA_VARIANTS(m32n8k16, bf16)}}};
21215 case NVPTX::BI__mma_tf32_m16n16k8_mma_f32:
21216 return {4, 4, 8, 8, {{
MMA_VARIANTS(m16n16k8, tf32)}}};
21218 llvm_unreachable(
"Unexpected builtin ID.");
21221#undef MMA_SATF_VARIANTS
21222#undef MMA_VARIANTS_I4
21223#undef MMA_VARIANTS_B1_AND
21224#undef MMA_VARIANTS_B1_XOR
21233 return CGF.
Builder.CreateCall(
21235 {Ptr, ConstantInt::get(CGF.Builder.getInt32Ty(), Align.getQuantity())});
21247 MDNode *MD = MDNode::get(CGF.
Builder.getContext(), {});
21248 LD->setMetadata(LLVMContext::MD_invariant_load, MD);
21256 llvm::Type *ElemTy =
21258 return CGF.
Builder.CreateCall(
21260 {Ptr, CGF.EmitScalarExpr(E->getArg(1))});
21263static Value *MakeCpAsync(
unsigned IntrinsicID,
unsigned IntrinsicIDS,
21266 return E->getNumArgs() == 3
21268 {CGF.EmitScalarExpr(E->getArg(0)),
21269 CGF.EmitScalarExpr(E->getArg(1)),
21270 CGF.EmitScalarExpr(E->getArg(2))})
21272 {CGF.EmitScalarExpr(E->getArg(0)),
21273 CGF.EmitScalarExpr(E->getArg(1))});
21276static Value *MakeHalfType(
unsigned IntrinsicID,
unsigned BuiltinID,
21279 if (!(
C.getLangOpts().NativeHalfType ||
21280 !
C.getTargetInfo().useFP16ConversionIntrinsics())) {
21282 " requires native half type support.");
21286 if (BuiltinID == NVPTX::BI__nvvm_ldg_h || BuiltinID == NVPTX::BI__nvvm_ldg_h2)
21287 return MakeLdg(CGF,
E);
21289 if (IntrinsicID == Intrinsic::nvvm_ldu_global_f)
21290 return MakeLdu(IntrinsicID, CGF,
E);
21294 auto *FTy = F->getFunctionType();
21295 unsigned ICEArguments = 0;
21297 C.GetBuiltinType(BuiltinID, Error, &ICEArguments);
21299 for (
unsigned i = 0, e =
E->getNumArgs(); i != e; ++i) {
21300 assert((ICEArguments & (1 << i)) == 0);
21302 auto *PTy = FTy->getParamType(i);
21303 if (PTy != ArgValue->
getType())
21304 ArgValue = CGF.
Builder.CreateBitCast(ArgValue, PTy);
21305 Args.push_back(ArgValue);
21308 return CGF.
Builder.CreateCall(F, Args);
21314 switch (BuiltinID) {
21315 case NVPTX::BI__nvvm_atom_add_gen_i:
21316 case NVPTX::BI__nvvm_atom_add_gen_l:
21317 case NVPTX::BI__nvvm_atom_add_gen_ll:
21320 case NVPTX::BI__nvvm_atom_sub_gen_i:
21321 case NVPTX::BI__nvvm_atom_sub_gen_l:
21322 case NVPTX::BI__nvvm_atom_sub_gen_ll:
21325 case NVPTX::BI__nvvm_atom_and_gen_i:
21326 case NVPTX::BI__nvvm_atom_and_gen_l:
21327 case NVPTX::BI__nvvm_atom_and_gen_ll:
21330 case NVPTX::BI__nvvm_atom_or_gen_i:
21331 case NVPTX::BI__nvvm_atom_or_gen_l:
21332 case NVPTX::BI__nvvm_atom_or_gen_ll:
21335 case NVPTX::BI__nvvm_atom_xor_gen_i:
21336 case NVPTX::BI__nvvm_atom_xor_gen_l:
21337 case NVPTX::BI__nvvm_atom_xor_gen_ll:
21340 case NVPTX::BI__nvvm_atom_xchg_gen_i:
21341 case NVPTX::BI__nvvm_atom_xchg_gen_l:
21342 case NVPTX::BI__nvvm_atom_xchg_gen_ll:
21345 case NVPTX::BI__nvvm_atom_max_gen_i:
21346 case NVPTX::BI__nvvm_atom_max_gen_l:
21347 case NVPTX::BI__nvvm_atom_max_gen_ll:
21350 case NVPTX::BI__nvvm_atom_max_gen_ui:
21351 case NVPTX::BI__nvvm_atom_max_gen_ul:
21352 case NVPTX::BI__nvvm_atom_max_gen_ull:
21355 case NVPTX::BI__nvvm_atom_min_gen_i:
21356 case NVPTX::BI__nvvm_atom_min_gen_l:
21357 case NVPTX::BI__nvvm_atom_min_gen_ll:
21360 case NVPTX::BI__nvvm_atom_min_gen_ui:
21361 case NVPTX::BI__nvvm_atom_min_gen_ul:
21362 case NVPTX::BI__nvvm_atom_min_gen_ull:
21365 case NVPTX::BI__nvvm_atom_cas_gen_us:
21366 case NVPTX::BI__nvvm_atom_cas_gen_i:
21367 case NVPTX::BI__nvvm_atom_cas_gen_l:
21368 case NVPTX::BI__nvvm_atom_cas_gen_ll:
21373 case NVPTX::BI__nvvm_atom_add_gen_f:
21374 case NVPTX::BI__nvvm_atom_add_gen_d: {
21379 AtomicOrdering::SequentiallyConsistent);
21382 case NVPTX::BI__nvvm_atom_inc_gen_ui: {
21387 return Builder.CreateCall(FnALI32, {Ptr, Val});
21390 case NVPTX::BI__nvvm_atom_dec_gen_ui: {
21395 return Builder.CreateCall(FnALD32, {Ptr, Val});
21398 case NVPTX::BI__nvvm_ldg_c:
21399 case NVPTX::BI__nvvm_ldg_sc:
21400 case NVPTX::BI__nvvm_ldg_c2:
21401 case NVPTX::BI__nvvm_ldg_sc2:
21402 case NVPTX::BI__nvvm_ldg_c4:
21403 case NVPTX::BI__nvvm_ldg_sc4:
21404 case NVPTX::BI__nvvm_ldg_s:
21405 case NVPTX::BI__nvvm_ldg_s2:
21406 case NVPTX::BI__nvvm_ldg_s4:
21407 case NVPTX::BI__nvvm_ldg_i:
21408 case NVPTX::BI__nvvm_ldg_i2:
21409 case NVPTX::BI__nvvm_ldg_i4:
21410 case NVPTX::BI__nvvm_ldg_l:
21411 case NVPTX::BI__nvvm_ldg_l2:
21412 case NVPTX::BI__nvvm_ldg_ll:
21413 case NVPTX::BI__nvvm_ldg_ll2:
21414 case NVPTX::BI__nvvm_ldg_uc:
21415 case NVPTX::BI__nvvm_ldg_uc2:
21416 case NVPTX::BI__nvvm_ldg_uc4:
21417 case NVPTX::BI__nvvm_ldg_us:
21418 case NVPTX::BI__nvvm_ldg_us2:
21419 case NVPTX::BI__nvvm_ldg_us4:
21420 case NVPTX::BI__nvvm_ldg_ui:
21421 case NVPTX::BI__nvvm_ldg_ui2:
21422 case NVPTX::BI__nvvm_ldg_ui4:
21423 case NVPTX::BI__nvvm_ldg_ul:
21424 case NVPTX::BI__nvvm_ldg_ul2:
21425 case NVPTX::BI__nvvm_ldg_ull:
21426 case NVPTX::BI__nvvm_ldg_ull2:
21427 case NVPTX::BI__nvvm_ldg_f:
21428 case NVPTX::BI__nvvm_ldg_f2:
21429 case NVPTX::BI__nvvm_ldg_f4:
21430 case NVPTX::BI__nvvm_ldg_d:
21431 case NVPTX::BI__nvvm_ldg_d2:
21435 return MakeLdg(*
this,
E);
21437 case NVPTX::BI__nvvm_ldu_c:
21438 case NVPTX::BI__nvvm_ldu_sc:
21439 case NVPTX::BI__nvvm_ldu_c2:
21440 case NVPTX::BI__nvvm_ldu_sc2:
21441 case NVPTX::BI__nvvm_ldu_c4:
21442 case NVPTX::BI__nvvm_ldu_sc4:
21443 case NVPTX::BI__nvvm_ldu_s:
21444 case NVPTX::BI__nvvm_ldu_s2:
21445 case NVPTX::BI__nvvm_ldu_s4:
21446 case NVPTX::BI__nvvm_ldu_i:
21447 case NVPTX::BI__nvvm_ldu_i2:
21448 case NVPTX::BI__nvvm_ldu_i4:
21449 case NVPTX::BI__nvvm_ldu_l:
21450 case NVPTX::BI__nvvm_ldu_l2:
21451 case NVPTX::BI__nvvm_ldu_ll:
21452 case NVPTX::BI__nvvm_ldu_ll2:
21453 case NVPTX::BI__nvvm_ldu_uc:
21454 case NVPTX::BI__nvvm_ldu_uc2:
21455 case NVPTX::BI__nvvm_ldu_uc4:
21456 case NVPTX::BI__nvvm_ldu_us:
21457 case NVPTX::BI__nvvm_ldu_us2:
21458 case NVPTX::BI__nvvm_ldu_us4:
21459 case NVPTX::BI__nvvm_ldu_ui:
21460 case NVPTX::BI__nvvm_ldu_ui2:
21461 case NVPTX::BI__nvvm_ldu_ui4:
21462 case NVPTX::BI__nvvm_ldu_ul:
21463 case NVPTX::BI__nvvm_ldu_ul2:
21464 case NVPTX::BI__nvvm_ldu_ull:
21465 case NVPTX::BI__nvvm_ldu_ull2:
21466 return MakeLdu(Intrinsic::nvvm_ldu_global_i, *
this,
E);
21467 case NVPTX::BI__nvvm_ldu_f:
21468 case NVPTX::BI__nvvm_ldu_f2:
21469 case NVPTX::BI__nvvm_ldu_f4:
21470 case NVPTX::BI__nvvm_ldu_d:
21471 case NVPTX::BI__nvvm_ldu_d2:
21472 return MakeLdu(Intrinsic::nvvm_ldu_global_f, *
this,
E);
21474 case NVPTX::BI__nvvm_atom_cta_add_gen_i:
21475 case NVPTX::BI__nvvm_atom_cta_add_gen_l:
21476 case NVPTX::BI__nvvm_atom_cta_add_gen_ll:
21477 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_cta, *
this,
E);
21478 case NVPTX::BI__nvvm_atom_sys_add_gen_i:
21479 case NVPTX::BI__nvvm_atom_sys_add_gen_l:
21480 case NVPTX::BI__nvvm_atom_sys_add_gen_ll:
21481 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_sys, *
this,
E);
21482 case NVPTX::BI__nvvm_atom_cta_add_gen_f:
21483 case NVPTX::BI__nvvm_atom_cta_add_gen_d:
21484 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_cta, *
this,
E);
21485 case NVPTX::BI__nvvm_atom_sys_add_gen_f:
21486 case NVPTX::BI__nvvm_atom_sys_add_gen_d:
21487 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_sys, *
this,
E);
21488 case NVPTX::BI__nvvm_atom_cta_xchg_gen_i:
21489 case NVPTX::BI__nvvm_atom_cta_xchg_gen_l:
21490 case NVPTX::BI__nvvm_atom_cta_xchg_gen_ll:
21491 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_cta, *
this,
E);
21492 case NVPTX::BI__nvvm_atom_sys_xchg_gen_i:
21493 case NVPTX::BI__nvvm_atom_sys_xchg_gen_l:
21494 case NVPTX::BI__nvvm_atom_sys_xchg_gen_ll:
21495 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_sys, *
this,
E);
21496 case NVPTX::BI__nvvm_atom_cta_max_gen_i:
21497 case NVPTX::BI__nvvm_atom_cta_max_gen_ui:
21498 case NVPTX::BI__nvvm_atom_cta_max_gen_l:
21499 case NVPTX::BI__nvvm_atom_cta_max_gen_ul:
21500 case NVPTX::BI__nvvm_atom_cta_max_gen_ll:
21501 case NVPTX::BI__nvvm_atom_cta_max_gen_ull:
21502 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_cta, *
this,
E);
21503 case NVPTX::BI__nvvm_atom_sys_max_gen_i:
21504 case NVPTX::BI__nvvm_atom_sys_max_gen_ui:
21505 case NVPTX::BI__nvvm_atom_sys_max_gen_l:
21506 case NVPTX::BI__nvvm_atom_sys_max_gen_ul:
21507 case NVPTX::BI__nvvm_atom_sys_max_gen_ll:
21508 case NVPTX::BI__nvvm_atom_sys_max_gen_ull:
21509 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_sys, *
this,
E);
21510 case NVPTX::BI__nvvm_atom_cta_min_gen_i:
21511 case NVPTX::BI__nvvm_atom_cta_min_gen_ui:
21512 case NVPTX::BI__nvvm_atom_cta_min_gen_l:
21513 case NVPTX::BI__nvvm_atom_cta_min_gen_ul:
21514 case NVPTX::BI__nvvm_atom_cta_min_gen_ll:
21515 case NVPTX::BI__nvvm_atom_cta_min_gen_ull:
21516 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_cta, *
this,
E);
21517 case NVPTX::BI__nvvm_atom_sys_min_gen_i:
21518 case NVPTX::BI__nvvm_atom_sys_min_gen_ui:
21519 case NVPTX::BI__nvvm_atom_sys_min_gen_l:
21520 case NVPTX::BI__nvvm_atom_sys_min_gen_ul:
21521 case NVPTX::BI__nvvm_atom_sys_min_gen_ll:
21522 case NVPTX::BI__nvvm_atom_sys_min_gen_ull:
21523 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_sys, *
this,
E);
21524 case NVPTX::BI__nvvm_atom_cta_inc_gen_ui:
21525 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_cta, *
this,
E);
21526 case NVPTX::BI__nvvm_atom_cta_dec_gen_ui:
21527 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_cta, *
this,
E);
21528 case NVPTX::BI__nvvm_atom_sys_inc_gen_ui:
21529 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_sys, *
this,
E);
21530 case NVPTX::BI__nvvm_atom_sys_dec_gen_ui:
21531 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_sys, *
this,
E);
21532 case NVPTX::BI__nvvm_atom_cta_and_gen_i:
21533 case NVPTX::BI__nvvm_atom_cta_and_gen_l:
21534 case NVPTX::BI__nvvm_atom_cta_and_gen_ll:
21535 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_cta, *
this,
E);
21536 case NVPTX::BI__nvvm_atom_sys_and_gen_i:
21537 case NVPTX::BI__nvvm_atom_sys_and_gen_l:
21538 case NVPTX::BI__nvvm_atom_sys_and_gen_ll:
21539 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_sys, *
this,
E);
21540 case NVPTX::BI__nvvm_atom_cta_or_gen_i:
21541 case NVPTX::BI__nvvm_atom_cta_or_gen_l:
21542 case NVPTX::BI__nvvm_atom_cta_or_gen_ll:
21543 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_cta, *
this,
E);
21544 case NVPTX::BI__nvvm_atom_sys_or_gen_i:
21545 case NVPTX::BI__nvvm_atom_sys_or_gen_l:
21546 case NVPTX::BI__nvvm_atom_sys_or_gen_ll:
21547 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_sys, *
this,
E);
21548 case NVPTX::BI__nvvm_atom_cta_xor_gen_i:
21549 case NVPTX::BI__nvvm_atom_cta_xor_gen_l:
21550 case NVPTX::BI__nvvm_atom_cta_xor_gen_ll:
21551 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_cta, *
this,
E);
21552 case NVPTX::BI__nvvm_atom_sys_xor_gen_i:
21553 case NVPTX::BI__nvvm_atom_sys_xor_gen_l:
21554 case NVPTX::BI__nvvm_atom_sys_xor_gen_ll:
21555 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_sys, *
this,
E);
21556 case NVPTX::BI__nvvm_atom_cta_cas_gen_us:
21557 case NVPTX::BI__nvvm_atom_cta_cas_gen_i:
21558 case NVPTX::BI__nvvm_atom_cta_cas_gen_l:
21559 case NVPTX::BI__nvvm_atom_cta_cas_gen_ll: {
21561 llvm::Type *ElemTy =
21565 Intrinsic::nvvm_atomic_cas_gen_i_cta, {ElemTy, Ptr->getType()}),
21566 {Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))});
21568 case NVPTX::BI__nvvm_atom_sys_cas_gen_us:
21569 case NVPTX::BI__nvvm_atom_sys_cas_gen_i:
21570 case NVPTX::BI__nvvm_atom_sys_cas_gen_l:
21571 case NVPTX::BI__nvvm_atom_sys_cas_gen_ll: {
21573 llvm::Type *ElemTy =
21577 Intrinsic::nvvm_atomic_cas_gen_i_sys, {ElemTy, Ptr->getType()}),
21578 {Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))});
21580 case NVPTX::BI__nvvm_match_all_sync_i32p:
21581 case NVPTX::BI__nvvm_match_all_sync_i64p: {
21587 ? Intrinsic::nvvm_match_all_sync_i32p
21588 : Intrinsic::nvvm_match_all_sync_i64p),
21593 return Builder.CreateExtractValue(ResultPair, 0);
21597 case NVPTX::BI__hmma_m16n16k16_ld_a:
21598 case NVPTX::BI__hmma_m16n16k16_ld_b:
21599 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
21600 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
21601 case NVPTX::BI__hmma_m32n8k16_ld_a:
21602 case NVPTX::BI__hmma_m32n8k16_ld_b:
21603 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
21604 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
21605 case NVPTX::BI__hmma_m8n32k16_ld_a:
21606 case NVPTX::BI__hmma_m8n32k16_ld_b:
21607 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
21608 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
21610 case NVPTX::BI__imma_m16n16k16_ld_a_s8:
21611 case NVPTX::BI__imma_m16n16k16_ld_a_u8:
21612 case NVPTX::BI__imma_m16n16k16_ld_b_s8:
21613 case NVPTX::BI__imma_m16n16k16_ld_b_u8:
21614 case NVPTX::BI__imma_m16n16k16_ld_c:
21615 case NVPTX::BI__imma_m32n8k16_ld_a_s8:
21616 case NVPTX::BI__imma_m32n8k16_ld_a_u8:
21617 case NVPTX::BI__imma_m32n8k16_ld_b_s8:
21618 case NVPTX::BI__imma_m32n8k16_ld_b_u8:
21619 case NVPTX::BI__imma_m32n8k16_ld_c:
21620 case NVPTX::BI__imma_m8n32k16_ld_a_s8:
21621 case NVPTX::BI__imma_m8n32k16_ld_a_u8:
21622 case NVPTX::BI__imma_m8n32k16_ld_b_s8:
21623 case NVPTX::BI__imma_m8n32k16_ld_b_u8:
21624 case NVPTX::BI__imma_m8n32k16_ld_c:
21626 case NVPTX::BI__imma_m8n8k32_ld_a_s4:
21627 case NVPTX::BI__imma_m8n8k32_ld_a_u4:
21628 case NVPTX::BI__imma_m8n8k32_ld_b_s4:
21629 case NVPTX::BI__imma_m8n8k32_ld_b_u4:
21630 case NVPTX::BI__imma_m8n8k32_ld_c:
21631 case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
21632 case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
21633 case NVPTX::BI__bmma_m8n8k128_ld_c:
21635 case NVPTX::BI__dmma_m8n8k4_ld_a:
21636 case NVPTX::BI__dmma_m8n8k4_ld_b:
21637 case NVPTX::BI__dmma_m8n8k4_ld_c:
21639 case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
21640 case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
21641 case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
21642 case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
21643 case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
21644 case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
21645 case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
21646 case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
21647 case NVPTX::BI__mma_tf32_m16n16k8_ld_c: {
21651 std::optional<llvm::APSInt> isColMajorArg =
21653 if (!isColMajorArg)
21655 bool isColMajor = isColMajorArg->getSExtValue();
21656 NVPTXMmaLdstInfo II = getNVPTXMmaLdstInfo(BuiltinID);
21657 unsigned IID = isColMajor ? II.IID_col : II.IID_row;
21665 assert(II.NumResults);
21666 if (II.NumResults == 1) {
21670 for (
unsigned i = 0; i < II.NumResults; ++i) {
21675 llvm::ConstantInt::get(
IntTy, i)),
21682 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
21683 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
21684 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
21685 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
21686 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
21687 case NVPTX::BI__hmma_m8n32k16_st_c_f32:
21688 case NVPTX::BI__imma_m16n16k16_st_c_i32:
21689 case NVPTX::BI__imma_m32n8k16_st_c_i32:
21690 case NVPTX::BI__imma_m8n32k16_st_c_i32:
21691 case NVPTX::BI__imma_m8n8k32_st_c_i32:
21692 case NVPTX::BI__bmma_m8n8k128_st_c_i32:
21693 case NVPTX::BI__dmma_m8n8k4_st_c_f64:
21694 case NVPTX::BI__mma_m16n16k8_st_c_f32: {
21698 std::optional<llvm::APSInt> isColMajorArg =
21700 if (!isColMajorArg)
21702 bool isColMajor = isColMajorArg->getSExtValue();
21703 NVPTXMmaLdstInfo II = getNVPTXMmaLdstInfo(BuiltinID);
21704 unsigned IID = isColMajor ? II.IID_col : II.IID_row;
21709 llvm::Type *ParamType = Intrinsic->getFunctionType()->getParamType(1);
21711 for (
unsigned i = 0; i < II.NumResults; ++i) {
21715 llvm::ConstantInt::get(
IntTy, i)),
21717 Values.push_back(
Builder.CreateBitCast(
V, ParamType));
21719 Values.push_back(Ldm);
21726 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
21727 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
21728 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
21729 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
21730 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
21731 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
21732 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
21733 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
21734 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
21735 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
21736 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
21737 case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
21738 case NVPTX::BI__imma_m16n16k16_mma_s8:
21739 case NVPTX::BI__imma_m16n16k16_mma_u8:
21740 case NVPTX::BI__imma_m32n8k16_mma_s8:
21741 case NVPTX::BI__imma_m32n8k16_mma_u8:
21742 case NVPTX::BI__imma_m8n32k16_mma_s8:
21743 case NVPTX::BI__imma_m8n32k16_mma_u8:
21744 case NVPTX::BI__imma_m8n8k32_mma_s4:
21745 case NVPTX::BI__imma_m8n8k32_mma_u4:
21746 case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1:
21747 case NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1:
21748 case NVPTX::BI__dmma_m8n8k4_mma_f64:
21749 case NVPTX::BI__mma_bf16_m16n16k16_mma_f32:
21750 case NVPTX::BI__mma_bf16_m8n32k16_mma_f32:
21751 case NVPTX::BI__mma_bf16_m32n8k16_mma_f32:
21752 case NVPTX::BI__mma_tf32_m16n16k8_mma_f32: {
21757 std::optional<llvm::APSInt> LayoutArg =
21761 int Layout = LayoutArg->getSExtValue();
21762 if (Layout < 0 || Layout > 3)
21764 llvm::APSInt SatfArg;
21765 if (BuiltinID == NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1 ||
21766 BuiltinID == NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1)
21768 else if (std::optional<llvm::APSInt> OptSatfArg =
21770 SatfArg = *OptSatfArg;
21773 bool Satf = SatfArg.getSExtValue();
21774 NVPTXMmaInfo MI = getNVPTXMmaInfo(BuiltinID);
21775 unsigned IID = MI.getMMAIntrinsic(Layout, Satf);
21781 llvm::Type *AType = Intrinsic->getFunctionType()->getParamType(0);
21783 for (
unsigned i = 0; i < MI.NumEltsA; ++i) {
21787 llvm::ConstantInt::get(
IntTy, i)),
21789 Values.push_back(
Builder.CreateBitCast(
V, AType));
21792 llvm::Type *BType = Intrinsic->getFunctionType()->getParamType(MI.NumEltsA);
21793 for (
unsigned i = 0; i < MI.NumEltsB; ++i) {
21797 llvm::ConstantInt::get(
IntTy, i)),
21799 Values.push_back(
Builder.CreateBitCast(
V, BType));
21802 llvm::Type *CType =
21803 Intrinsic->getFunctionType()->getParamType(MI.NumEltsA + MI.NumEltsB);
21804 for (
unsigned i = 0; i < MI.NumEltsC; ++i) {
21808 llvm::ConstantInt::get(
IntTy, i)),
21810 Values.push_back(
Builder.CreateBitCast(
V, CType));
21814 for (
unsigned i = 0; i < MI.NumEltsD; ++i)
21818 llvm::ConstantInt::get(
IntTy, i)),
21823 case NVPTX::BI__nvvm_ex2_approx_f16:
21824 return MakeHalfType(Intrinsic::nvvm_ex2_approx_f16, BuiltinID,
E, *
this);
21825 case NVPTX::BI__nvvm_ex2_approx_f16x2:
21826 return MakeHalfType(Intrinsic::nvvm_ex2_approx_f16x2, BuiltinID,
E, *
this);
21827 case NVPTX::BI__nvvm_ff2f16x2_rn:
21828 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rn, BuiltinID,
E, *
this);
21829 case NVPTX::BI__nvvm_ff2f16x2_rn_relu:
21830 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rn_relu, BuiltinID,
E, *
this);
21831 case NVPTX::BI__nvvm_ff2f16x2_rz:
21832 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rz, BuiltinID,
E, *
this);
21833 case NVPTX::BI__nvvm_ff2f16x2_rz_relu:
21834 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rz_relu, BuiltinID,
E, *
this);
21835 case NVPTX::BI__nvvm_fma_rn_f16:
21836 return MakeHalfType(Intrinsic::nvvm_fma_rn_f16, BuiltinID,
E, *
this);
21837 case NVPTX::BI__nvvm_fma_rn_f16x2:
21838 return MakeHalfType(Intrinsic::nvvm_fma_rn_f16x2, BuiltinID,
E, *
this);
21839 case NVPTX::BI__nvvm_fma_rn_ftz_f16:
21840 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_f16, BuiltinID,
E, *
this);
21841 case NVPTX::BI__nvvm_fma_rn_ftz_f16x2:
21842 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_f16x2, BuiltinID,
E, *
this);
21843 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16:
21844 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_relu_f16, BuiltinID,
E,
21846 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16x2:
21847 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_relu_f16x2, BuiltinID,
E,
21849 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16:
21850 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_sat_f16, BuiltinID,
E,
21852 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16x2:
21853 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_sat_f16x2, BuiltinID,
E,
21855 case NVPTX::BI__nvvm_fma_rn_relu_f16:
21856 return MakeHalfType(Intrinsic::nvvm_fma_rn_relu_f16, BuiltinID,
E, *
this);
21857 case NVPTX::BI__nvvm_fma_rn_relu_f16x2:
21858 return MakeHalfType(Intrinsic::nvvm_fma_rn_relu_f16x2, BuiltinID,
E, *
this);
21859 case NVPTX::BI__nvvm_fma_rn_sat_f16:
21860 return MakeHalfType(Intrinsic::nvvm_fma_rn_sat_f16, BuiltinID,
E, *
this);
21861 case NVPTX::BI__nvvm_fma_rn_sat_f16x2:
21862 return MakeHalfType(Intrinsic::nvvm_fma_rn_sat_f16x2, BuiltinID,
E, *
this);
21863 case NVPTX::BI__nvvm_fmax_f16:
21864 return MakeHalfType(Intrinsic::nvvm_fmax_f16, BuiltinID,
E, *
this);
21865 case NVPTX::BI__nvvm_fmax_f16x2:
21866 return MakeHalfType(Intrinsic::nvvm_fmax_f16x2, BuiltinID,
E, *
this);
21867 case NVPTX::BI__nvvm_fmax_ftz_f16:
21868 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_f16, BuiltinID,
E, *
this);
21869 case NVPTX::BI__nvvm_fmax_ftz_f16x2:
21870 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_f16x2, BuiltinID,
E, *
this);
21871 case NVPTX::BI__nvvm_fmax_ftz_nan_f16:
21872 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_f16, BuiltinID,
E, *
this);
21873 case NVPTX::BI__nvvm_fmax_ftz_nan_f16x2:
21874 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_f16x2, BuiltinID,
E,
21876 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16:
21877 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f16, BuiltinID,
21879 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16x2:
21880 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f16x2,
21881 BuiltinID,
E, *
this);
21882 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16:
21883 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_xorsign_abs_f16, BuiltinID,
E,
21885 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16x2:
21886 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_xorsign_abs_f16x2, BuiltinID,
21888 case NVPTX::BI__nvvm_fmax_nan_f16:
21889 return MakeHalfType(Intrinsic::nvvm_fmax_nan_f16, BuiltinID,
E, *
this);
21890 case NVPTX::BI__nvvm_fmax_nan_f16x2:
21891 return MakeHalfType(Intrinsic::nvvm_fmax_nan_f16x2, BuiltinID,
E, *
this);
21892 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16:
21893 return MakeHalfType(Intrinsic::nvvm_fmax_nan_xorsign_abs_f16, BuiltinID,
E,
21895 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16x2:
21896 return MakeHalfType(Intrinsic::nvvm_fmax_nan_xorsign_abs_f16x2, BuiltinID,
21898 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16:
21899 return MakeHalfType(Intrinsic::nvvm_fmax_xorsign_abs_f16, BuiltinID,
E,
21901 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16x2:
21902 return MakeHalfType(Intrinsic::nvvm_fmax_xorsign_abs_f16x2, BuiltinID,
E,
21904 case NVPTX::BI__nvvm_fmin_f16:
21905 return MakeHalfType(Intrinsic::nvvm_fmin_f16, BuiltinID,
E, *
this);
21906 case NVPTX::BI__nvvm_fmin_f16x2:
21907 return MakeHalfType(Intrinsic::nvvm_fmin_f16x2, BuiltinID,
E, *
this);
21908 case NVPTX::BI__nvvm_fmin_ftz_f16:
21909 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_f16, BuiltinID,
E, *
this);
21910 case NVPTX::BI__nvvm_fmin_ftz_f16x2:
21911 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_f16x2, BuiltinID,
E, *
this);
21912 case NVPTX::BI__nvvm_fmin_ftz_nan_f16:
21913 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_f16, BuiltinID,
E, *
this);
21914 case NVPTX::BI__nvvm_fmin_ftz_nan_f16x2:
21915 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_f16x2, BuiltinID,
E,
21917 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16:
21918 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f16, BuiltinID,
21920 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16x2:
21921 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f16x2,
21922 BuiltinID,
E, *
this);
21923 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16:
21924 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_xorsign_abs_f16, BuiltinID,
E,
21926 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16x2:
21927 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_xorsign_abs_f16x2, BuiltinID,
21929 case NVPTX::BI__nvvm_fmin_nan_f16:
21930 return MakeHalfType(Intrinsic::nvvm_fmin_nan_f16, BuiltinID,
E, *
this);
21931 case NVPTX::BI__nvvm_fmin_nan_f16x2:
21932 return MakeHalfType(Intrinsic::nvvm_fmin_nan_f16x2, BuiltinID,
E, *
this);
21933 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16:
21934 return MakeHalfType(Intrinsic::nvvm_fmin_nan_xorsign_abs_f16, BuiltinID,
E,
21936 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16x2:
21937 return MakeHalfType(Intrinsic::nvvm_fmin_nan_xorsign_abs_f16x2, BuiltinID,
21939 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16:
21940 return MakeHalfType(Intrinsic::nvvm_fmin_xorsign_abs_f16, BuiltinID,
E,
21942 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16x2:
21943 return MakeHalfType(Intrinsic::nvvm_fmin_xorsign_abs_f16x2, BuiltinID,
E,
21945 case NVPTX::BI__nvvm_ldg_h:
21946 case NVPTX::BI__nvvm_ldg_h2:
21947 return MakeHalfType(Intrinsic::not_intrinsic, BuiltinID,
E, *
this);
21948 case NVPTX::BI__nvvm_ldu_h:
21949 case NVPTX::BI__nvvm_ldu_h2:
21950 return MakeHalfType(Intrinsic::nvvm_ldu_global_f, BuiltinID,
E, *
this);
21951 case NVPTX::BI__nvvm_cp_async_ca_shared_global_4:
21952 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_4,
21953 Intrinsic::nvvm_cp_async_ca_shared_global_4_s, *
this,
E,
21955 case NVPTX::BI__nvvm_cp_async_ca_shared_global_8:
21956 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_8,
21957 Intrinsic::nvvm_cp_async_ca_shared_global_8_s, *
this,
E,
21959 case NVPTX::BI__nvvm_cp_async_ca_shared_global_16:
21960 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_16,
21961 Intrinsic::nvvm_cp_async_ca_shared_global_16_s, *
this,
E,
21963 case NVPTX::BI__nvvm_cp_async_cg_shared_global_16:
21964 return MakeCpAsync(Intrinsic::nvvm_cp_async_cg_shared_global_16,
21965 Intrinsic::nvvm_cp_async_cg_shared_global_16_s, *
this,
E,
21967 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_x:
21970 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_y:
21973 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_z:
21976 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_w:
21979 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_x:
21982 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_y:
21985 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_z:
21988 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_w:
21991 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_x:
21994 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_y:
21997 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_z:
22000 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_w:
22003 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_x:
22006 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_y:
22009 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_z:
22012 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_w:
22015 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctarank:
22018 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctarank:
22021 case NVPTX::BI__nvvm_is_explicit_cluster:
22024 case NVPTX::BI__nvvm_isspacep_shared_cluster:
22028 case NVPTX::BI__nvvm_mapa:
22031 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
22032 case NVPTX::BI__nvvm_mapa_shared_cluster:
22035 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
22036 case NVPTX::BI__nvvm_getctarank:
22040 case NVPTX::BI__nvvm_getctarank_shared_cluster:
22044 case NVPTX::BI__nvvm_barrier_cluster_arrive:
22047 case NVPTX::BI__nvvm_barrier_cluster_arrive_relaxed:
22050 case NVPTX::BI__nvvm_barrier_cluster_wait:
22053 case NVPTX::BI__nvvm_fence_sc_cluster:
22062struct BuiltinAlignArgs {
22063 llvm::Value *Src =
nullptr;
22064 llvm::Type *SrcType =
nullptr;
22065 llvm::Value *Alignment =
nullptr;
22066 llvm::Value *Mask =
nullptr;
22067 llvm::IntegerType *IntType =
nullptr;
22075 SrcType = Src->getType();
22076 if (SrcType->isPointerTy()) {
22077 IntType = IntegerType::get(
22081 assert(SrcType->isIntegerTy());
22082 IntType = cast<llvm::IntegerType>(SrcType);
22085 Alignment = CGF.
Builder.CreateZExtOrTrunc(Alignment, IntType,
"alignment");
22086 auto *One = llvm::ConstantInt::get(IntType, 1);
22087 Mask = CGF.
Builder.CreateSub(Alignment, One,
"mask");
22094 BuiltinAlignArgs Args(
E, *
this);
22095 llvm::Value *SrcAddress = Args.Src;
22096 if (Args.SrcType->isPointerTy())
22098 Builder.CreateBitOrPointerCast(Args.Src, Args.IntType,
"src_addr");
22100 Builder.CreateAnd(SrcAddress, Args.Mask,
"set_bits"),
22101 llvm::Constant::getNullValue(Args.IntType),
"is_aligned"));
22108 BuiltinAlignArgs Args(
E, *
this);
22109 llvm::Value *SrcForMask = Args.Src;
22115 if (Args.Src->getType()->isPointerTy()) {
22125 SrcForMask =
Builder.CreateAdd(SrcForMask, Args.Mask,
"over_boundary");
22129 llvm::Value *InvertedMask =
Builder.CreateNot(Args.Mask,
"inverted_mask");
22130 llvm::Value *
Result =
nullptr;
22131 if (Args.Src->getType()->isPointerTy()) {
22133 Intrinsic::ptrmask, {Args.SrcType, Args.IntType},
22134 {SrcForMask, InvertedMask},
nullptr,
"aligned_result");
22136 Result =
Builder.CreateAnd(SrcForMask, InvertedMask,
"aligned_result");
22138 assert(
Result->getType() == Args.SrcType);
22144 switch (BuiltinID) {
22145 case WebAssembly::BI__builtin_wasm_memory_size: {
22150 return Builder.CreateCall(Callee, I);
22152 case WebAssembly::BI__builtin_wasm_memory_grow: {
22158 return Builder.CreateCall(Callee, Args);
22160 case WebAssembly::BI__builtin_wasm_tls_size: {
22163 return Builder.CreateCall(Callee);
22165 case WebAssembly::BI__builtin_wasm_tls_align: {
22168 return Builder.CreateCall(Callee);
22170 case WebAssembly::BI__builtin_wasm_tls_base: {
22172 return Builder.CreateCall(Callee);
22174 case WebAssembly::BI__builtin_wasm_throw: {
22178 return Builder.CreateCall(Callee, {
Tag, Obj});
22180 case WebAssembly::BI__builtin_wasm_rethrow: {
22182 return Builder.CreateCall(Callee);
22184 case WebAssembly::BI__builtin_wasm_memory_atomic_wait32: {
22191 case WebAssembly::BI__builtin_wasm_memory_atomic_wait64: {
22198 case WebAssembly::BI__builtin_wasm_memory_atomic_notify: {
22202 return Builder.CreateCall(Callee, {Addr, Count});
22204 case WebAssembly::BI__builtin_wasm_trunc_s_i32_f32:
22205 case WebAssembly::BI__builtin_wasm_trunc_s_i32_f64:
22206 case WebAssembly::BI__builtin_wasm_trunc_s_i64_f32:
22207 case WebAssembly::BI__builtin_wasm_trunc_s_i64_f64: {
22212 return Builder.CreateCall(Callee, {Src});
22214 case WebAssembly::BI__builtin_wasm_trunc_u_i32_f32:
22215 case WebAssembly::BI__builtin_wasm_trunc_u_i32_f64:
22216 case WebAssembly::BI__builtin_wasm_trunc_u_i64_f32:
22217 case WebAssembly::BI__builtin_wasm_trunc_u_i64_f64: {
22222 return Builder.CreateCall(Callee, {Src});
22224 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32_f32:
22225 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32_f64:
22226 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i64_f32:
22227 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i64_f64:
22228 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i16x8_f16x8:
22229 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32x4_f32x4: {
22234 return Builder.CreateCall(Callee, {Src});
22236 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32_f32:
22237 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32_f64:
22238 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i64_f32:
22239 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i64_f64:
22240 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i16x8_f16x8:
22241 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32x4_f32x4: {
22246 return Builder.CreateCall(Callee, {Src});
22248 case WebAssembly::BI__builtin_wasm_min_f32:
22249 case WebAssembly::BI__builtin_wasm_min_f64:
22250 case WebAssembly::BI__builtin_wasm_min_f16x8:
22251 case WebAssembly::BI__builtin_wasm_min_f32x4:
22252 case WebAssembly::BI__builtin_wasm_min_f64x2: {
22257 return Builder.CreateCall(Callee, {LHS, RHS});
22259 case WebAssembly::BI__builtin_wasm_max_f32:
22260 case WebAssembly::BI__builtin_wasm_max_f64:
22261 case WebAssembly::BI__builtin_wasm_max_f16x8:
22262 case WebAssembly::BI__builtin_wasm_max_f32x4:
22263 case WebAssembly::BI__builtin_wasm_max_f64x2: {
22268 return Builder.CreateCall(Callee, {LHS, RHS});
22270 case WebAssembly::BI__builtin_wasm_pmin_f16x8:
22271 case WebAssembly::BI__builtin_wasm_pmin_f32x4:
22272 case WebAssembly::BI__builtin_wasm_pmin_f64x2: {
22277 return Builder.CreateCall(Callee, {LHS, RHS});
22279 case WebAssembly::BI__builtin_wasm_pmax_f16x8:
22280 case WebAssembly::BI__builtin_wasm_pmax_f32x4:
22281 case WebAssembly::BI__builtin_wasm_pmax_f64x2: {
22286 return Builder.CreateCall(Callee, {LHS, RHS});
22288 case WebAssembly::BI__builtin_wasm_ceil_f16x8:
22289 case WebAssembly::BI__builtin_wasm_floor_f16x8:
22290 case WebAssembly::BI__builtin_wasm_trunc_f16x8:
22291 case WebAssembly::BI__builtin_wasm_nearest_f16x8:
22292 case WebAssembly::BI__builtin_wasm_ceil_f32x4:
22293 case WebAssembly::BI__builtin_wasm_floor_f32x4:
22294 case WebAssembly::BI__builtin_wasm_trunc_f32x4:
22295 case WebAssembly::BI__builtin_wasm_nearest_f32x4:
22296 case WebAssembly::BI__builtin_wasm_ceil_f64x2:
22297 case WebAssembly::BI__builtin_wasm_floor_f64x2:
22298 case WebAssembly::BI__builtin_wasm_trunc_f64x2:
22299 case WebAssembly::BI__builtin_wasm_nearest_f64x2: {
22301 switch (BuiltinID) {
22302 case WebAssembly::BI__builtin_wasm_ceil_f16x8:
22303 case WebAssembly::BI__builtin_wasm_ceil_f32x4:
22304 case WebAssembly::BI__builtin_wasm_ceil_f64x2:
22305 IntNo = Intrinsic::ceil;
22307 case WebAssembly::BI__builtin_wasm_floor_f16x8:
22308 case WebAssembly::BI__builtin_wasm_floor_f32x4:
22309 case WebAssembly::BI__builtin_wasm_floor_f64x2:
22310 IntNo = Intrinsic::floor;
22312 case WebAssembly::BI__builtin_wasm_trunc_f16x8:
22313 case WebAssembly::BI__builtin_wasm_trunc_f32x4:
22314 case WebAssembly::BI__builtin_wasm_trunc_f64x2:
22315 IntNo = Intrinsic::trunc;
22317 case WebAssembly::BI__builtin_wasm_nearest_f16x8:
22318 case WebAssembly::BI__builtin_wasm_nearest_f32x4:
22319 case WebAssembly::BI__builtin_wasm_nearest_f64x2:
22320 IntNo = Intrinsic::nearbyint;
22323 llvm_unreachable(
"unexpected builtin ID");
22329 case WebAssembly::BI__builtin_wasm_ref_null_extern: {
22331 return Builder.CreateCall(Callee);
22333 case WebAssembly::BI__builtin_wasm_ref_null_func: {
22335 return Builder.CreateCall(Callee);
22337 case WebAssembly::BI__builtin_wasm_swizzle_i8x16: {
22341 return Builder.CreateCall(Callee, {Src, Indices});
22343 case WebAssembly::BI__builtin_wasm_abs_i8x16:
22344 case WebAssembly::BI__builtin_wasm_abs_i16x8:
22345 case WebAssembly::BI__builtin_wasm_abs_i32x4:
22346 case WebAssembly::BI__builtin_wasm_abs_i64x2: {
22349 Constant *
Zero = llvm::Constant::getNullValue(Vec->
getType());
22350 Value *ICmp =
Builder.CreateICmpSLT(Vec, Zero,
"abscond");
22351 return Builder.CreateSelect(ICmp, Neg, Vec,
"abs");
22353 case WebAssembly::BI__builtin_wasm_avgr_u_i8x16:
22354 case WebAssembly::BI__builtin_wasm_avgr_u_i16x8: {
22359 return Builder.CreateCall(Callee, {LHS, RHS});
22361 case WebAssembly::BI__builtin_wasm_q15mulr_sat_s_i16x8: {
22365 return Builder.CreateCall(Callee, {LHS, RHS});
22367 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_s_i16x8:
22368 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_u_i16x8:
22369 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_s_i32x4:
22370 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_u_i32x4: {
22373 switch (BuiltinID) {
22374 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_s_i16x8:
22375 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_s_i32x4:
22376 IntNo = Intrinsic::wasm_extadd_pairwise_signed;
22378 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_u_i16x8:
22379 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_u_i32x4:
22380 IntNo = Intrinsic::wasm_extadd_pairwise_unsigned;
22383 llvm_unreachable(
"unexpected builtin ID");
22387 return Builder.CreateCall(Callee, Vec);
22389 case WebAssembly::BI__builtin_wasm_bitselect: {
22395 return Builder.CreateCall(Callee, {V1, V2,
C});
22397 case WebAssembly::BI__builtin_wasm_dot_s_i32x4_i16x8: {
22401 return Builder.CreateCall(Callee, {LHS, RHS});
22403 case WebAssembly::BI__builtin_wasm_any_true_v128:
22404 case WebAssembly::BI__builtin_wasm_all_true_i8x16:
22405 case WebAssembly::BI__builtin_wasm_all_true_i16x8:
22406 case WebAssembly::BI__builtin_wasm_all_true_i32x4:
22407 case WebAssembly::BI__builtin_wasm_all_true_i64x2: {
22409 switch (BuiltinID) {
22410 case WebAssembly::BI__builtin_wasm_any_true_v128:
22411 IntNo = Intrinsic::wasm_anytrue;
22413 case WebAssembly::BI__builtin_wasm_all_true_i8x16:
22414 case WebAssembly::BI__builtin_wasm_all_true_i16x8:
22415 case WebAssembly::BI__builtin_wasm_all_true_i32x4:
22416 case WebAssembly::BI__builtin_wasm_all_true_i64x2:
22417 IntNo = Intrinsic::wasm_alltrue;
22420 llvm_unreachable(
"unexpected builtin ID");
22424 return Builder.CreateCall(Callee, {Vec});
22426 case WebAssembly::BI__builtin_wasm_bitmask_i8x16:
22427 case WebAssembly::BI__builtin_wasm_bitmask_i16x8:
22428 case WebAssembly::BI__builtin_wasm_bitmask_i32x4:
22429 case WebAssembly::BI__builtin_wasm_bitmask_i64x2: {
22433 return Builder.CreateCall(Callee, {Vec});
22435 case WebAssembly::BI__builtin_wasm_abs_f16x8:
22436 case WebAssembly::BI__builtin_wasm_abs_f32x4:
22437 case WebAssembly::BI__builtin_wasm_abs_f64x2: {
22440 return Builder.CreateCall(Callee, {Vec});
22442 case WebAssembly::BI__builtin_wasm_sqrt_f16x8:
22443 case WebAssembly::BI__builtin_wasm_sqrt_f32x4:
22444 case WebAssembly::BI__builtin_wasm_sqrt_f64x2: {
22447 return Builder.CreateCall(Callee, {Vec});
22449 case WebAssembly::BI__builtin_wasm_narrow_s_i8x16_i16x8:
22450 case WebAssembly::BI__builtin_wasm_narrow_u_i8x16_i16x8:
22451 case WebAssembly::BI__builtin_wasm_narrow_s_i16x8_i32x4:
22452 case WebAssembly::BI__builtin_wasm_narrow_u_i16x8_i32x4: {
22456 switch (BuiltinID) {
22457 case WebAssembly::BI__builtin_wasm_narrow_s_i8x16_i16x8:
22458 case WebAssembly::BI__builtin_wasm_narrow_s_i16x8_i32x4:
22459 IntNo = Intrinsic::wasm_narrow_signed;
22461 case WebAssembly::BI__builtin_wasm_narrow_u_i8x16_i16x8:
22462 case WebAssembly::BI__builtin_wasm_narrow_u_i16x8_i32x4:
22463 IntNo = Intrinsic::wasm_narrow_unsigned;
22466 llvm_unreachable(
"unexpected builtin ID");
22470 return Builder.CreateCall(Callee, {Low, High});
22472 case WebAssembly::BI__builtin_wasm_trunc_sat_s_zero_f64x2_i32x4:
22473 case WebAssembly::BI__builtin_wasm_trunc_sat_u_zero_f64x2_i32x4: {
22476 switch (BuiltinID) {
22477 case WebAssembly::BI__builtin_wasm_trunc_sat_s_zero_f64x2_i32x4:
22478 IntNo = Intrinsic::fptosi_sat;
22480 case WebAssembly::BI__builtin_wasm_trunc_sat_u_zero_f64x2_i32x4:
22481 IntNo = Intrinsic::fptoui_sat;
22484 llvm_unreachable(
"unexpected builtin ID");
22486 llvm::Type *SrcT = Vec->
getType();
22487 llvm::Type *TruncT = SrcT->getWithNewType(
Builder.getInt32Ty());
22490 Value *Splat = Constant::getNullValue(TruncT);
22493 case WebAssembly::BI__builtin_wasm_shuffle_i8x16: {
22498 while (OpIdx < 18) {
22499 std::optional<llvm::APSInt> LaneConst =
22501 assert(LaneConst &&
"Constant arg isn't actually constant?");
22502 Ops[OpIdx++] = llvm::ConstantInt::get(
getLLVMContext(), *LaneConst);
22505 return Builder.CreateCall(Callee, Ops);
22507 case WebAssembly::BI__builtin_wasm_relaxed_madd_f16x8:
22508 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f16x8:
22509 case WebAssembly::BI__builtin_wasm_relaxed_madd_f32x4:
22510 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f32x4:
22511 case WebAssembly::BI__builtin_wasm_relaxed_madd_f64x2:
22512 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f64x2: {
22517 switch (BuiltinID) {
22518 case WebAssembly::BI__builtin_wasm_relaxed_madd_f16x8:
22519 case WebAssembly::BI__builtin_wasm_relaxed_madd_f32x4:
22520 case WebAssembly::BI__builtin_wasm_relaxed_madd_f64x2:
22521 IntNo = Intrinsic::wasm_relaxed_madd;
22523 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f16x8:
22524 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f32x4:
22525 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f64x2:
22526 IntNo = Intrinsic::wasm_relaxed_nmadd;
22529 llvm_unreachable(
"unexpected builtin ID");
22532 return Builder.CreateCall(Callee, {A, B,
C});
22534 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i8x16:
22535 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i16x8:
22536 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i32x4:
22537 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i64x2: {
22543 return Builder.CreateCall(Callee, {A, B,
C});
22545 case WebAssembly::BI__builtin_wasm_relaxed_swizzle_i8x16: {
22549 return Builder.CreateCall(Callee, {Src, Indices});
22551 case WebAssembly::BI__builtin_wasm_relaxed_min_f32x4:
22552 case WebAssembly::BI__builtin_wasm_relaxed_max_f32x4:
22553 case WebAssembly::BI__builtin_wasm_relaxed_min_f64x2:
22554 case WebAssembly::BI__builtin_wasm_relaxed_max_f64x2: {
22558 switch (BuiltinID) {
22559 case WebAssembly::BI__builtin_wasm_relaxed_min_f32x4:
22560 case WebAssembly::BI__builtin_wasm_relaxed_min_f64x2:
22561 IntNo = Intrinsic::wasm_relaxed_min;
22563 case WebAssembly::BI__builtin_wasm_relaxed_max_f32x4:
22564 case WebAssembly::BI__builtin_wasm_relaxed_max_f64x2:
22565 IntNo = Intrinsic::wasm_relaxed_max;
22568 llvm_unreachable(
"unexpected builtin ID");
22571 return Builder.CreateCall(Callee, {LHS, RHS});
22573 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_i32x4_f32x4:
22574 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_i32x4_f32x4:
22575 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_zero_i32x4_f64x2:
22576 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_zero_i32x4_f64x2: {
22579 switch (BuiltinID) {
22580 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_i32x4_f32x4:
22581 IntNo = Intrinsic::wasm_relaxed_trunc_signed;
22583 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_i32x4_f32x4:
22584 IntNo = Intrinsic::wasm_relaxed_trunc_unsigned;
22586 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_zero_i32x4_f64x2:
22587 IntNo = Intrinsic::wasm_relaxed_trunc_signed_zero;
22589 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_zero_i32x4_f64x2:
22590 IntNo = Intrinsic::wasm_relaxed_trunc_unsigned_zero;
22593 llvm_unreachable(
"unexpected builtin ID");
22596 return Builder.CreateCall(Callee, {Vec});
22598 case WebAssembly::BI__builtin_wasm_relaxed_q15mulr_s_i16x8: {
22602 return Builder.CreateCall(Callee, {LHS, RHS});
22604 case WebAssembly::BI__builtin_wasm_relaxed_dot_i8x16_i7x16_s_i16x8: {
22609 return Builder.CreateCall(Callee, {LHS, RHS});
22611 case WebAssembly::BI__builtin_wasm_relaxed_dot_i8x16_i7x16_add_s_i32x4: {
22616 CGM.
getIntrinsic(Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed);
22617 return Builder.CreateCall(Callee, {LHS, RHS, Acc});
22619 case WebAssembly::BI__builtin_wasm_relaxed_dot_bf16x8_add_f32_f32x4: {
22625 return Builder.CreateCall(Callee, {LHS, RHS, Acc});
22627 case WebAssembly::BI__builtin_wasm_loadf16_f32: {
22630 return Builder.CreateCall(Callee, {Addr});
22632 case WebAssembly::BI__builtin_wasm_storef16_f32: {
22636 return Builder.CreateCall(Callee, {Val, Addr});
22638 case WebAssembly::BI__builtin_wasm_splat_f16x8: {
22641 return Builder.CreateCall(Callee, {Val});
22643 case WebAssembly::BI__builtin_wasm_extract_lane_f16x8: {
22649 case WebAssembly::BI__builtin_wasm_replace_lane_f16x8: {
22656 case WebAssembly::BI__builtin_wasm_table_get: {
22667 "Unexpected reference type for __builtin_wasm_table_get");
22668 return Builder.CreateCall(Callee, {Table, Index});
22670 case WebAssembly::BI__builtin_wasm_table_set: {
22682 "Unexpected reference type for __builtin_wasm_table_set");
22683 return Builder.CreateCall(Callee, {Table, Index, Val});
22685 case WebAssembly::BI__builtin_wasm_table_size: {
22691 case WebAssembly::BI__builtin_wasm_table_grow: {
22704 "Unexpected reference type for __builtin_wasm_table_grow");
22706 return Builder.CreateCall(Callee, {Table, Val, NElems});
22708 case WebAssembly::BI__builtin_wasm_table_fill: {
22722 "Unexpected reference type for __builtin_wasm_table_fill");
22724 return Builder.CreateCall(Callee, {Table, Index, Val, NElems});
22726 case WebAssembly::BI__builtin_wasm_table_copy: {
22736 return Builder.CreateCall(Callee, {TableX, TableY, SrcIdx, DstIdx, NElems});
22743static std::pair<Intrinsic::ID, unsigned>
22746 unsigned BuiltinID;
22747 Intrinsic::ID IntrinsicID;
22750 static Info Infos[] = {
22751#define CUSTOM_BUILTIN_MAPPING(x,s) \
22752 { Hexagon::BI__builtin_HEXAGON_##x, Intrinsic::hexagon_##x, s },
22784#include "clang/Basic/BuiltinsHexagonMapCustomDep.def"
22785#undef CUSTOM_BUILTIN_MAPPING
22788 auto CmpInfo = [] (Info A, Info B) {
return A.BuiltinID < B.BuiltinID; };
22789 static const bool SortOnce = (llvm::sort(Infos, CmpInfo),
true);
22792 const Info *F = llvm::lower_bound(Infos, Info{BuiltinID, 0, 0}, CmpInfo);
22793 if (F == std::end(Infos) || F->BuiltinID != BuiltinID)
22794 return {Intrinsic::not_intrinsic, 0};
22796 return {F->IntrinsicID, F->VecLen};
22805 auto MakeCircOp = [
this,
E](
unsigned IntID,
bool IsLoad) {
22819 for (
unsigned i = 1, e =
E->getNumArgs(); i != e; ++i)
22825 llvm::Value *NewBase = IsLoad ?
Builder.CreateExtractValue(
Result, 1)
22829 llvm::Value *RetVal =
22839 auto MakeBrevLd = [
this,
E](
unsigned IntID, llvm::Type *DestTy) {
22856 CGM.
getIntrinsic(IntID), {BaseAddress, EmitScalarExpr(E->getArg(2))});
22859 llvm::Value *DestVal =
Builder.CreateExtractValue(
Result, 0);
22864 DestVal =
Builder.CreateTrunc(DestVal, DestTy);
22871 auto V2Q = [
this, VecLen] (llvm::Value *Vec) {
22872 Intrinsic::ID
ID = VecLen == 128 ? Intrinsic::hexagon_V6_vandvrt_128B
22873 : Intrinsic::hexagon_V6_vandvrt;
22875 {Vec,
Builder.getInt32(-1)});
22877 auto Q2V = [
this, VecLen] (llvm::Value *Pred) {
22878 Intrinsic::ID
ID = VecLen == 128 ? Intrinsic::hexagon_V6_vandqrt_128B
22879 : Intrinsic::hexagon_V6_vandqrt;
22881 {Pred,
Builder.getInt32(-1)});
22884 switch (BuiltinID) {
22888 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarry:
22889 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarry_128B:
22890 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarry:
22891 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarry_128B: {
22898 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1)), PredIn});
22900 llvm::Value *PredOut =
Builder.CreateExtractValue(
Result, 1);
22908 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarryo:
22909 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarryo_128B:
22910 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarryo:
22911 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarryo_128B: {
22917 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
22919 llvm::Value *PredOut =
Builder.CreateExtractValue(
Result, 1);
22925 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstoreq:
22926 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorenq:
22927 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentq:
22928 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentnq:
22929 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstoreq_128B:
22930 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorenq_128B:
22931 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentq_128B:
22932 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentnq_128B: {
22934 const Expr *PredOp =
E->getArg(0);
22936 if (
auto *Cast = dyn_cast<ImplicitCastExpr>(PredOp)) {
22937 if (
Cast->getCastKind() == CK_BitCast)
22938 PredOp =
Cast->getSubExpr();
22941 for (
int i = 1, e =
E->getNumArgs(); i != e; ++i)
22946 case Hexagon::BI__builtin_HEXAGON_L2_loadrub_pci:
22947 case Hexagon::BI__builtin_HEXAGON_L2_loadrb_pci:
22948 case Hexagon::BI__builtin_HEXAGON_L2_loadruh_pci:
22949 case Hexagon::BI__builtin_HEXAGON_L2_loadrh_pci:
22950 case Hexagon::BI__builtin_HEXAGON_L2_loadri_pci:
22951 case Hexagon::BI__builtin_HEXAGON_L2_loadrd_pci:
22952 case Hexagon::BI__builtin_HEXAGON_L2_loadrub_pcr:
22953 case Hexagon::BI__builtin_HEXAGON_L2_loadrb_pcr:
22954 case Hexagon::BI__builtin_HEXAGON_L2_loadruh_pcr:
22955 case Hexagon::BI__builtin_HEXAGON_L2_loadrh_pcr:
22956 case Hexagon::BI__builtin_HEXAGON_L2_loadri_pcr:
22957 case Hexagon::BI__builtin_HEXAGON_L2_loadrd_pcr:
22958 return MakeCircOp(ID,
true);
22959 case Hexagon::BI__builtin_HEXAGON_S2_storerb_pci:
22960 case Hexagon::BI__builtin_HEXAGON_S2_storerh_pci:
22961 case Hexagon::BI__builtin_HEXAGON_S2_storerf_pci:
22962 case Hexagon::BI__builtin_HEXAGON_S2_storeri_pci:
22963 case Hexagon::BI__builtin_HEXAGON_S2_storerd_pci:
22964 case Hexagon::BI__builtin_HEXAGON_S2_storerb_pcr:
22965 case Hexagon::BI__builtin_HEXAGON_S2_storerh_pcr:
22966 case Hexagon::BI__builtin_HEXAGON_S2_storerf_pcr:
22967 case Hexagon::BI__builtin_HEXAGON_S2_storeri_pcr:
22968 case Hexagon::BI__builtin_HEXAGON_S2_storerd_pcr:
22969 return MakeCircOp(ID,
false);
22970 case Hexagon::BI__builtin_brev_ldub:
22971 return MakeBrevLd(Intrinsic::hexagon_L2_loadrub_pbr,
Int8Ty);
22972 case Hexagon::BI__builtin_brev_ldb:
22973 return MakeBrevLd(Intrinsic::hexagon_L2_loadrb_pbr,
Int8Ty);
22974 case Hexagon::BI__builtin_brev_lduh:
22975 return MakeBrevLd(Intrinsic::hexagon_L2_loadruh_pbr,
Int16Ty);
22976 case Hexagon::BI__builtin_brev_ldh:
22977 return MakeBrevLd(Intrinsic::hexagon_L2_loadrh_pbr,
Int16Ty);
22978 case Hexagon::BI__builtin_brev_ldw:
22979 return MakeBrevLd(Intrinsic::hexagon_L2_loadri_pbr,
Int32Ty);
22980 case Hexagon::BI__builtin_brev_ldd:
22981 return MakeBrevLd(Intrinsic::hexagon_L2_loadrd_pbr,
Int64Ty);
22989 StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
22997 llvm::Constant *RISCVCPUModel =
22999 cast<llvm::GlobalValue>(RISCVCPUModel)->setDSOLocal(
true);
23001 auto loadRISCVCPUID = [&](
unsigned Index) {
23004 Ptr, llvm::MaybeAlign());
23008 const llvm::RISCV::CPUModel Model = llvm::RISCV::getCPUModel(CPUStr);
23011 Value *VendorID = loadRISCVCPUID(0);
23013 Builder.CreateICmpEQ(VendorID,
Builder.getInt32(Model.MVendorID));
23016 Value *ArchID = loadRISCVCPUID(1);
23021 Value *ImpID = loadRISCVCPUID(2);
23032 if (BuiltinID == Builtin::BI__builtin_cpu_supports)
23034 if (BuiltinID == Builtin::BI__builtin_cpu_init)
23036 if (BuiltinID == Builtin::BI__builtin_cpu_is)
23043 unsigned ICEArguments = 0;
23051 if (BuiltinID == RISCVVector::BI__builtin_rvv_vget_v ||
23052 BuiltinID == RISCVVector::BI__builtin_rvv_vset_v)
23053 ICEArguments = 1 << 1;
23058 if (BuiltinID == RISCV::BI__builtin_riscv_ntl_load)
23059 ICEArguments |= (1 << 1);
23060 if (BuiltinID == RISCV::BI__builtin_riscv_ntl_store)
23061 ICEArguments |= (1 << 2);
23063 for (
unsigned i = 0, e =
E->getNumArgs(); i != e; i++) {
23068 Ops.push_back(AggValue);
23074 Intrinsic::ID
ID = Intrinsic::not_intrinsic;
23077 constexpr unsigned RVV_VTA = 0x1;
23078 constexpr unsigned RVV_VMA = 0x2;
23079 int PolicyAttrs = 0;
23080 bool IsMasked =
false;
23082 unsigned SegInstSEW = 8;
23086 switch (BuiltinID) {
23087 default: llvm_unreachable(
"unexpected builtin ID");
23088 case RISCV::BI__builtin_riscv_orc_b_32:
23089 case RISCV::BI__builtin_riscv_orc_b_64:
23090 case RISCV::BI__builtin_riscv_clmul_32:
23091 case RISCV::BI__builtin_riscv_clmul_64:
23092 case RISCV::BI__builtin_riscv_clmulh_32:
23093 case RISCV::BI__builtin_riscv_clmulh_64:
23094 case RISCV::BI__builtin_riscv_clmulr_32:
23095 case RISCV::BI__builtin_riscv_clmulr_64:
23096 case RISCV::BI__builtin_riscv_xperm4_32:
23097 case RISCV::BI__builtin_riscv_xperm4_64:
23098 case RISCV::BI__builtin_riscv_xperm8_32:
23099 case RISCV::BI__builtin_riscv_xperm8_64:
23100 case RISCV::BI__builtin_riscv_brev8_32:
23101 case RISCV::BI__builtin_riscv_brev8_64:
23102 case RISCV::BI__builtin_riscv_zip_32:
23103 case RISCV::BI__builtin_riscv_unzip_32: {
23104 switch (BuiltinID) {
23105 default: llvm_unreachable(
"unexpected builtin ID");
23107 case RISCV::BI__builtin_riscv_orc_b_32:
23108 case RISCV::BI__builtin_riscv_orc_b_64:
23109 ID = Intrinsic::riscv_orc_b;
23113 case RISCV::BI__builtin_riscv_clmul_32:
23114 case RISCV::BI__builtin_riscv_clmul_64:
23115 ID = Intrinsic::riscv_clmul;
23117 case RISCV::BI__builtin_riscv_clmulh_32:
23118 case RISCV::BI__builtin_riscv_clmulh_64:
23119 ID = Intrinsic::riscv_clmulh;
23121 case RISCV::BI__builtin_riscv_clmulr_32:
23122 case RISCV::BI__builtin_riscv_clmulr_64:
23123 ID = Intrinsic::riscv_clmulr;
23127 case RISCV::BI__builtin_riscv_xperm8_32:
23128 case RISCV::BI__builtin_riscv_xperm8_64:
23129 ID = Intrinsic::riscv_xperm8;
23131 case RISCV::BI__builtin_riscv_xperm4_32:
23132 case RISCV::BI__builtin_riscv_xperm4_64:
23133 ID = Intrinsic::riscv_xperm4;
23137 case RISCV::BI__builtin_riscv_brev8_32:
23138 case RISCV::BI__builtin_riscv_brev8_64:
23139 ID = Intrinsic::riscv_brev8;
23141 case RISCV::BI__builtin_riscv_zip_32:
23142 ID = Intrinsic::riscv_zip;
23144 case RISCV::BI__builtin_riscv_unzip_32:
23145 ID = Intrinsic::riscv_unzip;
23149 IntrinsicTypes = {ResultType};
23156 case RISCV::BI__builtin_riscv_sha256sig0:
23157 ID = Intrinsic::riscv_sha256sig0;
23159 case RISCV::BI__builtin_riscv_sha256sig1:
23160 ID = Intrinsic::riscv_sha256sig1;
23162 case RISCV::BI__builtin_riscv_sha256sum0:
23163 ID = Intrinsic::riscv_sha256sum0;
23165 case RISCV::BI__builtin_riscv_sha256sum1:
23166 ID = Intrinsic::riscv_sha256sum1;
23170 case RISCV::BI__builtin_riscv_sm4ks:
23171 ID = Intrinsic::riscv_sm4ks;
23173 case RISCV::BI__builtin_riscv_sm4ed:
23174 ID = Intrinsic::riscv_sm4ed;
23178 case RISCV::BI__builtin_riscv_sm3p0:
23179 ID = Intrinsic::riscv_sm3p0;
23181 case RISCV::BI__builtin_riscv_sm3p1:
23182 ID = Intrinsic::riscv_sm3p1;
23185 case RISCV::BI__builtin_riscv_clz_32:
23186 case RISCV::BI__builtin_riscv_clz_64: {
23189 if (
Result->getType() != ResultType)
23194 case RISCV::BI__builtin_riscv_ctz_32:
23195 case RISCV::BI__builtin_riscv_ctz_64: {
23198 if (
Result->getType() != ResultType)
23205 case RISCV::BI__builtin_riscv_ntl_load: {
23207 unsigned DomainVal = 5;
23208 if (Ops.size() == 2)
23209 DomainVal = cast<ConstantInt>(Ops[1])->getZExtValue();
23211 llvm::MDNode *RISCVDomainNode = llvm::MDNode::get(
23213 llvm::ConstantAsMetadata::get(
Builder.getInt32(DomainVal)));
23214 llvm::MDNode *NontemporalNode = llvm::MDNode::get(
23218 if(ResTy->isScalableTy()) {
23219 const ScalableVectorType *SVTy = cast<ScalableVectorType>(ResTy);
23220 llvm::Type *ScalarTy = ResTy->getScalarType();
23221 Width = ScalarTy->getPrimitiveSizeInBits() *
23222 SVTy->getElementCount().getKnownMinValue();
23224 Width = ResTy->getPrimitiveSizeInBits();
23228 Load->setMetadata(llvm::LLVMContext::MD_nontemporal, NontemporalNode);
23229 Load->setMetadata(
CGM.
getModule().getMDKindID(
"riscv-nontemporal-domain"),
23234 case RISCV::BI__builtin_riscv_ntl_store: {
23235 unsigned DomainVal = 5;
23236 if (Ops.size() == 3)
23237 DomainVal = cast<ConstantInt>(Ops[2])->getZExtValue();
23239 llvm::MDNode *RISCVDomainNode = llvm::MDNode::get(
23241 llvm::ConstantAsMetadata::get(
Builder.getInt32(DomainVal)));
23242 llvm::MDNode *NontemporalNode = llvm::MDNode::get(
23246 Store->setMetadata(llvm::LLVMContext::MD_nontemporal, NontemporalNode);
23253 case RISCV::BI__builtin_riscv_cv_alu_addN:
23254 ID = Intrinsic::riscv_cv_alu_addN;
23256 case RISCV::BI__builtin_riscv_cv_alu_addRN:
23257 ID = Intrinsic::riscv_cv_alu_addRN;
23259 case RISCV::BI__builtin_riscv_cv_alu_adduN:
23260 ID = Intrinsic::riscv_cv_alu_adduN;
23262 case RISCV::BI__builtin_riscv_cv_alu_adduRN:
23263 ID = Intrinsic::riscv_cv_alu_adduRN;
23265 case RISCV::BI__builtin_riscv_cv_alu_clip:
23266 ID = Intrinsic::riscv_cv_alu_clip;
23268 case RISCV::BI__builtin_riscv_cv_alu_clipu:
23269 ID = Intrinsic::riscv_cv_alu_clipu;
23271 case RISCV::BI__builtin_riscv_cv_alu_extbs:
23274 case RISCV::BI__builtin_riscv_cv_alu_extbz:
23277 case RISCV::BI__builtin_riscv_cv_alu_exths:
23280 case RISCV::BI__builtin_riscv_cv_alu_exthz:
23283 case RISCV::BI__builtin_riscv_cv_alu_slet:
23286 case RISCV::BI__builtin_riscv_cv_alu_sletu:
23289 case RISCV::BI__builtin_riscv_cv_alu_subN:
23290 ID = Intrinsic::riscv_cv_alu_subN;
23292 case RISCV::BI__builtin_riscv_cv_alu_subRN:
23293 ID = Intrinsic::riscv_cv_alu_subRN;
23295 case RISCV::BI__builtin_riscv_cv_alu_subuN:
23296 ID = Intrinsic::riscv_cv_alu_subuN;
23298 case RISCV::BI__builtin_riscv_cv_alu_subuRN:
23299 ID = Intrinsic::riscv_cv_alu_subuRN;
23303#include "clang/Basic/riscv_vector_builtin_cg.inc"
23306#include "clang/Basic/riscv_sifive_vector_builtin_cg.inc"
23309 assert(ID != Intrinsic::not_intrinsic);
23312 return Builder.CreateCall(F, Ops,
"");
Defines the clang::ASTContext interface.
#define PPC_LNX_FEATURE(NAME, DESC, ENUMNAME, ENUMVAL, HWCAPN)
static constexpr SparcCPUInfo CPUInfo[]
#define X86_CPU_SUBTYPE(ENUM, STR)
#define X86_CPU_SUBTYPE_ALIAS(ENUM, ALIAS)
#define X86_VENDOR(ENUM, STRING)
#define X86_CPU_TYPE_ALIAS(ENUM, ALIAS)
#define X86_CPU_TYPE(ENUM, STR)
static constexpr Builtin::Info BuiltinInfo[]
static void Accumulate(SMap &SM, CFGBlock *B)
static Value * EmitSpecialRegisterBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Type *RegisterType, llvm::Type *ValueType, SpecialRegisterAccessKind AccessKind, StringRef SysReg="")
static llvm::Value * ARMMVEVectorReinterpret(CGBuilderTy &Builder, CodeGenFunction *CGF, llvm::Value *V, llvm::Type *DestType)
static Value * MakeBinaryAtomicValue(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E, AtomicOrdering Ordering=AtomicOrdering::SequentiallyConsistent)
Utility to insert an atomic instruction based on Intrinsic::ID and the expression node.
static char bitActionToX86BTCode(BitTest::ActionKind A)
#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier)
static Value * EmitAtomicCmpXchg128ForMSIntrin(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering SuccessOrdering)
#define INTRINSIC_X86_XSAVE_ID(NAME)
static CanQualType getOSLogArgType(ASTContext &C, int Size)
Get the argument type for arguments to os_log_helper.
static Value * EmitOverflowCheckedAbs(CodeGenFunction &CGF, const CallExpr *E, bool SanitizeOverflow)
static llvm::VectorType * GetFloatNeonType(CodeGenFunction *CGF, NeonTypeFlags IntTypeFlags)
static Value * tryUseTestFPKind(CodeGenFunction &CGF, unsigned BuiltinID, Value *V)
static llvm::Value * MVEImmediateShr(CGBuilderTy &Builder, llvm::Value *V, uint32_t Shift, bool Unsigned)
static bool areBOSTypesCompatible(int From, int To)
Checks if using the result of __builtin_object_size(p, From) in place of __builtin_object_size(p,...
static llvm::Value * SignOrZeroExtend(CGBuilderTy &Builder, llvm::Value *V, llvm::Type *T, bool Unsigned)
static void InsertExplicitZeroOperand(CGBuilderTy &Builder, llvm::Type *Ty, SmallVectorImpl< Value * > &Ops)
static const ARMVectorIntrinsicInfo AArch64SMEIntrinsicMap[]
static Value * EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E, SmallVectorImpl< Value * > &Ops, llvm::Triple::ArchType Arch)
#define MMA_VARIANTS_B1_AND(geom, type)
static void swapCommutativeSMEOperands(unsigned BuiltinID, SmallVectorImpl< Value * > &Ops)
static bool AArch64SISDIntrinsicsProvenSorted
static Value * EmitX86CompressExpand(CodeGenFunction &CGF, ArrayRef< Value * > Ops, bool IsCompress)
static const ARMVectorIntrinsicInfo AArch64SVEIntrinsicMap[]
static bool HasExtraNeonArgument(unsigned BuiltinID)
Return true if BuiltinID is an overloaded Neon intrinsic with an extra argument that specifies the ve...
static bool TypeRequiresBuiltinLaunderImp(const ASTContext &Ctx, QualType Ty, llvm::SmallPtrSetImpl< const Decl * > &Seen)
static Value * EmitAtomicIncrementValue(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering Ordering=AtomicOrdering::SequentiallyConsistent)
static std::pair< Intrinsic::ID, unsigned > getIntrinsicForHexagonNonClangBuiltin(unsigned BuiltinID)
static Value * emitRangedBuiltin(CodeGenFunction &CGF, unsigned IntrinsicID, int low, int high)
#define MMA_INTR(geom_op_type, layout)
static Value * EmitX86MaskedCompare(CodeGenFunction &CGF, unsigned CC, bool Signed, ArrayRef< Value * > Ops)
static Value * emitUnaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
static Intrinsic::ID getDotProductIntrinsic(CGHLSLRuntime &RT, QualType QT)
#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier)
static bool AArch64SVEIntrinsicsProvenSorted
static RValue EmitMSVCRTSetJmp(CodeGenFunction &CGF, MSVCSetJmpKind SJKind, const CallExpr *E)
MSVC handles setjmp a bit differently on different platforms.
static const ARMVectorIntrinsicInfo * findARMVectorIntrinsicInMap(ArrayRef< ARMVectorIntrinsicInfo > IntrinsicMap, unsigned BuiltinID, bool &MapProvenSorted)
static Value * EmitScalarFMAExpr(CodeGenFunction &CGF, const CallExpr *E, MutableArrayRef< Value * > Ops, Value *Upper, bool ZeroMask=false, unsigned PTIdx=0, bool NegAcc=false)
static Value * loadRISCVFeatureBits(unsigned Index, CGBuilderTy &Builder, CodeGenModule &CGM)
#define MUTATE_LDBL(func)
static Value * EmitX86ExpandLoad(CodeGenFunction &CGF, ArrayRef< Value * > Ops)
static unsigned CountCountedByAttrs(const RecordDecl *RD)
static Value * emitMaybeConstrainedFPToIntRoundBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
static Value * EmitX86MaskedLoad(CodeGenFunction &CGF, ArrayRef< Value * > Ops, Align Alignment)
static bool TypeRequiresBuiltinLaunder(CodeGenModule &CGM, QualType Ty)
Determine if the specified type requires laundering by checking if it is a dynamic class type or cont...
static void InsertExplicitUndefOperand(CGBuilderTy &Builder, llvm::Type *Ty, SmallVectorImpl< Value * > &Ops)
static Value * EmitISOVolatileLoad(CodeGenFunction &CGF, const CallExpr *E)
static struct WidthAndSignedness EncompassingIntegerType(ArrayRef< struct WidthAndSignedness > Types)
static Integer GetIntegerConstantValue(const Expr *E, ASTContext &Context)
#define MMA_VARIANTS(geom, type)
static bool AArch64SMEIntrinsicsProvenSorted
static llvm::Value * VectorZip(CGBuilderTy &Builder, llvm::Value *V0, llvm::Value *V1)
static Value * EmitTargetArchBuiltinExpr(CodeGenFunction *CGF, unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
constexpr unsigned SVEBitsPerBlock
static std::optional< CodeGenFunction::MSVCIntrin > translateX86ToMsvcIntrin(unsigned BuiltinID)
static const std::pair< unsigned, unsigned > NEONEquivalentIntrinsicMap[]
#define NEONMAP0(NameBase)
static Value * EmitX86MaskedStore(CodeGenFunction &CGF, ArrayRef< Value * > Ops, Align Alignment)
static Value * handleHlslSplitdouble(const CallExpr *E, CodeGenFunction *CGF)
static Value * emitBinaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E, Instruction::BinaryOps Op, bool Invert=false)
Utility to insert an atomic instruction based Intrinsic::ID and the expression node,...
static bool HasNoIndirectArgumentsOrResults(CGFunctionInfo const &FnInfo)
Checks no arguments or results are passed indirectly in the ABI (i.e.
static Value * EmitX86Muldq(CodeGenFunction &CGF, bool IsSigned, ArrayRef< Value * > Ops)
Value * readX18AsPtr(CodeGenFunction &CGF)
Helper for the read/write/add/inc X18 builtins: read the X18 register and return it as an i8 pointer.
static llvm::Value * ARMMVEVectorElementReverse(CGBuilderTy &Builder, llvm::Value *V, unsigned ReverseWidth)
#define MMA_SATF_VARIANTS(geom, type)
static std::optional< CodeGenFunction::MSVCIntrin > translateAarch64ToMsvcIntrin(unsigned BuiltinID)
static std::optional< CodeGenFunction::MSVCIntrin > translateArmToMsvcIntrin(unsigned BuiltinID)
static llvm::Value * EmitBitTestIntrinsic(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E)
Emit a _bittest* intrinsic.
static Value * emitBuiltinWithOneOverloadedType(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, llvm::StringRef Name="")
static const ARMVectorIntrinsicInfo ARMSIMDIntrinsicMap[]
static Value * EmitSignBit(CodeGenFunction &CGF, Value *V)
Emit the computation of the sign bit for a floating point value.
static Value * EmitFAbs(CodeGenFunction &CGF, Value *V)
EmitFAbs - Emit a call to @llvm.fabs().
#define CUSTOM_BUILTIN_MAPPING(x, s)
static Value * EmitX86CvtF16ToFloatExpr(CodeGenFunction &CGF, ArrayRef< Value * > Ops, llvm::Type *DstTy)
static bool isSpecialUnsignedMultiplySignedResult(unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info, WidthAndSignedness ResultInfo)
static llvm::Value * getDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType)
static llvm::Value * emitPPCLoadReserveIntrinsic(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E)
static llvm::Value * VectorUnzip(CGBuilderTy &Builder, llvm::Value *V, bool Odd)
static Value * EmitX86SExtMask(CodeGenFunction &CGF, Value *Op, llvm::Type *DstTy)
static Value * emitTernaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
static WidthAndSignedness getIntegerWidthAndSignedness(const clang::ASTContext &context, const clang::QualType Type)
static Value * EmitX86FunnelShift(CodeGenFunction &CGF, Value *Op0, Value *Op1, Value *Amt, bool IsRight)
static RValue EmitCheckedMixedSignMultiply(CodeGenFunction &CGF, const clang::Expr *Op1, WidthAndSignedness Op1Info, const clang::Expr *Op2, WidthAndSignedness Op2Info, const clang::Expr *ResultArg, QualType ResultQTy, WidthAndSignedness ResultInfo)
Emit a checked mixed-sign multiply.
static llvm::ScalableVectorType * getSVEVectorForElementType(llvm::Type *EltTy)
static unsigned mutateLongDoubleBuiltin(unsigned BuiltinID)
#define INTRINSIC_WITH_CC(NAME)
static llvm::FixedVectorType * GetNeonType(CodeGenFunction *CGF, NeonTypeFlags TypeFlags, bool HasLegalHalfType=true, bool V1Ty=false, bool AllowBFloatArgsAndRet=true)
static RValue EmitBinaryAtomic(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E)
static llvm::Value * ARMMVEConstantSplat(CGBuilderTy &Builder, llvm::Type *VT)
static Value * EmitX86MaskLogic(CodeGenFunction &CGF, Instruction::BinaryOps Opc, ArrayRef< Value * > Ops, bool InvertLHS=false)
static Value * EmitFromInt(CodeGenFunction &CGF, llvm::Value *V, QualType T, llvm::Type *ResultType)
static Value * EmitAMDGCNBallotForExec(CodeGenFunction &CGF, const CallExpr *E, llvm::Type *RegisterType, llvm::Type *ValueType, bool isExecHi)
static void emitSincosBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Intrinsic::ID IntrinsicID)
static void initializeAlloca(CodeGenFunction &CGF, AllocaInst *AI, Value *Size, Align AlignmentInBytes)
static Value * EmitX86Select(CodeGenFunction &CGF, Value *Mask, Value *Op0, Value *Op1)
SpecialRegisterAccessKind
static Value * EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering SuccessOrdering=AtomicOrdering::SequentiallyConsistent)
This function should be invoked to emit atomic cmpxchg for Microsoft's _InterlockedCompareExchange* i...
static Address CheckAtomicAlignment(CodeGenFunction &CGF, const CallExpr *E)
static Value * EmitX86FMAExpr(CodeGenFunction &CGF, const CallExpr *E, ArrayRef< Value * > Ops, unsigned BuiltinID, bool IsAddSub)
static Value * getMaskVecValue(CodeGenFunction &CGF, Value *Mask, unsigned NumElts)
static bool isSpecialMixedSignMultiply(unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info, WidthAndSignedness ResultInfo)
Determine if a binop is a checked mixed-sign multiply we can specialize.
static Value * MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E, bool ReturnBool)
Utility to insert an atomic cmpxchg instruction.
static Value * emitBinaryExpMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Intrinsic::ID IntrinsicID, llvm::Intrinsic::ID ConstrainedIntrinsicID)
static Value * EmitToInt(CodeGenFunction &CGF, llvm::Value *V, QualType T, llvm::IntegerType *IntType)
Emit the conversions required to turn the given value into an integer of the given size.
static llvm::Value * ARMMVEVectorSplat(CGBuilderTy &Builder, llvm::Value *V)
static Value * EmitX86MaskedCompareResult(CodeGenFunction &CGF, Value *Cmp, unsigned NumElts, Value *MaskIn)
static Value * EmitX86CompressStore(CodeGenFunction &CGF, ArrayRef< Value * > Ops)
static bool NEONSIMDIntrinsicsProvenSorted
static const ARMVectorIntrinsicInfo AArch64SIMDIntrinsicMap[]
static Value * EmitNontemporalStore(CodeGenFunction &CGF, const CallExpr *E)
static Intrinsic::ID getFirstBitHighIntrinsic(CGHLSLRuntime &RT, QualType QT)
static llvm::Value * EmitOverflowIntrinsic(CodeGenFunction &CGF, const llvm::Intrinsic::ID IntrinsicID, llvm::Value *X, llvm::Value *Y, llvm::Value *&Carry)
Emit a call to llvm.
static Value * EmitCommonNeonSISDBuiltinExpr(CodeGenFunction &CGF, const ARMVectorIntrinsicInfo &SISDInfo, SmallVectorImpl< Value * > &Ops, const CallExpr *E)
static Value * emitFPIntBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
#define MMA_LDST(n, geom_op_type)
static Value * EmitX86vpcom(CodeGenFunction &CGF, ArrayRef< Value * > Ops, bool IsSigned)
static Value * emitFrexpBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Intrinsic::ID IntrinsicID)
@ _InterlockedExchangeAdd_rel
@ _InterlockedIncrement_acq
@ _InterlockedExchange_nf
@ _InterlockedIncrement_nf
@ _InterlockedExchange_acq
@ _InterlockedCompareExchange128_rel
@ _InterlockedCompareExchange128_acq
@ _InterlockedCompareExchange_acq
@ _InterlockedExchangeAdd_nf
@ _InterlockedCompareExchange_nf
@ _InterlockedDecrement_rel
@ _InterlockedExchangeSub
@ _InterlockedExchangeAdd_acq
@ _InterlockedIncrement_rel
@ _InterlockedCompareExchange128_nf
@ _InterlockedCompareExchange128
@ _InterlockedExchange_rel
@ _InterlockedCompareExchange
@ _InterlockedDecrement_nf
@ _InterlockedExchangeAdd
@ _InterlockedDecrement_acq
@ _InterlockedCompareExchange_rel
static Value * EmitX86ConvertToMask(CodeGenFunction &CGF, Value *In)
static Value * EmitISOVolatileStore(CodeGenFunction &CGF, const CallExpr *E)
static Value * EmitSystemZIntrinsicWithCC(CodeGenFunction &CGF, unsigned IntrinsicID, const CallExpr *E)
Handle a SystemZ function in which the final argument is a pointer to an int that receives the post-i...
static RValue EmitHipStdParUnsupportedBuiltin(CodeGenFunction *CGF, const FunctionDecl *FD)
static llvm::Value * EmitX86BitTestIntrinsic(CodeGenFunction &CGF, BitTest BT, const CallExpr *E, Value *BitBase, Value *BitPos)
static RValue EmitCheckedUnsignedMultiplySignedResult(CodeGenFunction &CGF, const clang::Expr *Op1, WidthAndSignedness Op1Info, const clang::Expr *Op2, WidthAndSignedness Op2Info, const clang::Expr *ResultArg, QualType ResultQTy, WidthAndSignedness ResultInfo)
static Value * emitCallMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID, llvm::Type *Ty, ArrayRef< Value * > Args)
static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *FD, const CallExpr *E, llvm::Constant *calleeValue)
static Value * handleAsDoubleBuiltin(CodeGenFunction &CGF, const CallExpr *E)
static Value * EmitNontemporalLoad(CodeGenFunction &CGF, const CallExpr *E)
static Value * EmitX86Ternlog(CodeGenFunction &CGF, bool ZeroMask, ArrayRef< Value * > Ops)
static llvm::AtomicOrdering getBitTestAtomicOrdering(BitTest::InterlockingKind I)
#define MMA_VARIANTS_B1_XOR(geom, type)
#define MMA_VARIANTS_I4(geom, type)
static Value * EmitX86ConvertIntToFp(CodeGenFunction &CGF, const CallExpr *E, ArrayRef< Value * > Ops, bool IsSigned)
static Value * packTBLDVectorList(CodeGenFunction &CGF, ArrayRef< Value * > Ops, Value *ExtOp, Value *IndexOp, llvm::Type *ResTy, unsigned IntID, const char *Name)
static Value * EmitAbs(CodeGenFunction &CGF, Value *ArgValue, bool HasNSW)
static Value * EmitX86ScalarSelect(CodeGenFunction &CGF, Value *Mask, Value *Op0, Value *Op1)
static Value * EmitAtomicDecrementValue(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering Ordering=AtomicOrdering::SequentiallyConsistent)
static Value * handleHlslClip(const CallExpr *E, CodeGenFunction *CGF)
static bool AArch64SIMDIntrinsicsProvenSorted
static const ARMVectorIntrinsicInfo AArch64SISDIntrinsicMap[]
CodeGenFunction::ComplexPairTy ComplexPairTy
#define ALIAS(NAME, TOK, FLAGS)
llvm::MachO::Record Record
static std::string getName(const CallEvent &Call)
static std::string toString(const clang::SanitizerSet &Sanitizers)
Produce a string containing comma-separated names of sanitizers in Sanitizers set.
HLSLResourceBindingAttr::RegisterType RegisterType
static QualType getPointeeType(const MemRegion *R)
Enumerates target-specific builtins in their own namespaces within namespace clang.
Defines the clang::TargetOptions class.
C Language Family Type Representation.
__DEVICE__ float modf(float __x, float *__iptr)
__DEVICE__ double nan(const char *)
__device__ __2f16 float __ockl_bool s
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
CharUnits getTypeAlignInChars(QualType T) const
Return the ABI-specified alignment of a (complete) type T, in characters.
unsigned getIntWidth(QualType T) const
const ASTRecordLayout & getASTRecordLayout(const RecordDecl *D) const
Get or compute information about the layout of the specified record (struct/union/class) D,...
Builtin::Context & BuiltinInfo
QualType getConstantArrayType(QualType EltTy, const llvm::APInt &ArySize, const Expr *SizeExpr, ArraySizeModifier ASM, unsigned IndexTypeQuals) const
Return the unique reference to the type for a constant array of the specified element type.
QualType getBaseElementType(const ArrayType *VAT) const
Return the innermost element type of an array type.
QualType getObjCIdType() const
Represents the Objective-CC id type.
bool hasSameUnqualifiedType(QualType T1, QualType T2) const
Determine whether the given types are equivalent after cvr-qualifiers have been removed.
const ArrayType * getAsArrayType(QualType T) const
Type Query functions.
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
QualType GetBuiltinType(unsigned ID, GetBuiltinTypeError &Error, unsigned *IntegerConstantArgs=nullptr) const
Return the type for the specified builtin.
const TargetInfo & getTargetInfo() const
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
unsigned getTargetAddressSpace(LangAS AS) const
@ GE_Missing_type
Missing a type.
ASTRecordLayout - This class contains layout information for one RecordDecl, which is a struct/union/...
uint64_t getFieldOffset(unsigned FieldNo) const
getFieldOffset - Get the offset of the given field index, in bits.
Represents an array type, per C99 6.7.5.2 - Array Declarators.
QualType getElementType() const
static std::unique_ptr< AtomicScopeModel > create(AtomicScopeModelKind K)
Create an atomic scope model by AtomicScopeModelKind.
bool isLibFunction(unsigned ID) const
Return true if this is a builtin for a libc/libm function, with a "__builtin_" prefix (e....
llvm::StringRef getName(unsigned ID) const
Return the identifier name for the specified builtin, e.g.
bool isConstWithoutErrnoAndExceptions(unsigned ID) const
Return true if this function has no side effects and doesn't read memory, except for possibly errno o...
bool isConstWithoutExceptions(unsigned ID) const
bool isConst(unsigned ID) const
Return true if this function has no side effects and doesn't read memory.
CallExpr - Represents a function call (C99 6.5.2.2, C++ [expr.call]).
CharUnits - This is an opaque type for sizes expressed in character units.
llvm::Align getAsAlign() const
getAsAlign - Returns Quantity as a valid llvm::Align, Beware llvm::Align assumes power of two 8-bit b...
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
static CharUnits One()
One - Construct a CharUnits quantity of one.
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
XRayInstrSet XRayInstrumentationBundle
Set of XRay instrumentation kinds to emit.
ABIArgInfo - Helper class to encapsulate information about how a specific C type should be passed to ...
Like RawAddress, an abstract representation of an aligned address, but the pointer contained in this ...
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
Return the pointer contained in this class after authenticating it and adding offset to it if necessa...
CharUnits getAlignment() const
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
Address withAlignment(CharUnits NewAlignment) const
Return address with different alignment, but same pointer and element type.
llvm::PointerType * getType() const
Return the type of the pointer value.
Address getAddress() const
A scoped helper to set the current debug location to the specified location or preferred location of ...
static ApplyDebugLocation CreateArtificial(CodeGenFunction &CGF)
Apply TemporaryLocation if it is valid.
static ApplyDebugLocation CreateEmpty(CodeGenFunction &CGF)
Set the IRBuilder to not attach debug locations.
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
llvm::StoreInst * CreateAlignedStore(llvm::Value *Val, llvm::Value *Addr, CharUnits Align, bool IsVolatile=false)
Address CreateGEP(CodeGenFunction &CGF, Address Addr, llvm::Value *Index, const llvm::Twine &Name="")
llvm::CallInst * CreateMemMove(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
llvm::CallInst * CreateMemCpyInline(Address Dest, Address Src, uint64_t Size)
llvm::AtomicRMWInst * CreateAtomicRMW(llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val, llvm::AtomicOrdering Ordering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
llvm::CallInst * CreateMemSetInline(Address Dest, llvm::Value *Value, uint64_t Size)
llvm::StoreInst * CreateDefaultAlignedStore(llvm::Value *Val, llvm::Value *Addr, bool IsVolatile=false)
llvm::CallInst * CreateMemSet(Address Dest, llvm::Value *Value, llvm::Value *Size, bool IsVolatile=false)
Address CreateStructGEP(Address Addr, unsigned Index, const llvm::Twine &Name="")
llvm::AtomicCmpXchgInst * CreateAtomicCmpXchg(Address Addr, llvm::Value *Cmp, llvm::Value *New, llvm::AtomicOrdering SuccessOrdering, llvm::AtomicOrdering FailureOrdering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
Address CreateConstByteGEP(Address Addr, CharUnits Offset, const llvm::Twine &Name="")
Address CreateLaunderInvariantGroup(Address Addr)
llvm::CallInst * CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
llvm::LoadInst * CreateAlignedLoad(llvm::Type *Ty, llvm::Value *Addr, CharUnits Align, const llvm::Twine &Name="")
Address CreateAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
Address CreateConstInBoundsGEP(Address Addr, uint64_t Index, const llvm::Twine &Name="")
Given addr = T* ... produce name = getelementptr inbounds addr, i64 index where i64 is actually the t...
Address CreateInBoundsGEP(Address Addr, ArrayRef< llvm::Value * > IdxList, llvm::Type *ElementType, CharUnits Align, const Twine &Name="")
virtual std::string getDeviceSideName(const NamedDecl *ND)=0
Returns function or variable name on device side even if the current compilation is for host.
virtual llvm::GlobalVariable * getThrowInfo(QualType T)
All available information about a concrete callee.
static CGCallee forDirect(llvm::Constant *functionPtr, const CGCalleeInfo &abstractInfo=CGCalleeInfo())
llvm::DIType * getOrCreateStandaloneType(QualType Ty, SourceLocation Loc)
Emit standalone debug info for a type.
llvm::DILocation * CreateTrapFailureMessageFor(llvm::DebugLoc TrapLocation, StringRef Category, StringRef FailureMsg)
Create a debug location from TrapLocation that adds an artificial inline frame where the frame name i...
CGFunctionInfo - Class to encapsulate the information about a function definition.
ABIArgInfo & getReturnInfo()
MutableArrayRef< ArgInfo > arguments()
virtual void EmitGCMemmoveCollectable(CodeGen::CodeGenFunction &CGF, Address DestPtr, Address SrcPtr, llvm::Value *Size)=0
EnqueuedBlockInfo emitOpenCLEnqueuedBlock(CodeGenFunction &CGF, const Expr *E)
CallArgList - Type for representing both the value and type of arguments in a call.
void add(RValue rvalue, QualType type)
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
llvm::Value * EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E, llvm::Triple::ArchType Arch)
llvm::Value * EmitFromMemory(llvm::Value *Value, QualType Ty)
EmitFromMemory - Change a scalar value from its memory representation to its value representation.
llvm::Value * EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID, const CallExpr *E)
void FinishFunction(SourceLocation EndLoc=SourceLocation())
FinishFunction - Complete IR generation of the current function.
llvm::Value * EmitLifetimeStart(llvm::TypeSize Size, llvm::Value *Addr)
std::pair< RValue, llvm::Value * > EmitAtomicCompareExchange(LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, llvm::AtomicOrdering Success=llvm::AtomicOrdering::SequentiallyConsistent, llvm::AtomicOrdering Failure=llvm::AtomicOrdering::SequentiallyConsistent, bool IsWeak=false, AggValueSlot Slot=AggValueSlot::ignored())
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
llvm::Value * EmitSVEPredicateCast(llvm::Value *Pred, llvm::ScalableVectorType *VTy)
llvm::CallInst * EmitTrapCall(llvm::Intrinsic::ID IntrID)
Emit a call to trap or debugtrap and attach function attribute "trap-func-name" if specified.
SanitizerSet SanOpts
Sanitizers enabled for this function.
RValue EmitBuiltinIsAligned(const CallExpr *E)
Emit IR for __builtin_is_aligned.
LValue EmitAggExprToLValue(const Expr *E)
EmitAggExprToLValue - Emit the computation of the specified expression of aggregate type into a tempo...
void EmitNonNullArgCheck(RValue RV, QualType ArgType, SourceLocation ArgLoc, AbstractCallee AC, unsigned ParmNum)
Create a check for a function parameter that may potentially be declared as non-null.
llvm::Value * EmitHLSLBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
llvm::Value * EmitHexagonBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
void GetAArch64SVEProcessedOperands(unsigned BuiltinID, const CallExpr *E, SmallVectorImpl< llvm::Value * > &Ops, SVETypeFlags TypeFlags)
llvm::Value * EmitAMDGPUBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
void EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr)
void pushLifetimeExtendedDestroy(CleanupKind kind, Address addr, QualType type, Destroyer *destroyer, bool useEHCleanupForArray)
void EmitTrapCheck(llvm::Value *Checked, SanitizerHandler CheckHandlerID, bool NoMerge=false)
Create a basic block that will call the trap intrinsic, and emit a conditional branch to it,...
llvm::Value * EmitCheckedArgForBuiltin(const Expr *E, BuiltinCheckKind Kind)
Emits an argument for a call to a builtin.
llvm::Value * EmitSVEGatherLoad(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
LValue EmitHLSLOutArgExpr(const HLSLOutArgExpr *E, CallArgList &Args, QualType Ty)
CleanupKind getARCCleanupKind()
Retrieves the default cleanup kind for an ARC cleanup.
llvm::Value * EmitRISCVCpuSupports(const CallExpr *E)
llvm::Value * EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitAArch64SVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
RValue emitBuiltinOSLogFormat(const CallExpr &E)
Emit IR for __builtin_os_log_format.
LValue EmitLValue(const Expr *E, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitLValue - Emit code to compute a designator that specifies the location of the expression.
llvm::Value * EmitVAStartEnd(llvm::Value *ArgValue, bool IsStart)
Emits a call to an LLVM variable-argument intrinsic, either llvm.va_start or llvm....
llvm::Value * EmitSVEMaskedStore(const CallExpr *, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
llvm::Value * EmitSVEReinterpret(llvm::Value *Val, llvm::Type *Ty)
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
llvm::Value * EmitSEHExceptionInfo()
RValue EmitBuiltinAlignTo(const CallExpr *E, bool AlignUp)
Emit IR for __builtin_align_up/__builtin_align_down.
const LangOptions & getLangOpts() const
llvm::Value * EmitLoadOfCountedByField(const Expr *Base, const FieldDecl *FAMDecl, const FieldDecl *CountDecl)
Build an expression accessing the "counted_by" field.
void ProcessOrderScopeAMDGCN(llvm::Value *Order, llvm::Value *Scope, llvm::AtomicOrdering &AO, llvm::SyncScope::ID &SSID)
llvm::Constant * EmitCheckTypeDescriptor(QualType T)
Emit a description of a type in a format suitable for passing to a runtime sanitizer handler.
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
void EmitUnreachable(SourceLocation Loc)
Emit a reached-unreachable diagnostic if Loc is valid and runtime checking is enabled.
llvm::Value * EmitSVETupleCreate(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType, ArrayRef< llvm::Value * > Ops)
Address makeNaturalAddressForPointer(llvm::Value *Ptr, QualType T, CharUnits Alignment=CharUnits::Zero(), bool ForPointeeType=false, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
Construct an address with the natural alignment of T.
llvm::Value * EmitCheckedArgForAssume(const Expr *E)
Emits an argument for a call to a __builtin_assume.
ComplexPairTy EmitComplexExpr(const Expr *E, bool IgnoreReal=false, bool IgnoreImag=false)
EmitComplexExpr - Emit the computation of the specified expression of complex type,...
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
TypeCheckKind
Situations in which we might emit a check for the suitability of a pointer or glvalue.
@ TCK_Store
Checking the destination of a store. Must be suitably sized and aligned.
@ TCK_Load
Checking the operand of a load. Must be suitably sized and aligned.
llvm::Value * EmitSMELdrStr(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::Value * EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx, const llvm::ElementCount &Count)
llvm::Type * ConvertTypeForMem(QualType T)
llvm::Value * EmitSVEMaskedLoad(const CallExpr *, llvm::Type *ReturnTy, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID, bool IsZExtReturn)
bool AlwaysEmitXRayCustomEvents() const
AlwaysEmitXRayCustomEvents - Return true if we must unconditionally emit XRay custom event handling c...
llvm::Value * EmitSVEDupX(llvm::Value *Scalar)
RawAddress CreateMemTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
@ Default
! No language constraints on evaluation order.
const TargetInfo & getTarget() const
llvm::Value * vectorWrapScalar16(llvm::Value *Op)
llvm::Function * LookupNeonLLVMIntrinsic(unsigned IntrinsicID, unsigned Modifier, llvm::Type *ArgTy, const CallExpr *E)
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
llvm::Value * EmitSEHExceptionCode()
Address EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitPointerWithAlignment - Given an expression with a pointer type, emit the value and compute our be...
llvm::Value * EmitTargetBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
EmitTargetBuiltinExpr - Emit the given builtin call.
RValue EmitAnyExprToTemp(const Expr *E)
EmitAnyExprToTemp - Similarly to EmitAnyExpr(), however, the result will always be accessible even if...
RValue EmitCoroutineIntrinsic(const CallExpr *E, unsigned int IID)
llvm::Value * EmitAArch64SMEBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
RValue EmitAMDGPUDevicePrintfCallExpr(const CallExpr *E)
Address EmitArrayToPointerDecay(const Expr *Array, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
void EmitCheck(ArrayRef< std::pair< llvm::Value *, SanitizerMask > > Checked, SanitizerHandler Check, ArrayRef< llvm::Constant * > StaticArgs, ArrayRef< llvm::Value * > DynamicArgs)
Create a basic block that will either trap or call a handler function in the UBSan runtime with the p...
RValue EmitBuiltinNewDeleteCall(const FunctionProtoType *Type, const CallExpr *TheCallExpr, bool IsDelete)
llvm::Value * EmitRISCVCpuInit()
RValue EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
RValue EmitAnyExpr(const Expr *E, AggValueSlot aggSlot=AggValueSlot::ignored(), bool ignoreResult=false)
EmitAnyExpr - Emit code to compute the specified expression which can have any type.
llvm::Value * EmitSVEPrefetchLoad(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
llvm::Value * EmitAArch64CompareBuiltinExpr(llvm::Value *Op, llvm::Type *Ty, const llvm::CmpInst::Predicate Fp, const llvm::CmpInst::Predicate Ip, const llvm::Twine &Name="")
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
AggValueSlot CreateAggTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateAggTemp - Create a temporary memory object for the given aggregate type.
llvm::ScalableVectorType * getSVEType(const SVETypeFlags &TypeFlags)
RValue emitRotate(const CallExpr *E, bool IsRotateRight)
llvm::Constant * EmitCheckSourceLocation(SourceLocation Loc)
Emit a description of a source location in a format suitable for passing to a runtime sanitizer handl...
void ErrorUnsupported(const Stmt *S, const char *Type)
ErrorUnsupported - Print out an error that codegen doesn't support the specified stmt yet.
CGDebugInfo * getDebugInfo()
const FieldDecl * FindFlexibleArrayMemberFieldAndOffset(ASTContext &Ctx, const RecordDecl *RD, const FieldDecl *FAMDecl, uint64_t &Offset)
llvm::Value * EmitRISCVCpuIs(const CallExpr *E)
Address EmitVAListRef(const Expr *E)
llvm::Value * EmitNeonShiftVector(llvm::Value *V, llvm::Type *Ty, bool negateForRightShift)
llvm::Value * EmitSVEMovl(const SVETypeFlags &TypeFlags, llvm::ArrayRef< llvm::Value * > Ops, unsigned BuiltinID)
void emitAlignmentAssumption(llvm::Value *PtrValue, QualType Ty, SourceLocation Loc, SourceLocation AssumptionLoc, llvm::Value *Alignment, llvm::Value *OffsetValue=nullptr)
const TargetCodeGenInfo & getTargetHooks() const
llvm::Value * EmitARMBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
llvm::Type * getEltType(const SVETypeFlags &TypeFlags)
void EmitAggExpr(const Expr *E, AggValueSlot AS)
EmitAggExpr - Emit the computation of the specified expression of aggregate type.
bool ShouldXRayInstrumentFunction() const
ShouldXRayInstrument - Return true if the current function should be instrumented with XRay nop sleds...
llvm::Value * EmitSVEPMull(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
llvm::Value * EmitToMemory(llvm::Value *Value, QualType Ty)
EmitToMemory - Change a scalar value from its value representation to its in-memory representation.
bool IsInPreservedAIRegion
True if CodeGen currently emits code inside presereved access index region.
llvm::Value * EmitARCRetain(QualType type, llvm::Value *value)
bool AlwaysEmitXRayTypedEvents() const
AlwaysEmitXRayTypedEvents - Return true if clang must unconditionally emit XRay typed event handling ...
void SetSqrtFPAccuracy(llvm::Value *Val)
Set the minimum required accuracy of the given sqrt operation based on CodeGenOpts.
RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee, ReturnValueSlot ReturnValue, const CallArgList &Args, llvm::CallBase **CallOrInvoke, bool IsMustTail, SourceLocation Loc, bool IsVirtualFunctionPointerThunk=false)
EmitCall - Generate a call of the given function, expecting the given result type,...
llvm::Value * EmitSVEScatterStore(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::CallInst * EmitNounwindRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
ASTContext & getContext() const
llvm::Function * generateBuiltinOSLogHelperFunction(const analyze_os_log::OSLogBufferLayout &Layout, CharUnits BufferAlignment)
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
CGCallee EmitCallee(const Expr *E)
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
llvm::Value * EmitScalarOrConstFoldImmArg(unsigned ICEArguments, unsigned Idx, const CallExpr *E)
void checkTargetFeatures(const CallExpr *E, const FunctionDecl *TargetDecl)
llvm::Value * BuildVector(ArrayRef< llvm::Value * > Ops)
llvm::Value * EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitARMCDEBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
llvm::Value * GetCountedByFieldExprGEP(const Expr *Base, const FieldDecl *FAMDecl, const FieldDecl *CountDecl)
llvm::CallInst * EmitRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
llvm::Value * EmitSVEPredicateTupleCast(llvm::Value *PredTuple, llvm::StructType *Ty)
llvm::Type * ConvertType(QualType T)
void EmitWritebacks(const CallArgList &Args)
EmitWriteback - Emit callbacks for function.
llvm::CallBase * EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee, ArrayRef< llvm::Value * > args, const Twine &name="")
llvm::Value * EmitSystemZBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitSMEReadWrite(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, LValue LV, QualType Type, SanitizerSet SkippedChecks=SanitizerSet(), llvm::Value *ArraySize=nullptr)
llvm::Value * EmitSMELd1St1(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::Value * EmitPPCBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
static Destroyer destroyARCStrongPrecise
void EmitARCIntrinsicUse(ArrayRef< llvm::Value * > values)
RValue EmitNVPTXDevicePrintfCallExpr(const CallExpr *E)
llvm::Value * EvaluateExprAsBool(const Expr *E)
EvaluateExprAsBool - Perform the usual unary conversions on the specified expression and compare the ...
llvm::Value * EmitSVEStructLoad(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Address EmitMSVAListRef(const Expr *E)
Emit a "reference" to a __builtin_ms_va_list; this is always the value of the expression,...
llvm::Value * EmitCheckedInBoundsGEP(llvm::Type *ElemTy, llvm::Value *Ptr, ArrayRef< llvm::Value * > IdxList, bool SignedIndices, bool IsSubtraction, SourceLocation Loc, const Twine &Name="")
Same as IRBuilder::CreateInBoundsGEP, but additionally emits a check to detect undefined behavior whe...
llvm::Value * EmitNeonRShiftImm(llvm::Value *Vec, llvm::Value *Amt, llvm::Type *Ty, bool usgn, const char *name)
SmallVector< llvm::Type *, 2 > getSVEOverloadTypes(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType, ArrayRef< llvm::Value * > Ops)
static bool hasAggregateEvaluationKind(QualType T)
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
llvm::Value * EmitARMMVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
llvm::Value * EmitSVEStructStore(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
llvm::Value * EmitSEHAbnormalTermination()
llvm::Value * EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E)
Address ReturnValue
ReturnValue - The temporary alloca to hold the return value.
llvm::Value * EmitSVEAllTruePred(const SVETypeFlags &TypeFlags)
RValue GetUndefRValue(QualType Ty)
GetUndefRValue - Get an appropriate 'undef' rvalue for the given type.
llvm::Type * SVEBuiltinMemEltTy(const SVETypeFlags &TypeFlags)
SVEBuiltinMemEltTy - Returns the memory element type for this memory access builtin.
llvm::LLVMContext & getLLVMContext()
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
void AddAMDGPUFenceAddressSpaceMMRA(llvm::Instruction *Inst, const CallExpr *E)
llvm::Value * EmitSMEZero(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::Value * EmitRISCVBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
llvm::Value * EmitCommonNeonBuiltinExpr(unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic, const char *NameHint, unsigned Modifier, const CallExpr *E, SmallVectorImpl< llvm::Value * > &Ops, Address PtrOp0, Address PtrOp1, llvm::Triple::ArchType Arch)
llvm::Value * EmitNeonCall(llvm::Function *F, SmallVectorImpl< llvm::Value * > &O, const char *name, unsigned shift=0, bool rightshift=false)
llvm::Value * EmitAnnotationCall(llvm::Function *AnnotationFn, llvm::Value *AnnotatedVal, StringRef AnnotationStr, SourceLocation Location, const AnnotateAttr *Attr)
Emit an annotation call (intrinsic).
llvm::ScalableVectorType * getSVEPredType(const SVETypeFlags &TypeFlags)
llvm::Value * EmitSVEGatherPrefetch(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
llvm::Value * EmitSPIRVBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitBPFBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitSVETupleSetOrGet(const SVETypeFlags &TypeFlags, ArrayRef< llvm::Value * > Ops)
This class organizes the cross-function state that is used while generating LLVM code.
CGHLSLRuntime & getHLSLRuntime()
Return a reference to the configured HLSL runtime.
llvm::Module & getModule() const
llvm::FunctionCallee CreateRuntimeFunction(llvm::FunctionType *Ty, StringRef Name, llvm::AttributeList ExtraAttrs=llvm::AttributeList(), bool Local=false, bool AssumeConvergent=false)
Create or return a runtime function declaration with the specified type and name.
llvm::Constant * getRawFunctionPointer(GlobalDecl GD, llvm::Type *Ty=nullptr)
Return a function pointer for a reference to the given function.
llvm::Constant * getBuiltinLibFunction(const FunctionDecl *FD, unsigned BuiltinID)
Given a builtin id for a function like "__builtin_fabsf", return a Function* for "fabsf".
DiagnosticsEngine & getDiags() const
void ErrorUnsupported(const Stmt *S, const char *Type)
Print out an error that codegen doesn't support the specified stmt yet.
const LangOptions & getLangOpts() const
CGCUDARuntime & getCUDARuntime()
Return a reference to the configured CUDA runtime.
CodeGenTypes & getTypes()
CGOpenCLRuntime & getOpenCLRuntime()
Return a reference to the configured OpenCL runtime.
const TargetInfo & getTarget() const
const llvm::DataLayout & getDataLayout() const
void Error(SourceLocation loc, StringRef error)
Emit a general error that something can't be done.
CGCXXABI & getCXXABI() const
llvm::Constant * GetFunctionStart(const ValueDecl *Decl)
const llvm::Triple & getTriple() const
void DecorateInstructionWithTBAA(llvm::Instruction *Inst, TBAAAccessInfo TBAAInfo)
DecorateInstructionWithTBAA - Decorate the instruction with a TBAA tag.
llvm::Constant * CreateRuntimeVariable(llvm::Type *Ty, StringRef Name)
Create a new runtime global variable with the specified type and name.
TBAAAccessInfo getTBAAAccessInfo(QualType AccessType)
getTBAAAccessInfo - Get TBAA information that describes an access to an object of the given type.
ASTContext & getContext() const
const CodeGenOptions & getCodeGenOpts() const
StringRef getMangledName(GlobalDecl GD)
CharUnits getNaturalPointeeTypeAlignment(QualType T, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
llvm::LLVMContext & getLLVMContext()
llvm::Function * getIntrinsic(unsigned IID, ArrayRef< llvm::Type * > Tys={})
CGObjCRuntime & getObjCRuntime()
Return a reference to the configured Objective-C runtime.
void SetLLVMFunctionAttributes(GlobalDecl GD, const CGFunctionInfo &Info, llvm::Function *F, bool IsThunk)
Set the LLVM function attributes (sext, zext, etc).
void SetLLVMFunctionAttributesForDefinition(const Decl *D, llvm::Function *F)
Set the LLVM function attributes which only apply to a function definition.
ConstantAddress GetAddrOfConstantCString(const std::string &Str, const char *GlobalName=nullptr)
Returns a pointer to a character array containing the literal and a terminating '\0' character.
llvm::Type * ConvertType(QualType T)
ConvertType - Convert type T into a llvm::Type.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
const CGFunctionInfo & arrangeBuiltinFunctionCall(QualType resultType, const CallArgList &args)
llvm::Constant * emitAbstract(const Expr *E, QualType T)
Emit the result of the given expression as an abstract constant, asserting that it succeeded.
Information for lazily generating a cleanup.
FunctionArgList - Type for representing both the decl and type of parameters to a function.
LValue - This represents an lvalue references.
llvm::Value * getRawBitFieldPointer(CodeGenFunction &CGF) const
llvm::Value * getPointer(CodeGenFunction &CGF) const
Address getAddress() const
void setNontemporal(bool Value)
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
llvm::Value * getAggregatePointer(QualType PointeeType, CodeGenFunction &CGF) const
static RValue getIgnored()
static RValue get(llvm::Value *V)
static RValue getAggregate(Address addr, bool isVolatile=false)
Convert an Address to an RValue.
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
An abstract representation of an aligned address.
llvm::Value * getPointer() const
static RawAddress invalid()
ReturnValueSlot - Contains the address where the return value of a function can be stored,...
virtual llvm::Value * encodeReturnAddress(CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const
Performs the code-generation required to convert the address of an instruction into a return address ...
virtual llvm::Value * decodeReturnAddress(CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const
Performs the code-generation required to convert a return address as stored by the system into the ac...
const T & getABIInfo() const
virtual int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const
Determines the DWARF register number for the stack pointer, for exception-handling purposes.
virtual llvm::Value * testFPKind(llvm::Value *V, unsigned BuiltinID, CGBuilderTy &Builder, CodeGenModule &CGM) const
Performs a target specific test of a floating point value for things like IsNaN, Infinity,...
Complex values, per C99 6.2.5p11.
Represents a concrete matrix type with constant number of rows and columns.
Represents a sugar type with __counted_by or __sized_by annotations, including their _or_null variant...
DynamicCountPointerKind getKind() const
RecordDecl * getOuterLexicalRecordContext()
Retrieve the outermost lexically enclosing record context.
bool isImplicit() const
isImplicit - Indicates whether the declaration was implicitly generated by the implementation.
FunctionDecl * getAsFunction() LLVM_READONLY
Returns the function itself, or the templated function if this is a function template.
DeclContext * getDeclContext()
static bool isFlexibleArrayMemberLike(ASTContext &Context, const Decl *D, QualType Ty, LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel, bool IgnoreTemplateOrMacroSubstitution)
Whether it resembles a flexible array member.
Concrete class used by the front-end to report problems and issues.
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
This represents one expression.
bool EvaluateAsInt(EvalResult &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer,...
Expr * IgnoreParenCasts() LLVM_READONLY
Skip past any parentheses and casts which might surround this expression until reaching a fixed point...
llvm::APSInt EvaluateKnownConstInt(const ASTContext &Ctx, SmallVectorImpl< PartialDiagnosticAt > *Diag=nullptr) const
EvaluateKnownConstInt - Call EvaluateAsRValue and return the folded integer.
Expr * IgnoreParenImpCasts() LLVM_READONLY
Skip past any parentheses and implicit casts which might surround this expression until reaching a fi...
bool EvaluateAsFloat(llvm::APFloat &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsFloat - Return true if this is a constant which we can fold and convert to a floating point...
Expr * IgnoreParens() LLVM_READONLY
Skip past any parentheses which might surround this expression until reaching a fixed point.
@ NPC_ValueDependentIsNotNull
Specifies that a value-dependent expression should be considered to never be a null pointer constant.
ExprObjectKind getObjectKind() const
getObjectKind - The object kind that this expression produces.
bool EvaluateAsRValue(EvalResult &Result, const ASTContext &Ctx, bool InConstantContext=false) const
EvaluateAsRValue - Return true if this is a constant which we can fold to an rvalue using any crazy t...
bool HasSideEffects(const ASTContext &Ctx, bool IncludePossibleEffects=true) const
HasSideEffects - This routine returns true for all those expressions which have any effect other than...
std::optional< std::string > tryEvaluateString(ASTContext &Ctx) const
If the current Expr can be evaluated to a pointer to a null-terminated constant string,...
Expr * IgnoreImpCasts() LLVM_READONLY
Skip past any implicit casts which might surround this expression until reaching a fixed point.
NullPointerConstantKind isNullPointerConstant(ASTContext &Ctx, NullPointerConstantValueDependence NPC) const
isNullPointerConstant - C99 6.3.2.3p3 - Test if this reduces down to a Null pointer constant.
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
std::optional< llvm::APSInt > getIntegerConstantExpr(const ASTContext &Ctx, SourceLocation *Loc=nullptr) const
isIntegerConstantExpr - Return the value if this expression is a valid integer constant expression.
bool tryEvaluateObjectSize(uint64_t &Result, ASTContext &Ctx, unsigned Type) const
If the current Expr is a pointer, this will try to statically determine the number of bytes available...
const ValueDecl * getAsBuiltinConstantDeclRef(const ASTContext &Context) const
If this expression is an unambiguous reference to a single declaration, in the style of __builtin_fun...
Represents difference between two FPOptions values.
Represents a member of a struct/union/class.
const FieldDecl * findCountedByField() const
Find the FieldDecl specified in a FAM's "counted_by" attribute.
Represents a function declaration or definition.
const ParmVarDecl * getParamDecl(unsigned i) const
unsigned getBuiltinID(bool ConsiderWrapperFunctions=false) const
Returns a value indicating whether this function corresponds to a builtin function.
Represents a prototype with parameter type info, e.g.
@ SME_PStateSMEnabledMask
@ SME_PStateSMCompatibleMask
GlobalDecl - represents a global declaration.
const Decl * getDecl() const
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
StrictFlexArraysLevelKind
MemberExpr - [C99 6.5.2.3] Structure and Union Members.
StringRef getName() const
Get the name of identifier for this declaration as a StringRef.
std::string getNameAsString() const
Get a human-readable name for the declaration, even if it is one of the special kinds of names (C++ c...
Flags to identify the types for overloaded Neon builtins.
EltType getEltType() const
PointerType - C99 6.7.5.1 - Pointer Declarators.
QualType getPointeeType() const
A (possibly-)qualified type.
bool isVolatileQualified() const
Determine whether this type is volatile-qualified.
bool isWebAssemblyFuncrefType() const
Returns true if it is a WebAssembly Funcref Type.
LangAS getAddressSpace() const
Return the address space of this type.
bool isWebAssemblyExternrefType() const
Returns true if it is a WebAssembly Externref Type.
The collection of all-type qualifiers we support.
Represents a struct/union/class.
field_range fields() const
Flags to identify the types for overloaded SVE builtins.
bool isZExtReturn() const
bool isReverseUSDOT() const
bool isOverloadNone() const
MemEltType getMemEltType() const
bool isGatherLoad() const
bool isOverloadCvt() const
EltType getEltType() const
bool isOverloadDefault() const
bool isOverloadWhileRW() const
bool isReverseMergeAnyAccOp() const
bool isReductionQV() const
bool isInsertOp1SVALL() const
bool isAppendSVALL() const
bool isReverseMergeAnyBinOp() const
bool isStructStore() const
bool isTupleCreate() const
bool isGatherPrefetch() const
bool hasSplatOperand() const
MergeType getMergeType() const
bool isByteIndexed() const
bool isStructLoad() const
bool isOverloadWhileOrMultiVecCvt() const
unsigned getSplatOperand() const
bool isScatterStore() const
bool isReverseCompare() const
Scope - A scope is a transient data structure that is used while parsing the program.
Encodes a location in the source.
SourceLocation getBeginLoc() const LLVM_READONLY
bool areArgsDestroyedLeftToRightInCallee() const
Are arguments to a call destroyed left to right in the callee? This is a fundamental language change,...
Exposes information about the current target.
TargetOptions & getTargetOpts() const
Retrieve the target options.
virtual bool hasLegalHalfType() const
Determine whether _Float16 is supported on this target.
const llvm::Triple & getTriple() const
Returns the target triple of the primary target.
bool isLittleEndian() const
unsigned getMaxOpenCLWorkGroupSize() const
TargetCXXABI getCXXABI() const
Get the C++ ABI currently in use.
virtual bool checkArithmeticFenceSupported() const
Controls if __arithmetic_fence is supported in the targeted backend.
unsigned getSuitableAlign() const
Return the alignment that is the largest alignment ever used for any scalar/SIMD data type on the tar...
virtual std::string_view getClobbers() const =0
Returns a string of target-specific clobbers, in LLVM format.
llvm::CodeObjectVersionKind CodeObjectVersion
Code object version for AMDGPU.
The base class of the type hierarchy.
CXXRecordDecl * getAsCXXRecordDecl() const
Retrieves the CXXRecordDecl that this type refers to, either because the type is a RecordType or beca...
bool isBlockPointerType() const
bool isSignedIntegerType() const
Return true if this is an integer type that is signed, according to C99 6.2.5p4 [char,...
bool isComplexType() const
isComplexType() does not include complex integers (a GCC extension).
bool hasIntegerRepresentation() const
Determine whether this type has an integer representation of some sort, e.g., it is an integer type o...
bool isCountAttributedType() const
bool isPointerType() const
bool isIntegerType() const
isIntegerType() does not include complex integers (a GCC extension).
const T * castAs() const
Member-template castAs<specific type>.
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
bool isIntegralOrEnumerationType() const
Determine whether this type is an integral or enumeration type.
bool hasUnsignedIntegerRepresentation() const
Determine whether this type has an unsigned integer representation of some sort, e....
bool hasSignedIntegerRepresentation() const
Determine whether this type has an signed integer representation of some sort, e.g....
bool isObjCObjectPointerType() const
bool hasFloatingRepresentation() const
Determine whether this type has a floating-point representation of some sort, e.g....
bool isVectorType() const
bool isFloatingType() const
bool isUnsignedIntegerType() const
Return true if this is an integer type that is unsigned, according to C99 6.2.5p6 [which returns true...
const T * getAs() const
Member-template getAs<specific type>'.
bool isRecordType() const
bool isSizelessVectorType() const
Returns true for all scalable vector types.
QualType getSizelessVectorEltType(const ASTContext &Ctx) const
Returns the representative type for the element of a sizeless vector builtin type.
RecordDecl * getAsRecordDecl() const
Retrieves the RecordDecl this type refers to.
Represent the declaration of a variable (in which case it is an lvalue) a function (in which case it ...
Represents a GCC generic vector type.
unsigned getNumElements() const
QualType getElementType() const
SmallVector< OSLogBufferItem, 4 > Items
unsigned char getNumArgsByte() const
unsigned char getSummaryByte() const
Defines the clang::TargetInfo interface.
@ Type
The l-value was considered opaque, so the alignment was determined from a type.
llvm::Constant * initializationPatternFor(CodeGenModule &, llvm::Type *)
TypeEvaluationKind
The kind of evaluation to perform on values of a particular type.
@ EHCleanup
Denotes a cleanup that should run when a scope is exited using exceptional control flow (a throw stat...
constexpr XRayInstrMask Typed
constexpr XRayInstrMask Custom
bool computeOSLogBufferLayout(clang::ASTContext &Ctx, const clang::CallExpr *E, OSLogBufferLayout &layout)
const void * Store
Store - This opaque type encapsulates an immutable mapping from locations to values.
bool Dup(InterpState &S, CodePtr OpPC)
bool Zero(InterpState &S, CodePtr OpPC)
bool Mul(InterpState &S, CodePtr OpPC)
bool Neg(InterpState &S, CodePtr OpPC)
bool Load(InterpState &S, CodePtr OpPC)
bool Cast(InterpState &S, CodePtr OpPC)
bool Ret(InterpState &S, CodePtr &PC)
The JSON file list parser is used to communicate input to InstallAPI.
@ OK_BitField
A bitfield object is a bitfield on a C or C++ record.
@ Vector
'vector' clause, allowed on 'loop', Combined, and 'routine' directives.
@ DType
'dtype' clause, an alias for 'device_type', stored separately for diagnostic purposes.
bool operator<(DeclarationName LHS, DeclarationName RHS)
Ordering on two declaration names.
@ Asm
Assembly: we accept this only so that we can preprocess it.
@ Result
The result type of a method or function.
LangAS
Defines the address space values used by the address space qualifier of QualType.
const FunctionProtoType * T
SyncScope
Defines synch scope values used internally by clang.
llvm::StringRef getAsString(SyncScope S)
@ Success
Template argument deduction was successful.
@ Other
Other implicit parameter.
Diagnostic wrappers for TextAPI types for error reporting.
llvm::PointerType * VoidPtrTy
llvm::IntegerType * Int64Ty
llvm::PointerType * ConstGlobalsPtrTy
void* in the address space for constant globals
CharUnits getIntAlign() const
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
llvm::Type * HalfTy
half, bfloat, float, double
llvm::IntegerType * SizeTy
llvm::IntegerType * Int32Ty
llvm::IntegerType * IntPtrTy
llvm::IntegerType * IntTy
int
llvm::IntegerType * Int16Ty
llvm::PointerType * Int8PtrTy
llvm::PointerType * UnqualPtrTy
llvm::PointerType * AllocaInt8PtrTy
LangAS getASTAllocaAddressSpace() const
EvalResult is a struct with detailed info about an evaluated expression.
APValue Val
Val - This is the value the expression can be folded to.
void clear(SanitizerMask K=SanitizerKind::All)
Disable the sanitizers specified in K.
void set(SanitizerMask K, bool Value)
Enable or disable a certain (single) sanitizer.
bool has(SanitizerMask K) const
Check if a certain (single) sanitizer is enabled.
bool has(XRayInstrMask K) const
#define scalbln(__x, __y)
#define copysign(__x, __y)
#define remquo(__x, __y, __z)
#define nextafter(__x, __y)
#define nexttoward(__x, __y)
#define remainder(__x, __y)
#define fma(__x, __y, __z)