549 NEONMAP1(__a32_vcvt_bf16_f32, arm_neon_vcvtfp2bf, 0),
557 NEONMAP1(vabsq_v, arm_neon_vabs, 0),
561 NEONMAP1(vaesdq_u8, arm_neon_aesd, 0),
562 NEONMAP1(vaeseq_u8, arm_neon_aese, 0),
563 NEONMAP1(vaesimcq_u8, arm_neon_aesimc, 0),
564 NEONMAP1(vaesmcq_u8, arm_neon_aesmc, 0),
565 NEONMAP1(vbfdot_f32, arm_neon_bfdot, 0),
566 NEONMAP1(vbfdotq_f32, arm_neon_bfdot, 0),
567 NEONMAP1(vbfmlalbq_f32, arm_neon_bfmlalb, 0),
568 NEONMAP1(vbfmlaltq_f32, arm_neon_bfmlalt, 0),
569 NEONMAP1(vbfmmlaq_f32, arm_neon_bfmmla, 0),
582 NEONMAP1(vcage_v, arm_neon_vacge, 0),
583 NEONMAP1(vcageq_v, arm_neon_vacge, 0),
584 NEONMAP1(vcagt_v, arm_neon_vacgt, 0),
585 NEONMAP1(vcagtq_v, arm_neon_vacgt, 0),
586 NEONMAP1(vcale_v, arm_neon_vacge, 0),
587 NEONMAP1(vcaleq_v, arm_neon_vacge, 0),
588 NEONMAP1(vcalt_v, arm_neon_vacgt, 0),
589 NEONMAP1(vcaltq_v, arm_neon_vacgt, 0),
609 NEONMAP1(vcvt_n_f16_s16, arm_neon_vcvtfxs2fp, 0),
610 NEONMAP1(vcvt_n_f16_u16, arm_neon_vcvtfxu2fp, 0),
611 NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
612 NEONMAP1(vcvt_n_s16_f16, arm_neon_vcvtfp2fxs, 0),
613 NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0),
614 NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0),
615 NEONMAP1(vcvt_n_u16_f16, arm_neon_vcvtfp2fxu, 0),
616 NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0),
617 NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0),
624 NEONMAP1(vcvta_s16_f16, arm_neon_vcvtas, 0),
625 NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0),
626 NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0),
627 NEONMAP1(vcvta_u16_f16, arm_neon_vcvtau, 0),
628 NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0),
629 NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0),
630 NEONMAP1(vcvtaq_s16_f16, arm_neon_vcvtas, 0),
631 NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0),
632 NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0),
633 NEONMAP1(vcvtaq_u16_f16, arm_neon_vcvtau, 0),
634 NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0),
635 NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0),
636 NEONMAP1(vcvth_bf16_f32, arm_neon_vcvtbfp2bf, 0),
637 NEONMAP1(vcvtm_s16_f16, arm_neon_vcvtms, 0),
638 NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0),
639 NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0),
640 NEONMAP1(vcvtm_u16_f16, arm_neon_vcvtmu, 0),
641 NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0),
642 NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0),
643 NEONMAP1(vcvtmq_s16_f16, arm_neon_vcvtms, 0),
644 NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0),
645 NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0),
646 NEONMAP1(vcvtmq_u16_f16, arm_neon_vcvtmu, 0),
647 NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0),
648 NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0),
649 NEONMAP1(vcvtn_s16_f16, arm_neon_vcvtns, 0),
650 NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0),
651 NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0),
652 NEONMAP1(vcvtn_u16_f16, arm_neon_vcvtnu, 0),
653 NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0),
654 NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0),
655 NEONMAP1(vcvtnq_s16_f16, arm_neon_vcvtns, 0),
656 NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0),
657 NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0),
658 NEONMAP1(vcvtnq_u16_f16, arm_neon_vcvtnu, 0),
659 NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0),
660 NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0),
661 NEONMAP1(vcvtp_s16_f16, arm_neon_vcvtps, 0),
662 NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0),
663 NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0),
664 NEONMAP1(vcvtp_u16_f16, arm_neon_vcvtpu, 0),
665 NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0),
666 NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0),
667 NEONMAP1(vcvtpq_s16_f16, arm_neon_vcvtps, 0),
668 NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0),
669 NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0),
670 NEONMAP1(vcvtpq_u16_f16, arm_neon_vcvtpu, 0),
671 NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0),
672 NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0),
676 NEONMAP1(vcvtq_n_f16_s16, arm_neon_vcvtfxs2fp, 0),
677 NEONMAP1(vcvtq_n_f16_u16, arm_neon_vcvtfxu2fp, 0),
678 NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
679 NEONMAP1(vcvtq_n_s16_f16, arm_neon_vcvtfp2fxs, 0),
680 NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0),
681 NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0),
682 NEONMAP1(vcvtq_n_u16_f16, arm_neon_vcvtfp2fxu, 0),
683 NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0),
684 NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0),
691 NEONMAP1(vdot_s32, arm_neon_sdot, 0),
692 NEONMAP1(vdot_u32, arm_neon_udot, 0),
693 NEONMAP1(vdotq_s32, arm_neon_sdot, 0),
694 NEONMAP1(vdotq_u32, arm_neon_udot, 0),
705 NEONMAP1(vld1_x2_v, arm_neon_vld1x2, 0),
706 NEONMAP1(vld1_x3_v, arm_neon_vld1x3, 0),
707 NEONMAP1(vld1_x4_v, arm_neon_vld1x4, 0),
709 NEONMAP1(vld1q_v, arm_neon_vld1, 0),
710 NEONMAP1(vld1q_x2_v, arm_neon_vld1x2, 0),
711 NEONMAP1(vld1q_x3_v, arm_neon_vld1x3, 0),
712 NEONMAP1(vld1q_x4_v, arm_neon_vld1x4, 0),
713 NEONMAP1(vld2_dup_v, arm_neon_vld2dup, 0),
714 NEONMAP1(vld2_lane_v, arm_neon_vld2lane, 0),
716 NEONMAP1(vld2q_dup_v, arm_neon_vld2dup, 0),
717 NEONMAP1(vld2q_lane_v, arm_neon_vld2lane, 0),
718 NEONMAP1(vld2q_v, arm_neon_vld2, 0),
719 NEONMAP1(vld3_dup_v, arm_neon_vld3dup, 0),
720 NEONMAP1(vld3_lane_v, arm_neon_vld3lane, 0),
722 NEONMAP1(vld3q_dup_v, arm_neon_vld3dup, 0),
723 NEONMAP1(vld3q_lane_v, arm_neon_vld3lane, 0),
724 NEONMAP1(vld3q_v, arm_neon_vld3, 0),
725 NEONMAP1(vld4_dup_v, arm_neon_vld4dup, 0),
726 NEONMAP1(vld4_lane_v, arm_neon_vld4lane, 0),
728 NEONMAP1(vld4q_dup_v, arm_neon_vld4dup, 0),
729 NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0),
730 NEONMAP1(vld4q_v, arm_neon_vld4, 0),
739 NEONMAP1(vmmlaq_s32, arm_neon_smmla, 0),
740 NEONMAP1(vmmlaq_u32, arm_neon_ummla, 0),
758 NEONMAP2(vqdmlal_v, arm_neon_vqdmull, sadd_sat, 0),
759 NEONMAP2(vqdmlsl_v, arm_neon_vqdmull, ssub_sat, 0),
783 NEONMAP1(vqshlu_n_v, arm_neon_vqshiftsu, 0),
784 NEONMAP1(vqshluq_n_v, arm_neon_vqshiftsu, 0),
788 NEONMAP2(vrecpe_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
789 NEONMAP2(vrecpeq_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
812 NEONMAP2(vrsqrte_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
813 NEONMAP2(vrsqrteq_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
817 NEONMAP1(vsha1su0q_u32, arm_neon_sha1su0, 0),
818 NEONMAP1(vsha1su1q_u32, arm_neon_sha1su1, 0),
819 NEONMAP1(vsha256h2q_u32, arm_neon_sha256h2, 0),
820 NEONMAP1(vsha256hq_u32, arm_neon_sha256h, 0),
821 NEONMAP1(vsha256su0q_u32, arm_neon_sha256su0, 0),
822 NEONMAP1(vsha256su1q_u32, arm_neon_sha256su1, 0),
832 NEONMAP1(vst1_x2_v, arm_neon_vst1x2, 0),
833 NEONMAP1(vst1_x3_v, arm_neon_vst1x3, 0),
834 NEONMAP1(vst1_x4_v, arm_neon_vst1x4, 0),
835 NEONMAP1(vst1q_v, arm_neon_vst1, 0),
836 NEONMAP1(vst1q_x2_v, arm_neon_vst1x2, 0),
837 NEONMAP1(vst1q_x3_v, arm_neon_vst1x3, 0),
838 NEONMAP1(vst1q_x4_v, arm_neon_vst1x4, 0),
839 NEONMAP1(vst2_lane_v, arm_neon_vst2lane, 0),
841 NEONMAP1(vst2q_lane_v, arm_neon_vst2lane, 0),
842 NEONMAP1(vst2q_v, arm_neon_vst2, 0),
843 NEONMAP1(vst3_lane_v, arm_neon_vst3lane, 0),
845 NEONMAP1(vst3q_lane_v, arm_neon_vst3lane, 0),
846 NEONMAP1(vst3q_v, arm_neon_vst3, 0),
847 NEONMAP1(vst4_lane_v, arm_neon_vst4lane, 0),
849 NEONMAP1(vst4q_lane_v, arm_neon_vst4lane, 0),
850 NEONMAP1(vst4q_v, arm_neon_vst4, 0),
856 NEONMAP1(vusdot_s32, arm_neon_usdot, 0),
857 NEONMAP1(vusdotq_s32, arm_neon_usdot, 0),
858 NEONMAP1(vusmmlaq_s32, arm_neon_usmmla, 0),
1129 unsigned BuiltinID,
unsigned LLVMIntrinsic,
unsigned AltLLVMIntrinsic,
1130 const char *NameHint,
unsigned Modifier,
const CallExpr *E,
1132 llvm::Triple::ArchType
Arch) {
1138 std::optional<llvm::APSInt> NeonTypeConst =
1145 const bool Usgn =
Type.isUnsigned();
1146 const bool Quad =
Type.isQuad();
1147 const bool Floating =
Type.isFloatingPoint();
1149 const bool AllowBFloatArgsAndRet =
1152 llvm::FixedVectorType *VTy =
1153 GetNeonType(
this,
Type, HasFastHalfType,
false, AllowBFloatArgsAndRet);
1154 llvm::Type *Ty = VTy;
1158 auto getAlignmentValue32 = [&](
Address addr) ->
Value* {
1159 return Builder.getInt32(addr.getAlignment().getQuantity());
1162 unsigned Int = LLVMIntrinsic;
1164 Int = AltLLVMIntrinsic;
1166 switch (BuiltinID) {
1168 case NEON::BI__builtin_neon_splat_lane_v:
1169 case NEON::BI__builtin_neon_splat_laneq_v:
1170 case NEON::BI__builtin_neon_splatq_lane_v:
1171 case NEON::BI__builtin_neon_splatq_laneq_v: {
1172 auto NumElements = VTy->getElementCount();
1173 if (BuiltinID == NEON::BI__builtin_neon_splatq_lane_v)
1174 NumElements = NumElements * 2;
1175 if (BuiltinID == NEON::BI__builtin_neon_splat_laneq_v)
1176 NumElements = NumElements.divideCoefficientBy(2);
1178 Ops[0] =
Builder.CreateBitCast(Ops[0], VTy);
1181 case NEON::BI__builtin_neon_vpadd_v:
1182 case NEON::BI__builtin_neon_vpaddq_v:
1184 if (VTy->getElementType()->isFloatingPointTy() &&
1185 Int == Intrinsic::aarch64_neon_addp)
1186 Int = Intrinsic::aarch64_neon_faddp;
1188 case NEON::BI__builtin_neon_vabs_v:
1189 case NEON::BI__builtin_neon_vabsq_v:
1190 if (VTy->getElementType()->isFloatingPointTy())
1191 return EmitNeonCall(
CGM.getIntrinsic(Intrinsic::fabs, Ty), Ops,
"vabs");
1192 return EmitNeonCall(
CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops,
"vabs");
1193 case NEON::BI__builtin_neon_vadd_v:
1194 case NEON::BI__builtin_neon_vaddq_v: {
1195 llvm::Type *VTy = llvm::FixedVectorType::get(
Int8Ty, Quad ? 16 : 8);
1196 Ops[0] =
Builder.CreateBitCast(Ops[0], VTy);
1197 Ops[1] =
Builder.CreateBitCast(Ops[1], VTy);
1198 Ops[0] =
Builder.CreateXor(Ops[0], Ops[1]);
1199 return Builder.CreateBitCast(Ops[0], Ty);
1201 case NEON::BI__builtin_neon_vaddhn_v: {
1202 llvm::FixedVectorType *SrcTy =
1203 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
1206 Ops[0] =
Builder.CreateBitCast(Ops[0], SrcTy);
1207 Ops[1] =
Builder.CreateBitCast(Ops[1], SrcTy);
1208 Ops[0] =
Builder.CreateAdd(Ops[0], Ops[1],
"vaddhn");
1212 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
1213 Ops[0] =
Builder.CreateLShr(Ops[0], ShiftAmt,
"vaddhn");
1216 return Builder.CreateTrunc(Ops[0], VTy,
"vaddhn");
1218 case NEON::BI__builtin_neon_vcale_v:
1219 case NEON::BI__builtin_neon_vcaleq_v:
1220 case NEON::BI__builtin_neon_vcalt_v:
1221 case NEON::BI__builtin_neon_vcaltq_v:
1222 std::swap(Ops[0], Ops[1]);
1224 case NEON::BI__builtin_neon_vcage_v:
1225 case NEON::BI__builtin_neon_vcageq_v:
1226 case NEON::BI__builtin_neon_vcagt_v:
1227 case NEON::BI__builtin_neon_vcagtq_v: {
1229 switch (VTy->getScalarSizeInBits()) {
1230 default: llvm_unreachable(
"unexpected type");
1241 auto *VecFlt = llvm::FixedVectorType::get(Ty, VTy->getNumElements());
1242 llvm::Type *Tys[] = { VTy, VecFlt };
1243 Function *F =
CGM.getIntrinsic(LLVMIntrinsic, Tys);
1246 case NEON::BI__builtin_neon_vceqz_v:
1247 case NEON::BI__builtin_neon_vceqzq_v:
1249 Ops[0], Ty, Floating ? ICmpInst::FCMP_OEQ : ICmpInst::ICMP_EQ,
"vceqz");
1250 case NEON::BI__builtin_neon_vcgez_v:
1251 case NEON::BI__builtin_neon_vcgezq_v:
1253 Ops[0], Ty, Floating ? ICmpInst::FCMP_OGE : ICmpInst::ICMP_SGE,
1255 case NEON::BI__builtin_neon_vclez_v:
1256 case NEON::BI__builtin_neon_vclezq_v:
1258 Ops[0], Ty, Floating ? ICmpInst::FCMP_OLE : ICmpInst::ICMP_SLE,
1260 case NEON::BI__builtin_neon_vcgtz_v:
1261 case NEON::BI__builtin_neon_vcgtzq_v:
1263 Ops[0], Ty, Floating ? ICmpInst::FCMP_OGT : ICmpInst::ICMP_SGT,
1265 case NEON::BI__builtin_neon_vcltz_v:
1266 case NEON::BI__builtin_neon_vcltzq_v:
1268 Ops[0], Ty, Floating ? ICmpInst::FCMP_OLT : ICmpInst::ICMP_SLT,
1270 case NEON::BI__builtin_neon_vclz_v:
1271 case NEON::BI__builtin_neon_vclzq_v:
1276 case NEON::BI__builtin_neon_vcvt_f32_v:
1277 case NEON::BI__builtin_neon_vcvtq_f32_v:
1278 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
1281 return Usgn ?
Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
1282 :
Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
1283 case NEON::BI__builtin_neon_vcvt_f16_s16:
1284 case NEON::BI__builtin_neon_vcvt_f16_u16:
1285 case NEON::BI__builtin_neon_vcvtq_f16_s16:
1286 case NEON::BI__builtin_neon_vcvtq_f16_u16:
1287 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
1290 return Usgn ?
Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
1291 :
Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
1292 case NEON::BI__builtin_neon_vcvt_n_f16_s16:
1293 case NEON::BI__builtin_neon_vcvt_n_f16_u16:
1294 case NEON::BI__builtin_neon_vcvtq_n_f16_s16:
1295 case NEON::BI__builtin_neon_vcvtq_n_f16_u16: {
1300 case NEON::BI__builtin_neon_vcvt_n_f32_v:
1301 case NEON::BI__builtin_neon_vcvt_n_f64_v:
1302 case NEON::BI__builtin_neon_vcvtq_n_f32_v:
1303 case NEON::BI__builtin_neon_vcvtq_n_f64_v: {
1305 Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
1309 case NEON::BI__builtin_neon_vcvt_n_s16_f16:
1310 case NEON::BI__builtin_neon_vcvt_n_s32_v:
1311 case NEON::BI__builtin_neon_vcvt_n_u16_f16:
1312 case NEON::BI__builtin_neon_vcvt_n_u32_v:
1313 case NEON::BI__builtin_neon_vcvt_n_s64_v:
1314 case NEON::BI__builtin_neon_vcvt_n_u64_v:
1315 case NEON::BI__builtin_neon_vcvtq_n_s16_f16:
1316 case NEON::BI__builtin_neon_vcvtq_n_s32_v:
1317 case NEON::BI__builtin_neon_vcvtq_n_u16_f16:
1318 case NEON::BI__builtin_neon_vcvtq_n_u32_v:
1319 case NEON::BI__builtin_neon_vcvtq_n_s64_v:
1320 case NEON::BI__builtin_neon_vcvtq_n_u64_v: {
1322 Function *F =
CGM.getIntrinsic(LLVMIntrinsic, Tys);
1325 case NEON::BI__builtin_neon_vcvt_s32_v:
1326 case NEON::BI__builtin_neon_vcvt_u32_v:
1327 case NEON::BI__builtin_neon_vcvt_s64_v:
1328 case NEON::BI__builtin_neon_vcvt_u64_v:
1329 case NEON::BI__builtin_neon_vcvt_s16_f16:
1330 case NEON::BI__builtin_neon_vcvt_u16_f16:
1331 case NEON::BI__builtin_neon_vcvtq_s32_v:
1332 case NEON::BI__builtin_neon_vcvtq_u32_v:
1333 case NEON::BI__builtin_neon_vcvtq_s64_v:
1334 case NEON::BI__builtin_neon_vcvtq_u64_v:
1335 case NEON::BI__builtin_neon_vcvtq_s16_f16:
1336 case NEON::BI__builtin_neon_vcvtq_u16_f16: {
1340 if (!
Builder.getIsFPConstrained())
1341 Int = Usgn ? Intrinsic::fptoui_sat : Intrinsic::fptosi_sat;
1342 llvm::Type *Tys[2] = {Ty, Ops[0]->getType()};
1347 return Usgn ?
Builder.CreateFPToUI(Ops[0], Ty,
"vcvt")
1348 :
Builder.CreateFPToSI(Ops[0], Ty,
"vcvt");
1350 case NEON::BI__builtin_neon_vcvta_s16_f16:
1351 case NEON::BI__builtin_neon_vcvta_s32_v:
1352 case NEON::BI__builtin_neon_vcvta_s64_v:
1353 case NEON::BI__builtin_neon_vcvta_u16_f16:
1354 case NEON::BI__builtin_neon_vcvta_u32_v:
1355 case NEON::BI__builtin_neon_vcvta_u64_v:
1356 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
1357 case NEON::BI__builtin_neon_vcvtaq_s32_v:
1358 case NEON::BI__builtin_neon_vcvtaq_s64_v:
1359 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
1360 case NEON::BI__builtin_neon_vcvtaq_u32_v:
1361 case NEON::BI__builtin_neon_vcvtaq_u64_v:
1362 case NEON::BI__builtin_neon_vcvtn_s16_f16:
1363 case NEON::BI__builtin_neon_vcvtn_s32_v:
1364 case NEON::BI__builtin_neon_vcvtn_s64_v:
1365 case NEON::BI__builtin_neon_vcvtn_u16_f16:
1366 case NEON::BI__builtin_neon_vcvtn_u32_v:
1367 case NEON::BI__builtin_neon_vcvtn_u64_v:
1368 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
1369 case NEON::BI__builtin_neon_vcvtnq_s32_v:
1370 case NEON::BI__builtin_neon_vcvtnq_s64_v:
1371 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
1372 case NEON::BI__builtin_neon_vcvtnq_u32_v:
1373 case NEON::BI__builtin_neon_vcvtnq_u64_v:
1374 case NEON::BI__builtin_neon_vcvtp_s16_f16:
1375 case NEON::BI__builtin_neon_vcvtp_s32_v:
1376 case NEON::BI__builtin_neon_vcvtp_s64_v:
1377 case NEON::BI__builtin_neon_vcvtp_u16_f16:
1378 case NEON::BI__builtin_neon_vcvtp_u32_v:
1379 case NEON::BI__builtin_neon_vcvtp_u64_v:
1380 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
1381 case NEON::BI__builtin_neon_vcvtpq_s32_v:
1382 case NEON::BI__builtin_neon_vcvtpq_s64_v:
1383 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
1384 case NEON::BI__builtin_neon_vcvtpq_u32_v:
1385 case NEON::BI__builtin_neon_vcvtpq_u64_v:
1386 case NEON::BI__builtin_neon_vcvtm_s16_f16:
1387 case NEON::BI__builtin_neon_vcvtm_s32_v:
1388 case NEON::BI__builtin_neon_vcvtm_s64_v:
1389 case NEON::BI__builtin_neon_vcvtm_u16_f16:
1390 case NEON::BI__builtin_neon_vcvtm_u32_v:
1391 case NEON::BI__builtin_neon_vcvtm_u64_v:
1392 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
1393 case NEON::BI__builtin_neon_vcvtmq_s32_v:
1394 case NEON::BI__builtin_neon_vcvtmq_s64_v:
1395 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
1396 case NEON::BI__builtin_neon_vcvtmq_u32_v:
1397 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
1399 return EmitNeonCall(
CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint);
1401 case NEON::BI__builtin_neon_vcvtx_f32_v: {
1402 llvm::Type *Tys[2] = { VTy->getTruncatedElementVectorType(VTy), Ty};
1403 return EmitNeonCall(
CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint);
1406 case NEON::BI__builtin_neon_vext_v:
1407 case NEON::BI__builtin_neon_vextq_v: {
1410 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
1411 Indices.push_back(i+CV);
1413 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
1414 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
1415 return Builder.CreateShuffleVector(Ops[0], Ops[1], Indices,
"vext");
1417 case NEON::BI__builtin_neon_vfma_v:
1418 case NEON::BI__builtin_neon_vfmaq_v: {
1419 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
1420 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
1421 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
1425 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
1426 {Ops[1], Ops[2], Ops[0]});
1428 case NEON::BI__builtin_neon_vld1_x2_v:
1429 case NEON::BI__builtin_neon_vld1q_x2_v:
1430 case NEON::BI__builtin_neon_vld1_x3_v:
1431 case NEON::BI__builtin_neon_vld1q_x3_v:
1432 case NEON::BI__builtin_neon_vld1_x4_v:
1433 case NEON::BI__builtin_neon_vld1q_x4_v: {
1435 Function *F =
CGM.getIntrinsic(LLVMIntrinsic, Tys);
1436 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld1xN");
1437 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
1439 case NEON::BI__builtin_neon_vld1_v:
1440 case NEON::BI__builtin_neon_vld1q_v: {
1442 Ops.push_back(getAlignmentValue32(PtrOp0));
1443 return EmitNeonCall(
CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops,
"vld1");
1445 case NEON::BI__builtin_neon_vld2_v:
1446 case NEON::BI__builtin_neon_vld2q_v:
1447 case NEON::BI__builtin_neon_vld3_v:
1448 case NEON::BI__builtin_neon_vld3q_v:
1449 case NEON::BI__builtin_neon_vld4_v:
1450 case NEON::BI__builtin_neon_vld4q_v:
1451 case NEON::BI__builtin_neon_vld2_dup_v:
1452 case NEON::BI__builtin_neon_vld2q_dup_v:
1453 case NEON::BI__builtin_neon_vld3_dup_v:
1454 case NEON::BI__builtin_neon_vld3q_dup_v:
1455 case NEON::BI__builtin_neon_vld4_dup_v:
1456 case NEON::BI__builtin_neon_vld4q_dup_v: {
1458 Function *F =
CGM.getIntrinsic(LLVMIntrinsic, Tys);
1459 Value *Align = getAlignmentValue32(PtrOp1);
1460 Ops[1] =
Builder.CreateCall(F, {Ops[1], Align}, NameHint);
1461 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
1463 case NEON::BI__builtin_neon_vld1_dup_v:
1464 case NEON::BI__builtin_neon_vld1q_dup_v: {
1465 Value *
V = PoisonValue::get(Ty);
1467 LoadInst *Ld =
Builder.CreateLoad(PtrOp0);
1468 llvm::Constant *CI = ConstantInt::get(
SizeTy, 0);
1469 Ops[0] =
Builder.CreateInsertElement(
V, Ld, CI);
1472 case NEON::BI__builtin_neon_vld2_lane_v:
1473 case NEON::BI__builtin_neon_vld2q_lane_v:
1474 case NEON::BI__builtin_neon_vld3_lane_v:
1475 case NEON::BI__builtin_neon_vld3q_lane_v:
1476 case NEON::BI__builtin_neon_vld4_lane_v:
1477 case NEON::BI__builtin_neon_vld4q_lane_v: {
1479 Function *F =
CGM.getIntrinsic(LLVMIntrinsic, Tys);
1480 for (
unsigned I = 2; I < Ops.size() - 1; ++I)
1481 Ops[I] =
Builder.CreateBitCast(Ops[I], Ty);
1482 Ops.push_back(getAlignmentValue32(PtrOp1));
1484 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
1486 case NEON::BI__builtin_neon_vmovl_v: {
1487 llvm::FixedVectorType *DTy =
1488 llvm::FixedVectorType::getTruncatedElementVectorType(VTy);
1489 Ops[0] =
Builder.CreateBitCast(Ops[0], DTy);
1491 return Builder.CreateZExt(Ops[0], Ty,
"vmovl");
1492 return Builder.CreateSExt(Ops[0], Ty,
"vmovl");
1494 case NEON::BI__builtin_neon_vmovn_v: {
1495 llvm::FixedVectorType *QTy =
1496 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
1497 Ops[0] =
Builder.CreateBitCast(Ops[0], QTy);
1498 return Builder.CreateTrunc(Ops[0], Ty,
"vmovn");
1500 case NEON::BI__builtin_neon_vmull_v:
1506 Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls;
1507 Int =
Type.isPoly() ? (unsigned)Intrinsic::arm_neon_vmullp :
Int;
1509 case NEON::BI__builtin_neon_vpadal_v:
1510 case NEON::BI__builtin_neon_vpadalq_v: {
1512 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
1516 llvm::FixedVectorType::get(EltTy, VTy->getNumElements() * 2);
1517 llvm::Type *Tys[2] = { Ty, NarrowTy };
1520 case NEON::BI__builtin_neon_vpaddl_v:
1521 case NEON::BI__builtin_neon_vpaddlq_v: {
1523 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
1524 llvm::Type *EltTy = llvm::IntegerType::get(
getLLVMContext(), EltBits / 2);
1526 llvm::FixedVectorType::get(EltTy, VTy->getNumElements() * 2);
1527 llvm::Type *Tys[2] = { Ty, NarrowTy };
1530 case NEON::BI__builtin_neon_vqdmlal_v:
1531 case NEON::BI__builtin_neon_vqdmlsl_v: {
1536 return EmitNeonCall(
CGM.getIntrinsic(AltLLVMIntrinsic, Ty), Ops, NameHint);
1538 case NEON::BI__builtin_neon_vqdmulhq_lane_v:
1539 case NEON::BI__builtin_neon_vqdmulh_lane_v:
1540 case NEON::BI__builtin_neon_vqrdmulhq_lane_v:
1541 case NEON::BI__builtin_neon_vqrdmulh_lane_v: {
1543 if (BuiltinID == NEON::BI__builtin_neon_vqdmulhq_lane_v ||
1544 BuiltinID == NEON::BI__builtin_neon_vqrdmulhq_lane_v)
1545 RTy = llvm::FixedVectorType::get(RTy->getElementType(),
1546 RTy->getNumElements() * 2);
1547 llvm::Type *Tys[2] = {
1552 case NEON::BI__builtin_neon_vqdmulhq_laneq_v:
1553 case NEON::BI__builtin_neon_vqdmulh_laneq_v:
1554 case NEON::BI__builtin_neon_vqrdmulhq_laneq_v:
1555 case NEON::BI__builtin_neon_vqrdmulh_laneq_v: {
1556 llvm::Type *Tys[2] = {
1561 case NEON::BI__builtin_neon_vqshl_n_v:
1562 case NEON::BI__builtin_neon_vqshlq_n_v:
1565 case NEON::BI__builtin_neon_vqshlu_n_v:
1566 case NEON::BI__builtin_neon_vqshluq_n_v:
1569 case NEON::BI__builtin_neon_vrecpe_v:
1570 case NEON::BI__builtin_neon_vrecpeq_v:
1571 case NEON::BI__builtin_neon_vrsqrte_v:
1572 case NEON::BI__builtin_neon_vrsqrteq_v:
1573 Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic;
1575 case NEON::BI__builtin_neon_vrndi_v:
1576 case NEON::BI__builtin_neon_vrndiq_v:
1578 ? Intrinsic::experimental_constrained_nearbyint
1579 : Intrinsic::nearbyint;
1581 case NEON::BI__builtin_neon_vrshr_n_v:
1582 case NEON::BI__builtin_neon_vrshrq_n_v:
1585 case NEON::BI__builtin_neon_vsha512hq_u64:
1586 case NEON::BI__builtin_neon_vsha512h2q_u64:
1587 case NEON::BI__builtin_neon_vsha512su0q_u64:
1588 case NEON::BI__builtin_neon_vsha512su1q_u64: {
1592 case NEON::BI__builtin_neon_vshl_n_v:
1593 case NEON::BI__builtin_neon_vshlq_n_v:
1595 return Builder.CreateShl(
Builder.CreateBitCast(Ops[0],Ty), Ops[1],
1597 case NEON::BI__builtin_neon_vshll_n_v: {
1598 llvm::FixedVectorType *SrcTy =
1599 llvm::FixedVectorType::getTruncatedElementVectorType(VTy);
1600 Ops[0] =
Builder.CreateBitCast(Ops[0], SrcTy);
1602 Ops[0] =
Builder.CreateZExt(Ops[0], VTy);
1604 Ops[0] =
Builder.CreateSExt(Ops[0], VTy);
1606 return Builder.CreateShl(Ops[0], Ops[1],
"vshll_n");
1608 case NEON::BI__builtin_neon_vshrn_n_v: {
1609 llvm::FixedVectorType *SrcTy =
1610 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
1611 Ops[0] =
Builder.CreateBitCast(Ops[0], SrcTy);
1614 Ops[0] =
Builder.CreateLShr(Ops[0], Ops[1]);
1616 Ops[0] =
Builder.CreateAShr(Ops[0], Ops[1]);
1617 return Builder.CreateTrunc(Ops[0], Ty,
"vshrn_n");
1619 case NEON::BI__builtin_neon_vshr_n_v:
1620 case NEON::BI__builtin_neon_vshrq_n_v:
1622 case NEON::BI__builtin_neon_vst1_v:
1623 case NEON::BI__builtin_neon_vst1q_v:
1624 case NEON::BI__builtin_neon_vst2_v:
1625 case NEON::BI__builtin_neon_vst2q_v:
1626 case NEON::BI__builtin_neon_vst3_v:
1627 case NEON::BI__builtin_neon_vst3q_v:
1628 case NEON::BI__builtin_neon_vst4_v:
1629 case NEON::BI__builtin_neon_vst4q_v:
1630 case NEON::BI__builtin_neon_vst2_lane_v:
1631 case NEON::BI__builtin_neon_vst2q_lane_v:
1632 case NEON::BI__builtin_neon_vst3_lane_v:
1633 case NEON::BI__builtin_neon_vst3q_lane_v:
1634 case NEON::BI__builtin_neon_vst4_lane_v:
1635 case NEON::BI__builtin_neon_vst4q_lane_v: {
1637 Ops.push_back(getAlignmentValue32(PtrOp0));
1640 case NEON::BI__builtin_neon_vsm3partw1q_u32:
1641 case NEON::BI__builtin_neon_vsm3partw2q_u32:
1642 case NEON::BI__builtin_neon_vsm3ss1q_u32:
1643 case NEON::BI__builtin_neon_vsm4ekeyq_u32:
1644 case NEON::BI__builtin_neon_vsm4eq_u32: {
1648 case NEON::BI__builtin_neon_vsm3tt1aq_u32:
1649 case NEON::BI__builtin_neon_vsm3tt1bq_u32:
1650 case NEON::BI__builtin_neon_vsm3tt2aq_u32:
1651 case NEON::BI__builtin_neon_vsm3tt2bq_u32: {
1656 case NEON::BI__builtin_neon_vst1_x2_v:
1657 case NEON::BI__builtin_neon_vst1q_x2_v:
1658 case NEON::BI__builtin_neon_vst1_x3_v:
1659 case NEON::BI__builtin_neon_vst1q_x3_v:
1660 case NEON::BI__builtin_neon_vst1_x4_v:
1661 case NEON::BI__builtin_neon_vst1q_x4_v: {
1664 if (
Arch == llvm::Triple::aarch64 ||
Arch == llvm::Triple::aarch64_be ||
1665 Arch == llvm::Triple::aarch64_32) {
1667 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
1673 case NEON::BI__builtin_neon_vsubhn_v: {
1674 llvm::FixedVectorType *SrcTy =
1675 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
1678 Ops[0] =
Builder.CreateBitCast(Ops[0], SrcTy);
1679 Ops[1] =
Builder.CreateBitCast(Ops[1], SrcTy);
1680 Ops[0] =
Builder.CreateSub(Ops[0], Ops[1],
"vsubhn");
1684 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
1685 Ops[0] =
Builder.CreateLShr(Ops[0], ShiftAmt,
"vsubhn");
1688 return Builder.CreateTrunc(Ops[0], VTy,
"vsubhn");
1690 case NEON::BI__builtin_neon_vtrn_v:
1691 case NEON::BI__builtin_neon_vtrnq_v: {
1692 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
1693 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
1694 Value *SV =
nullptr;
1696 for (
unsigned vi = 0; vi != 2; ++vi) {
1698 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
1699 Indices.push_back(i+vi);
1700 Indices.push_back(i+e+vi);
1703 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vtrn");
1708 case NEON::BI__builtin_neon_vtst_v:
1709 case NEON::BI__builtin_neon_vtstq_v: {
1710 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
1711 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
1712 Ops[0] =
Builder.CreateAnd(Ops[0], Ops[1]);
1713 Ops[0] =
Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
1714 ConstantAggregateZero::get(Ty));
1715 return Builder.CreateSExt(Ops[0], Ty,
"vtst");
1717 case NEON::BI__builtin_neon_vuzp_v:
1718 case NEON::BI__builtin_neon_vuzpq_v: {
1719 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
1720 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
1721 Value *SV =
nullptr;
1723 for (
unsigned vi = 0; vi != 2; ++vi) {
1725 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
1726 Indices.push_back(2*i+vi);
1729 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vuzp");
1734 case NEON::BI__builtin_neon_vxarq_u64: {
1739 case NEON::BI__builtin_neon_vzip_v:
1740 case NEON::BI__builtin_neon_vzipq_v: {
1741 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
1742 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
1743 Value *SV =
nullptr;
1745 for (
unsigned vi = 0; vi != 2; ++vi) {
1747 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
1748 Indices.push_back((i + vi*e) >> 1);
1749 Indices.push_back(((i + vi*e) >> 1)+e);
1752 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vzip");
1757 case NEON::BI__builtin_neon_vdot_s32:
1758 case NEON::BI__builtin_neon_vdot_u32:
1759 case NEON::BI__builtin_neon_vdotq_s32:
1760 case NEON::BI__builtin_neon_vdotq_u32: {
1762 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
1763 llvm::Type *Tys[2] = { Ty, InputTy };
1766 case NEON::BI__builtin_neon_vfmlal_low_f16:
1767 case NEON::BI__builtin_neon_vfmlalq_low_f16: {
1769 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
1770 llvm::Type *Tys[2] = { Ty, InputTy };
1773 case NEON::BI__builtin_neon_vfmlsl_low_f16:
1774 case NEON::BI__builtin_neon_vfmlslq_low_f16: {
1776 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
1777 llvm::Type *Tys[2] = { Ty, InputTy };
1780 case NEON::BI__builtin_neon_vfmlal_high_f16:
1781 case NEON::BI__builtin_neon_vfmlalq_high_f16: {
1783 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
1784 llvm::Type *Tys[2] = { Ty, InputTy };
1787 case NEON::BI__builtin_neon_vfmlsl_high_f16:
1788 case NEON::BI__builtin_neon_vfmlslq_high_f16: {
1790 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
1791 llvm::Type *Tys[2] = { Ty, InputTy };
1794 case NEON::BI__builtin_neon_vmmlaq_s32:
1795 case NEON::BI__builtin_neon_vmmlaq_u32: {
1797 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
1798 llvm::Type *Tys[2] = { Ty, InputTy };
1799 return EmitNeonCall(
CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops,
"vmmla");
1801 case NEON::BI__builtin_neon_vmmlaq_f16:
1802 case NEON::BI__builtin_neon_vmmlaq_f32_f16: {
1804 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
1805 llvm::Type *Tys[2] = {Ty, InputTy};
1808 case NEON::BI__builtin_neon_vusmmlaq_s32: {
1810 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
1811 llvm::Type *Tys[2] = { Ty, InputTy };
1814 case NEON::BI__builtin_neon_vusdot_s32:
1815 case NEON::BI__builtin_neon_vusdotq_s32: {
1817 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
1818 llvm::Type *Tys[2] = { Ty, InputTy };
1821 case NEON::BI__builtin_neon_vbfdot_f32:
1822 case NEON::BI__builtin_neon_vbfdotq_f32: {
1823 llvm::Type *InputTy =
1824 llvm::FixedVectorType::get(
BFloatTy, Ty->getPrimitiveSizeInBits() / 16);
1825 llvm::Type *Tys[2] = { Ty, InputTy };
1828 case NEON::BI__builtin_neon___a32_vcvt_bf16_f32: {
1829 llvm::Type *Tys[1] = { Ty };
1836 assert(
Int &&
"Expected valid intrinsic number");
2192 llvm::Triple::ArchType
Arch) {
2193 if (
auto Hint = GetValueForARMHint(BuiltinID))
2196 if (BuiltinID == clang::ARM::BI__emit) {
2198 llvm::FunctionType *FTy =
2199 llvm::FunctionType::get(
VoidTy,
false);
2203 llvm_unreachable(
"Sema will ensure that the parameter is constant");
2206 uint64_t ZExtValue =
Value.zextOrTrunc(IsThumb ? 16 : 32).getZExtValue();
2208 llvm::InlineAsm *Emit =
2209 IsThumb ? InlineAsm::get(FTy,
".inst.n 0x" + utohexstr(ZExtValue),
"",
2211 : InlineAsm::get(FTy,
".inst 0x" + utohexstr(ZExtValue),
"",
2214 return Builder.CreateCall(Emit);
2217 if (BuiltinID == clang::ARM::BI__builtin_arm_dbg) {
2219 return Builder.CreateCall(
CGM.getIntrinsic(Intrinsic::arm_dbg), Option);
2222 if (BuiltinID == clang::ARM::BI__builtin_arm_prefetch) {
2234 if (BuiltinID == clang::ARM::BI__builtin_arm_rbit) {
2237 CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
2240 if (BuiltinID == clang::ARM::BI__builtin_arm_clz ||
2241 BuiltinID == clang::ARM::BI__builtin_arm_clz64) {
2243 Function *F =
CGM.getIntrinsic(Intrinsic::ctlz, Arg->getType());
2245 if (BuiltinID == clang::ARM::BI__builtin_arm_clz64)
2251 if (BuiltinID == clang::ARM::BI__builtin_arm_cls) {
2253 return Builder.CreateCall(
CGM.getIntrinsic(Intrinsic::arm_cls), Arg,
"cls");
2255 if (BuiltinID == clang::ARM::BI__builtin_arm_cls64) {
2257 return Builder.CreateCall(
CGM.getIntrinsic(Intrinsic::arm_cls64), Arg,
2261 if (BuiltinID == clang::ARM::BI__clear_cache) {
2264 Function *F =
CGM.getIntrinsic(Intrinsic::clear_cache, {
CGM.DefaultPtrTy});
2265 return Builder.CreateCall(F, {Begin, End});
2268 if (BuiltinID == clang::ARM::BI__builtin_arm_mcrr ||
2269 BuiltinID == clang::ARM::BI__builtin_arm_mcrr2) {
2272 switch (BuiltinID) {
2273 default: llvm_unreachable(
"unexpected builtin");
2274 case clang::ARM::BI__builtin_arm_mcrr:
2275 F =
CGM.getIntrinsic(Intrinsic::arm_mcrr);
2277 case clang::ARM::BI__builtin_arm_mcrr2:
2278 F =
CGM.getIntrinsic(Intrinsic::arm_mcrr2);
2299 return Builder.CreateCall(F, {Coproc, Opc1, Rt, Rt2, CRm});
2302 if (BuiltinID == clang::ARM::BI__builtin_arm_mrrc ||
2303 BuiltinID == clang::ARM::BI__builtin_arm_mrrc2) {
2306 switch (BuiltinID) {
2307 default: llvm_unreachable(
"unexpected builtin");
2308 case clang::ARM::BI__builtin_arm_mrrc:
2309 F =
CGM.getIntrinsic(Intrinsic::arm_mrrc);
2311 case clang::ARM::BI__builtin_arm_mrrc2:
2312 F =
CGM.getIntrinsic(Intrinsic::arm_mrrc2);
2319 Value *RtAndRt2 =
Builder.CreateCall(F, {Coproc, Opc1, CRm});
2329 Value *ShiftCast = llvm::ConstantInt::get(
Int64Ty, 32);
2330 RtAndRt2 =
Builder.CreateShl(Rt, ShiftCast,
"shl",
true);
2331 RtAndRt2 =
Builder.CreateOr(RtAndRt2, Rt1);
2336 if (BuiltinID == clang::ARM::BI__builtin_arm_ldrexd ||
2337 ((BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
2338 BuiltinID == clang::ARM::BI__builtin_arm_ldaex) &&
2340 BuiltinID == clang::ARM::BI__ldrexd) {
2343 switch (BuiltinID) {
2344 default: llvm_unreachable(
"unexpected builtin");
2345 case clang::ARM::BI__builtin_arm_ldaex:
2346 F =
CGM.getIntrinsic(Intrinsic::arm_ldaexd);
2348 case clang::ARM::BI__builtin_arm_ldrexd:
2349 case clang::ARM::BI__builtin_arm_ldrex:
2350 case clang::ARM::BI__ldrexd:
2351 F =
CGM.getIntrinsic(Intrinsic::arm_ldrexd);
2364 Val =
Builder.CreateShl(Val0, ShiftCst,
"shl",
true );
2365 Val =
Builder.CreateOr(Val, Val1);
2369 if (BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
2370 BuiltinID == clang::ARM::BI__builtin_arm_ldaex) {
2379 BuiltinID == clang::ARM::BI__builtin_arm_ldaex ? Intrinsic::arm_ldaex
2380 : Intrinsic::arm_ldrex,
2382 CallInst *Val =
Builder.CreateCall(F, LoadAddr,
"ldrex");
2386 if (RealResTy->isPointerTy())
2387 return Builder.CreateIntToPtr(Val, RealResTy);
2389 llvm::Type *IntResTy = llvm::IntegerType::get(
2391 return Builder.CreateBitCast(
Builder.CreateTruncOrBitCast(Val, IntResTy),
2396 if (BuiltinID == clang::ARM::BI__builtin_arm_strexd ||
2397 ((BuiltinID == clang::ARM::BI__builtin_arm_stlex ||
2398 BuiltinID == clang::ARM::BI__builtin_arm_strex) &&
2401 BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlexd
2402 : Intrinsic::arm_strexd);
2407 Builder.CreateStore(Val, Tmp);
2410 Val =
Builder.CreateLoad(LdPtr);
2415 return Builder.CreateCall(F, {Arg0, Arg1, StPtr},
"strexd");
2418 if (BuiltinID == clang::ARM::BI__builtin_arm_strex ||
2419 BuiltinID == clang::ARM::BI__builtin_arm_stlex) {
2424 llvm::Type *StoreTy =
2427 if (StoreVal->
getType()->isPointerTy())
2430 llvm::Type *
IntTy = llvm::IntegerType::get(
2432 CGM.getDataLayout().getTypeSizeInBits(StoreVal->
getType()));
2438 BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlex
2439 : Intrinsic::arm_strex,
2442 CallInst *CI =
Builder.CreateCall(F, {StoreVal, StoreAddr},
"strex");
2444 1, Attribute::get(
getLLVMContext(), Attribute::ElementType, StoreTy));
2448 if (BuiltinID == clang::ARM::BI__builtin_arm_clrex) {
2449 Function *F =
CGM.getIntrinsic(Intrinsic::arm_clrex);
2454 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
2455 switch (BuiltinID) {
2456 case clang::ARM::BI__builtin_arm_crc32b:
2457 CRCIntrinsicID = Intrinsic::arm_crc32b;
break;
2458 case clang::ARM::BI__builtin_arm_crc32cb:
2459 CRCIntrinsicID = Intrinsic::arm_crc32cb;
break;
2460 case clang::ARM::BI__builtin_arm_crc32h:
2461 CRCIntrinsicID = Intrinsic::arm_crc32h;
break;
2462 case clang::ARM::BI__builtin_arm_crc32ch:
2463 CRCIntrinsicID = Intrinsic::arm_crc32ch;
break;
2464 case clang::ARM::BI__builtin_arm_crc32w:
2465 case clang::ARM::BI__builtin_arm_crc32d:
2466 CRCIntrinsicID = Intrinsic::arm_crc32w;
break;
2467 case clang::ARM::BI__builtin_arm_crc32cw:
2468 case clang::ARM::BI__builtin_arm_crc32cd:
2469 CRCIntrinsicID = Intrinsic::arm_crc32cw;
break;
2472 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
2478 if (BuiltinID == clang::ARM::BI__builtin_arm_crc32d ||
2479 BuiltinID == clang::ARM::BI__builtin_arm_crc32cd) {
2487 return Builder.CreateCall(F, {Res, Arg1b});
2492 return Builder.CreateCall(F, {Arg0, Arg1});
2496 if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
2497 BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
2498 BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
2499 BuiltinID == clang::ARM::BI__builtin_arm_wsr ||
2500 BuiltinID == clang::ARM::BI__builtin_arm_wsr64 ||
2501 BuiltinID == clang::ARM::BI__builtin_arm_wsrp) {
2504 if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
2505 BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
2506 BuiltinID == clang::ARM::BI__builtin_arm_rsrp)
2509 bool IsPointerBuiltin = BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
2510 BuiltinID == clang::ARM::BI__builtin_arm_wsrp;
2512 bool Is64Bit = BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
2513 BuiltinID == clang::ARM::BI__builtin_arm_wsr64;
2515 llvm::Type *ValueType;
2517 if (IsPointerBuiltin) {
2520 }
else if (Is64Bit) {
2530 if (BuiltinID == ARM::BI__builtin_sponentry) {
2549 return P.first == BuiltinID;
2552 BuiltinID = It->second;
2556 unsigned ICEArguments = 0;
2561 auto getAlignmentValue32 = [&](
Address addr) ->
Value* {
2562 return Builder.getInt32(addr.getAlignment().getQuantity());
2569 unsigned NumArgs = E->
getNumArgs() - (HasExtraArg ? 1 : 0);
2570 for (
unsigned i = 0, e = NumArgs; i != e; i++) {
2572 switch (BuiltinID) {
2573 case NEON::BI__builtin_neon_vld1_v:
2574 case NEON::BI__builtin_neon_vld1q_v:
2575 case NEON::BI__builtin_neon_vld1q_lane_v:
2576 case NEON::BI__builtin_neon_vld1_lane_v:
2577 case NEON::BI__builtin_neon_vld1_dup_v:
2578 case NEON::BI__builtin_neon_vld1q_dup_v:
2579 case NEON::BI__builtin_neon_vst1_v:
2580 case NEON::BI__builtin_neon_vst1q_v:
2581 case NEON::BI__builtin_neon_vst1q_lane_v:
2582 case NEON::BI__builtin_neon_vst1_lane_v:
2583 case NEON::BI__builtin_neon_vst2_v:
2584 case NEON::BI__builtin_neon_vst2q_v:
2585 case NEON::BI__builtin_neon_vst2_lane_v:
2586 case NEON::BI__builtin_neon_vst2q_lane_v:
2587 case NEON::BI__builtin_neon_vst3_v:
2588 case NEON::BI__builtin_neon_vst3q_v:
2589 case NEON::BI__builtin_neon_vst3_lane_v:
2590 case NEON::BI__builtin_neon_vst3q_lane_v:
2591 case NEON::BI__builtin_neon_vst4_v:
2592 case NEON::BI__builtin_neon_vst4q_v:
2593 case NEON::BI__builtin_neon_vst4_lane_v:
2594 case NEON::BI__builtin_neon_vst4q_lane_v:
2603 switch (BuiltinID) {
2604 case NEON::BI__builtin_neon_vld2_v:
2605 case NEON::BI__builtin_neon_vld2q_v:
2606 case NEON::BI__builtin_neon_vld3_v:
2607 case NEON::BI__builtin_neon_vld3q_v:
2608 case NEON::BI__builtin_neon_vld4_v:
2609 case NEON::BI__builtin_neon_vld4q_v:
2610 case NEON::BI__builtin_neon_vld2_lane_v:
2611 case NEON::BI__builtin_neon_vld2q_lane_v:
2612 case NEON::BI__builtin_neon_vld3_lane_v:
2613 case NEON::BI__builtin_neon_vld3q_lane_v:
2614 case NEON::BI__builtin_neon_vld4_lane_v:
2615 case NEON::BI__builtin_neon_vld4q_lane_v:
2616 case NEON::BI__builtin_neon_vld2_dup_v:
2617 case NEON::BI__builtin_neon_vld2q_dup_v:
2618 case NEON::BI__builtin_neon_vld3_dup_v:
2619 case NEON::BI__builtin_neon_vld3q_dup_v:
2620 case NEON::BI__builtin_neon_vld4_dup_v:
2621 case NEON::BI__builtin_neon_vld4q_dup_v:
2633 switch (BuiltinID) {
2636 case NEON::BI__builtin_neon_vget_lane_i8:
2637 case NEON::BI__builtin_neon_vget_lane_i16:
2638 case NEON::BI__builtin_neon_vget_lane_i32:
2639 case NEON::BI__builtin_neon_vget_lane_i64:
2640 case NEON::BI__builtin_neon_vget_lane_bf16:
2641 case NEON::BI__builtin_neon_vget_lane_f32:
2642 case NEON::BI__builtin_neon_vgetq_lane_i8:
2643 case NEON::BI__builtin_neon_vgetq_lane_i16:
2644 case NEON::BI__builtin_neon_vgetq_lane_i32:
2645 case NEON::BI__builtin_neon_vgetq_lane_i64:
2646 case NEON::BI__builtin_neon_vgetq_lane_bf16:
2647 case NEON::BI__builtin_neon_vgetq_lane_f32:
2648 case NEON::BI__builtin_neon_vduph_lane_bf16:
2649 case NEON::BI__builtin_neon_vduph_laneq_bf16:
2650 return Builder.CreateExtractElement(Ops[0], Ops[1],
"vget_lane");
2652 case NEON::BI__builtin_neon_vrndns_f32: {
2654 llvm::Type *Tys[] = {Arg->
getType()};
2655 Function *F =
CGM.getIntrinsic(Intrinsic::roundeven, Tys);
2656 return Builder.CreateCall(F, {Arg},
"vrndn"); }
2658 case NEON::BI__builtin_neon_vset_lane_i8:
2659 case NEON::BI__builtin_neon_vset_lane_i16:
2660 case NEON::BI__builtin_neon_vset_lane_i32:
2661 case NEON::BI__builtin_neon_vset_lane_i64:
2662 case NEON::BI__builtin_neon_vset_lane_bf16:
2663 case NEON::BI__builtin_neon_vset_lane_f32:
2664 case NEON::BI__builtin_neon_vsetq_lane_i8:
2665 case NEON::BI__builtin_neon_vsetq_lane_i16:
2666 case NEON::BI__builtin_neon_vsetq_lane_i32:
2667 case NEON::BI__builtin_neon_vsetq_lane_i64:
2668 case NEON::BI__builtin_neon_vsetq_lane_bf16:
2669 case NEON::BI__builtin_neon_vsetq_lane_f32:
2670 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
2672 case NEON::BI__builtin_neon_vsha1h_u32:
2675 case NEON::BI__builtin_neon_vsha1cq_u32:
2678 case NEON::BI__builtin_neon_vsha1pq_u32:
2681 case NEON::BI__builtin_neon_vsha1mq_u32:
2685 case NEON::BI__builtin_neon_vcvth_bf16_f32:
2686 return EmitNeonCall(
CGM.getIntrinsic(Intrinsic::arm_neon_vcvtbfp2bf), Ops,
2688 case NEON::BI__builtin_neon_vcvt_f16_f32:
2689 return EmitNeonCall(
CGM.getIntrinsic(Intrinsic::arm_neon_vcvtfp2hf), Ops,
2691 case NEON::BI__builtin_neon_vcvt_f32_f16:
2692 return EmitNeonCall(
CGM.getIntrinsic(Intrinsic::arm_neon_vcvthf2fp), Ops,
2697 case clang::ARM::BI_MoveToCoprocessor:
2698 case clang::ARM::BI_MoveToCoprocessor2: {
2699 Function *F =
CGM.getIntrinsic(BuiltinID == clang::ARM::BI_MoveToCoprocessor
2700 ? Intrinsic::arm_mcr
2701 : Intrinsic::arm_mcr2);
2702 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0],
2703 Ops[3], Ops[4], Ops[5]});
2708 assert(HasExtraArg);
2710 std::optional<llvm::APSInt>
Result =
2715 if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f ||
2716 BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_d) {
2719 if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f)
2725 bool usgn =
Result->getZExtValue() == 1;
2726 unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr;
2730 return Builder.CreateCall(F, Ops,
"vcvtr");
2735 bool usgn =
Type.isUnsigned();
2736 bool rightShift =
false;
2738 llvm::FixedVectorType *VTy =
2741 llvm::Type *Ty = VTy;
2756 switch (BuiltinID) {
2757 default:
return nullptr;
2758 case NEON::BI__builtin_neon_vld1q_lane_v:
2761 if (VTy->getElementType()->isIntegerTy(64)) {
2763 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
2765 Value *SV = llvm::ConstantVector::get(ConstantInt::get(
Int32Ty, 1-Lane));
2766 Ops[1] =
Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
2768 Ty = llvm::FixedVectorType::get(VTy->getElementType(), 1);
2770 Function *F =
CGM.getIntrinsic(Intrinsic::arm_neon_vld1, Tys);
2771 Value *Align = getAlignmentValue32(PtrOp0);
2774 int Indices[] = {1 - Lane, Lane};
2775 return Builder.CreateShuffleVector(Ops[1], Ld, Indices,
"vld1q_lane");
2778 case NEON::BI__builtin_neon_vld1_lane_v: {
2779 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
2782 return Builder.CreateInsertElement(Ops[1], Ld, Ops[2],
"vld1_lane");
2784 case NEON::BI__builtin_neon_vqrshrn_n_v:
2786 usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns;
2789 case NEON::BI__builtin_neon_vqrshrun_n_v:
2790 return EmitNeonCall(
CGM.getIntrinsic(Intrinsic::arm_neon_vqrshiftnsu, Ty),
2791 Ops,
"vqrshrun_n", 1,
true);
2792 case NEON::BI__builtin_neon_vqshrn_n_v:
2793 Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns;
2796 case NEON::BI__builtin_neon_vqshrun_n_v:
2797 return EmitNeonCall(
CGM.getIntrinsic(Intrinsic::arm_neon_vqshiftnsu, Ty),
2798 Ops,
"vqshrun_n", 1,
true);
2799 case NEON::BI__builtin_neon_vrecpe_v:
2800 case NEON::BI__builtin_neon_vrecpeq_v:
2803 case NEON::BI__builtin_neon_vrshrn_n_v:
2804 return EmitNeonCall(
CGM.getIntrinsic(Intrinsic::arm_neon_vrshiftn, Ty),
2805 Ops,
"vrshrn_n", 1,
true);
2806 case NEON::BI__builtin_neon_vrsra_n_v:
2807 case NEON::BI__builtin_neon_vrsraq_n_v:
2808 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
2809 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
2811 Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
2812 Ops[1] =
Builder.CreateCall(
CGM.getIntrinsic(
Int, Ty), {Ops[1], Ops[2]});
2813 return Builder.CreateAdd(Ops[0], Ops[1],
"vrsra_n");
2814 case NEON::BI__builtin_neon_vsri_n_v:
2815 case NEON::BI__builtin_neon_vsriq_n_v:
2818 case NEON::BI__builtin_neon_vsli_n_v:
2819 case NEON::BI__builtin_neon_vsliq_n_v:
2821 return EmitNeonCall(
CGM.getIntrinsic(Intrinsic::arm_neon_vshiftins, Ty),
2823 case NEON::BI__builtin_neon_vsra_n_v:
2824 case NEON::BI__builtin_neon_vsraq_n_v:
2825 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
2827 return Builder.CreateAdd(Ops[0], Ops[1]);
2828 case NEON::BI__builtin_neon_vst1q_lane_v:
2831 if (VTy->getElementType()->isIntegerTy(64)) {
2832 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
2834 Ops[1] =
Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
2835 Ops[2] = getAlignmentValue32(PtrOp0);
2836 llvm::Type *Tys[] = {
Int8PtrTy, Ops[1]->getType()};
2837 return Builder.CreateCall(
CGM.getIntrinsic(Intrinsic::arm_neon_vst1,
2841 case NEON::BI__builtin_neon_vst1_lane_v: {
2842 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
2843 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2]);
2844 return Builder.CreateStore(Ops[1],
2847 case NEON::BI__builtin_neon_vtbl1_v:
2850 case NEON::BI__builtin_neon_vtbl2_v:
2853 case NEON::BI__builtin_neon_vtbl3_v:
2856 case NEON::BI__builtin_neon_vtbl4_v:
2859 case NEON::BI__builtin_neon_vtbx1_v:
2862 case NEON::BI__builtin_neon_vtbx2_v:
2865 case NEON::BI__builtin_neon_vtbx3_v:
2868 case NEON::BI__builtin_neon_vtbx4_v:
4499 llvm::Triple::ArchType
Arch) {
4508 if (BuiltinID == Builtin::BI__builtin_cpu_supports)
4509 return EmitAArch64CpuSupports(E);
4511 unsigned HintID =
static_cast<unsigned>(-1);
4512 switch (BuiltinID) {
4514 case clang::AArch64::BI__builtin_arm_nop:
4517 case clang::AArch64::BI__builtin_arm_yield:
4518 case clang::AArch64::BI__yield:
4521 case clang::AArch64::BI__builtin_arm_wfe:
4522 case clang::AArch64::BI__wfe:
4525 case clang::AArch64::BI__builtin_arm_wfi:
4526 case clang::AArch64::BI__wfi:
4529 case clang::AArch64::BI__builtin_arm_sev:
4530 case clang::AArch64::BI__sev:
4533 case clang::AArch64::BI__builtin_arm_sevl:
4534 case clang::AArch64::BI__sevl:
4539 if (HintID !=
static_cast<unsigned>(-1)) {
4540 Function *F =
CGM.getIntrinsic(Intrinsic::aarch64_hint);
4541 return Builder.CreateCall(F, llvm::ConstantInt::get(
Int32Ty, HintID));
4544 if (BuiltinID == clang::AArch64::BI__builtin_arm_trap) {
4545 Function *F =
CGM.getIntrinsic(Intrinsic::aarch64_break);
4550 if (BuiltinID == clang::AArch64::BI__builtin_arm_get_sme_state) {
4553 llvm::FunctionType::get(StructType::get(
CGM.Int64Ty,
CGM.Int64Ty), {},
4555 "__arm_sme_state"));
4557 "aarch64_pstate_sm_compatible");
4558 CI->setAttributes(Attrs);
4561 AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2);
4568 if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit) {
4570 "rbit of unusual size!");
4573 CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
4575 if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit64) {
4577 "rbit of unusual size!");
4580 CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
4583 if (BuiltinID == clang::AArch64::BI__builtin_arm_clz ||
4584 BuiltinID == clang::AArch64::BI__builtin_arm_clz64) {
4586 Function *F =
CGM.getIntrinsic(Intrinsic::ctlz, Arg->getType());
4588 if (BuiltinID == clang::AArch64::BI__builtin_arm_clz64)
4593 if (BuiltinID == clang::AArch64::BI__builtin_arm_cls) {
4595 return Builder.CreateCall(
CGM.getIntrinsic(Intrinsic::aarch64_cls), Arg,
4598 if (BuiltinID == clang::AArch64::BI__builtin_arm_cls64) {
4600 return Builder.CreateCall(
CGM.getIntrinsic(Intrinsic::aarch64_cls64), Arg,
4604 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint32zf ||
4605 BuiltinID == clang::AArch64::BI__builtin_arm_rint32z) {
4607 llvm::Type *Ty = Arg->getType();
4608 return Builder.CreateCall(
CGM.getIntrinsic(Intrinsic::aarch64_frint32z, Ty),
4612 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint64zf ||
4613 BuiltinID == clang::AArch64::BI__builtin_arm_rint64z) {
4615 llvm::Type *Ty = Arg->getType();
4616 return Builder.CreateCall(
CGM.getIntrinsic(Intrinsic::aarch64_frint64z, Ty),
4620 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint32xf ||
4621 BuiltinID == clang::AArch64::BI__builtin_arm_rint32x) {
4623 llvm::Type *Ty = Arg->getType();
4624 return Builder.CreateCall(
CGM.getIntrinsic(Intrinsic::aarch64_frint32x, Ty),
4628 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint64xf ||
4629 BuiltinID == clang::AArch64::BI__builtin_arm_rint64x) {
4631 llvm::Type *Ty = Arg->getType();
4632 return Builder.CreateCall(
CGM.getIntrinsic(Intrinsic::aarch64_frint64x, Ty),
4636 if (BuiltinID == clang::AArch64::BI__builtin_arm_jcvt) {
4638 "__jcvt of unusual size!");
4641 CGM.getIntrinsic(Intrinsic::aarch64_fjcvtzs), Arg);
4644 if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b ||
4645 BuiltinID == clang::AArch64::BI__builtin_arm_st64b ||
4646 BuiltinID == clang::AArch64::BI__builtin_arm_st64bv ||
4647 BuiltinID == clang::AArch64::BI__builtin_arm_st64bv0) {
4651 if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b) {
4654 Function *F =
CGM.getIntrinsic(Intrinsic::aarch64_ld64b);
4655 llvm::Value *Val =
Builder.CreateCall(F, MemAddr);
4657 for (
size_t i = 0; i < 8; i++) {
4658 llvm::Value *ValOffsetPtr =
4670 Args.push_back(MemAddr);
4671 for (
size_t i = 0; i < 8; i++) {
4672 llvm::Value *ValOffsetPtr =
4678 auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_st64b
4679 ? Intrinsic::aarch64_st64b
4680 : BuiltinID == clang::AArch64::BI__builtin_arm_st64bv
4681 ? Intrinsic::aarch64_st64bv
4682 : Intrinsic::aarch64_st64bv0);
4684 return Builder.CreateCall(F, Args);
4687 if (BuiltinID == clang::AArch64::BI__builtin_arm_rndr ||
4688 BuiltinID == clang::AArch64::BI__builtin_arm_rndrrs) {
4690 auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_rndr
4691 ? Intrinsic::aarch64_rndr
4692 : Intrinsic::aarch64_rndrrs);
4694 llvm::Value *Val =
Builder.CreateCall(F);
4695 Value *RandomValue =
Builder.CreateExtractValue(Val, 0);
4699 Builder.CreateStore(RandomValue, MemAddress);
4704 if (BuiltinID == clang::AArch64::BI__clear_cache) {
4707 Function *F =
CGM.getIntrinsic(Intrinsic::clear_cache, {
CGM.DefaultPtrTy});
4708 return Builder.CreateCall(F, {Begin, End});
4711 if ((BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
4712 BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) &&
4715 CGM.getIntrinsic(BuiltinID == clang::AArch64::BI__builtin_arm_ldaex
4716 ? Intrinsic::aarch64_ldaxp
4717 : Intrinsic::aarch64_ldxp);
4724 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
4725 Val0 =
Builder.CreateZExt(Val0, Int128Ty);
4726 Val1 =
Builder.CreateZExt(Val1, Int128Ty);
4728 Value *ShiftCst = llvm::ConstantInt::get(Int128Ty, 64);
4729 Val =
Builder.CreateShl(Val0, ShiftCst,
"shl",
true );
4730 Val =
Builder.CreateOr(Val, Val1);
4732 }
else if (BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
4733 BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) {
4742 CGM.getIntrinsic(BuiltinID == clang::AArch64::BI__builtin_arm_ldaex
4743 ? Intrinsic::aarch64_ldaxr
4744 : Intrinsic::aarch64_ldxr,
4746 CallInst *Val =
Builder.CreateCall(F, LoadAddr,
"ldxr");
4750 if (RealResTy->isPointerTy())
4751 return Builder.CreateIntToPtr(Val, RealResTy);
4753 llvm::Type *IntResTy = llvm::IntegerType::get(
4755 return Builder.CreateBitCast(
Builder.CreateTruncOrBitCast(Val, IntResTy),
4759 if ((BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
4760 BuiltinID == clang::AArch64::BI__builtin_arm_stlex) &&
4763 CGM.getIntrinsic(BuiltinID == clang::AArch64::BI__builtin_arm_stlex
4764 ? Intrinsic::aarch64_stlxp
4765 : Intrinsic::aarch64_stxp);
4772 llvm::Value *Val =
Builder.CreateLoad(Tmp);
4777 return Builder.CreateCall(F, {Arg0, Arg1, StPtr},
"stxp");
4780 if (BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
4781 BuiltinID == clang::AArch64::BI__builtin_arm_stlex) {
4786 llvm::Type *StoreTy =
4789 if (StoreVal->
getType()->isPointerTy())
4792 llvm::Type *
IntTy = llvm::IntegerType::get(
4794 CGM.getDataLayout().getTypeSizeInBits(StoreVal->
getType()));
4800 CGM.getIntrinsic(BuiltinID == clang::AArch64::BI__builtin_arm_stlex
4801 ? Intrinsic::aarch64_stlxr
4802 : Intrinsic::aarch64_stxr,
4804 CallInst *CI =
Builder.CreateCall(F, {StoreVal, StoreAddr},
"stxr");
4806 1, Attribute::get(
getLLVMContext(), Attribute::ElementType, StoreTy));
4810 if (BuiltinID == clang::AArch64::BI__getReg ||
4811 BuiltinID == clang::AArch64::BI__setReg) {
4814 llvm_unreachable(
"Sema will ensure that the parameter is constant");
4817 LLVMContext &Context =
CGM.getLLVMContext();
4820 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, Reg)};
4821 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
4822 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
4825 if (BuiltinID == clang::AArch64::BI__getReg) {
4827 CGM.getIntrinsic(Intrinsic::read_volatile_register, {
Int64Ty});
4828 CI =
Builder.CreateCall(F, Metadata);
4831 CGM.getIntrinsic(Intrinsic::write_volatile_register, {
Int64Ty});
4837 if (BuiltinID == clang::AArch64::BI__getRegFp ||
4838 BuiltinID == clang::AArch64::BI__setRegFp) {
4841 llvm_unreachable(
"Sema will ensure that the parameter is constant");
4844 LLVMContext &Context =
CGM.getLLVMContext();
4847 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, Reg)};
4848 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
4849 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
4852 if (BuiltinID == clang::AArch64::BI__getRegFp) {
4854 CGM.getIntrinsic(Intrinsic::read_volatile_register, {
Int64Ty});
4855 llvm::Value *Bits =
Builder.CreateCall(F, Metadata);
4856 Ret =
Builder.CreateBitCast(Bits, llvm::Type::getDoubleTy(Context));
4861 CGM.getIntrinsic(Intrinsic::write_volatile_register, {
Int64Ty});
4862 Ret =
Builder.CreateCall(F, {Metadata, Bits});
4867 if (BuiltinID == clang::AArch64::BI__break) {
4870 llvm_unreachable(
"Sema will ensure that the parameter is constant");
4872 llvm::Function *F =
CGM.getIntrinsic(Intrinsic::aarch64_break);
4876 if (BuiltinID == clang::AArch64::BI__builtin_arm_clrex) {
4877 Function *F =
CGM.getIntrinsic(Intrinsic::aarch64_clrex);
4882 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
4883 switch (BuiltinID) {
4884 case clang::AArch64::BI__builtin_arm_crc32b:
4885 CRCIntrinsicID = Intrinsic::aarch64_crc32b;
break;
4886 case clang::AArch64::BI__builtin_arm_crc32cb:
4887 CRCIntrinsicID = Intrinsic::aarch64_crc32cb;
break;
4888 case clang::AArch64::BI__builtin_arm_crc32h:
4889 CRCIntrinsicID = Intrinsic::aarch64_crc32h;
break;
4890 case clang::AArch64::BI__builtin_arm_crc32ch:
4891 CRCIntrinsicID = Intrinsic::aarch64_crc32ch;
break;
4892 case clang::AArch64::BI__builtin_arm_crc32w:
4893 CRCIntrinsicID = Intrinsic::aarch64_crc32w;
break;
4894 case clang::AArch64::BI__builtin_arm_crc32cw:
4895 CRCIntrinsicID = Intrinsic::aarch64_crc32cw;
break;
4896 case clang::AArch64::BI__builtin_arm_crc32d:
4897 CRCIntrinsicID = Intrinsic::aarch64_crc32x;
break;
4898 case clang::AArch64::BI__builtin_arm_crc32cd:
4899 CRCIntrinsicID = Intrinsic::aarch64_crc32cx;
break;
4902 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
4907 llvm::Type *DataTy = F->getFunctionType()->getParamType(1);
4908 Arg1 =
Builder.CreateZExtOrBitCast(Arg1, DataTy);
4910 return Builder.CreateCall(F, {Arg0, Arg1});
4914 if (BuiltinID == AArch64::BI__builtin_arm_mops_memset_tag) {
4921 CGM.getIntrinsic(Intrinsic::aarch64_mops_memset_tag), {Dst, Val, Size});
4924 if (BuiltinID == AArch64::BI__builtin_arm_range_prefetch ||
4925 BuiltinID == AArch64::BI__builtin_arm_range_prefetch_x)
4928 switch (BuiltinID) {
4931 case AArch64::BI__builtin_arm_atomic_store_with_hint:
4933 case AArch64::BI__builtin_arm_atomic_fetch_add_with_hint:
4935 case AArch64::BI__builtin_arm_atomic_fetch_sub_with_hint:
4937 case AArch64::BI__builtin_arm_atomic_fetch_and_with_hint:
4939 case AArch64::BI__builtin_arm_atomic_fetch_xor_with_hint:
4941 case AArch64::BI__builtin_arm_atomic_fetch_or_with_hint:
4946 Intrinsic::ID MTEIntrinsicID = Intrinsic::not_intrinsic;
4947 switch (BuiltinID) {
4948 case clang::AArch64::BI__builtin_arm_irg:
4949 MTEIntrinsicID = Intrinsic::aarch64_irg;
break;
4950 case clang::AArch64::BI__builtin_arm_addg:
4951 MTEIntrinsicID = Intrinsic::aarch64_addg;
break;
4952 case clang::AArch64::BI__builtin_arm_gmi:
4953 MTEIntrinsicID = Intrinsic::aarch64_gmi;
break;
4954 case clang::AArch64::BI__builtin_arm_ldg:
4955 MTEIntrinsicID = Intrinsic::aarch64_ldg;
break;
4956 case clang::AArch64::BI__builtin_arm_stg:
4957 MTEIntrinsicID = Intrinsic::aarch64_stg;
break;
4958 case clang::AArch64::BI__builtin_arm_subp:
4959 MTEIntrinsicID = Intrinsic::aarch64_subp;
break;
4962 if (BuiltinID == clang::AArch64::BI__arm_set_fpm_lscale) {
4970 return Builder.CreateOr(MaskedFPM, ShiftedScale);
4972 if (BuiltinID == clang::AArch64::BI__arm_set_fpm_nscale) {
4980 return Builder.CreateOr(MaskedFPM, ShiftedScale);
4982 if (BuiltinID == clang::AArch64::BI__arm_set_fpm_lscale2) {
4990 return Builder.CreateOr(LowFPM, ShiftedScale);
4993 if (MTEIntrinsicID != Intrinsic::not_intrinsic) {
4994 if (MTEIntrinsicID == Intrinsic::aarch64_irg) {
4997 assert(Mask->
getType()->getScalarSizeInBits() == 64 &&
4998 "SemaARM::BuiltinARMMemoryTaggingCall() enforces this");
4999 return Builder.CreateCall(
CGM.getIntrinsic(MTEIntrinsicID),
5002 if (MTEIntrinsicID == Intrinsic::aarch64_addg) {
5007 return Builder.CreateCall(
CGM.getIntrinsic(MTEIntrinsicID),
5008 {Pointer, TagOffset});
5010 if (MTEIntrinsicID == Intrinsic::aarch64_gmi) {
5013 assert(ExcludedMask->
getType()->getScalarSizeInBits() == 64 &&
5014 "SemaARM::BuiltinARMMemoryTaggingCall() enforces this");
5015 return Builder.CreateCall(
CGM.getIntrinsic(MTEIntrinsicID),
5016 {Pointer, ExcludedMask});
5021 if (MTEIntrinsicID == Intrinsic::aarch64_ldg) {
5023 return Builder.CreateCall(
CGM.getIntrinsic(MTEIntrinsicID),
5024 {TagAddress, TagAddress});
5029 if (MTEIntrinsicID == Intrinsic::aarch64_stg) {
5031 return Builder.CreateCall(
CGM.getIntrinsic(MTEIntrinsicID),
5032 {TagAddress, TagAddress});
5034 if (MTEIntrinsicID == Intrinsic::aarch64_subp) {
5038 CGM.getIntrinsic(MTEIntrinsicID), {PointerA, PointerB});
5042 if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
5043 BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
5044 BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
5045 BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
5046 BuiltinID == clang::AArch64::BI__builtin_arm_wsr ||
5047 BuiltinID == clang::AArch64::BI__builtin_arm_wsr64 ||
5048 BuiltinID == clang::AArch64::BI__builtin_arm_wsr128 ||
5049 BuiltinID == clang::AArch64::BI__builtin_arm_wsrp) {
5052 if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
5053 BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
5054 BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
5055 BuiltinID == clang::AArch64::BI__builtin_arm_rsrp)
5058 bool IsPointerBuiltin = BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
5059 BuiltinID == clang::AArch64::BI__builtin_arm_wsrp;
5061 bool Is32Bit = BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
5062 BuiltinID == clang::AArch64::BI__builtin_arm_wsr;
5064 bool Is128Bit = BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
5065 BuiltinID == clang::AArch64::BI__builtin_arm_wsr128;
5067 llvm::Type *ValueType;
5071 }
else if (Is128Bit) {
5072 llvm::Type *Int128Ty =
5073 llvm::IntegerType::getInt128Ty(
CGM.getLLVMContext());
5074 ValueType = Int128Ty;
5076 }
else if (IsPointerBuiltin) {
5086 if (BuiltinID == clang::AArch64::BI_ReadStatusReg ||
5087 BuiltinID == clang::AArch64::BI_WriteStatusReg) {
5088 LLVMContext &Context =
CGM.getLLVMContext();
5093 std::string SysRegStr;
5094 llvm::raw_string_ostream(SysRegStr)
5095 << (0b10 | SysReg >> 14) <<
":" << ((SysReg >> 11) & 7) <<
":"
5096 << ((SysReg >> 7) & 15) <<
":" << ((SysReg >> 3) & 15) <<
":"
5099 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysRegStr) };
5100 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
5101 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
5106 if (BuiltinID == clang::AArch64::BI_ReadStatusReg) {
5107 llvm::Function *F =
CGM.getIntrinsic(Intrinsic::read_register, Types);
5109 return Builder.CreateCall(F, Metadata);
5112 llvm::Function *F =
CGM.getIntrinsic(Intrinsic::write_register, Types);
5114 llvm::Value *
Result =
Builder.CreateCall(F, {Metadata, ArgValue});
5119 if (BuiltinID == clang::AArch64::BI__sys) {
5122 const unsigned Op1 = SysReg >> 11;
5123 const unsigned CRn = (SysReg >> 7) & 0xf;
5124 const unsigned CRm = (SysReg >> 3) & 0xf;
5125 const unsigned Op2 = SysReg & 0x7;
5127 Builder.CreateCall(
CGM.getIntrinsic(Intrinsic::aarch64_sys),
5128 {Builder.getInt32(Op1), Builder.getInt32(CRn),
5129 Builder.getInt32(CRm), Builder.getInt32(Op2),
5130 EmitScalarExpr(E->getArg(1))});
5134 return ConstantInt::get(
Builder.getInt32Ty(), 0);
5137 if (BuiltinID == clang::AArch64::BI_AddressOfReturnAddress) {
5143 if (BuiltinID == clang::AArch64::BI__builtin_sponentry) {
5148 if (BuiltinID == clang::AArch64::BI__mulh ||
5149 BuiltinID == clang::AArch64::BI__umulh) {
5151 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
5153 bool IsSigned = BuiltinID == clang::AArch64::BI__mulh;
5159 Value *MulResult, *HigherBits;
5161 MulResult =
Builder.CreateNSWMul(LHS, RHS);
5162 HigherBits =
Builder.CreateAShr(MulResult, 64);
5164 MulResult =
Builder.CreateNUWMul(LHS, RHS);
5165 HigherBits =
Builder.CreateLShr(MulResult, 64);
5167 HigherBits =
Builder.CreateIntCast(HigherBits, ResType, IsSigned);
5172 if (BuiltinID == AArch64::BI__writex18byte ||
5173 BuiltinID == AArch64::BI__writex18word ||
5174 BuiltinID == AArch64::BI__writex18dword ||
5175 BuiltinID == AArch64::BI__writex18qword) {
5191 if (BuiltinID == AArch64::BI__readx18byte ||
5192 BuiltinID == AArch64::BI__readx18word ||
5193 BuiltinID == AArch64::BI__readx18dword ||
5194 BuiltinID == AArch64::BI__readx18qword) {
5209 if (BuiltinID == AArch64::BI__addx18byte ||
5210 BuiltinID == AArch64::BI__addx18word ||
5211 BuiltinID == AArch64::BI__addx18dword ||
5212 BuiltinID == AArch64::BI__addx18qword ||
5213 BuiltinID == AArch64::BI__incx18byte ||
5214 BuiltinID == AArch64::BI__incx18word ||
5215 BuiltinID == AArch64::BI__incx18dword ||
5216 BuiltinID == AArch64::BI__incx18qword) {
5219 switch (BuiltinID) {
5220 case AArch64::BI__incx18byte:
5224 case AArch64::BI__incx18word:
5228 case AArch64::BI__incx18dword:
5232 case AArch64::BI__incx18qword:
5238 isIncrement =
false;
5263 if (BuiltinID == AArch64::BI_CopyDoubleFromInt64 ||
5264 BuiltinID == AArch64::BI_CopyFloatFromInt32 ||
5265 BuiltinID == AArch64::BI_CopyInt32FromFloat ||
5266 BuiltinID == AArch64::BI_CopyInt64FromDouble) {
5269 return Builder.CreateBitCast(Arg, RetTy);
5272 if (BuiltinID == AArch64::BI_CountLeadingOnes ||
5273 BuiltinID == AArch64::BI_CountLeadingOnes64 ||
5274 BuiltinID == AArch64::BI_CountLeadingZeros ||
5275 BuiltinID == AArch64::BI_CountLeadingZeros64) {
5279 if (BuiltinID == AArch64::BI_CountLeadingOnes ||
5280 BuiltinID == AArch64::BI_CountLeadingOnes64)
5281 Arg =
Builder.CreateXor(Arg, Constant::getAllOnesValue(
ArgType));
5286 if (BuiltinID == AArch64::BI_CountLeadingOnes64 ||
5287 BuiltinID == AArch64::BI_CountLeadingZeros64)
5292 if (BuiltinID == AArch64::BI_CountLeadingSigns ||
5293 BuiltinID == AArch64::BI_CountLeadingSigns64) {
5296 Function *F = (BuiltinID == AArch64::BI_CountLeadingSigns)
5297 ?
CGM.getIntrinsic(Intrinsic::aarch64_cls)
5298 :
CGM.getIntrinsic(Intrinsic::aarch64_cls64);
5301 if (BuiltinID == AArch64::BI_CountLeadingSigns64)
5306 if (BuiltinID == AArch64::BI_CountOneBits ||
5307 BuiltinID == AArch64::BI_CountOneBits64) {
5313 if (BuiltinID == AArch64::BI_CountOneBits64)
5318 if (BuiltinID == AArch64::BI_CountTrailingZeros ||
5319 BuiltinID == AArch64::BI_CountTrailingZeros64) {
5326 if (BuiltinID == AArch64::BI_CountTrailingZeros64)
5331 if (BuiltinID == AArch64::BI__prefetch) {
5340 if (BuiltinID == AArch64::BI__prefetch2) {
5348 uint64_t Op = PrfOp.getZExtValue();
5349 uint64_t
Type = (Op >> 3) & 0x3;
5350 uint64_t
Target = (Op >> 1) & 0x3;
5351 uint64_t Policy = Op & 0x1;
5356 Function *F =
CGM.getIntrinsic(Intrinsic::aarch64_prefetch);
5360 if (BuiltinID == AArch64::BI__hlt) {
5361 Function *F =
CGM.getIntrinsic(Intrinsic::aarch64_hlt);
5367 return ConstantInt::get(
Builder.getInt32Ty(), 0);
5370 if (BuiltinID == AArch64::BI__hvc || BuiltinID == AArch64::BI__svc) {
5371 unsigned IID = BuiltinID == AArch64::BI__svc ? Intrinsic::aarch64_svc
5372 : Intrinsic::aarch64_hvc;
5380 for (
unsigned I = 1, N = E->
getNumArgs(); I < N; ++I) {
5382 llvm::Type *ArgTy = Arg->
getType();
5383 if (ArgTy->isPointerTy())
5385 else if (ArgTy->isFloatingPointTy())
5388 Arg =
Builder.CreateZExtOrTrunc(
5390 Arg,
Builder.getIntNTy(ArgTy->getPrimitiveSizeInBits())),
5395 Args.push_back(Arg);
5397 while (Args.size() < 5)
5398 Args.push_back(llvm::PoisonValue::get(
Int64Ty));
5404 if (BuiltinID == NEON::BI__builtin_neon_vcvth_bf16_f32)
5412 if (std::optional<MSVCIntrin> MsvcIntId =
5418 return P.first == BuiltinID;
5421 BuiltinID = It->second;
5427 bool IsSISD = (
Builtin !=
nullptr);
5431 unsigned ICEArguments = 0;
5442 unsigned NumArgs = E->
getNumArgs() - (HasExtraArg ? 1 : 0);
5443 for (
unsigned i = 0, e = NumArgs; i != e; i++) {
5445 switch (BuiltinID) {
5446 case NEON::BI__builtin_neon_vld1_v:
5447 case NEON::BI__builtin_neon_vld1q_v:
5448 case NEON::BI__builtin_neon_vld1_dup_v:
5449 case NEON::BI__builtin_neon_vld1q_dup_v:
5450 case NEON::BI__builtin_neon_vld1_lane_v:
5451 case NEON::BI__builtin_neon_vld1q_lane_v:
5452 case NEON::BI__builtin_neon_vst1_v:
5453 case NEON::BI__builtin_neon_vst1q_v:
5454 case NEON::BI__builtin_neon_vst1_lane_v:
5455 case NEON::BI__builtin_neon_vst1q_lane_v:
5456 case NEON::BI__builtin_neon_vldap1_lane_s64:
5457 case NEON::BI__builtin_neon_vldap1q_lane_s64:
5458 case NEON::BI__builtin_neon_vstl1_lane_s64:
5459 case NEON::BI__builtin_neon_vstl1q_lane_s64:
5472 assert(
Result &&
"SISD intrinsic should have been handled");
5478 if (std::optional<llvm::APSInt>
Result =
5483 bool usgn =
Type.isUnsigned();
5484 bool quad =
Type.isQuad();
5503 switch (BuiltinID) {
5505 case NEON::BI__builtin_neon_vabsh_f16:
5507 case NEON::BI__builtin_neon_vaddq_p128: {
5509 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
5510 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
5511 Ops[0] =
Builder.CreateXor(Ops[0], Ops[1]);
5512 llvm::Type *Int128Ty = llvm::Type::getIntNTy(
getLLVMContext(), 128);
5513 return Builder.CreateBitCast(Ops[0], Int128Ty);
5515 case NEON::BI__builtin_neon_vldrq_p128: {
5516 llvm::Type *Int128Ty = llvm::Type::getIntNTy(
getLLVMContext(), 128);
5517 return Builder.CreateAlignedLoad(Int128Ty, Ops[0],
5520 case NEON::BI__builtin_neon_vstrq_p128: {
5521 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
5523 case NEON::BI__builtin_neon_vcvts_f32_u32:
5524 case NEON::BI__builtin_neon_vcvtd_f64_u64:
5527 case NEON::BI__builtin_neon_vcvts_f32_s32:
5528 case NEON::BI__builtin_neon_vcvtd_f64_s64: {
5529 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
5532 Ops[0] =
Builder.CreateBitCast(Ops[0], InTy);
5534 return Builder.CreateUIToFP(Ops[0], FTy);
5535 return Builder.CreateSIToFP(Ops[0], FTy);
5537 case NEON::BI__builtin_neon_vcvth_f16_u16:
5538 case NEON::BI__builtin_neon_vcvth_f16_u32:
5539 case NEON::BI__builtin_neon_vcvth_f16_u64:
5542 case NEON::BI__builtin_neon_vcvth_f16_s16:
5543 case NEON::BI__builtin_neon_vcvth_f16_s32:
5544 case NEON::BI__builtin_neon_vcvth_f16_s64: {
5545 llvm::Type *FTy =
HalfTy;
5547 if (Ops[0]->
getType()->getPrimitiveSizeInBits() == 64)
5549 else if (Ops[0]->
getType()->getPrimitiveSizeInBits() == 32)
5553 Ops[0] =
Builder.CreateBitCast(Ops[0], InTy);
5555 return Builder.CreateUIToFP(Ops[0], FTy);
5556 return Builder.CreateSIToFP(Ops[0], FTy);
5558 case NEON::BI__builtin_neon_vcvtah_u16_f16:
5559 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
5560 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
5561 case NEON::BI__builtin_neon_vcvtph_u16_f16:
5562 case NEON::BI__builtin_neon_vcvtah_s16_f16:
5563 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
5564 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
5565 case NEON::BI__builtin_neon_vcvtph_s16_f16: {
5567 llvm::Type* FTy =
HalfTy;
5568 llvm::Type *Tys[2] = {InTy, FTy};
5569 switch (BuiltinID) {
5570 default: llvm_unreachable(
"missing builtin ID in switch!");
5571 case NEON::BI__builtin_neon_vcvtah_u16_f16:
5572 Int = Intrinsic::aarch64_neon_fcvtau;
break;
5573 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
5574 Int = Intrinsic::aarch64_neon_fcvtmu;
break;
5575 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
5576 Int = Intrinsic::aarch64_neon_fcvtnu;
break;
5577 case NEON::BI__builtin_neon_vcvtph_u16_f16:
5578 Int = Intrinsic::aarch64_neon_fcvtpu;
break;
5579 case NEON::BI__builtin_neon_vcvtah_s16_f16:
5580 Int = Intrinsic::aarch64_neon_fcvtas;
break;
5581 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
5582 Int = Intrinsic::aarch64_neon_fcvtms;
break;
5583 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
5584 Int = Intrinsic::aarch64_neon_fcvtns;
break;
5585 case NEON::BI__builtin_neon_vcvtph_s16_f16:
5586 Int = Intrinsic::aarch64_neon_fcvtps;
break;
5590 case NEON::BI__builtin_neon_vcaleh_f16:
5591 case NEON::BI__builtin_neon_vcalth_f16:
5592 case NEON::BI__builtin_neon_vcageh_f16:
5593 case NEON::BI__builtin_neon_vcagth_f16: {
5595 llvm::Type* FTy =
HalfTy;
5596 llvm::Type *Tys[2] = {InTy, FTy};
5597 switch (BuiltinID) {
5598 default: llvm_unreachable(
"missing builtin ID in switch!");
5599 case NEON::BI__builtin_neon_vcageh_f16:
5600 Int = Intrinsic::aarch64_neon_facge;
break;
5601 case NEON::BI__builtin_neon_vcagth_f16:
5602 Int = Intrinsic::aarch64_neon_facgt;
break;
5603 case NEON::BI__builtin_neon_vcaleh_f16:
5604 Int = Intrinsic::aarch64_neon_facge; std::swap(Ops[0], Ops[1]);
break;
5605 case NEON::BI__builtin_neon_vcalth_f16:
5606 Int = Intrinsic::aarch64_neon_facgt; std::swap(Ops[0], Ops[1]);
break;
5611 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
5612 case NEON::BI__builtin_neon_vcvth_n_u16_f16: {
5614 llvm::Type* FTy =
HalfTy;
5615 llvm::Type *Tys[2] = {InTy, FTy};
5616 switch (BuiltinID) {
5617 default: llvm_unreachable(
"missing builtin ID in switch!");
5618 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
5619 Int = Intrinsic::aarch64_neon_vcvtfp2fxs;
break;
5620 case NEON::BI__builtin_neon_vcvth_n_u16_f16:
5621 Int = Intrinsic::aarch64_neon_vcvtfp2fxu;
break;
5626 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
5627 case NEON::BI__builtin_neon_vcvth_n_f16_u16: {
5628 llvm::Type* FTy =
HalfTy;
5630 llvm::Type *Tys[2] = {FTy, InTy};
5631 switch (BuiltinID) {
5632 default: llvm_unreachable(
"missing builtin ID in switch!");
5633 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
5634 Int = Intrinsic::aarch64_neon_vcvtfxs2fp;
5635 Ops[0] =
Builder.CreateSExt(Ops[0], InTy,
"sext");
5637 case NEON::BI__builtin_neon_vcvth_n_f16_u16:
5638 Int = Intrinsic::aarch64_neon_vcvtfxu2fp;
5639 Ops[0] =
Builder.CreateZExt(Ops[0], InTy);
5644 case NEON::BI__builtin_neon_vpaddd_s64: {
5647 auto *Ty = llvm::FixedVectorType::get(
Int64Ty, 2);
5649 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty,
"v2i64");
5650 llvm::Value *Idx0 = llvm::ConstantInt::get(
SizeTy, 0);
5651 llvm::Value *Idx1 = llvm::ConstantInt::get(
SizeTy, 1);
5652 Value *Op0 =
Builder.CreateExtractElement(Ops[0], Idx0,
"lane0");
5653 Value *Op1 =
Builder.CreateExtractElement(Ops[0], Idx1,
"lane1");
5655 return Builder.CreateAdd(Op0, Op1,
"vpaddd");
5657 case NEON::BI__builtin_neon_vpaddd_f64: {
5658 auto *Ty = llvm::FixedVectorType::get(
DoubleTy, 2);
5660 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty,
"v2f64");
5661 llvm::Value *Idx0 = llvm::ConstantInt::get(
SizeTy, 0);
5662 llvm::Value *Idx1 = llvm::ConstantInt::get(
SizeTy, 1);
5663 Value *Op0 =
Builder.CreateExtractElement(Ops[0], Idx0,
"lane0");
5664 Value *Op1 =
Builder.CreateExtractElement(Ops[0], Idx1,
"lane1");
5666 return Builder.CreateFAdd(Op0, Op1,
"vpaddd");
5668 case NEON::BI__builtin_neon_vpadds_f32: {
5669 auto *Ty = llvm::FixedVectorType::get(
FloatTy, 2);
5671 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty,
"v2f32");
5672 llvm::Value *Idx0 = llvm::ConstantInt::get(
SizeTy, 0);
5673 llvm::Value *Idx1 = llvm::ConstantInt::get(
SizeTy, 1);
5674 Value *Op0 =
Builder.CreateExtractElement(Ops[0], Idx0,
"lane0");
5675 Value *Op1 =
Builder.CreateExtractElement(Ops[0], Idx1,
"lane1");
5677 return Builder.CreateFAdd(Op0, Op1,
"vpaddd");
5679 case NEON::BI__builtin_neon_vceqzd_s64:
5682 ICmpInst::ICMP_EQ,
"vceqz");
5683 case NEON::BI__builtin_neon_vceqzd_f64:
5684 case NEON::BI__builtin_neon_vceqzs_f32:
5685 case NEON::BI__builtin_neon_vceqzh_f16:
5688 ICmpInst::FCMP_OEQ,
"vceqz");
5689 case NEON::BI__builtin_neon_vcgezd_s64:
5692 ICmpInst::ICMP_SGE,
"vcgez");
5693 case NEON::BI__builtin_neon_vcgezd_f64:
5694 case NEON::BI__builtin_neon_vcgezs_f32:
5695 case NEON::BI__builtin_neon_vcgezh_f16:
5698 ICmpInst::FCMP_OGE,
"vcgez");
5699 case NEON::BI__builtin_neon_vclezd_s64:
5702 ICmpInst::ICMP_SLE,
"vclez");
5703 case NEON::BI__builtin_neon_vclezd_f64:
5704 case NEON::BI__builtin_neon_vclezs_f32:
5705 case NEON::BI__builtin_neon_vclezh_f16:
5708 ICmpInst::FCMP_OLE,
"vclez");
5709 case NEON::BI__builtin_neon_vcgtzd_s64:
5712 ICmpInst::ICMP_SGT,
"vcgtz");
5713 case NEON::BI__builtin_neon_vcgtzd_f64:
5714 case NEON::BI__builtin_neon_vcgtzs_f32:
5715 case NEON::BI__builtin_neon_vcgtzh_f16:
5718 ICmpInst::FCMP_OGT,
"vcgtz");
5719 case NEON::BI__builtin_neon_vcltzd_s64:
5722 ICmpInst::ICMP_SLT,
"vcltz");
5724 case NEON::BI__builtin_neon_vcltzd_f64:
5725 case NEON::BI__builtin_neon_vcltzs_f32:
5726 case NEON::BI__builtin_neon_vcltzh_f16:
5729 ICmpInst::FCMP_OLT,
"vcltz");
5731 case NEON::BI__builtin_neon_vceqzd_u64: {
5734 ICmpInst::ICMP_EQ,
"vceqzd");
5736 case NEON::BI__builtin_neon_vceqd_f64:
5737 case NEON::BI__builtin_neon_vcled_f64:
5738 case NEON::BI__builtin_neon_vcltd_f64:
5739 case NEON::BI__builtin_neon_vcged_f64:
5740 case NEON::BI__builtin_neon_vcgtd_f64: {
5741 llvm::CmpInst::Predicate P;
5742 switch (BuiltinID) {
5743 default: llvm_unreachable(
"missing builtin ID in switch!");
5744 case NEON::BI__builtin_neon_vceqd_f64: P = llvm::FCmpInst::FCMP_OEQ;
break;
5745 case NEON::BI__builtin_neon_vcled_f64: P = llvm::FCmpInst::FCMP_OLE;
break;
5746 case NEON::BI__builtin_neon_vcltd_f64: P = llvm::FCmpInst::FCMP_OLT;
break;
5747 case NEON::BI__builtin_neon_vcged_f64: P = llvm::FCmpInst::FCMP_OGE;
break;
5748 case NEON::BI__builtin_neon_vcgtd_f64: P = llvm::FCmpInst::FCMP_OGT;
break;
5752 if (P == llvm::FCmpInst::FCMP_OEQ)
5753 Ops[0] =
Builder.CreateFCmp(P, Ops[0], Ops[1]);
5755 Ops[0] =
Builder.CreateFCmpS(P, Ops[0], Ops[1]);
5758 case NEON::BI__builtin_neon_vceqs_f32:
5759 case NEON::BI__builtin_neon_vcles_f32:
5760 case NEON::BI__builtin_neon_vclts_f32:
5761 case NEON::BI__builtin_neon_vcges_f32:
5762 case NEON::BI__builtin_neon_vcgts_f32: {
5763 llvm::CmpInst::Predicate P;
5764 switch (BuiltinID) {
5765 default: llvm_unreachable(
"missing builtin ID in switch!");
5766 case NEON::BI__builtin_neon_vceqs_f32: P = llvm::FCmpInst::FCMP_OEQ;
break;
5767 case NEON::BI__builtin_neon_vcles_f32: P = llvm::FCmpInst::FCMP_OLE;
break;
5768 case NEON::BI__builtin_neon_vclts_f32: P = llvm::FCmpInst::FCMP_OLT;
break;
5769 case NEON::BI__builtin_neon_vcges_f32: P = llvm::FCmpInst::FCMP_OGE;
break;
5770 case NEON::BI__builtin_neon_vcgts_f32: P = llvm::FCmpInst::FCMP_OGT;
break;
5774 if (P == llvm::FCmpInst::FCMP_OEQ)
5775 Ops[0] =
Builder.CreateFCmp(P, Ops[0], Ops[1]);
5777 Ops[0] =
Builder.CreateFCmpS(P, Ops[0], Ops[1]);
5780 case NEON::BI__builtin_neon_vceqh_f16:
5781 case NEON::BI__builtin_neon_vcleh_f16:
5782 case NEON::BI__builtin_neon_vclth_f16:
5783 case NEON::BI__builtin_neon_vcgeh_f16:
5784 case NEON::BI__builtin_neon_vcgth_f16: {
5785 llvm::CmpInst::Predicate P;
5786 switch (BuiltinID) {
5787 default: llvm_unreachable(
"missing builtin ID in switch!");
5788 case NEON::BI__builtin_neon_vceqh_f16: P = llvm::FCmpInst::FCMP_OEQ;
break;
5789 case NEON::BI__builtin_neon_vcleh_f16: P = llvm::FCmpInst::FCMP_OLE;
break;
5790 case NEON::BI__builtin_neon_vclth_f16: P = llvm::FCmpInst::FCMP_OLT;
break;
5791 case NEON::BI__builtin_neon_vcgeh_f16: P = llvm::FCmpInst::FCMP_OGE;
break;
5792 case NEON::BI__builtin_neon_vcgth_f16: P = llvm::FCmpInst::FCMP_OGT;
break;
5796 if (P == llvm::FCmpInst::FCMP_OEQ)
5797 Ops[0] =
Builder.CreateFCmp(P, Ops[0], Ops[1]);
5799 Ops[0] =
Builder.CreateFCmpS(P, Ops[0], Ops[1]);
5802 case NEON::BI__builtin_neon_vceqd_s64:
5803 case NEON::BI__builtin_neon_vceqd_u64:
5804 case NEON::BI__builtin_neon_vcgtd_s64:
5805 case NEON::BI__builtin_neon_vcgtd_u64:
5806 case NEON::BI__builtin_neon_vcltd_s64:
5807 case NEON::BI__builtin_neon_vcltd_u64:
5808 case NEON::BI__builtin_neon_vcged_u64:
5809 case NEON::BI__builtin_neon_vcged_s64:
5810 case NEON::BI__builtin_neon_vcled_u64:
5811 case NEON::BI__builtin_neon_vcled_s64: {
5812 llvm::CmpInst::Predicate P;
5813 switch (BuiltinID) {
5814 default: llvm_unreachable(
"missing builtin ID in switch!");
5815 case NEON::BI__builtin_neon_vceqd_s64:
5816 case NEON::BI__builtin_neon_vceqd_u64:P = llvm::ICmpInst::ICMP_EQ;
break;
5817 case NEON::BI__builtin_neon_vcgtd_s64:P = llvm::ICmpInst::ICMP_SGT;
break;
5818 case NEON::BI__builtin_neon_vcgtd_u64:P = llvm::ICmpInst::ICMP_UGT;
break;
5819 case NEON::BI__builtin_neon_vcltd_s64:P = llvm::ICmpInst::ICMP_SLT;
break;
5820 case NEON::BI__builtin_neon_vcltd_u64:P = llvm::ICmpInst::ICMP_ULT;
break;
5821 case NEON::BI__builtin_neon_vcged_u64:P = llvm::ICmpInst::ICMP_UGE;
break;
5822 case NEON::BI__builtin_neon_vcged_s64:P = llvm::ICmpInst::ICMP_SGE;
break;
5823 case NEON::BI__builtin_neon_vcled_u64:P = llvm::ICmpInst::ICMP_ULE;
break;
5824 case NEON::BI__builtin_neon_vcled_s64:P = llvm::ICmpInst::ICMP_SLE;
break;
5828 Ops[0] =
Builder.CreateICmp(P, Ops[0], Ops[1]);
5831 case NEON::BI__builtin_neon_vnegd_s64:
5832 return Builder.CreateNeg(Ops[0],
"vnegd");
5833 case NEON::BI__builtin_neon_vnegh_f16:
5834 return Builder.CreateFNeg(Ops[0],
"vnegh");
5835 case NEON::BI__builtin_neon_vtstd_s64:
5836 case NEON::BI__builtin_neon_vtstd_u64: {
5839 Ops[0] =
Builder.CreateAnd(Ops[0], Ops[1]);
5840 Ops[0] =
Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
5841 llvm::Constant::getNullValue(
Int64Ty));
5844 case NEON::BI__builtin_neon_vset_lane_i8:
5845 case NEON::BI__builtin_neon_vset_lane_i16:
5846 case NEON::BI__builtin_neon_vset_lane_i32:
5847 case NEON::BI__builtin_neon_vset_lane_i64:
5848 case NEON::BI__builtin_neon_vset_lane_bf16:
5849 case NEON::BI__builtin_neon_vset_lane_f32:
5850 case NEON::BI__builtin_neon_vsetq_lane_i8:
5851 case NEON::BI__builtin_neon_vsetq_lane_i16:
5852 case NEON::BI__builtin_neon_vsetq_lane_i32:
5853 case NEON::BI__builtin_neon_vsetq_lane_i64:
5854 case NEON::BI__builtin_neon_vsetq_lane_bf16:
5855 case NEON::BI__builtin_neon_vsetq_lane_f32:
5856 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
5857 case NEON::BI__builtin_neon_vset_lane_f64:
5860 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(
DoubleTy, 1));
5861 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
5862 case NEON::BI__builtin_neon_vset_lane_mf8:
5863 case NEON::BI__builtin_neon_vsetq_lane_mf8:
5867 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
5868 case NEON::BI__builtin_neon_vsetq_lane_f64:
5871 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(
DoubleTy, 2));
5872 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
5874 case NEON::BI__builtin_neon_vget_lane_i8:
5875 case NEON::BI__builtin_neon_vdupb_lane_i8:
5876 return Builder.CreateExtractElement(Ops[0], Ops[1],
"vget_lane");
5877 case NEON::BI__builtin_neon_vgetq_lane_i8:
5878 case NEON::BI__builtin_neon_vdupb_laneq_i8:
5879 return Builder.CreateExtractElement(Ops[0], Ops[1],
"vgetq_lane");
5880 case NEON::BI__builtin_neon_vget_lane_mf8:
5881 case NEON::BI__builtin_neon_vdupb_lane_mf8:
5882 case NEON::BI__builtin_neon_vgetq_lane_mf8:
5883 case NEON::BI__builtin_neon_vdupb_laneq_mf8:
5884 return Builder.CreateExtractElement(Ops[0], Ops[1],
"vget_lane");
5885 case NEON::BI__builtin_neon_vget_lane_i16:
5886 case NEON::BI__builtin_neon_vduph_lane_i16:
5887 return Builder.CreateExtractElement(Ops[0], Ops[1],
"vget_lane");
5888 case NEON::BI__builtin_neon_vgetq_lane_i16:
5889 case NEON::BI__builtin_neon_vduph_laneq_i16:
5890 return Builder.CreateExtractElement(Ops[0], Ops[1],
"vgetq_lane");
5891 case NEON::BI__builtin_neon_vget_lane_i32:
5892 case NEON::BI__builtin_neon_vdups_lane_i32:
5893 return Builder.CreateExtractElement(Ops[0], Ops[1],
"vget_lane");
5894 case NEON::BI__builtin_neon_vdups_lane_f32:
5895 return Builder.CreateExtractElement(Ops[0], Ops[1],
"vdups_lane");
5896 case NEON::BI__builtin_neon_vgetq_lane_i32:
5897 case NEON::BI__builtin_neon_vdups_laneq_i32:
5898 return Builder.CreateExtractElement(Ops[0], Ops[1],
"vgetq_lane");
5899 case NEON::BI__builtin_neon_vget_lane_i64:
5900 case NEON::BI__builtin_neon_vdupd_lane_i64:
5901 return Builder.CreateExtractElement(Ops[0], Ops[1],
"vget_lane");
5902 case NEON::BI__builtin_neon_vdupd_lane_f64:
5903 return Builder.CreateExtractElement(Ops[0], Ops[1],
"vdupd_lane");
5904 case NEON::BI__builtin_neon_vgetq_lane_i64:
5905 case NEON::BI__builtin_neon_vdupd_laneq_i64:
5906 return Builder.CreateExtractElement(Ops[0], Ops[1],
"vgetq_lane");
5907 case NEON::BI__builtin_neon_vget_lane_f32:
5908 return Builder.CreateExtractElement(Ops[0], Ops[1],
"vget_lane");
5909 case NEON::BI__builtin_neon_vget_lane_f64:
5910 return Builder.CreateExtractElement(Ops[0], Ops[1],
"vget_lane");
5911 case NEON::BI__builtin_neon_vgetq_lane_f32:
5912 case NEON::BI__builtin_neon_vdups_laneq_f32:
5913 return Builder.CreateExtractElement(Ops[0], Ops[1],
"vgetq_lane");
5914 case NEON::BI__builtin_neon_vgetq_lane_f64:
5915 case NEON::BI__builtin_neon_vdupd_laneq_f64:
5916 return Builder.CreateExtractElement(Ops[0], Ops[1],
"vgetq_lane");
5917 case NEON::BI__builtin_neon_vaddh_f16:
5918 return Builder.CreateFAdd(Ops[0], Ops[1],
"vaddh");
5919 case NEON::BI__builtin_neon_vsubh_f16:
5920 return Builder.CreateFSub(Ops[0], Ops[1],
"vsubh");
5921 case NEON::BI__builtin_neon_vmulh_f16:
5922 return Builder.CreateFMul(Ops[0], Ops[1],
"vmulh");
5923 case NEON::BI__builtin_neon_vdivh_f16:
5924 return Builder.CreateFDiv(Ops[0], Ops[1],
"vdivh");
5925 case NEON::BI__builtin_neon_vfmah_f16:
5928 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma,
HalfTy,
5929 {Ops[1], Ops[2], Ops[0]});
5930 case NEON::BI__builtin_neon_vfmsh_f16: {
5935 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma,
HalfTy,
5936 {Neg, Ops[2], Ops[0]});
5938 case NEON::BI__builtin_neon_vaddd_s64:
5939 case NEON::BI__builtin_neon_vaddd_u64:
5940 return Builder.CreateAdd(Ops[0], Ops[1],
"vaddd");
5941 case NEON::BI__builtin_neon_vsubd_s64:
5942 case NEON::BI__builtin_neon_vsubd_u64:
5943 return Builder.CreateSub(Ops[0], Ops[1],
"vsubd");
5944 case NEON::BI__builtin_neon_vqdmlalh_s16:
5945 case NEON::BI__builtin_neon_vqdmlslh_s16: {
5949 auto *VTy = llvm::FixedVectorType::get(
Int32Ty, 4);
5950 Ops[1] =
EmitNeonCall(
CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy),
5951 ProductOps,
"vqdmlXl");
5953 Ops[1] =
Builder.CreateExtractElement(Ops[1], CI,
"lane0");
5955 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlalh_s16
5956 ? Intrinsic::aarch64_neon_sqadd
5957 : Intrinsic::aarch64_neon_sqsub;
5962 case NEON::BI__builtin_neon_vqshlud_n_s64: {
5967 case NEON::BI__builtin_neon_vqshld_n_u64:
5968 case NEON::BI__builtin_neon_vqshld_n_s64: {
5969 Int = BuiltinID == NEON::BI__builtin_neon_vqshld_n_u64
5970 ? Intrinsic::aarch64_neon_uqshl
5971 : Intrinsic::aarch64_neon_sqshl;
5975 case NEON::BI__builtin_neon_vrshrd_n_u64:
5976 case NEON::BI__builtin_neon_vrshrd_n_s64: {
5977 Int = BuiltinID == NEON::BI__builtin_neon_vrshrd_n_u64
5978 ? Intrinsic::aarch64_neon_urshl
5979 : Intrinsic::aarch64_neon_srshl;
5981 Ops[1] = ConstantInt::get(
Int64Ty, -SV);
5984 case NEON::BI__builtin_neon_vrsrad_n_u64:
5985 case NEON::BI__builtin_neon_vrsrad_n_s64: {
5986 Int = BuiltinID == NEON::BI__builtin_neon_vrsrad_n_u64
5987 ? Intrinsic::aarch64_neon_urshl
5988 : Intrinsic::aarch64_neon_srshl;
5990 Ops[2] =
Builder.CreateNeg(Ops[2]);
5992 {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)});
5995 case NEON::BI__builtin_neon_vshld_n_s64:
5996 case NEON::BI__builtin_neon_vshld_n_u64: {
5999 Ops[0], ConstantInt::get(
Int64Ty, Amt->getZExtValue()),
"shld_n");
6001 case NEON::BI__builtin_neon_vshrd_n_s64: {
6004 Ops[0], ConstantInt::get(
Int64Ty, std::min(
static_cast<uint64_t
>(63),
6005 Amt->getZExtValue())),
6008 case NEON::BI__builtin_neon_vshrd_n_u64: {
6010 uint64_t ShiftAmt = Amt->getZExtValue();
6013 return ConstantInt::get(
Int64Ty, 0);
6014 return Builder.CreateLShr(Ops[0], ConstantInt::get(
Int64Ty, ShiftAmt),
6017 case NEON::BI__builtin_neon_vsrad_n_s64: {
6020 Ops[1], ConstantInt::get(
Int64Ty, std::min(
static_cast<uint64_t
>(63),
6021 Amt->getZExtValue())),
6023 return Builder.CreateAdd(Ops[0], Ops[1]);
6025 case NEON::BI__builtin_neon_vsrad_n_u64: {
6027 uint64_t ShiftAmt = Amt->getZExtValue();
6032 Ops[1] =
Builder.CreateLShr(Ops[1], ConstantInt::get(
Int64Ty, ShiftAmt),
6034 return Builder.CreateAdd(Ops[0], Ops[1]);
6036 case NEON::BI__builtin_neon_vqdmlalh_lane_s16:
6037 case NEON::BI__builtin_neon_vqdmlalh_laneq_s16:
6038 case NEON::BI__builtin_neon_vqdmlslh_lane_s16:
6039 case NEON::BI__builtin_neon_vqdmlslh_laneq_s16: {
6040 Ops[2] =
Builder.CreateExtractElement(Ops[2], Ops[3],
"lane");
6044 auto *VTy = llvm::FixedVectorType::get(
Int32Ty, 4);
6045 Ops[1] =
EmitNeonCall(
CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy),
6046 ProductOps,
"vqdmlXl");
6048 Ops[1] =
Builder.CreateExtractElement(Ops[1], CI,
"lane0");
6053 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlalh_lane_s16 ||
6054 BuiltinID == NEON::BI__builtin_neon_vqdmlalh_laneq_s16)
6055 ? Intrinsic::aarch64_neon_sqadd
6056 : Intrinsic::aarch64_neon_sqsub;
6059 case NEON::BI__builtin_neon_vqdmlals_s32:
6060 case NEON::BI__builtin_neon_vqdmlsls_s32: {
6062 ProductOps.push_back(Ops[1]);
6063 ProductOps.push_back(Ops[2]);
6065 EmitNeonCall(
CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar),
6066 ProductOps,
"vqdmlXl");
6068 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlals_s32
6069 ? Intrinsic::aarch64_neon_sqadd
6070 : Intrinsic::aarch64_neon_sqsub;
6075 case NEON::BI__builtin_neon_vqdmlals_lane_s32:
6076 case NEON::BI__builtin_neon_vqdmlals_laneq_s32:
6077 case NEON::BI__builtin_neon_vqdmlsls_lane_s32:
6078 case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: {
6079 Ops[2] =
Builder.CreateExtractElement(Ops[2], Ops[3],
"lane");
6081 ProductOps.push_back(Ops[1]);
6082 ProductOps.push_back(Ops[2]);
6084 EmitNeonCall(
CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar),
6085 ProductOps,
"vqdmlXl");
6090 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlals_lane_s32 ||
6091 BuiltinID == NEON::BI__builtin_neon_vqdmlals_laneq_s32)
6092 ? Intrinsic::aarch64_neon_sqadd
6093 : Intrinsic::aarch64_neon_sqsub;
6096 case NEON::BI__builtin_neon_vget_lane_bf16:
6097 case NEON::BI__builtin_neon_vduph_lane_bf16:
6098 case NEON::BI__builtin_neon_vduph_lane_f16: {
6099 return Builder.CreateExtractElement(Ops[0], Ops[1],
"vget_lane");
6101 case NEON::BI__builtin_neon_vgetq_lane_bf16:
6102 case NEON::BI__builtin_neon_vduph_laneq_bf16:
6103 case NEON::BI__builtin_neon_vduph_laneq_f16: {
6104 return Builder.CreateExtractElement(Ops[0], Ops[1],
"vgetq_lane");
6106 case NEON::BI__builtin_neon_vcvt_bf16_f32: {
6107 llvm::Type *V4F32 = FixedVectorType::get(
Builder.getFloatTy(), 4);
6108 llvm::Type *V4BF16 = FixedVectorType::get(
Builder.getBFloatTy(), 4);
6109 return Builder.CreateFPTrunc(
Builder.CreateBitCast(Ops[0], V4F32), V4BF16);
6111 case NEON::BI__builtin_neon_vcvtq_low_bf16_f32: {
6113 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
6114 llvm::Type *V4F32 = FixedVectorType::get(
Builder.getFloatTy(), 4);
6115 llvm::Type *V4BF16 = FixedVectorType::get(
Builder.getBFloatTy(), 4);
6116 llvm::Value *Trunc =
6117 Builder.CreateFPTrunc(
Builder.CreateBitCast(Ops[0], V4F32), V4BF16);
6118 return Builder.CreateShuffleVector(
6119 Trunc, ConstantAggregateZero::get(V4BF16), ConcatMask);
6121 case NEON::BI__builtin_neon_vcvtq_high_bf16_f32: {
6123 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
6125 std::iota(LoMask.begin(), LoMask.end(), 0);
6126 llvm::Type *V4F32 = FixedVectorType::get(
Builder.getFloatTy(), 4);
6127 llvm::Type *V4BF16 = FixedVectorType::get(
Builder.getBFloatTy(), 4);
6128 llvm::Type *V8BF16 = FixedVectorType::get(
Builder.getBFloatTy(), 8);
6129 llvm::Value *Inactive =
Builder.CreateShuffleVector(
6130 Builder.CreateBitCast(Ops[0], V8BF16), LoMask);
6131 llvm::Value *Trunc =
6132 Builder.CreateFPTrunc(
Builder.CreateBitCast(Ops[1], V4F32), V4BF16);
6133 return Builder.CreateShuffleVector(Inactive, Trunc, ConcatMask);
6135 case NEON::BI__builtin_neon_vcvt_f16_f32: {
6136 llvm::Type *V4F32 = FixedVectorType::get(
Builder.getFloatTy(), 4);
6137 llvm::Type *V4F16 = FixedVectorType::get(
Builder.getHalfTy(), 4);
6138 return Builder.CreateFPTrunc(
Builder.CreateBitCast(Ops[0], V4F32), V4F16);
6140 case NEON::BI__builtin_neon_vcvt_f32_f16: {
6141 llvm::Type *V4F32 = FixedVectorType::get(
Builder.getFloatTy(), 4);
6142 llvm::Type *V4F16 = FixedVectorType::get(
Builder.getHalfTy(), 4);
6143 return Builder.CreateFPExt(
Builder.CreateBitCast(Ops[0], V4F16), V4F32);
6146 case clang::AArch64::BI_InterlockedAdd:
6147 case clang::AArch64::BI_InterlockedAdd_acq:
6148 case clang::AArch64::BI_InterlockedAdd_rel:
6149 case clang::AArch64::BI_InterlockedAdd_nf:
6150 case clang::AArch64::BI_InterlockedAdd64:
6151 case clang::AArch64::BI_InterlockedAdd64_acq:
6152 case clang::AArch64::BI_InterlockedAdd64_rel:
6153 case clang::AArch64::BI_InterlockedAdd64_nf: {
6155 Value *Val = Ops[1];
6156 llvm::AtomicOrdering Ordering;
6157 switch (BuiltinID) {
6158 case clang::AArch64::BI_InterlockedAdd:
6159 case clang::AArch64::BI_InterlockedAdd64:
6160 Ordering = llvm::AtomicOrdering::SequentiallyConsistent;
6162 case clang::AArch64::BI_InterlockedAdd_acq:
6163 case clang::AArch64::BI_InterlockedAdd64_acq:
6164 Ordering = llvm::AtomicOrdering::Acquire;
6166 case clang::AArch64::BI_InterlockedAdd_rel:
6167 case clang::AArch64::BI_InterlockedAdd64_rel:
6168 Ordering = llvm::AtomicOrdering::Release;
6170 case clang::AArch64::BI_InterlockedAdd_nf:
6171 case clang::AArch64::BI_InterlockedAdd64_nf:
6172 Ordering = llvm::AtomicOrdering::Monotonic;
6175 llvm_unreachable(
"missing builtin ID in switch!");
6177 AtomicRMWInst *RMWI =
6178 Builder.CreateAtomicRMW(AtomicRMWInst::Add, DestAddr, Val, Ordering);
6179 return Builder.CreateAdd(RMWI, Val);
6184 llvm::Type *Ty = VTy;
6188 bool ExtractLow =
false;
6189 bool ExtendLaneArg =
false;
6190 switch (BuiltinID) {
6191 default:
return nullptr;
6192 case NEON::BI__builtin_neon_vbsl_v:
6193 case NEON::BI__builtin_neon_vbslq_v: {
6194 llvm::Type *BitTy = llvm::VectorType::getInteger(VTy);
6195 Ops[0] =
Builder.CreateBitCast(Ops[0], BitTy,
"vbsl");
6196 Ops[1] =
Builder.CreateBitCast(Ops[1], BitTy,
"vbsl");
6197 Ops[2] =
Builder.CreateBitCast(Ops[2], BitTy,
"vbsl");
6199 Ops[1] =
Builder.CreateAnd(Ops[0], Ops[1],
"vbsl");
6200 Ops[2] =
Builder.CreateAnd(
Builder.CreateNot(Ops[0]), Ops[2],
"vbsl");
6201 Ops[0] =
Builder.CreateOr(Ops[1], Ops[2],
"vbsl");
6202 return Builder.CreateBitCast(Ops[0], Ty);
6204 case NEON::BI__builtin_neon_vfma_lane_v:
6205 case NEON::BI__builtin_neon_vfmaq_lane_v: {
6208 Value *Addend = Ops[0];
6209 Value *Multiplicand = Ops[1];
6210 Value *LaneSource = Ops[2];
6211 Ops[0] = Multiplicand;
6212 Ops[1] = LaneSource;
6216 auto *SourceTy = BuiltinID == NEON::BI__builtin_neon_vfmaq_lane_v
6217 ? llvm::FixedVectorType::get(VTy->getElementType(),
6218 VTy->getNumElements() / 2)
6221 Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(), cst);
6222 Ops[1] =
Builder.CreateBitCast(Ops[1], SourceTy);
6223 Ops[1] =
Builder.CreateShuffleVector(Ops[1], Ops[1], SV,
"lane");
6226 Int =
Builder.getIsFPConstrained() ? Intrinsic::experimental_constrained_fma
6230 case NEON::BI__builtin_neon_vfma_laneq_v: {
6233 if (VTy && VTy->getElementType() ==
DoubleTy) {
6236 llvm::FixedVectorType *VTy =
6238 Ops[2] =
Builder.CreateBitCast(Ops[2], VTy);
6239 Ops[2] =
Builder.CreateExtractElement(Ops[2], Ops[3],
"extract");
6242 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma,
6243 DoubleTy, {Ops[1], Ops[2], Ops[0]});
6246 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
6247 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
6249 auto *STy = llvm::FixedVectorType::get(VTy->getElementType(),
6250 VTy->getNumElements() * 2);
6251 Ops[2] =
Builder.CreateBitCast(Ops[2], STy);
6252 Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(),
6254 Ops[2] =
Builder.CreateShuffleVector(Ops[2], Ops[2], SV,
"lane");
6257 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
6258 {Ops[2], Ops[1], Ops[0]});
6260 case NEON::BI__builtin_neon_vfmaq_laneq_v: {
6261 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
6262 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
6264 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
6267 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
6268 {Ops[2], Ops[1], Ops[0]});
6270 case NEON::BI__builtin_neon_vfmah_lane_f16:
6271 case NEON::BI__builtin_neon_vfmas_lane_f32:
6272 case NEON::BI__builtin_neon_vfmah_laneq_f16:
6273 case NEON::BI__builtin_neon_vfmas_laneq_f32:
6274 case NEON::BI__builtin_neon_vfmad_lane_f64:
6275 case NEON::BI__builtin_neon_vfmad_laneq_f64: {
6277 Ops[2] =
Builder.CreateExtractElement(Ops[2], Ops[3],
"extract");
6279 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
6280 {Ops[1], Ops[2], Ops[0]});
6282 case NEON::BI__builtin_neon_vmull_v:
6284 Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull;
6285 if (
Type.isPoly())
Int = Intrinsic::aarch64_neon_pmull;
6287 case NEON::BI__builtin_neon_vmax_v:
6288 case NEON::BI__builtin_neon_vmaxq_v:
6290 Int = usgn ? Intrinsic::umax : Intrinsic::smax;
6291 if (Ty->isFPOrFPVectorTy())
Int = Intrinsic::aarch64_neon_fmax;
6293 case NEON::BI__builtin_neon_vmaxh_f16: {
6294 Int = Intrinsic::aarch64_neon_fmax;
6297 case NEON::BI__builtin_neon_vmin_v:
6298 case NEON::BI__builtin_neon_vminq_v:
6300 Int = usgn ? Intrinsic::umin : Intrinsic::smin;
6301 if (Ty->isFPOrFPVectorTy())
Int = Intrinsic::aarch64_neon_fmin;
6303 case NEON::BI__builtin_neon_vminh_f16: {
6304 Int = Intrinsic::aarch64_neon_fmin;
6307 case NEON::BI__builtin_neon_vabd_v:
6308 case NEON::BI__builtin_neon_vabdq_v:
6310 Int = usgn ? Intrinsic::aarch64_neon_uabd : Intrinsic::aarch64_neon_sabd;
6311 if (Ty->isFPOrFPVectorTy())
Int = Intrinsic::aarch64_neon_fabd;
6313 case NEON::BI__builtin_neon_vpadal_v:
6314 case NEON::BI__builtin_neon_vpadalq_v: {
6315 unsigned ArgElts = VTy->getNumElements();
6317 unsigned BitWidth = EltTy->getBitWidth();
6318 auto *ArgTy = llvm::FixedVectorType::get(
6319 llvm::IntegerType::get(
getLLVMContext(), BitWidth / 2), 2 * ArgElts);
6320 llvm::Type* Tys[2] = { VTy, ArgTy };
6321 Int = usgn ? Intrinsic::aarch64_neon_uaddlp : Intrinsic::aarch64_neon_saddlp;
6323 TmpOps.push_back(Ops[1]);
6326 llvm::Value *addend =
Builder.CreateBitCast(Ops[0], tmp->getType());
6327 return Builder.CreateAdd(tmp, addend);
6329 case NEON::BI__builtin_neon_vpmin_v:
6330 case NEON::BI__builtin_neon_vpminq_v:
6332 Int = usgn ? Intrinsic::aarch64_neon_uminp : Intrinsic::aarch64_neon_sminp;
6333 if (Ty->isFPOrFPVectorTy())
Int = Intrinsic::aarch64_neon_fminp;
6335 case NEON::BI__builtin_neon_vpmax_v:
6336 case NEON::BI__builtin_neon_vpmaxq_v:
6338 Int = usgn ? Intrinsic::aarch64_neon_umaxp : Intrinsic::aarch64_neon_smaxp;
6339 if (Ty->isFPOrFPVectorTy())
Int = Intrinsic::aarch64_neon_fmaxp;
6341 case NEON::BI__builtin_neon_vminnm_v:
6342 case NEON::BI__builtin_neon_vminnmq_v:
6343 Int = Intrinsic::aarch64_neon_fminnm;
6345 case NEON::BI__builtin_neon_vminnmh_f16:
6346 Int = Intrinsic::aarch64_neon_fminnm;
6348 case NEON::BI__builtin_neon_vmaxnm_v:
6349 case NEON::BI__builtin_neon_vmaxnmq_v:
6350 Int = Intrinsic::aarch64_neon_fmaxnm;
6352 case NEON::BI__builtin_neon_vmaxnmh_f16:
6353 Int = Intrinsic::aarch64_neon_fmaxnm;
6355 case NEON::BI__builtin_neon_vrecpss_f32: {
6359 case NEON::BI__builtin_neon_vrecpsd_f64:
6362 case NEON::BI__builtin_neon_vrecpsh_f16:
6365 case NEON::BI__builtin_neon_vqshrun_n_v:
6366 Int = Intrinsic::aarch64_neon_sqshrun;
6368 case NEON::BI__builtin_neon_vqrshrun_n_v:
6369 Int = Intrinsic::aarch64_neon_sqrshrun;
6371 case NEON::BI__builtin_neon_vqshrn_n_v:
6372 Int = usgn ? Intrinsic::aarch64_neon_uqshrn : Intrinsic::aarch64_neon_sqshrn;
6374 case NEON::BI__builtin_neon_vrshrn_n_v:
6375 Int = Intrinsic::aarch64_neon_rshrn;
6377 case NEON::BI__builtin_neon_vqrshrn_n_v:
6378 Int = usgn ? Intrinsic::aarch64_neon_uqrshrn : Intrinsic::aarch64_neon_sqrshrn;
6380 case NEON::BI__builtin_neon_vrndah_f16: {
6382 ? Intrinsic::experimental_constrained_round
6386 case NEON::BI__builtin_neon_vrnda_v:
6387 case NEON::BI__builtin_neon_vrndaq_v: {
6389 ? Intrinsic::experimental_constrained_round
6393 case NEON::BI__builtin_neon_vrndih_f16: {
6395 ? Intrinsic::experimental_constrained_nearbyint
6396 : Intrinsic::nearbyint;
6399 case NEON::BI__builtin_neon_vrndmh_f16: {
6401 ? Intrinsic::experimental_constrained_floor
6405 case NEON::BI__builtin_neon_vrndm_v:
6406 case NEON::BI__builtin_neon_vrndmq_v: {
6408 ? Intrinsic::experimental_constrained_floor
6412 case NEON::BI__builtin_neon_vrndnh_f16: {
6414 ? Intrinsic::experimental_constrained_roundeven
6415 : Intrinsic::roundeven;
6418 case NEON::BI__builtin_neon_vrndn_v:
6419 case NEON::BI__builtin_neon_vrndnq_v: {
6421 ? Intrinsic::experimental_constrained_roundeven
6422 : Intrinsic::roundeven;
6425 case NEON::BI__builtin_neon_vrndns_f32: {
6427 ? Intrinsic::experimental_constrained_roundeven
6428 : Intrinsic::roundeven;
6431 case NEON::BI__builtin_neon_vrndph_f16: {
6433 ? Intrinsic::experimental_constrained_ceil
6437 case NEON::BI__builtin_neon_vrndp_v:
6438 case NEON::BI__builtin_neon_vrndpq_v: {
6440 ? Intrinsic::experimental_constrained_ceil
6444 case NEON::BI__builtin_neon_vrndxh_f16: {
6446 ? Intrinsic::experimental_constrained_rint
6450 case NEON::BI__builtin_neon_vrndx_v:
6451 case NEON::BI__builtin_neon_vrndxq_v: {
6453 ? Intrinsic::experimental_constrained_rint
6457 case NEON::BI__builtin_neon_vrndh_f16: {
6459 ? Intrinsic::experimental_constrained_trunc
6463 case NEON::BI__builtin_neon_vrnd_v:
6464 case NEON::BI__builtin_neon_vrndq_v: {
6466 ? Intrinsic::experimental_constrained_trunc
6470 case NEON::BI__builtin_neon_vcvt_f64_v:
6471 case NEON::BI__builtin_neon_vcvtq_f64_v:
6472 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
6474 return usgn ?
Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
6475 :
Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
6476 case NEON::BI__builtin_neon_vcvt_f64_f32: {
6478 "unexpected vcvt_f64_f32 builtin");
6482 return Builder.CreateFPExt(Ops[0], Ty,
"vcvt");
6484 case NEON::BI__builtin_neon_vcvt_f32_f64: {
6486 "unexpected vcvt_f32_f64 builtin");
6490 return Builder.CreateFPTrunc(Ops[0], Ty,
"vcvt");
6492 case NEON::BI__builtin_neon_vcvta_s16_f16:
6493 case NEON::BI__builtin_neon_vcvta_u16_f16:
6494 case NEON::BI__builtin_neon_vcvta_s32_v:
6495 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
6496 case NEON::BI__builtin_neon_vcvtaq_s32_v:
6497 case NEON::BI__builtin_neon_vcvta_u32_v:
6498 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
6499 case NEON::BI__builtin_neon_vcvtaq_u32_v:
6500 case NEON::BI__builtin_neon_vcvta_s64_v:
6501 case NEON::BI__builtin_neon_vcvtaq_s64_v:
6502 case NEON::BI__builtin_neon_vcvta_u64_v:
6503 case NEON::BI__builtin_neon_vcvtaq_u64_v: {
6504 Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas;
6508 case NEON::BI__builtin_neon_vcvtm_s16_f16:
6509 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
6510 case NEON::BI__builtin_neon_vcvtm_u16_f16:
6511 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
6512 case NEON::BI__builtin_neon_vcvtm_s32_v:
6513 case NEON::BI__builtin_neon_vcvtmq_s32_v:
6514 case NEON::BI__builtin_neon_vcvtm_u32_v:
6515 case NEON::BI__builtin_neon_vcvtmq_u32_v:
6516 case NEON::BI__builtin_neon_vcvtm_s64_v:
6517 case NEON::BI__builtin_neon_vcvtmq_s64_v:
6518 case NEON::BI__builtin_neon_vcvtm_u64_v:
6519 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
6520 Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms;
6524 case NEON::BI__builtin_neon_vcvtn_s16_f16:
6525 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
6526 case NEON::BI__builtin_neon_vcvtn_u16_f16:
6527 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
6528 case NEON::BI__builtin_neon_vcvtn_s32_v:
6529 case NEON::BI__builtin_neon_vcvtnq_s32_v:
6530 case NEON::BI__builtin_neon_vcvtn_u32_v:
6531 case NEON::BI__builtin_neon_vcvtnq_u32_v:
6532 case NEON::BI__builtin_neon_vcvtn_s64_v:
6533 case NEON::BI__builtin_neon_vcvtnq_s64_v:
6534 case NEON::BI__builtin_neon_vcvtn_u64_v:
6535 case NEON::BI__builtin_neon_vcvtnq_u64_v: {
6536 Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns;
6540 case NEON::BI__builtin_neon_vcvtp_s16_f16:
6541 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
6542 case NEON::BI__builtin_neon_vcvtp_u16_f16:
6543 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
6544 case NEON::BI__builtin_neon_vcvtp_s32_v:
6545 case NEON::BI__builtin_neon_vcvtpq_s32_v:
6546 case NEON::BI__builtin_neon_vcvtp_u32_v:
6547 case NEON::BI__builtin_neon_vcvtpq_u32_v:
6548 case NEON::BI__builtin_neon_vcvtp_s64_v:
6549 case NEON::BI__builtin_neon_vcvtpq_s64_v:
6550 case NEON::BI__builtin_neon_vcvtp_u64_v:
6551 case NEON::BI__builtin_neon_vcvtpq_u64_v: {
6552 Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps;
6556 case NEON::BI__builtin_neon_vmulx_v:
6557 case NEON::BI__builtin_neon_vmulxq_v: {
6558 Int = Intrinsic::aarch64_neon_fmulx;
6561 case NEON::BI__builtin_neon_vmulxh_lane_f16:
6562 case NEON::BI__builtin_neon_vmulxh_laneq_f16: {
6565 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2],
"extract");
6567 Int = Intrinsic::aarch64_neon_fmulx;
6570 case NEON::BI__builtin_neon_vmul_lane_v:
6571 case NEON::BI__builtin_neon_vmul_laneq_v: {
6574 if (BuiltinID == NEON::BI__builtin_neon_vmul_laneq_v)
6577 llvm::FixedVectorType *VTy =
6579 Ops[1] =
Builder.CreateBitCast(Ops[1], VTy);
6580 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2],
"extract");
6584 case NEON::BI__builtin_neon_vpmaxnm_v:
6585 case NEON::BI__builtin_neon_vpmaxnmq_v: {
6586 Int = Intrinsic::aarch64_neon_fmaxnmp;
6589 case NEON::BI__builtin_neon_vpminnm_v:
6590 case NEON::BI__builtin_neon_vpminnmq_v: {
6591 Int = Intrinsic::aarch64_neon_fminnmp;
6594 case NEON::BI__builtin_neon_vsqrth_f16: {
6596 ? Intrinsic::experimental_constrained_sqrt
6600 case NEON::BI__builtin_neon_vsqrt_v:
6601 case NEON::BI__builtin_neon_vsqrtq_v: {
6603 ? Intrinsic::experimental_constrained_sqrt
6605 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
6608 case NEON::BI__builtin_neon_vrbit_v:
6609 case NEON::BI__builtin_neon_vrbitq_v: {
6610 Int = Intrinsic::bitreverse;
6613 case NEON::BI__builtin_neon_vmaxv_f16: {
6614 Int = Intrinsic::aarch64_neon_fmaxv;
6616 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
6617 llvm::Type *Tys[2] = {Ty, VTy};
6620 case NEON::BI__builtin_neon_vmaxvq_f16: {
6621 Int = Intrinsic::aarch64_neon_fmaxv;
6623 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
6624 llvm::Type *Tys[2] = {Ty, VTy};
6627 case NEON::BI__builtin_neon_vminv_f16: {
6628 Int = Intrinsic::aarch64_neon_fminv;
6630 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
6631 llvm::Type *Tys[2] = {Ty, VTy};
6634 case NEON::BI__builtin_neon_vminvq_f16: {
6635 Int = Intrinsic::aarch64_neon_fminv;
6637 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
6638 llvm::Type *Tys[2] = {Ty, VTy};
6641 case NEON::BI__builtin_neon_vmaxnmv_f16: {
6642 Int = Intrinsic::aarch64_neon_fmaxnmv;
6644 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
6645 llvm::Type *Tys[2] = {Ty, VTy};
6648 case NEON::BI__builtin_neon_vmaxnmvq_f16: {
6649 Int = Intrinsic::aarch64_neon_fmaxnmv;
6651 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
6652 llvm::Type *Tys[2] = {Ty, VTy};
6655 case NEON::BI__builtin_neon_vminnmv_f16: {
6656 Int = Intrinsic::aarch64_neon_fminnmv;
6658 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
6659 llvm::Type *Tys[2] = {Ty, VTy};
6662 case NEON::BI__builtin_neon_vminnmvq_f16: {
6663 Int = Intrinsic::aarch64_neon_fminnmv;
6665 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
6666 llvm::Type *Tys[2] = {Ty, VTy};
6669 case NEON::BI__builtin_neon_vmul_n_f64: {
6672 return Builder.CreateFMul(Ops[0], RHS);
6674 case NEON::BI__builtin_neon_vaddlv_u8:
6675 case NEON::BI__builtin_neon_vaddlvq_u8:
6676 case NEON::BI__builtin_neon_vaddlv_u16:
6677 case NEON::BI__builtin_neon_vaddlvq_u16: {
6678 Int = Intrinsic::aarch64_neon_uaddlv;
6681 llvm::Type *Tys[2] = {Ty, VTy};
6683 if (VTy->getElementType()->getPrimitiveSizeInBits() == 8)
6687 case NEON::BI__builtin_neon_vaddlv_s8:
6688 case NEON::BI__builtin_neon_vaddlvq_s8:
6689 case NEON::BI__builtin_neon_vaddlv_s16:
6690 case NEON::BI__builtin_neon_vaddlvq_s16: {
6691 Int = Intrinsic::aarch64_neon_saddlv;
6694 llvm::Type *Tys[2] = {Ty, VTy};
6696 if (VTy->getElementType()->getPrimitiveSizeInBits() == 8)
6700 case NEON::BI__builtin_neon_vsri_n_v:
6701 case NEON::BI__builtin_neon_vsriq_n_v: {
6702 Int = Intrinsic::aarch64_neon_vsri;
6703 llvm::Function *Intrin =
CGM.getIntrinsic(
Int, Ty);
6706 case NEON::BI__builtin_neon_vsli_n_v:
6707 case NEON::BI__builtin_neon_vsliq_n_v: {
6708 Int = Intrinsic::aarch64_neon_vsli;
6709 llvm::Function *Intrin =
CGM.getIntrinsic(
Int, Ty);
6712 case NEON::BI__builtin_neon_vsra_n_v:
6713 case NEON::BI__builtin_neon_vsraq_n_v:
6714 Ops[0] =
Builder.CreateBitCast(Ops[0], Ty);
6716 return Builder.CreateAdd(Ops[0], Ops[1]);
6717 case NEON::BI__builtin_neon_vrsra_n_v:
6718 case NEON::BI__builtin_neon_vrsraq_n_v: {
6719 Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl;
6721 TmpOps.push_back(Ops[1]);
6722 TmpOps.push_back(Ops[2]);
6724 llvm::Value *tmp =
EmitNeonCall(F, TmpOps,
"vrshr_n", 1,
true);
6725 Ops[0] =
Builder.CreateBitCast(Ops[0], VTy);
6726 return Builder.CreateAdd(Ops[0], tmp);
6728 case NEON::BI__builtin_neon_vld1_v:
6729 case NEON::BI__builtin_neon_vld1q_v: {
6732 case NEON::BI__builtin_neon_vst1_v:
6733 case NEON::BI__builtin_neon_vst1q_v:
6734 Ops[1] =
Builder.CreateBitCast(Ops[1], VTy);
6736 case NEON::BI__builtin_neon_vld1_lane_v:
6737 case NEON::BI__builtin_neon_vld1q_lane_v: {
6738 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
6739 Ops[0] =
Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0],
6741 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vld1_lane");
6743 case NEON::BI__builtin_neon_vldap1_lane_s64:
6744 case NEON::BI__builtin_neon_vldap1q_lane_s64: {
6745 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
6746 llvm::LoadInst *LI =
Builder.CreateAlignedLoad(
6748 LI->setAtomic(llvm::AtomicOrdering::Acquire);
6750 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vldap1_lane");
6752 case NEON::BI__builtin_neon_vld1_dup_v:
6753 case NEON::BI__builtin_neon_vld1q_dup_v: {
6754 Value *
V = PoisonValue::get(Ty);
6755 Ops[0] =
Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0],
6757 llvm::Constant *CI = ConstantInt::get(
Int32Ty, 0);
6758 Ops[0] =
Builder.CreateInsertElement(
V, Ops[0], CI);
6761 case NEON::BI__builtin_neon_vst1_lane_v:
6762 case NEON::BI__builtin_neon_vst1q_lane_v:
6763 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
6764 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2]);
6766 case NEON::BI__builtin_neon_vstl1_lane_s64:
6767 case NEON::BI__builtin_neon_vstl1q_lane_s64: {
6768 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
6769 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2]);
6770 llvm::StoreInst *SI =
6772 SI->setAtomic(llvm::AtomicOrdering::Release);
6775 case NEON::BI__builtin_neon_vld2_v:
6776 case NEON::BI__builtin_neon_vld2q_v: {
6778 Function *F =
CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2, Tys);
6779 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld2");
6780 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6782 case NEON::BI__builtin_neon_vld3_v:
6783 case NEON::BI__builtin_neon_vld3q_v: {
6785 Function *F =
CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3, Tys);
6786 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld3");
6787 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6789 case NEON::BI__builtin_neon_vld4_v:
6790 case NEON::BI__builtin_neon_vld4q_v: {
6792 Function *F =
CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4, Tys);
6793 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld4");
6794 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6796 case NEON::BI__builtin_neon_vld2_dup_v:
6797 case NEON::BI__builtin_neon_vld2q_dup_v: {
6799 Function *F =
CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2r, Tys);
6800 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld2");
6801 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6803 case NEON::BI__builtin_neon_vld3_dup_v:
6804 case NEON::BI__builtin_neon_vld3q_dup_v: {
6806 Function *F =
CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3r, Tys);
6807 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld3");
6808 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6810 case NEON::BI__builtin_neon_vld4_dup_v:
6811 case NEON::BI__builtin_neon_vld4q_dup_v: {
6813 Function *F =
CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4r, Tys);
6814 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld4");
6815 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6817 case NEON::BI__builtin_neon_vld2_lane_v:
6818 case NEON::BI__builtin_neon_vld2q_lane_v: {
6819 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
6820 Function *F =
CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2lane, Tys);
6821 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
6822 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
6823 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
6826 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6828 case NEON::BI__builtin_neon_vld3_lane_v:
6829 case NEON::BI__builtin_neon_vld3q_lane_v: {
6830 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
6831 Function *F =
CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3lane, Tys);
6832 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
6833 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
6834 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
6835 Ops[3] =
Builder.CreateBitCast(Ops[3], Ty);
6838 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6840 case NEON::BI__builtin_neon_vld4_lane_v:
6841 case NEON::BI__builtin_neon_vld4q_lane_v: {
6842 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
6843 Function *F =
CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4lane, Tys);
6844 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
6845 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
6846 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
6847 Ops[3] =
Builder.CreateBitCast(Ops[3], Ty);
6848 Ops[4] =
Builder.CreateBitCast(Ops[4], Ty);
6851 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6853 case NEON::BI__builtin_neon_vst2_v:
6854 case NEON::BI__builtin_neon_vst2q_v: {
6855 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
6856 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() };
6857 return EmitNeonCall(
CGM.getIntrinsic(Intrinsic::aarch64_neon_st2, Tys),
6860 case NEON::BI__builtin_neon_vst2_lane_v:
6861 case NEON::BI__builtin_neon_vst2q_lane_v: {
6862 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
6864 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
6865 return EmitNeonCall(
CGM.getIntrinsic(Intrinsic::aarch64_neon_st2lane, Tys),
6868 case NEON::BI__builtin_neon_vst3_v:
6869 case NEON::BI__builtin_neon_vst3q_v: {
6870 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
6871 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
6872 return EmitNeonCall(
CGM.getIntrinsic(Intrinsic::aarch64_neon_st3, Tys),
6875 case NEON::BI__builtin_neon_vst3_lane_v:
6876 case NEON::BI__builtin_neon_vst3q_lane_v: {
6877 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
6879 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
6880 return EmitNeonCall(
CGM.getIntrinsic(Intrinsic::aarch64_neon_st3lane, Tys),
6883 case NEON::BI__builtin_neon_vst4_v:
6884 case NEON::BI__builtin_neon_vst4q_v: {
6885 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
6886 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
6887 return EmitNeonCall(
CGM.getIntrinsic(Intrinsic::aarch64_neon_st4, Tys),
6890 case NEON::BI__builtin_neon_vst4_lane_v:
6891 case NEON::BI__builtin_neon_vst4q_lane_v: {
6892 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
6894 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() };
6895 return EmitNeonCall(
CGM.getIntrinsic(Intrinsic::aarch64_neon_st4lane, Tys),
6898 case NEON::BI__builtin_neon_vtrn_v:
6899 case NEON::BI__builtin_neon_vtrnq_v: {
6900 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
6901 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
6902 Value *SV =
nullptr;
6904 for (
unsigned vi = 0; vi != 2; ++vi) {
6906 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
6907 Indices.push_back(i+vi);
6908 Indices.push_back(i+e+vi);
6911 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vtrn");
6916 case NEON::BI__builtin_neon_vuzp_v:
6917 case NEON::BI__builtin_neon_vuzpq_v: {
6918 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
6919 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
6920 Value *SV =
nullptr;
6922 for (
unsigned vi = 0; vi != 2; ++vi) {
6924 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
6925 Indices.push_back(2*i+vi);
6928 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vuzp");
6933 case NEON::BI__builtin_neon_vzip_v:
6934 case NEON::BI__builtin_neon_vzipq_v: {
6935 Ops[1] =
Builder.CreateBitCast(Ops[1], Ty);
6936 Ops[2] =
Builder.CreateBitCast(Ops[2], Ty);
6937 Value *SV =
nullptr;
6939 for (
unsigned vi = 0; vi != 2; ++vi) {
6941 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
6942 Indices.push_back((i + vi*e) >> 1);
6943 Indices.push_back(((i + vi*e) >> 1)+e);
6946 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vzip");
6951 case NEON::BI__builtin_neon_vqtbl1q_v: {
6952 return EmitNeonCall(
CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl1, Ty),
6955 case NEON::BI__builtin_neon_vqtbl2q_v: {
6956 return EmitNeonCall(
CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl2, Ty),
6959 case NEON::BI__builtin_neon_vqtbl3q_v: {
6960 return EmitNeonCall(
CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl3, Ty),
6963 case NEON::BI__builtin_neon_vqtbl4q_v: {
6964 return EmitNeonCall(
CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl4, Ty),
6967 case NEON::BI__builtin_neon_vqtbx1q_v: {
6968 return EmitNeonCall(
CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx1, Ty),
6971 case NEON::BI__builtin_neon_vqtbx2q_v: {
6972 return EmitNeonCall(
CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx2, Ty),
6975 case NEON::BI__builtin_neon_vqtbx3q_v: {
6976 return EmitNeonCall(
CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx3, Ty),
6979 case NEON::BI__builtin_neon_vqtbx4q_v: {
6980 return EmitNeonCall(
CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx4, Ty),
6983 case NEON::BI__builtin_neon_vsqadd_v:
6984 case NEON::BI__builtin_neon_vsqaddq_v: {
6985 Int = Intrinsic::aarch64_neon_usqadd;
6988 case NEON::BI__builtin_neon_vuqadd_v:
6989 case NEON::BI__builtin_neon_vuqaddq_v: {
6990 Int = Intrinsic::aarch64_neon_suqadd;
6994 case NEON::BI__builtin_neon_vluti2_laneq_mf8:
6995 case NEON::BI__builtin_neon_vluti2_laneq_bf16:
6996 case NEON::BI__builtin_neon_vluti2_laneq_f16:
6997 case NEON::BI__builtin_neon_vluti2_laneq_p16:
6998 case NEON::BI__builtin_neon_vluti2_laneq_p8:
6999 case NEON::BI__builtin_neon_vluti2_laneq_s16:
7000 case NEON::BI__builtin_neon_vluti2_laneq_s8:
7001 case NEON::BI__builtin_neon_vluti2_laneq_u16:
7002 case NEON::BI__builtin_neon_vluti2_laneq_u8: {
7003 Int = Intrinsic::aarch64_neon_vluti2_laneq;
7010 case NEON::BI__builtin_neon_vluti2q_laneq_mf8:
7011 case NEON::BI__builtin_neon_vluti2q_laneq_bf16:
7012 case NEON::BI__builtin_neon_vluti2q_laneq_f16:
7013 case NEON::BI__builtin_neon_vluti2q_laneq_p16:
7014 case NEON::BI__builtin_neon_vluti2q_laneq_p8:
7015 case NEON::BI__builtin_neon_vluti2q_laneq_s16:
7016 case NEON::BI__builtin_neon_vluti2q_laneq_s8:
7017 case NEON::BI__builtin_neon_vluti2q_laneq_u16:
7018 case NEON::BI__builtin_neon_vluti2q_laneq_u8: {
7019 Int = Intrinsic::aarch64_neon_vluti2_laneq;
7026 case NEON::BI__builtin_neon_vluti2_lane_mf8:
7027 case NEON::BI__builtin_neon_vluti2_lane_bf16:
7028 case NEON::BI__builtin_neon_vluti2_lane_f16:
7029 case NEON::BI__builtin_neon_vluti2_lane_p16:
7030 case NEON::BI__builtin_neon_vluti2_lane_p8:
7031 case NEON::BI__builtin_neon_vluti2_lane_s16:
7032 case NEON::BI__builtin_neon_vluti2_lane_s8:
7033 case NEON::BI__builtin_neon_vluti2_lane_u16:
7034 case NEON::BI__builtin_neon_vluti2_lane_u8: {
7035 Int = Intrinsic::aarch64_neon_vluti2_lane;
7042 case NEON::BI__builtin_neon_vluti2q_lane_mf8:
7043 case NEON::BI__builtin_neon_vluti2q_lane_bf16:
7044 case NEON::BI__builtin_neon_vluti2q_lane_f16:
7045 case NEON::BI__builtin_neon_vluti2q_lane_p16:
7046 case NEON::BI__builtin_neon_vluti2q_lane_p8:
7047 case NEON::BI__builtin_neon_vluti2q_lane_s16:
7048 case NEON::BI__builtin_neon_vluti2q_lane_s8:
7049 case NEON::BI__builtin_neon_vluti2q_lane_u16:
7050 case NEON::BI__builtin_neon_vluti2q_lane_u8: {
7051 Int = Intrinsic::aarch64_neon_vluti2_lane;
7058 case NEON::BI__builtin_neon_vluti4q_lane_mf8:
7059 case NEON::BI__builtin_neon_vluti4q_lane_p8:
7060 case NEON::BI__builtin_neon_vluti4q_lane_s8:
7061 case NEON::BI__builtin_neon_vluti4q_lane_u8: {
7062 Int = Intrinsic::aarch64_neon_vluti4q_lane;
7065 case NEON::BI__builtin_neon_vluti4q_laneq_mf8:
7066 case NEON::BI__builtin_neon_vluti4q_laneq_p8:
7067 case NEON::BI__builtin_neon_vluti4q_laneq_s8:
7068 case NEON::BI__builtin_neon_vluti4q_laneq_u8: {
7069 Int = Intrinsic::aarch64_neon_vluti4q_laneq;
7072 case NEON::BI__builtin_neon_vluti4q_lane_bf16_x2:
7073 case NEON::BI__builtin_neon_vluti4q_lane_f16_x2:
7074 case NEON::BI__builtin_neon_vluti4q_lane_p16_x2:
7075 case NEON::BI__builtin_neon_vluti4q_lane_s16_x2:
7076 case NEON::BI__builtin_neon_vluti4q_lane_u16_x2: {
7077 Int = Intrinsic::aarch64_neon_vluti4q_lane_x2;
7080 case NEON::BI__builtin_neon_vluti4q_laneq_bf16_x2:
7081 case NEON::BI__builtin_neon_vluti4q_laneq_f16_x2:
7082 case NEON::BI__builtin_neon_vluti4q_laneq_p16_x2:
7083 case NEON::BI__builtin_neon_vluti4q_laneq_s16_x2:
7084 case NEON::BI__builtin_neon_vluti4q_laneq_u16_x2: {
7085 Int = Intrinsic::aarch64_neon_vluti4q_laneq_x2;
7088 case NEON::BI__builtin_neon_vmmlaq_f16_mf8_fpm:
7090 {llvm::FixedVectorType::get(
HalfTy, 8),
7091 llvm::FixedVectorType::get(
Int8Ty, 16)},
7093 case NEON::BI__builtin_neon_vmmlaq_f32_mf8_fpm:
7095 {llvm::FixedVectorType::get(
FloatTy, 4),
7096 llvm::FixedVectorType::get(
Int8Ty, 16)},
7098 case NEON::BI__builtin_neon_vcvt1_low_bf16_mf8_fpm:
7101 case NEON::BI__builtin_neon_vcvt1_bf16_mf8_fpm:
7102 case NEON::BI__builtin_neon_vcvt1_high_bf16_mf8_fpm:
7104 llvm::FixedVectorType::get(
BFloatTy, 8),
7105 Ops[0]->
getType(), ExtractLow, Ops, E,
"vbfcvt1");
7106 case NEON::BI__builtin_neon_vcvt2_low_bf16_mf8_fpm:
7109 case NEON::BI__builtin_neon_vcvt2_bf16_mf8_fpm:
7110 case NEON::BI__builtin_neon_vcvt2_high_bf16_mf8_fpm:
7112 llvm::FixedVectorType::get(
BFloatTy, 8),
7113 Ops[0]->
getType(), ExtractLow, Ops, E,
"vbfcvt2");
7114 case NEON::BI__builtin_neon_vcvt1_low_f16_mf8_fpm:
7117 case NEON::BI__builtin_neon_vcvt1_f16_mf8_fpm:
7118 case NEON::BI__builtin_neon_vcvt1_high_f16_mf8_fpm:
7120 llvm::FixedVectorType::get(
HalfTy, 8),
7121 Ops[0]->
getType(), ExtractLow, Ops, E,
"vbfcvt1");
7122 case NEON::BI__builtin_neon_vcvt2_low_f16_mf8_fpm:
7125 case NEON::BI__builtin_neon_vcvt2_f16_mf8_fpm:
7126 case NEON::BI__builtin_neon_vcvt2_high_f16_mf8_fpm:
7128 llvm::FixedVectorType::get(
HalfTy, 8),
7129 Ops[0]->
getType(), ExtractLow, Ops, E,
"vbfcvt2");
7130 case NEON::BI__builtin_neon_vcvt_mf8_f32_fpm:
7132 llvm::FixedVectorType::get(
Int8Ty, 8),
7133 Ops[0]->
getType(),
false, Ops, E,
"vfcvtn");
7134 case NEON::BI__builtin_neon_vcvt_mf8_f16_fpm:
7136 llvm::FixedVectorType::get(
Int8Ty, 8),
7137 llvm::FixedVectorType::get(
HalfTy, 4),
false, Ops,
7139 case NEON::BI__builtin_neon_vcvtq_mf8_f16_fpm:
7141 llvm::FixedVectorType::get(
Int8Ty, 16),
7142 llvm::FixedVectorType::get(
HalfTy, 8),
false, Ops,
7144 case NEON::BI__builtin_neon_vcvt_high_mf8_f32_fpm: {
7145 llvm::Type *Ty = llvm::FixedVectorType::get(
Int8Ty, 16);
7146 Ops[0] =
Builder.CreateInsertVector(Ty, PoisonValue::get(Ty), Ops[0],
7149 Ops[1]->
getType(),
false, Ops, E,
"vfcvtn2");
7152 case NEON::BI__builtin_neon_vdot_f16_mf8_fpm:
7153 case NEON::BI__builtin_neon_vdotq_f16_mf8_fpm:
7156 case NEON::BI__builtin_neon_vdot_lane_f16_mf8_fpm:
7157 case NEON::BI__builtin_neon_vdotq_lane_f16_mf8_fpm:
7158 ExtendLaneArg =
true;
7160 case NEON::BI__builtin_neon_vdot_laneq_f16_mf8_fpm:
7161 case NEON::BI__builtin_neon_vdotq_laneq_f16_mf8_fpm:
7163 ExtendLaneArg,
HalfTy, Ops, E,
"fdot2_lane");
7164 case NEON::BI__builtin_neon_vdot_f32_mf8_fpm:
7165 case NEON::BI__builtin_neon_vdotq_f32_mf8_fpm:
7168 case NEON::BI__builtin_neon_vdot_lane_f32_mf8_fpm:
7169 case NEON::BI__builtin_neon_vdotq_lane_f32_mf8_fpm:
7170 ExtendLaneArg =
true;
7172 case NEON::BI__builtin_neon_vdot_laneq_f32_mf8_fpm:
7173 case NEON::BI__builtin_neon_vdotq_laneq_f32_mf8_fpm:
7175 ExtendLaneArg,
FloatTy, Ops, E,
"fdot4_lane");
7177 case NEON::BI__builtin_neon_vdot_f32_f16:
7178 case NEON::BI__builtin_neon_vdotq_f32_f16: {
7179 llvm::Type *InputTy =
7180 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
7181 llvm::Type *Tys[2] = {Ty, InputTy};
7182 return EmitNeonCall(
CGM.getIntrinsic(Intrinsic::aarch64_neon_fdot, Tys),
7186 case NEON::BI__builtin_neon_vdot_lane_f32_f16:
7187 case NEON::BI__builtin_neon_vdot_laneq_f32_f16:
7188 case NEON::BI__builtin_neon_vdotq_lane_f32_f16:
7189 case NEON::BI__builtin_neon_vdotq_laneq_f32_f16: {
7190 llvm::FixedVectorType *InputTy =
7191 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
7192 llvm::FixedVectorType *LaneTy = llvm::FixedVectorType::get(
7196 Ops[2] =
Builder.CreateBitCast(Ops[2], LaneTy);
7198 InputTy->getElementCount());
7199 llvm::Type *Tys[2] = {Ty, InputTy};
7201 return EmitNeonCall(
CGM.getIntrinsic(Intrinsic::aarch64_neon_fdot, Tys),
7205 case NEON::BI__builtin_neon_vmlalbq_f16_mf8_fpm:
7207 {llvm::FixedVectorType::get(
HalfTy, 8)}, Ops, E,
7209 case NEON::BI__builtin_neon_vmlaltq_f16_mf8_fpm:
7211 {llvm::FixedVectorType::get(
HalfTy, 8)}, Ops, E,
7213 case NEON::BI__builtin_neon_vmlallbbq_f32_mf8_fpm:
7215 {llvm::FixedVectorType::get(
FloatTy, 4)}, Ops, E,
7217 case NEON::BI__builtin_neon_vmlallbtq_f32_mf8_fpm:
7219 {llvm::FixedVectorType::get(
FloatTy, 4)}, Ops, E,
7221 case NEON::BI__builtin_neon_vmlalltbq_f32_mf8_fpm:
7223 {llvm::FixedVectorType::get(
FloatTy, 4)}, Ops, E,
7225 case NEON::BI__builtin_neon_vmlallttq_f32_mf8_fpm:
7227 {llvm::FixedVectorType::get(
FloatTy, 4)}, Ops, E,
7229 case NEON::BI__builtin_neon_vmlalbq_lane_f16_mf8_fpm:
7230 ExtendLaneArg =
true;
7232 case NEON::BI__builtin_neon_vmlalbq_laneq_f16_mf8_fpm:
7234 ExtendLaneArg,
HalfTy, Ops, E,
"vmlal_lane");
7235 case NEON::BI__builtin_neon_vmlaltq_lane_f16_mf8_fpm:
7236 ExtendLaneArg =
true;
7238 case NEON::BI__builtin_neon_vmlaltq_laneq_f16_mf8_fpm:
7240 ExtendLaneArg,
HalfTy, Ops, E,
"vmlal_lane");
7241 case NEON::BI__builtin_neon_vmlallbbq_lane_f32_mf8_fpm:
7242 ExtendLaneArg =
true;
7244 case NEON::BI__builtin_neon_vmlallbbq_laneq_f32_mf8_fpm:
7246 ExtendLaneArg,
FloatTy, Ops, E,
"vmlall_lane");
7247 case NEON::BI__builtin_neon_vmlallbtq_lane_f32_mf8_fpm:
7248 ExtendLaneArg =
true;
7250 case NEON::BI__builtin_neon_vmlallbtq_laneq_f32_mf8_fpm:
7252 ExtendLaneArg,
FloatTy, Ops, E,
"vmlall_lane");
7253 case NEON::BI__builtin_neon_vmlalltbq_lane_f32_mf8_fpm:
7254 ExtendLaneArg =
true;
7256 case NEON::BI__builtin_neon_vmlalltbq_laneq_f32_mf8_fpm:
7258 ExtendLaneArg,
FloatTy, Ops, E,
"vmlall_lane");
7259 case NEON::BI__builtin_neon_vmlallttq_lane_f32_mf8_fpm:
7260 ExtendLaneArg =
true;
7262 case NEON::BI__builtin_neon_vmlallttq_laneq_f32_mf8_fpm:
7264 ExtendLaneArg,
FloatTy, Ops, E,
"vmlall_lane");
7265 case NEON::BI__builtin_neon_vamin_f16:
7266 case NEON::BI__builtin_neon_vaminq_f16:
7267 case NEON::BI__builtin_neon_vamin_f32:
7268 case NEON::BI__builtin_neon_vaminq_f32:
7269 case NEON::BI__builtin_neon_vaminq_f64: {
7270 Int = Intrinsic::aarch64_neon_famin;
7273 case NEON::BI__builtin_neon_vamax_f16:
7274 case NEON::BI__builtin_neon_vamaxq_f16:
7275 case NEON::BI__builtin_neon_vamax_f32:
7276 case NEON::BI__builtin_neon_vamaxq_f32:
7277 case NEON::BI__builtin_neon_vamaxq_f64: {
7278 Int = Intrinsic::aarch64_neon_famax;
7281 case NEON::BI__builtin_neon_vscale_f16:
7282 case NEON::BI__builtin_neon_vscaleq_f16:
7283 case NEON::BI__builtin_neon_vscale_f32:
7284 case NEON::BI__builtin_neon_vscaleq_f32:
7285 case NEON::BI__builtin_neon_vscaleq_f64: {
7286 Int = Intrinsic::aarch64_neon_fp8_fscale;