clang 23.0.0git
AArch64CodeGenUtils.h
Go to the documentation of this file.
1//===--- AArch64CodeGenUtils.h ----------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// Utilities used for generating code for AArch64 that are shared between the
11/// classic and ClangIR code-gen.
12///
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_CLANG_BASIC_AARCH64CODEGENUTILS_H
16#define LLVM_CLANG_BASIC_AARCH64CODEGENUTILS_H
17
18#include "clang/Basic/TargetBuiltins.h"
19#include "llvm/IR/Intrinsics.h"
20#include "llvm/IR/IntrinsicsAArch64.h"
21
22namespace clang {
23namespace aarch64 {
24
25//===----------------------------------------------------------------------===//
26// Intrinsics maps
27//
28// Maps that help automate code-generation.
29//===----------------------------------------------------------------------===//
30enum {
31 AddRetType = (1 << 0),
32 Add1ArgType = (1 << 1),
33 Add2ArgTypes = (1 << 2),
34
35 VectorizeRetType = (1 << 3),
36 VectorizeArgTypes = (1 << 4),
37
38 InventFloatType = (1 << 5),
39 UnsignedAlts = (1 << 6),
40
41 Use64BitVectors = (1 << 7),
42 Use128BitVectors = (1 << 8),
43
44 // Source operands are double-element-width of the result (e.g. vraddhn).
45 // ClangIR-only; classic code-gen derives this from the intrinsic's .td.
46 WidenArgs = (1 << 9),
47
48 Vectorize1ArgType = Add1ArgType | VectorizeArgTypes,
49 VectorRet = AddRetType | VectorizeRetType,
50 VectorRetGetArgs01 =
51 AddRetType | Add2ArgTypes | VectorizeRetType | VectorizeArgTypes,
52 FpCmpzModifiers =
53 AddRetType | VectorizeRetType | Add1ArgType | InventFloatType
54};
55
56/// Describes an ARM or AArch64 NEON intrinsic, or an AArch64 SISD intrinsic.
57///
58/// NEON and SISD code generation use NameHint and AltLLVMIntrinsic in addition
59/// to BuiltinID, LLVMIntrinsic, and TypeModifier. SVE and SME code generation
60/// does not use those fields, so AArch64SVEAndSMEVectorIntrinsicInfo omits
61/// them. On 64-bit hosts, the separate structure reduces each SVE and SME map
62/// entry from 32 to 16 bytes and avoids storing a name pointer for each entry.
63struct ARMNeonVectorIntrinsicInfo {
64 const char *NameHint;
65 unsigned BuiltinID;
66 unsigned LLVMIntrinsic;
67 unsigned AltLLVMIntrinsic;
68 uint64_t TypeModifier;
69
70 bool operator<(unsigned RHSBuiltinID) const {
71 return BuiltinID < RHSBuiltinID;
72 }
73 bool operator<(const ARMNeonVectorIntrinsicInfo &TE) const {
74 return BuiltinID < TE.BuiltinID;
75 }
76};
77
78static_assert(sizeof(ARMNeonVectorIntrinsicInfo) == 16 + 2 * sizeof(void *));
79
80/// Describes an AArch64 SVE or SME intrinsic.
81///
82/// See ARMNeonVectorIntrinsicInfo for the reason that SVE and SME use a
83/// separate structure.
84struct AArch64SVEAndSMEVectorIntrinsicInfo {
85 unsigned BuiltinID;
86 unsigned LLVMIntrinsic;
87 uint64_t TypeModifier;
88
89 bool operator<(unsigned RHSBuiltinID) const {
90 return BuiltinID < RHSBuiltinID;
91 }
92 bool operator<(const AArch64SVEAndSMEVectorIntrinsicInfo &TE) const {
93 return BuiltinID < TE.BuiltinID;
94 }
95};
96
97static_assert(sizeof(AArch64SVEAndSMEVectorIntrinsicInfo) == 16);
98
99#define NEONMAP0(NameBase) \
100 {#NameBase, NEON::BI__builtin_neon_##NameBase, 0, 0, 0}
101
102#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
103 {#NameBase, NEON::BI__builtin_neon_##NameBase, \
104 llvm::Intrinsic::LLVMIntrinsic, 0, TypeModifier}
105
106#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier) \
107 {#NameBase, NEON::BI__builtin_neon_##NameBase, \
108 llvm::Intrinsic::LLVMIntrinsic, llvm::Intrinsic::AltLLVMIntrinsic, \
109 TypeModifier}
110
111// clang-format off
112const inline ARMNeonVectorIntrinsicInfo AArch64SIMDIntrinsicMap [] = {
113 NEONMAP0(splat_lane_v),
114 NEONMAP0(splat_laneq_v),
115 NEONMAP0(splatq_lane_v),
116 NEONMAP0(splatq_laneq_v),
117 NEONMAP1(vabs_v, aarch64_neon_abs, 0),
118 NEONMAP1(vabsq_v, aarch64_neon_abs, 0),
119 NEONMAP0(vadd_v),
120 NEONMAP0(vaddhn_v),
121 NEONMAP0(vaddq_v),
122 NEONMAP1(vaesdq_u8, aarch64_crypto_aesd, 0),
123 NEONMAP1(vaeseq_u8, aarch64_crypto_aese, 0),
124 NEONMAP1(vaesimcq_u8, aarch64_crypto_aesimc, 0),
125 NEONMAP1(vaesmcq_u8, aarch64_crypto_aesmc, 0),
126 NEONMAP2(vbcaxq_s16, aarch64_crypto_bcaxu, aarch64_crypto_bcaxs, Add1ArgType | UnsignedAlts),
127 NEONMAP2(vbcaxq_s32, aarch64_crypto_bcaxu, aarch64_crypto_bcaxs, Add1ArgType | UnsignedAlts),
128 NEONMAP2(vbcaxq_s64, aarch64_crypto_bcaxu, aarch64_crypto_bcaxs, Add1ArgType | UnsignedAlts),
129 NEONMAP2(vbcaxq_s8, aarch64_crypto_bcaxu, aarch64_crypto_bcaxs, Add1ArgType | UnsignedAlts),
130 NEONMAP2(vbcaxq_u16, aarch64_crypto_bcaxu, aarch64_crypto_bcaxs, Add1ArgType | UnsignedAlts),
131 NEONMAP2(vbcaxq_u32, aarch64_crypto_bcaxu, aarch64_crypto_bcaxs, Add1ArgType | UnsignedAlts),
132 NEONMAP2(vbcaxq_u64, aarch64_crypto_bcaxu, aarch64_crypto_bcaxs, Add1ArgType | UnsignedAlts),
133 NEONMAP2(vbcaxq_u8, aarch64_crypto_bcaxu, aarch64_crypto_bcaxs, Add1ArgType | UnsignedAlts),
134 NEONMAP1(vbfdot_f32, aarch64_neon_bfdot, 0),
135 NEONMAP1(vbfdotq_f32, aarch64_neon_bfdot, 0),
136 NEONMAP1(vbfmlalbq_f32, aarch64_neon_bfmlalb, 0),
137 NEONMAP1(vbfmlaltq_f32, aarch64_neon_bfmlalt, 0),
138 NEONMAP1(vbfmmlaq_f32, aarch64_neon_bfmmla, 0),
139 NEONMAP1(vcadd_rot270_f16, aarch64_neon_vcadd_rot270, Add1ArgType),
140 NEONMAP1(vcadd_rot270_f32, aarch64_neon_vcadd_rot270, Add1ArgType),
141 NEONMAP1(vcadd_rot90_f16, aarch64_neon_vcadd_rot90, Add1ArgType),
142 NEONMAP1(vcadd_rot90_f32, aarch64_neon_vcadd_rot90, Add1ArgType),
143 NEONMAP1(vcaddq_rot270_f16, aarch64_neon_vcadd_rot270, Add1ArgType),
144 NEONMAP1(vcaddq_rot270_f32, aarch64_neon_vcadd_rot270, Add1ArgType),
145 NEONMAP1(vcaddq_rot270_f64, aarch64_neon_vcadd_rot270, Add1ArgType),
146 NEONMAP1(vcaddq_rot90_f16, aarch64_neon_vcadd_rot90, Add1ArgType),
147 NEONMAP1(vcaddq_rot90_f32, aarch64_neon_vcadd_rot90, Add1ArgType),
148 NEONMAP1(vcaddq_rot90_f64, aarch64_neon_vcadd_rot90, Add1ArgType),
149 NEONMAP1(vcage_v, aarch64_neon_facge, 0),
150 NEONMAP1(vcageq_v, aarch64_neon_facge, 0),
151 NEONMAP1(vcagt_v, aarch64_neon_facgt, 0),
152 NEONMAP1(vcagtq_v, aarch64_neon_facgt, 0),
153 NEONMAP1(vcale_v, aarch64_neon_facge, 0),
154 NEONMAP1(vcaleq_v, aarch64_neon_facge, 0),
155 NEONMAP1(vcalt_v, aarch64_neon_facgt, 0),
156 NEONMAP1(vcaltq_v, aarch64_neon_facgt, 0),
157 NEONMAP0(vceqz_v),
158 NEONMAP0(vceqzq_v),
159 NEONMAP0(vcgez_v),
160 NEONMAP0(vcgezq_v),
161 NEONMAP0(vcgtz_v),
162 NEONMAP0(vcgtzq_v),
163 NEONMAP0(vclez_v),
164 NEONMAP0(vclezq_v),
165 NEONMAP1(vcls_v, aarch64_neon_cls, Add1ArgType),
166 NEONMAP1(vclsq_v, aarch64_neon_cls, Add1ArgType),
167 NEONMAP0(vcltz_v),
168 NEONMAP0(vcltzq_v),
169 NEONMAP1(vclz_v, ctlz, Add1ArgType),
170 NEONMAP1(vclzq_v, ctlz, Add1ArgType),
171 NEONMAP1(vcmla_f16, aarch64_neon_vcmla_rot0, Add1ArgType),
172 NEONMAP1(vcmla_f32, aarch64_neon_vcmla_rot0, Add1ArgType),
173 NEONMAP1(vcmla_rot180_f16, aarch64_neon_vcmla_rot180, Add1ArgType),
174 NEONMAP1(vcmla_rot180_f32, aarch64_neon_vcmla_rot180, Add1ArgType),
175 NEONMAP1(vcmla_rot270_f16, aarch64_neon_vcmla_rot270, Add1ArgType),
176 NEONMAP1(vcmla_rot270_f32, aarch64_neon_vcmla_rot270, Add1ArgType),
177 NEONMAP1(vcmla_rot90_f16, aarch64_neon_vcmla_rot90, Add1ArgType),
178 NEONMAP1(vcmla_rot90_f32, aarch64_neon_vcmla_rot90, Add1ArgType),
179 NEONMAP1(vcmlaq_f16, aarch64_neon_vcmla_rot0, Add1ArgType),
180 NEONMAP1(vcmlaq_f32, aarch64_neon_vcmla_rot0, Add1ArgType),
181 NEONMAP1(vcmlaq_f64, aarch64_neon_vcmla_rot0, Add1ArgType),
182 NEONMAP1(vcmlaq_rot180_f16, aarch64_neon_vcmla_rot180, Add1ArgType),
183 NEONMAP1(vcmlaq_rot180_f32, aarch64_neon_vcmla_rot180, Add1ArgType),
184 NEONMAP1(vcmlaq_rot180_f64, aarch64_neon_vcmla_rot180, Add1ArgType),
185 NEONMAP1(vcmlaq_rot270_f16, aarch64_neon_vcmla_rot270, Add1ArgType),
186 NEONMAP1(vcmlaq_rot270_f32, aarch64_neon_vcmla_rot270, Add1ArgType),
187 NEONMAP1(vcmlaq_rot270_f64, aarch64_neon_vcmla_rot270, Add1ArgType),
188 NEONMAP1(vcmlaq_rot90_f16, aarch64_neon_vcmla_rot90, Add1ArgType),
189 NEONMAP1(vcmlaq_rot90_f32, aarch64_neon_vcmla_rot90, Add1ArgType),
190 NEONMAP1(vcmlaq_rot90_f64, aarch64_neon_vcmla_rot90, Add1ArgType),
191 NEONMAP1(vcnt_v, ctpop, Add1ArgType),
192 NEONMAP1(vcntq_v, ctpop, Add1ArgType),
193 NEONMAP0(vcvt_f16_s16),
194 NEONMAP0(vcvt_f16_u16),
195 NEONMAP0(vcvt_f32_v),
196 NEONMAP1(vcvt_n_f16_s16, aarch64_neon_vcvtfxs2fp, 0),
197 NEONMAP1(vcvt_n_f16_u16, aarch64_neon_vcvtfxu2fp, 0),
198 NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
199 NEONMAP2(vcvt_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
200 NEONMAP1(vcvt_n_s16_f16, aarch64_neon_vcvtfp2fxs, 0),
201 NEONMAP1(vcvt_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
202 NEONMAP1(vcvt_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
203 NEONMAP1(vcvt_n_u16_f16, aarch64_neon_vcvtfp2fxu, 0),
204 NEONMAP1(vcvt_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
205 NEONMAP1(vcvt_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
206 NEONMAP1(vcvt_s16_f16, aarch64_neon_fcvtzs, AddRetType | Add1ArgType),
207 NEONMAP1(vcvt_s32_v, aarch64_neon_fcvtzs, AddRetType | Add1ArgType),
208 NEONMAP1(vcvt_s64_v, aarch64_neon_fcvtzs, AddRetType | Add1ArgType),
209 NEONMAP1(vcvt_u16_f16, aarch64_neon_fcvtzu, AddRetType | Add1ArgType),
210 NEONMAP1(vcvt_u32_v, aarch64_neon_fcvtzu, AddRetType | Add1ArgType),
211 NEONMAP1(vcvt_u64_v, aarch64_neon_fcvtzu, AddRetType | Add1ArgType),
212 NEONMAP0(vcvtq_f16_s16),
213 NEONMAP0(vcvtq_f16_u16),
214 NEONMAP0(vcvtq_f32_v),
215 NEONMAP1(vcvtq_n_f16_s16, aarch64_neon_vcvtfxs2fp, 0),
216 NEONMAP1(vcvtq_n_f16_u16, aarch64_neon_vcvtfxu2fp, 0),
217 NEONMAP2(vcvtq_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
218 NEONMAP2(vcvtq_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
219 NEONMAP1(vcvtq_n_s16_f16, aarch64_neon_vcvtfp2fxs, 0),
220 NEONMAP1(vcvtq_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
221 NEONMAP1(vcvtq_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
222 NEONMAP1(vcvtq_n_u16_f16, aarch64_neon_vcvtfp2fxu, 0),
223 NEONMAP1(vcvtq_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
224 NEONMAP1(vcvtq_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
225 NEONMAP1(vcvtq_s16_f16, aarch64_neon_fcvtzs, AddRetType | Add1ArgType),
226 NEONMAP1(vcvtq_s32_v, aarch64_neon_fcvtzs, AddRetType | Add1ArgType),
227 NEONMAP1(vcvtq_s64_v, aarch64_neon_fcvtzs, AddRetType | Add1ArgType),
228 NEONMAP1(vcvtq_u16_f16, aarch64_neon_fcvtzu, AddRetType | Add1ArgType),
229 NEONMAP1(vcvtq_u32_v, aarch64_neon_fcvtzu, AddRetType | Add1ArgType),
230 NEONMAP1(vcvtq_u64_v, aarch64_neon_fcvtzu, AddRetType | Add1ArgType),
231 NEONMAP1(vcvtx_f32_v, aarch64_neon_fcvtxn, AddRetType | Add1ArgType),
232 NEONMAP1(vdot_s32, aarch64_neon_sdot, 0),
233 NEONMAP1(vdot_u32, aarch64_neon_udot, 0),
234 NEONMAP1(vdotq_s32, aarch64_neon_sdot, 0),
235 NEONMAP1(vdotq_u32, aarch64_neon_udot, 0),
236 NEONMAP2(veor3q_s16, aarch64_crypto_eor3u, aarch64_crypto_eor3s, Add1ArgType | UnsignedAlts),
237 NEONMAP2(veor3q_s32, aarch64_crypto_eor3u, aarch64_crypto_eor3s, Add1ArgType | UnsignedAlts),
238 NEONMAP2(veor3q_s64, aarch64_crypto_eor3u, aarch64_crypto_eor3s, Add1ArgType | UnsignedAlts),
239 NEONMAP2(veor3q_s8, aarch64_crypto_eor3u, aarch64_crypto_eor3s, Add1ArgType | UnsignedAlts),
240 NEONMAP2(veor3q_u16, aarch64_crypto_eor3u, aarch64_crypto_eor3s, Add1ArgType | UnsignedAlts),
241 NEONMAP2(veor3q_u32, aarch64_crypto_eor3u, aarch64_crypto_eor3s, Add1ArgType | UnsignedAlts),
242 NEONMAP2(veor3q_u64, aarch64_crypto_eor3u, aarch64_crypto_eor3s, Add1ArgType | UnsignedAlts),
243 NEONMAP2(veor3q_u8, aarch64_crypto_eor3u, aarch64_crypto_eor3s, Add1ArgType | UnsignedAlts),
244 NEONMAP0(vext_v),
245 NEONMAP0(vextq_v),
246 NEONMAP0(vfma_v),
247 NEONMAP0(vfmaq_v),
248 NEONMAP1(vfmlal_high_f16, aarch64_neon_fmlal2, 0),
249 NEONMAP1(vfmlal_low_f16, aarch64_neon_fmlal, 0),
250 NEONMAP1(vfmlalq_high_f16, aarch64_neon_fmlal2, 0),
251 NEONMAP1(vfmlalq_low_f16, aarch64_neon_fmlal, 0),
252 NEONMAP1(vfmlsl_high_f16, aarch64_neon_fmlsl2, 0),
253 NEONMAP1(vfmlsl_low_f16, aarch64_neon_fmlsl, 0),
254 NEONMAP1(vfmlslq_high_f16, aarch64_neon_fmlsl2, 0),
255 NEONMAP1(vfmlslq_low_f16, aarch64_neon_fmlsl, 0),
256 NEONMAP2(vhadd_v, aarch64_neon_uhadd, aarch64_neon_shadd, Add1ArgType | UnsignedAlts),
257 NEONMAP2(vhaddq_v, aarch64_neon_uhadd, aarch64_neon_shadd, Add1ArgType | UnsignedAlts),
258 NEONMAP2(vhsub_v, aarch64_neon_uhsub, aarch64_neon_shsub, Add1ArgType | UnsignedAlts),
259 NEONMAP2(vhsubq_v, aarch64_neon_uhsub, aarch64_neon_shsub, Add1ArgType | UnsignedAlts),
260 NEONMAP1(vld1_x2_v, aarch64_neon_ld1x2, 0),
261 NEONMAP1(vld1_x3_v, aarch64_neon_ld1x3, 0),
262 NEONMAP1(vld1_x4_v, aarch64_neon_ld1x4, 0),
263 NEONMAP1(vld1q_x2_v, aarch64_neon_ld1x2, 0),
264 NEONMAP1(vld1q_x3_v, aarch64_neon_ld1x3, 0),
265 NEONMAP1(vld1q_x4_v, aarch64_neon_ld1x4, 0),
266 NEONMAP1(vmmlaq_f16_f16, aarch64_neon_fmmla, 0),
267 NEONMAP1(vmmlaq_f32_f16, aarch64_neon_fmmla, 0),
268 NEONMAP1(vmmlaq_s32, aarch64_neon_smmla, 0),
269 NEONMAP1(vmmlaq_u32, aarch64_neon_ummla, 0),
270 NEONMAP0(vmovl_v),
271 NEONMAP0(vmovn_v),
272 NEONMAP1(vmul_v, aarch64_neon_pmul, Add1ArgType),
273 NEONMAP1(vmulq_v, aarch64_neon_pmul, Add1ArgType),
274 NEONMAP1(vpadd_v, aarch64_neon_addp, Add1ArgType),
275 NEONMAP2(vpaddl_v, aarch64_neon_uaddlp, aarch64_neon_saddlp, UnsignedAlts),
276 NEONMAP2(vpaddlq_v, aarch64_neon_uaddlp, aarch64_neon_saddlp, UnsignedAlts),
277 NEONMAP1(vpaddq_v, aarch64_neon_addp, Add1ArgType),
278 NEONMAP1(vqabs_v, aarch64_neon_sqabs, Add1ArgType),
279 NEONMAP1(vqabsq_v, aarch64_neon_sqabs, Add1ArgType),
280 NEONMAP2(vqadd_v, aarch64_neon_uqadd, aarch64_neon_sqadd, Add1ArgType | UnsignedAlts),
281 NEONMAP2(vqaddq_v, aarch64_neon_uqadd, aarch64_neon_sqadd, Add1ArgType | UnsignedAlts),
282 NEONMAP2(vqdmlal_v, aarch64_neon_sqdmull, aarch64_neon_sqadd, 0),
283 NEONMAP2(vqdmlsl_v, aarch64_neon_sqdmull, aarch64_neon_sqsub, 0),
284 NEONMAP1(vqdmulh_lane_v, aarch64_neon_sqdmulh_lane, 0),
285 NEONMAP1(vqdmulh_laneq_v, aarch64_neon_sqdmulh_laneq, 0),
286 NEONMAP1(vqdmulh_v, aarch64_neon_sqdmulh, Add1ArgType),
287 NEONMAP1(vqdmulhq_lane_v, aarch64_neon_sqdmulh_lane, 0),
288 NEONMAP1(vqdmulhq_laneq_v, aarch64_neon_sqdmulh_laneq, 0),
289 NEONMAP1(vqdmulhq_v, aarch64_neon_sqdmulh, Add1ArgType),
290 NEONMAP1(vqdmull_v, aarch64_neon_sqdmull, Add1ArgType),
291 NEONMAP2(vqmovn_v, aarch64_neon_uqxtn, aarch64_neon_sqxtn, Add1ArgType | UnsignedAlts),
292 NEONMAP1(vqmovun_v, aarch64_neon_sqxtun, Add1ArgType),
293 NEONMAP1(vqneg_v, aarch64_neon_sqneg, Add1ArgType),
294 NEONMAP1(vqnegq_v, aarch64_neon_sqneg, Add1ArgType),
295 NEONMAP1(vqrdmlah_s16, aarch64_neon_sqrdmlah, Add1ArgType),
296 NEONMAP1(vqrdmlah_s32, aarch64_neon_sqrdmlah, Add1ArgType),
297 NEONMAP1(vqrdmlahq_s16, aarch64_neon_sqrdmlah, Add1ArgType),
298 NEONMAP1(vqrdmlahq_s32, aarch64_neon_sqrdmlah, Add1ArgType),
299 NEONMAP1(vqrdmlsh_s16, aarch64_neon_sqrdmlsh, Add1ArgType),
300 NEONMAP1(vqrdmlsh_s32, aarch64_neon_sqrdmlsh, Add1ArgType),
301 NEONMAP1(vqrdmlshq_s16, aarch64_neon_sqrdmlsh, Add1ArgType),
302 NEONMAP1(vqrdmlshq_s32, aarch64_neon_sqrdmlsh, Add1ArgType),
303 NEONMAP1(vqrdmulh_lane_v, aarch64_neon_sqrdmulh_lane, 0),
304 NEONMAP1(vqrdmulh_laneq_v, aarch64_neon_sqrdmulh_laneq, 0),
305 NEONMAP1(vqrdmulh_v, aarch64_neon_sqrdmulh, Add1ArgType),
306 NEONMAP1(vqrdmulhq_lane_v, aarch64_neon_sqrdmulh_lane, 0),
307 NEONMAP1(vqrdmulhq_laneq_v, aarch64_neon_sqrdmulh_laneq, 0),
308 NEONMAP1(vqrdmulhq_v, aarch64_neon_sqrdmulh, Add1ArgType),
309 NEONMAP2(vqrshl_v, aarch64_neon_uqrshl, aarch64_neon_sqrshl, Add1ArgType | UnsignedAlts),
310 NEONMAP2(vqrshlq_v, aarch64_neon_uqrshl, aarch64_neon_sqrshl, Add1ArgType | UnsignedAlts),
311 NEONMAP2(vqshl_n_v, aarch64_neon_uqshl, aarch64_neon_sqshl, UnsignedAlts),
312 NEONMAP2(vqshl_v, aarch64_neon_uqshl, aarch64_neon_sqshl, Add1ArgType | UnsignedAlts),
313 NEONMAP2(vqshlq_n_v, aarch64_neon_uqshl, aarch64_neon_sqshl,UnsignedAlts),
314 NEONMAP2(vqshlq_v, aarch64_neon_uqshl, aarch64_neon_sqshl, Add1ArgType | UnsignedAlts),
315 NEONMAP1(vqshlu_n_v, aarch64_neon_sqshlu, 0),
316 NEONMAP1(vqshluq_n_v, aarch64_neon_sqshlu, 0),
317 NEONMAP2(vqsub_v, aarch64_neon_uqsub, aarch64_neon_sqsub, Add1ArgType | UnsignedAlts),
318 NEONMAP2(vqsubq_v, aarch64_neon_uqsub, aarch64_neon_sqsub, Add1ArgType | UnsignedAlts),
319 NEONMAP1(vraddhn_v, aarch64_neon_raddhn, Add1ArgType | WidenArgs),
320 NEONMAP1(vrax1q_u64, aarch64_crypto_rax1, 0),
321 NEONMAP2(vrecpe_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
322 NEONMAP2(vrecpeq_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
323 NEONMAP1(vrecps_v, aarch64_neon_frecps, Add1ArgType),
324 NEONMAP1(vrecpsq_v, aarch64_neon_frecps, Add1ArgType),
325 NEONMAP2(vrhadd_v, aarch64_neon_urhadd, aarch64_neon_srhadd, Add1ArgType | UnsignedAlts),
326 NEONMAP2(vrhaddq_v, aarch64_neon_urhadd, aarch64_neon_srhadd, Add1ArgType | UnsignedAlts),
327 NEONMAP1(vrnd32x_f32, aarch64_neon_frint32x, Add1ArgType),
328 NEONMAP1(vrnd32x_f64, aarch64_neon_frint32x, Add1ArgType),
329 NEONMAP1(vrnd32xq_f32, aarch64_neon_frint32x, Add1ArgType),
330 NEONMAP1(vrnd32xq_f64, aarch64_neon_frint32x, Add1ArgType),
331 NEONMAP1(vrnd32z_f32, aarch64_neon_frint32z, Add1ArgType),
332 NEONMAP1(vrnd32z_f64, aarch64_neon_frint32z, Add1ArgType),
333 NEONMAP1(vrnd32zq_f32, aarch64_neon_frint32z, Add1ArgType),
334 NEONMAP1(vrnd32zq_f64, aarch64_neon_frint32z, Add1ArgType),
335 NEONMAP1(vrnd64x_f32, aarch64_neon_frint64x, Add1ArgType),
336 NEONMAP1(vrnd64x_f64, aarch64_neon_frint64x, Add1ArgType),
337 NEONMAP1(vrnd64xq_f32, aarch64_neon_frint64x, Add1ArgType),
338 NEONMAP1(vrnd64xq_f64, aarch64_neon_frint64x, Add1ArgType),
339 NEONMAP1(vrnd64z_f32, aarch64_neon_frint64z, Add1ArgType),
340 NEONMAP1(vrnd64z_f64, aarch64_neon_frint64z, Add1ArgType),
341 NEONMAP1(vrnd64zq_f32, aarch64_neon_frint64z, Add1ArgType),
342 NEONMAP1(vrnd64zq_f64, aarch64_neon_frint64z, Add1ArgType),
343 NEONMAP0(vrndi_v),
344 NEONMAP0(vrndiq_v),
345 NEONMAP2(vrshl_v, aarch64_neon_urshl, aarch64_neon_srshl, Add1ArgType | UnsignedAlts),
346 NEONMAP2(vrshlq_v, aarch64_neon_urshl, aarch64_neon_srshl, Add1ArgType | UnsignedAlts),
347 NEONMAP2(vrshr_n_v, aarch64_neon_urshl, aarch64_neon_srshl, UnsignedAlts),
348 NEONMAP2(vrshrq_n_v, aarch64_neon_urshl, aarch64_neon_srshl, UnsignedAlts),
349 NEONMAP2(vrsqrte_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
350 NEONMAP2(vrsqrteq_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
351 NEONMAP1(vrsqrts_v, aarch64_neon_frsqrts, Add1ArgType),
352 NEONMAP1(vrsqrtsq_v, aarch64_neon_frsqrts, Add1ArgType),
353 NEONMAP1(vrsubhn_v, aarch64_neon_rsubhn, Add1ArgType | WidenArgs),
354 NEONMAP1(vsha1su0q_u32, aarch64_crypto_sha1su0, 0),
355 NEONMAP1(vsha1su1q_u32, aarch64_crypto_sha1su1, 0),
356 NEONMAP1(vsha256h2q_u32, aarch64_crypto_sha256h2, 0),
357 NEONMAP1(vsha256hq_u32, aarch64_crypto_sha256h, 0),
358 NEONMAP1(vsha256su0q_u32, aarch64_crypto_sha256su0, 0),
359 NEONMAP1(vsha256su1q_u32, aarch64_crypto_sha256su1, 0),
360 NEONMAP1(vsha512h2q_u64, aarch64_crypto_sha512h2, 0),
361 NEONMAP1(vsha512hq_u64, aarch64_crypto_sha512h, 0),
362 NEONMAP1(vsha512su0q_u64, aarch64_crypto_sha512su0, 0),
363 NEONMAP1(vsha512su1q_u64, aarch64_crypto_sha512su1, 0),
364 NEONMAP0(vshl_n_v),
365 NEONMAP2(vshl_v, aarch64_neon_ushl, aarch64_neon_sshl, Add1ArgType | UnsignedAlts),
366 NEONMAP0(vshll_n_v),
367 NEONMAP0(vshlq_n_v),
368 NEONMAP2(vshlq_v, aarch64_neon_ushl, aarch64_neon_sshl, Add1ArgType | UnsignedAlts),
369 NEONMAP0(vshr_n_v),
370 NEONMAP0(vshrn_n_v),
371 NEONMAP0(vshrq_n_v),
372 NEONMAP1(vsm3partw1q_u32, aarch64_crypto_sm3partw1, 0),
373 NEONMAP1(vsm3partw2q_u32, aarch64_crypto_sm3partw2, 0),
374 NEONMAP1(vsm3ss1q_u32, aarch64_crypto_sm3ss1, 0),
375 NEONMAP1(vsm3tt1aq_u32, aarch64_crypto_sm3tt1a, 0),
376 NEONMAP1(vsm3tt1bq_u32, aarch64_crypto_sm3tt1b, 0),
377 NEONMAP1(vsm3tt2aq_u32, aarch64_crypto_sm3tt2a, 0),
378 NEONMAP1(vsm3tt2bq_u32, aarch64_crypto_sm3tt2b, 0),
379 NEONMAP1(vsm4ekeyq_u32, aarch64_crypto_sm4ekey, 0),
380 NEONMAP1(vsm4eq_u32, aarch64_crypto_sm4e, 0),
381 NEONMAP1(vst1_x2_v, aarch64_neon_st1x2, 0),
382 NEONMAP1(vst1_x3_v, aarch64_neon_st1x3, 0),
383 NEONMAP1(vst1_x4_v, aarch64_neon_st1x4, 0),
384 NEONMAP1(vst1q_x2_v, aarch64_neon_st1x2, 0),
385 NEONMAP1(vst1q_x3_v, aarch64_neon_st1x3, 0),
386 NEONMAP1(vst1q_x4_v, aarch64_neon_st1x4, 0),
387 NEONMAP0(vsubhn_v),
388 NEONMAP0(vtst_v),
389 NEONMAP0(vtstq_v),
390 NEONMAP1(vusdot_s32, aarch64_neon_usdot, 0),
391 NEONMAP1(vusdotq_s32, aarch64_neon_usdot, 0),
392 NEONMAP1(vusmmlaq_s32, aarch64_neon_usmmla, 0),
393 NEONMAP1(vxarq_u64, aarch64_crypto_xar, 0),
394};
395
396
397// Single-Instruction-Single-Data (SISD) intrinsics.
398//
399// The name is somewhat misleading: not all intrinsics in this table are
400// strictly SISD. While many builtins operate on scalars,
401// * some take vector operands (e.g. reduction builtins such as
402// `vminvq_u16` and `vaddvq_s32`), and
403// * some take both scalar and vector operands (e.g. crypto builtins
404// such as `vsha1cq_u32`).
405//
406// TODO: Either rename this table to better reflect its contents, or
407// restrict it to true SISD intrinsics only.
408const inline ARMNeonVectorIntrinsicInfo AArch64SISDIntrinsicMap[] = {
409 NEONMAP1(vabdd_f64, aarch64_sisd_fabd, Add1ArgType),
410 NEONMAP1(vabds_f32, aarch64_sisd_fabd, Add1ArgType),
411 NEONMAP1(vabsd_s64, aarch64_neon_abs, Add1ArgType),
412 NEONMAP1(vaddlv_s32, aarch64_neon_saddlv, AddRetType | Add1ArgType),
413 NEONMAP1(vaddlv_u32, aarch64_neon_uaddlv, AddRetType | Add1ArgType),
414 NEONMAP1(vaddlvq_s32, aarch64_neon_saddlv, AddRetType | Add1ArgType),
415 NEONMAP1(vaddlvq_u32, aarch64_neon_uaddlv, AddRetType | Add1ArgType),
416 NEONMAP1(vaddv_f32, aarch64_neon_faddv, AddRetType | Add1ArgType),
417 NEONMAP1(vaddv_s16, vector_reduce_add, Add1ArgType),
418 NEONMAP1(vaddv_s32, vector_reduce_add, Add1ArgType),
419 NEONMAP1(vaddv_s8, vector_reduce_add, Add1ArgType),
420 NEONMAP1(vaddv_u16, vector_reduce_add, Add1ArgType),
421 NEONMAP1(vaddv_u32, vector_reduce_add, Add1ArgType),
422 NEONMAP1(vaddv_u8, vector_reduce_add, Add1ArgType),
423 NEONMAP1(vaddvq_f32, aarch64_neon_faddv, AddRetType | Add1ArgType),
424 NEONMAP1(vaddvq_f64, aarch64_neon_faddv, AddRetType | Add1ArgType),
425 NEONMAP1(vaddvq_s16, vector_reduce_add, Add1ArgType),
426 NEONMAP1(vaddvq_s32, vector_reduce_add, Add1ArgType),
427 NEONMAP1(vaddvq_s64, vector_reduce_add, Add1ArgType),
428 NEONMAP1(vaddvq_s8, vector_reduce_add, Add1ArgType),
429 NEONMAP1(vaddvq_u16, vector_reduce_add, Add1ArgType),
430 NEONMAP1(vaddvq_u32, vector_reduce_add, Add1ArgType),
431 NEONMAP1(vaddvq_u64, vector_reduce_add, Add1ArgType),
432 NEONMAP1(vaddvq_u8, vector_reduce_add, Add1ArgType),
433 NEONMAP1(vcaged_f64, aarch64_neon_facge, AddRetType | Add1ArgType),
434 NEONMAP1(vcages_f32, aarch64_neon_facge, AddRetType | Add1ArgType),
435 NEONMAP1(vcagtd_f64, aarch64_neon_facgt, AddRetType | Add1ArgType),
436 NEONMAP1(vcagts_f32, aarch64_neon_facgt, AddRetType | Add1ArgType),
437 NEONMAP1(vcaled_f64, aarch64_neon_facge, AddRetType | Add1ArgType),
438 NEONMAP1(vcales_f32, aarch64_neon_facge, AddRetType | Add1ArgType),
439 NEONMAP1(vcaltd_f64, aarch64_neon_facgt, AddRetType | Add1ArgType),
440 NEONMAP1(vcalts_f32, aarch64_neon_facgt, AddRetType | Add1ArgType),
441 NEONMAP1(vcvtad_s32_f64, aarch64_neon_fcvtas, AddRetType | Add1ArgType),
442 NEONMAP1(vcvtad_s64_f64, aarch64_neon_fcvtas, AddRetType | Add1ArgType),
443 NEONMAP1(vcvtad_u32_f64, aarch64_neon_fcvtau, AddRetType | Add1ArgType),
444 NEONMAP1(vcvtad_u64_f64, aarch64_neon_fcvtau, AddRetType | Add1ArgType),
445 NEONMAP1(vcvtas_s32_f32, aarch64_neon_fcvtas, AddRetType | Add1ArgType),
446 NEONMAP1(vcvtas_s64_f32, aarch64_neon_fcvtas, AddRetType | Add1ArgType),
447 NEONMAP1(vcvtas_u32_f32, aarch64_neon_fcvtau, AddRetType | Add1ArgType),
448 NEONMAP1(vcvtas_u64_f32, aarch64_neon_fcvtau, AddRetType | Add1ArgType),
449 NEONMAP1(vcvtd_n_f64_s64, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType),
450 NEONMAP1(vcvtd_n_f64_u64, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType),
451 NEONMAP1(vcvtd_n_s64_f64, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType),
452 NEONMAP1(vcvtd_n_u64_f64, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType),
453 NEONMAP1(vcvtd_s32_f64, aarch64_neon_fcvtzs, AddRetType | Add1ArgType),
454 NEONMAP1(vcvtd_s64_f64, aarch64_neon_fcvtzs, AddRetType | Add1ArgType),
455 NEONMAP1(vcvtd_u32_f64, aarch64_neon_fcvtzu, AddRetType | Add1ArgType),
456 NEONMAP1(vcvtd_u64_f64, aarch64_neon_fcvtzu, AddRetType | Add1ArgType),
457 NEONMAP0(vcvth_bf16_f32),
458 NEONMAP1(vcvtmd_s32_f64, aarch64_neon_fcvtms, AddRetType | Add1ArgType),
459 NEONMAP1(vcvtmd_s64_f64, aarch64_neon_fcvtms, AddRetType | Add1ArgType),
460 NEONMAP1(vcvtmd_u32_f64, aarch64_neon_fcvtmu, AddRetType | Add1ArgType),
461 NEONMAP1(vcvtmd_u64_f64, aarch64_neon_fcvtmu, AddRetType | Add1ArgType),
462 NEONMAP1(vcvtms_s32_f32, aarch64_neon_fcvtms, AddRetType | Add1ArgType),
463 NEONMAP1(vcvtms_s64_f32, aarch64_neon_fcvtms, AddRetType | Add1ArgType),
464 NEONMAP1(vcvtms_u32_f32, aarch64_neon_fcvtmu, AddRetType | Add1ArgType),
465 NEONMAP1(vcvtms_u64_f32, aarch64_neon_fcvtmu, AddRetType | Add1ArgType),
466 NEONMAP1(vcvtnd_s32_f64, aarch64_neon_fcvtns, AddRetType | Add1ArgType),
467 NEONMAP1(vcvtnd_s64_f64, aarch64_neon_fcvtns, AddRetType | Add1ArgType),
468 NEONMAP1(vcvtnd_u32_f64, aarch64_neon_fcvtnu, AddRetType | Add1ArgType),
469 NEONMAP1(vcvtnd_u64_f64, aarch64_neon_fcvtnu, AddRetType | Add1ArgType),
470 NEONMAP1(vcvtns_s32_f32, aarch64_neon_fcvtns, AddRetType | Add1ArgType),
471 NEONMAP1(vcvtns_s64_f32, aarch64_neon_fcvtns, AddRetType | Add1ArgType),
472 NEONMAP1(vcvtns_u32_f32, aarch64_neon_fcvtnu, AddRetType | Add1ArgType),
473 NEONMAP1(vcvtns_u64_f32, aarch64_neon_fcvtnu, AddRetType | Add1ArgType),
474 NEONMAP1(vcvtpd_s32_f64, aarch64_neon_fcvtps, AddRetType | Add1ArgType),
475 NEONMAP1(vcvtpd_s64_f64, aarch64_neon_fcvtps, AddRetType | Add1ArgType),
476 NEONMAP1(vcvtpd_u32_f64, aarch64_neon_fcvtpu, AddRetType | Add1ArgType),
477 NEONMAP1(vcvtpd_u64_f64, aarch64_neon_fcvtpu, AddRetType | Add1ArgType),
478 NEONMAP1(vcvtps_s32_f32, aarch64_neon_fcvtps, AddRetType | Add1ArgType),
479 NEONMAP1(vcvtps_s64_f32, aarch64_neon_fcvtps, AddRetType | Add1ArgType),
480 NEONMAP1(vcvtps_u32_f32, aarch64_neon_fcvtpu, AddRetType | Add1ArgType),
481 NEONMAP1(vcvtps_u64_f32, aarch64_neon_fcvtpu, AddRetType | Add1ArgType),
482 NEONMAP1(vcvts_n_f32_s32, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType),
483 NEONMAP1(vcvts_n_f32_u32, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType),
484 NEONMAP1(vcvts_n_s32_f32, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType),
485 NEONMAP1(vcvts_n_u32_f32, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType),
486 NEONMAP1(vcvts_s32_f32, aarch64_neon_fcvtzs, AddRetType | Add1ArgType),
487 NEONMAP1(vcvts_s64_f32, aarch64_neon_fcvtzs, AddRetType | Add1ArgType),
488 NEONMAP1(vcvts_u32_f32, aarch64_neon_fcvtzu, AddRetType | Add1ArgType),
489 NEONMAP1(vcvts_u64_f32, aarch64_neon_fcvtzu, AddRetType | Add1ArgType),
490 NEONMAP1(vcvtxd_f32_f64, aarch64_sisd_fcvtxn, 0),
491 NEONMAP1(vmaxnmv_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
492 NEONMAP1(vmaxnmvq_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
493 NEONMAP1(vmaxnmvq_f64, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
494 NEONMAP1(vmaxv_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
495 NEONMAP1(vmaxv_s16, vector_reduce_smax, Add1ArgType),
496 NEONMAP1(vmaxv_s32, vector_reduce_smax, Add1ArgType),
497 NEONMAP1(vmaxv_s8, vector_reduce_smax, Add1ArgType),
498 NEONMAP1(vmaxv_u16, vector_reduce_umax, Add1ArgType),
499 NEONMAP1(vmaxv_u32, vector_reduce_umax, Add1ArgType),
500 NEONMAP1(vmaxv_u8, vector_reduce_umax, Add1ArgType),
501 NEONMAP1(vmaxvq_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
502 NEONMAP1(vmaxvq_f64, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
503 NEONMAP1(vmaxvq_s16, vector_reduce_smax, Add1ArgType),
504 NEONMAP1(vmaxvq_s32, vector_reduce_smax, Add1ArgType),
505 NEONMAP1(vmaxvq_s8, vector_reduce_smax, Add1ArgType),
506 NEONMAP1(vmaxvq_u16, vector_reduce_umax, Add1ArgType),
507 NEONMAP1(vmaxvq_u32, vector_reduce_umax, Add1ArgType),
508 NEONMAP1(vmaxvq_u8, vector_reduce_umax, Add1ArgType),
509 NEONMAP1(vminnmv_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
510 NEONMAP1(vminnmvq_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
511 NEONMAP1(vminnmvq_f64, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
512 NEONMAP1(vminv_f32, aarch64_neon_fminv, AddRetType | Add1ArgType),
513 NEONMAP1(vminv_s16, vector_reduce_smin, Add1ArgType),
514 NEONMAP1(vminv_s32, vector_reduce_smin, Add1ArgType),
515 NEONMAP1(vminv_s8, vector_reduce_smin, Add1ArgType),
516 NEONMAP1(vminv_u16, vector_reduce_umin, Add1ArgType),
517 NEONMAP1(vminv_u32, vector_reduce_umin, Add1ArgType),
518 NEONMAP1(vminv_u8, vector_reduce_umin, Add1ArgType),
519 NEONMAP1(vminvq_f32, aarch64_neon_fminv, AddRetType | Add1ArgType),
520 NEONMAP1(vminvq_f64, aarch64_neon_fminv, AddRetType | Add1ArgType),
521 NEONMAP1(vminvq_s16, vector_reduce_smin, Add1ArgType),
522 NEONMAP1(vminvq_s32, vector_reduce_smin, Add1ArgType),
523 NEONMAP1(vminvq_s8, vector_reduce_smin, Add1ArgType),
524 NEONMAP1(vminvq_u16, vector_reduce_umin, Add1ArgType),
525 NEONMAP1(vminvq_u32, vector_reduce_umin, Add1ArgType),
526 NEONMAP1(vminvq_u8, vector_reduce_umin, Add1ArgType),
527 NEONMAP1(vmull_p64, aarch64_neon_pmull64, 0),
528 NEONMAP1(vmulxd_f64, aarch64_neon_fmulx, Add1ArgType),
529 NEONMAP1(vmulxs_f32, aarch64_neon_fmulx, Add1ArgType),
530 NEONMAP1(vpaddd_s64, vector_reduce_add, Add1ArgType),
531 NEONMAP1(vpaddd_u64, vector_reduce_add, Add1ArgType),
532 NEONMAP1(vpmaxnmqd_f64, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
533 NEONMAP1(vpmaxnms_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
534 NEONMAP1(vpmaxqd_f64, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
535 NEONMAP1(vpmaxs_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
536 NEONMAP1(vpminnmqd_f64, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
537 NEONMAP1(vpminnms_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
538 NEONMAP1(vpminqd_f64, aarch64_neon_fminv, AddRetType | Add1ArgType),
539 NEONMAP1(vpmins_f32, aarch64_neon_fminv, AddRetType | Add1ArgType),
540 NEONMAP1(vqabsb_s8, aarch64_neon_sqabs, Vectorize1ArgType | Use64BitVectors),
541 NEONMAP1(vqabsd_s64, aarch64_neon_sqabs, Add1ArgType),
542 NEONMAP1(vqabsh_s16, aarch64_neon_sqabs, Vectorize1ArgType | Use64BitVectors),
543 NEONMAP1(vqabss_s32, aarch64_neon_sqabs, Add1ArgType),
544 NEONMAP1(vqaddb_s8, aarch64_neon_sqadd, Vectorize1ArgType | Use64BitVectors),
545 NEONMAP1(vqaddb_u8, aarch64_neon_uqadd, Vectorize1ArgType | Use64BitVectors),
546 NEONMAP1(vqaddd_s64, aarch64_neon_sqadd, Add1ArgType),
547 NEONMAP1(vqaddd_u64, aarch64_neon_uqadd, Add1ArgType),
548 NEONMAP1(vqaddh_s16, aarch64_neon_sqadd, Vectorize1ArgType | Use64BitVectors),
549 NEONMAP1(vqaddh_u16, aarch64_neon_uqadd, Vectorize1ArgType | Use64BitVectors),
550 NEONMAP1(vqadds_s32, aarch64_neon_sqadd, Add1ArgType),
551 NEONMAP1(vqadds_u32, aarch64_neon_uqadd, Add1ArgType),
552 NEONMAP1(vqdmulhh_s16, aarch64_neon_sqdmulh, Vectorize1ArgType | Use64BitVectors),
553 NEONMAP1(vqdmulhs_s32, aarch64_neon_sqdmulh, Add1ArgType),
554 NEONMAP1(vqdmullh_s16, aarch64_neon_sqdmull, VectorRet | Use128BitVectors),
555 NEONMAP1(vqdmulls_s32, aarch64_neon_sqdmulls_scalar, 0),
556 NEONMAP1(vqmovnd_s64, aarch64_neon_scalar_sqxtn, AddRetType | Add1ArgType),
557 NEONMAP1(vqmovnd_u64, aarch64_neon_scalar_uqxtn, AddRetType | Add1ArgType),
558 NEONMAP1(vqmovnh_s16, aarch64_neon_sqxtn, VectorRet | Use64BitVectors),
559 NEONMAP1(vqmovnh_u16, aarch64_neon_uqxtn, VectorRet | Use64BitVectors),
560 NEONMAP1(vqmovns_s32, aarch64_neon_sqxtn, VectorRet | Use64BitVectors),
561 NEONMAP1(vqmovns_u32, aarch64_neon_uqxtn, VectorRet | Use64BitVectors),
562 NEONMAP1(vqmovund_s64, aarch64_neon_scalar_sqxtun, AddRetType | Add1ArgType),
563 NEONMAP1(vqmovunh_s16, aarch64_neon_sqxtun, VectorRet | Use64BitVectors),
564 NEONMAP1(vqmovuns_s32, aarch64_neon_sqxtun, VectorRet | Use64BitVectors),
565 NEONMAP1(vqnegb_s8, aarch64_neon_sqneg, Vectorize1ArgType | Use64BitVectors),
566 NEONMAP1(vqnegd_s64, aarch64_neon_sqneg, Add1ArgType),
567 NEONMAP1(vqnegh_s16, aarch64_neon_sqneg, Vectorize1ArgType | Use64BitVectors),
568 NEONMAP1(vqnegs_s32, aarch64_neon_sqneg, Add1ArgType),
569 NEONMAP1(vqrdmlahh_s16, aarch64_neon_sqrdmlah, Vectorize1ArgType | Use64BitVectors),
570 NEONMAP1(vqrdmlahs_s32, aarch64_neon_sqrdmlah, Add1ArgType),
571 NEONMAP1(vqrdmlshh_s16, aarch64_neon_sqrdmlsh, Vectorize1ArgType | Use64BitVectors),
572 NEONMAP1(vqrdmlshs_s32, aarch64_neon_sqrdmlsh, Add1ArgType),
573 NEONMAP1(vqrdmulhh_s16, aarch64_neon_sqrdmulh, Vectorize1ArgType | Use64BitVectors),
574 NEONMAP1(vqrdmulhs_s32, aarch64_neon_sqrdmulh, Add1ArgType),
575 NEONMAP1(vqrshlb_s8, aarch64_neon_sqrshl, Vectorize1ArgType | Use64BitVectors),
576 NEONMAP1(vqrshlb_u8, aarch64_neon_uqrshl, Vectorize1ArgType | Use64BitVectors),
577 NEONMAP1(vqrshld_s64, aarch64_neon_sqrshl, Add1ArgType),
578 NEONMAP1(vqrshld_u64, aarch64_neon_uqrshl, Add1ArgType),
579 NEONMAP1(vqrshlh_s16, aarch64_neon_sqrshl, Vectorize1ArgType | Use64BitVectors),
580 NEONMAP1(vqrshlh_u16, aarch64_neon_uqrshl, Vectorize1ArgType | Use64BitVectors),
581 NEONMAP1(vqrshls_s32, aarch64_neon_sqrshl, Add1ArgType),
582 NEONMAP1(vqrshls_u32, aarch64_neon_uqrshl, Add1ArgType),
583 NEONMAP1(vqrshrnd_n_s64, aarch64_neon_sqrshrn, AddRetType),
584 NEONMAP1(vqrshrnd_n_u64, aarch64_neon_uqrshrn, AddRetType),
585 NEONMAP1(vqrshrnh_n_s16, aarch64_neon_sqrshrn, VectorRet | Use64BitVectors),
586 NEONMAP1(vqrshrnh_n_u16, aarch64_neon_uqrshrn, VectorRet | Use64BitVectors),
587 NEONMAP1(vqrshrns_n_s32, aarch64_neon_sqrshrn, VectorRet | Use64BitVectors),
588 NEONMAP1(vqrshrns_n_u32, aarch64_neon_uqrshrn, VectorRet | Use64BitVectors),
589 NEONMAP1(vqrshrund_n_s64, aarch64_neon_sqrshrun, AddRetType),
590 NEONMAP1(vqrshrunh_n_s16, aarch64_neon_sqrshrun, VectorRet | Use64BitVectors),
591 NEONMAP1(vqrshruns_n_s32, aarch64_neon_sqrshrun, VectorRet | Use64BitVectors),
592 NEONMAP1(vqshlb_n_s8, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors),
593 NEONMAP1(vqshlb_n_u8, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors),
594 NEONMAP1(vqshlb_s8, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors),
595 NEONMAP1(vqshlb_u8, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors),
596 NEONMAP1(vqshld_s64, aarch64_neon_sqshl, Add1ArgType),
597 NEONMAP1(vqshld_u64, aarch64_neon_uqshl, Add1ArgType),
598 NEONMAP1(vqshlh_n_s16, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors),
599 NEONMAP1(vqshlh_n_u16, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors),
600 NEONMAP1(vqshlh_s16, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors),
601 NEONMAP1(vqshlh_u16, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors),
602 NEONMAP1(vqshls_n_s32, aarch64_neon_sqshl, Add1ArgType),
603 NEONMAP1(vqshls_n_u32, aarch64_neon_uqshl, Add1ArgType),
604 NEONMAP1(vqshls_s32, aarch64_neon_sqshl, Add1ArgType),
605 NEONMAP1(vqshls_u32, aarch64_neon_uqshl, Add1ArgType),
606 NEONMAP1(vqshlub_n_s8, aarch64_neon_sqshlu, Vectorize1ArgType | Use64BitVectors),
607 NEONMAP1(vqshluh_n_s16, aarch64_neon_sqshlu, Vectorize1ArgType | Use64BitVectors),
608 NEONMAP1(vqshlus_n_s32, aarch64_neon_sqshlu, Add1ArgType),
609 NEONMAP1(vqshrnd_n_s64, aarch64_neon_sqshrn, AddRetType),
610 NEONMAP1(vqshrnd_n_u64, aarch64_neon_uqshrn, AddRetType),
611 NEONMAP1(vqshrnh_n_s16, aarch64_neon_sqshrn, VectorRet | Use64BitVectors),
612 NEONMAP1(vqshrnh_n_u16, aarch64_neon_uqshrn, VectorRet | Use64BitVectors),
613 NEONMAP1(vqshrns_n_s32, aarch64_neon_sqshrn, VectorRet | Use64BitVectors),
614 NEONMAP1(vqshrns_n_u32, aarch64_neon_uqshrn, VectorRet | Use64BitVectors),
615 NEONMAP1(vqshrund_n_s64, aarch64_neon_sqshrun, AddRetType),
616 NEONMAP1(vqshrunh_n_s16, aarch64_neon_sqshrun, VectorRet | Use64BitVectors),
617 NEONMAP1(vqshruns_n_s32, aarch64_neon_sqshrun, VectorRet | Use64BitVectors),
618 NEONMAP1(vqsubb_s8, aarch64_neon_sqsub, Vectorize1ArgType | Use64BitVectors),
619 NEONMAP1(vqsubb_u8, aarch64_neon_uqsub, Vectorize1ArgType | Use64BitVectors),
620 NEONMAP1(vqsubd_s64, aarch64_neon_sqsub, Add1ArgType),
621 NEONMAP1(vqsubd_u64, aarch64_neon_uqsub, Add1ArgType),
622 NEONMAP1(vqsubh_s16, aarch64_neon_sqsub, Vectorize1ArgType | Use64BitVectors),
623 NEONMAP1(vqsubh_u16, aarch64_neon_uqsub, Vectorize1ArgType | Use64BitVectors),
624 NEONMAP1(vqsubs_s32, aarch64_neon_sqsub, Add1ArgType),
625 NEONMAP1(vqsubs_u32, aarch64_neon_uqsub, Add1ArgType),
626 NEONMAP1(vrecped_f64, aarch64_neon_frecpe, Add1ArgType),
627 NEONMAP1(vrecpes_f32, aarch64_neon_frecpe, Add1ArgType),
628 NEONMAP1(vrecpxd_f64, aarch64_neon_frecpx, Add1ArgType),
629 NEONMAP1(vrecpxs_f32, aarch64_neon_frecpx, Add1ArgType),
630 NEONMAP1(vrshld_s64, aarch64_neon_srshl, Add1ArgType),
631 NEONMAP1(vrshld_u64, aarch64_neon_urshl, Add1ArgType),
632 NEONMAP1(vrsqrted_f64, aarch64_neon_frsqrte, Add1ArgType),
633 NEONMAP1(vrsqrtes_f32, aarch64_neon_frsqrte, Add1ArgType),
634 NEONMAP1(vrsqrtsd_f64, aarch64_neon_frsqrts, Add1ArgType),
635 NEONMAP1(vrsqrtss_f32, aarch64_neon_frsqrts, Add1ArgType),
636 NEONMAP1(vsha1cq_u32, aarch64_crypto_sha1c, 0),
637 NEONMAP1(vsha1h_u32, aarch64_crypto_sha1h, 0),
638 NEONMAP1(vsha1mq_u32, aarch64_crypto_sha1m, 0),
639 NEONMAP1(vsha1pq_u32, aarch64_crypto_sha1p, 0),
640 NEONMAP1(vshld_s64, aarch64_neon_sshl, Add1ArgType),
641 NEONMAP1(vshld_u64, aarch64_neon_ushl, Add1ArgType),
642 NEONMAP1(vslid_n_s64, aarch64_neon_vsli, Vectorize1ArgType),
643 NEONMAP1(vslid_n_u64, aarch64_neon_vsli, Vectorize1ArgType),
644 NEONMAP1(vsqaddb_u8, aarch64_neon_usqadd, Vectorize1ArgType | Use64BitVectors),
645 NEONMAP1(vsqaddd_u64, aarch64_neon_usqadd, Add1ArgType),
646 NEONMAP1(vsqaddh_u16, aarch64_neon_usqadd, Vectorize1ArgType | Use64BitVectors),
647 NEONMAP1(vsqadds_u32, aarch64_neon_usqadd, Add1ArgType),
648 NEONMAP1(vsrid_n_s64, aarch64_neon_vsri, Vectorize1ArgType),
649 NEONMAP1(vsrid_n_u64, aarch64_neon_vsri, Vectorize1ArgType),
650 NEONMAP1(vuqaddb_s8, aarch64_neon_suqadd, Vectorize1ArgType | Use64BitVectors),
651 NEONMAP1(vuqaddd_s64, aarch64_neon_suqadd, Add1ArgType),
652 NEONMAP1(vuqaddh_s16, aarch64_neon_suqadd, Vectorize1ArgType | Use64BitVectors),
653 NEONMAP1(vuqadds_s32, aarch64_neon_suqadd, Add1ArgType),
654 // FP16 scalar intrinisics go here.
655 NEONMAP1(vabdh_f16, aarch64_sisd_fabd, Add1ArgType),
656 NEONMAP1(vcvtah_s32_f16, aarch64_neon_fcvtas, AddRetType | Add1ArgType),
657 NEONMAP1(vcvtah_s64_f16, aarch64_neon_fcvtas, AddRetType | Add1ArgType),
658 NEONMAP1(vcvtah_u32_f16, aarch64_neon_fcvtau, AddRetType | Add1ArgType),
659 NEONMAP1(vcvtah_u64_f16, aarch64_neon_fcvtau, AddRetType | Add1ArgType),
660 NEONMAP1(vcvth_n_f16_s32, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType),
661 NEONMAP1(vcvth_n_f16_s64, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType),
662 NEONMAP1(vcvth_n_f16_u32, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType),
663 NEONMAP1(vcvth_n_f16_u64, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType),
664 NEONMAP1(vcvth_n_s32_f16, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType),
665 NEONMAP1(vcvth_n_s64_f16, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType),
666 NEONMAP1(vcvth_n_u32_f16, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType),
667 NEONMAP1(vcvth_n_u64_f16, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType),
668 NEONMAP1(vcvth_s16_f16, aarch64_neon_fcvtzs, AddRetType | Add1ArgType),
669 NEONMAP1(vcvth_s32_f16, aarch64_neon_fcvtzs, AddRetType | Add1ArgType),
670 NEONMAP1(vcvth_s64_f16, aarch64_neon_fcvtzs, AddRetType | Add1ArgType),
671 NEONMAP1(vcvth_u16_f16, aarch64_neon_fcvtzu, AddRetType | Add1ArgType),
672 NEONMAP1(vcvth_u32_f16, aarch64_neon_fcvtzu, AddRetType | Add1ArgType),
673 NEONMAP1(vcvth_u64_f16, aarch64_neon_fcvtzu, AddRetType | Add1ArgType),
674 NEONMAP1(vcvtmh_s32_f16, aarch64_neon_fcvtms, AddRetType | Add1ArgType),
675 NEONMAP1(vcvtmh_s64_f16, aarch64_neon_fcvtms, AddRetType | Add1ArgType),
676 NEONMAP1(vcvtmh_u32_f16, aarch64_neon_fcvtmu, AddRetType | Add1ArgType),
677 NEONMAP1(vcvtmh_u64_f16, aarch64_neon_fcvtmu, AddRetType | Add1ArgType),
678 NEONMAP1(vcvtnh_s32_f16, aarch64_neon_fcvtns, AddRetType | Add1ArgType),
679 NEONMAP1(vcvtnh_s64_f16, aarch64_neon_fcvtns, AddRetType | Add1ArgType),
680 NEONMAP1(vcvtnh_u32_f16, aarch64_neon_fcvtnu, AddRetType | Add1ArgType),
681 NEONMAP1(vcvtnh_u64_f16, aarch64_neon_fcvtnu, AddRetType | Add1ArgType),
682 NEONMAP1(vcvtph_s32_f16, aarch64_neon_fcvtps, AddRetType | Add1ArgType),
683 NEONMAP1(vcvtph_s64_f16, aarch64_neon_fcvtps, AddRetType | Add1ArgType),
684 NEONMAP1(vcvtph_u32_f16, aarch64_neon_fcvtpu, AddRetType | Add1ArgType),
685 NEONMAP1(vcvtph_u64_f16, aarch64_neon_fcvtpu, AddRetType | Add1ArgType),
686 NEONMAP1(vmulxh_f16, aarch64_neon_fmulx, Add1ArgType),
687 NEONMAP1(vrecpeh_f16, aarch64_neon_frecpe, Add1ArgType),
688 NEONMAP1(vrecpxh_f16, aarch64_neon_frecpx, Add1ArgType),
689 NEONMAP1(vrsqrteh_f16, aarch64_neon_frsqrte, Add1ArgType),
690 NEONMAP1(vrsqrtsh_f16, aarch64_neon_frsqrts, Add1ArgType),
691};
692// clang-format on
693
694} // namespace aarch64
695} // namespace clang
696
697#endif
NEONMAP2
#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier)
Definition AArch64CodeGenUtils.h:106
NEONMAP1
#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier)
Definition AArch64CodeGenUtils.h:102
NEONMAP0
#define NEONMAP0(NameBase)
Definition AArch64CodeGenUtils.h:99
TargetBuiltins.h
Enumerates target-specific builtins in their own namespaces within namespace clang.
clang::aarch64::AArch64SISDIntrinsicMap
const ARMNeonVectorIntrinsicInfo AArch64SISDIntrinsicMap[]
Definition AArch64CodeGenUtils.h:408
clang::aarch64::AArch64SIMDIntrinsicMap
const ARMNeonVectorIntrinsicInfo AArch64SIMDIntrinsicMap[]
Definition AArch64CodeGenUtils.h:112
clang
The JSON file list parser is used to communicate input to InstallAPI.
Definition CalledOnceCheck.h:17
clang::aarch64::AArch64SVEAndSMEVectorIntrinsicInfo
Describes an AArch64 SVE or SME intrinsic.
Definition AArch64CodeGenUtils.h:84
clang::aarch64::AArch64SVEAndSMEVectorIntrinsicInfo::operator<
bool operator<(const AArch64SVEAndSMEVectorIntrinsicInfo &TE) const
Definition AArch64CodeGenUtils.h:92
clang::aarch64::ARMNeonVectorIntrinsicInfo
Describes an ARM or AArch64 NEON intrinsic, or an AArch64 SISD intrinsic.
Definition AArch64CodeGenUtils.h:63
clang::aarch64::ARMNeonVectorIntrinsicInfo::operator<
bool operator<(unsigned RHSBuiltinID) const
Definition AArch64CodeGenUtils.h:70
clang::aarch64::ARMNeonVectorIntrinsicInfo::operator<
bool operator<(const ARMNeonVectorIntrinsicInfo &TE) const
Definition AArch64CodeGenUtils.h:73
Generated on for clang by doxygen 1.14.0