clang 24.0.0git
AArch64CodeGenUtils.h
Go to the documentation of this file.
1//===--- AArch64CodeGenUtils.h ----------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// Utilities used for generating code for AArch64 that are shared between the
11/// classic and ClangIR code-gen.
12///
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_CLANG_BASIC_AARCH64CODEGENUTILS_H
16#define LLVM_CLANG_BASIC_AARCH64CODEGENUTILS_H
17
18#include "clang/Basic/TargetBuiltins.h"
19#include "llvm/IR/Intrinsics.h"
20#include "llvm/IR/IntrinsicsAArch64.h"
21
22namespace clang {
23namespace aarch64 {
24
25//===----------------------------------------------------------------------===//
26// Intrinsics maps
27//
28// Maps that help automate code-generation.
29//===----------------------------------------------------------------------===//
30enum {
31 AddRetType = (1 << 0),
32 Add1ArgType = (1 << 1),
33 Add2ArgTypes = (1 << 2),
34
35 VectorizeRetType = (1 << 3),
36 VectorizeArgTypes = (1 << 4),
37
38 InventFloatType = (1 << 5),
39 UnsignedAlts = (1 << 6),
40
41 Use64BitVectors = (1 << 7),
42 Use128BitVectors = (1 << 8),
43
44 // LLVM intrinsic source operands use double-width elements relative to the
45 // result (e.g. vraddhn). ClangIR-only; classic code-gen derives this from
46 // the intrinsic's .td.
47 ArgAsWidenedRetType = (1 << 9),
48
49 // The immediate operand is `LLVMMatchType<0>` in the intrinsic
50 // definition, so it is vectorized like a data operand (e.g. the `_n_`
51 // scalar saturating shifts: vqshlb_n_s8, vqshlub_n_s8, ...).
52 // ClangIR-only; classic code-gen derives this from the intrinsic's .td.
53 VectorizeImmArg = (1 << 10),
54
55 Vectorize1ArgType = Add1ArgType | VectorizeArgTypes,
56 VectorRet = AddRetType | VectorizeRetType,
57 VectorRetGetArgs01 = AddRetType | Add2ArgTypes | VectorizeRetType |
58 VectorizeArgTypes,
59 FpCmpzModifiers = AddRetType | VectorizeRetType | Add1ArgType |
60 InventFloatType
61};
62
63/// Describes an ARM or AArch64 NEON intrinsic, or an AArch64 SISD intrinsic.
64///
65/// NEON and SISD code generation use NameHint and AltLLVMIntrinsic in addition
66/// to BuiltinID, LLVMIntrinsic, and TypeModifier. SVE and SME code generation
67/// does not use those fields, so AArch64SVEAndSMEVectorIntrinsicInfo omits
68/// them. On 64-bit hosts, the separate structure reduces each SVE and SME map
69/// entry from 32 to 16 bytes and avoids storing a name pointer for each entry.
70struct ARMNeonVectorIntrinsicInfo {
71 const char *NameHint;
72 unsigned BuiltinID;
73 unsigned LLVMIntrinsic;
74 unsigned AltLLVMIntrinsic;
75 uint64_t TypeModifier;
76
77 bool operator<(unsigned RHSBuiltinID) const {
78 return BuiltinID < RHSBuiltinID;
79 }
80 bool operator<(const ARMNeonVectorIntrinsicInfo &TE) const {
81 return BuiltinID < TE.BuiltinID;
82 }
83};
84
85static_assert(sizeof(ARMNeonVectorIntrinsicInfo) == 16 + 2 * sizeof(void *));
86
87/// Describes an AArch64 SVE or SME intrinsic.
88///
89/// See ARMNeonVectorIntrinsicInfo for the reason that SVE and SME use a
90/// separate structure.
91struct AArch64SVEAndSMEVectorIntrinsicInfo {
92 unsigned BuiltinID;
93 unsigned LLVMIntrinsic;
94 uint64_t TypeModifier;
95
96 bool operator<(unsigned RHSBuiltinID) const {
97 return BuiltinID < RHSBuiltinID;
98 }
99 bool operator<(const AArch64SVEAndSMEVectorIntrinsicInfo &TE) const {
100 return BuiltinID < TE.BuiltinID;
101 }
102};
103
104static_assert(sizeof(AArch64SVEAndSMEVectorIntrinsicInfo) == 16);
105
106#define NEONMAP0(NameBase) \
107 {#NameBase, NEON::BI__builtin_neon_##NameBase, 0, 0, 0}
108
109#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
110 {#NameBase, NEON::BI__builtin_neon_##NameBase, \
111 llvm::Intrinsic::LLVMIntrinsic, 0, TypeModifier}
112
113#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier) \
114 {#NameBase, NEON::BI__builtin_neon_##NameBase, \
115 llvm::Intrinsic::LLVMIntrinsic, llvm::Intrinsic::AltLLVMIntrinsic, \
116 TypeModifier}
117
118// clang-format off
119const inline ARMNeonVectorIntrinsicInfo AArch64SIMDIntrinsicMap [] = {
120 NEONMAP0(splat_lane_v),
121 NEONMAP0(splat_laneq_v),
122 NEONMAP0(splatq_lane_v),
123 NEONMAP0(splatq_laneq_v),
124 NEONMAP1(vabs_v, aarch64_neon_abs, 0),
125 NEONMAP1(vabsq_v, aarch64_neon_abs, 0),
126 NEONMAP0(vadd_v),
127 NEONMAP0(vaddhn_v),
128 NEONMAP0(vaddq_v),
129 NEONMAP1(vaesdq_u8, aarch64_crypto_aesd, 0),
130 NEONMAP1(vaeseq_u8, aarch64_crypto_aese, 0),
131 NEONMAP1(vaesimcq_u8, aarch64_crypto_aesimc, 0),
132 NEONMAP1(vaesmcq_u8, aarch64_crypto_aesmc, 0),
133 NEONMAP2(vbcaxq_s16, aarch64_crypto_bcaxu, aarch64_crypto_bcaxs, Add1ArgType | UnsignedAlts),
134 NEONMAP2(vbcaxq_s32, aarch64_crypto_bcaxu, aarch64_crypto_bcaxs, Add1ArgType | UnsignedAlts),
135 NEONMAP2(vbcaxq_s64, aarch64_crypto_bcaxu, aarch64_crypto_bcaxs, Add1ArgType | UnsignedAlts),
136 NEONMAP2(vbcaxq_s8, aarch64_crypto_bcaxu, aarch64_crypto_bcaxs, Add1ArgType | UnsignedAlts),
137 NEONMAP2(vbcaxq_u16, aarch64_crypto_bcaxu, aarch64_crypto_bcaxs, Add1ArgType | UnsignedAlts),
138 NEONMAP2(vbcaxq_u32, aarch64_crypto_bcaxu, aarch64_crypto_bcaxs, Add1ArgType | UnsignedAlts),
139 NEONMAP2(vbcaxq_u64, aarch64_crypto_bcaxu, aarch64_crypto_bcaxs, Add1ArgType | UnsignedAlts),
140 NEONMAP2(vbcaxq_u8, aarch64_crypto_bcaxu, aarch64_crypto_bcaxs, Add1ArgType | UnsignedAlts),
141 NEONMAP1(vbfdot_f32, aarch64_neon_bfdot, 0),
142 NEONMAP1(vbfdotq_f32, aarch64_neon_bfdot, 0),
143 NEONMAP1(vbfmlalbq_f32, aarch64_neon_bfmlalb, 0),
144 NEONMAP1(vbfmlaltq_f32, aarch64_neon_bfmlalt, 0),
145 NEONMAP1(vbfmmlaq_f32, aarch64_neon_bfmmla, 0),
146 NEONMAP1(vcadd_rot270_f16, aarch64_neon_vcadd_rot270, Add1ArgType),
147 NEONMAP1(vcadd_rot270_f32, aarch64_neon_vcadd_rot270, Add1ArgType),
148 NEONMAP1(vcadd_rot90_f16, aarch64_neon_vcadd_rot90, Add1ArgType),
149 NEONMAP1(vcadd_rot90_f32, aarch64_neon_vcadd_rot90, Add1ArgType),
150 NEONMAP1(vcaddq_rot270_f16, aarch64_neon_vcadd_rot270, Add1ArgType),
151 NEONMAP1(vcaddq_rot270_f32, aarch64_neon_vcadd_rot270, Add1ArgType),
152 NEONMAP1(vcaddq_rot270_f64, aarch64_neon_vcadd_rot270, Add1ArgType),
153 NEONMAP1(vcaddq_rot90_f16, aarch64_neon_vcadd_rot90, Add1ArgType),
154 NEONMAP1(vcaddq_rot90_f32, aarch64_neon_vcadd_rot90, Add1ArgType),
155 NEONMAP1(vcaddq_rot90_f64, aarch64_neon_vcadd_rot90, Add1ArgType),
156 NEONMAP1(vcage_v, aarch64_neon_facge, 0),
157 NEONMAP1(vcageq_v, aarch64_neon_facge, 0),
158 NEONMAP1(vcagt_v, aarch64_neon_facgt, 0),
159 NEONMAP1(vcagtq_v, aarch64_neon_facgt, 0),
160 NEONMAP1(vcale_v, aarch64_neon_facge, 0),
161 NEONMAP1(vcaleq_v, aarch64_neon_facge, 0),
162 NEONMAP1(vcalt_v, aarch64_neon_facgt, 0),
163 NEONMAP1(vcaltq_v, aarch64_neon_facgt, 0),
164 NEONMAP0(vceqz_v),
165 NEONMAP0(vceqzq_v),
166 NEONMAP0(vcgez_v),
167 NEONMAP0(vcgezq_v),
168 NEONMAP0(vcgtz_v),
169 NEONMAP0(vcgtzq_v),
170 NEONMAP0(vclez_v),
171 NEONMAP0(vclezq_v),
172 NEONMAP1(vcls_v, aarch64_neon_cls, Add1ArgType),
173 NEONMAP1(vclsq_v, aarch64_neon_cls, Add1ArgType),
174 NEONMAP0(vcltz_v),
175 NEONMAP0(vcltzq_v),
176 NEONMAP1(vclz_v, ctlz, Add1ArgType),
177 NEONMAP1(vclzq_v, ctlz, Add1ArgType),
178 NEONMAP1(vcmla_f16, aarch64_neon_vcmla_rot0, Add1ArgType),
179 NEONMAP1(vcmla_f32, aarch64_neon_vcmla_rot0, Add1ArgType),
180 NEONMAP1(vcmla_rot180_f16, aarch64_neon_vcmla_rot180, Add1ArgType),
181 NEONMAP1(vcmla_rot180_f32, aarch64_neon_vcmla_rot180, Add1ArgType),
182 NEONMAP1(vcmla_rot270_f16, aarch64_neon_vcmla_rot270, Add1ArgType),
183 NEONMAP1(vcmla_rot270_f32, aarch64_neon_vcmla_rot270, Add1ArgType),
184 NEONMAP1(vcmla_rot90_f16, aarch64_neon_vcmla_rot90, Add1ArgType),
185 NEONMAP1(vcmla_rot90_f32, aarch64_neon_vcmla_rot90, Add1ArgType),
186 NEONMAP1(vcmlaq_f16, aarch64_neon_vcmla_rot0, Add1ArgType),
187 NEONMAP1(vcmlaq_f32, aarch64_neon_vcmla_rot0, Add1ArgType),
188 NEONMAP1(vcmlaq_f64, aarch64_neon_vcmla_rot0, Add1ArgType),
189 NEONMAP1(vcmlaq_rot180_f16, aarch64_neon_vcmla_rot180, Add1ArgType),
190 NEONMAP1(vcmlaq_rot180_f32, aarch64_neon_vcmla_rot180, Add1ArgType),
191 NEONMAP1(vcmlaq_rot180_f64, aarch64_neon_vcmla_rot180, Add1ArgType),
192 NEONMAP1(vcmlaq_rot270_f16, aarch64_neon_vcmla_rot270, Add1ArgType),
193 NEONMAP1(vcmlaq_rot270_f32, aarch64_neon_vcmla_rot270, Add1ArgType),
194 NEONMAP1(vcmlaq_rot270_f64, aarch64_neon_vcmla_rot270, Add1ArgType),
195 NEONMAP1(vcmlaq_rot90_f16, aarch64_neon_vcmla_rot90, Add1ArgType),
196 NEONMAP1(vcmlaq_rot90_f32, aarch64_neon_vcmla_rot90, Add1ArgType),
197 NEONMAP1(vcmlaq_rot90_f64, aarch64_neon_vcmla_rot90, Add1ArgType),
198 NEONMAP1(vcnt_v, ctpop, Add1ArgType),
199 NEONMAP1(vcntq_v, ctpop, Add1ArgType),
200 NEONMAP0(vcvt_f16_s16),
201 NEONMAP0(vcvt_f16_u16),
202 NEONMAP0(vcvt_f32_v),
203 NEONMAP1(vcvt_n_f16_s16, aarch64_neon_vcvtfxs2fp, 0),
204 NEONMAP1(vcvt_n_f16_u16, aarch64_neon_vcvtfxu2fp, 0),
205 NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
206 NEONMAP2(vcvt_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
207 NEONMAP1(vcvt_n_s16_f16, aarch64_neon_vcvtfp2fxs, 0),
208 NEONMAP1(vcvt_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
209 NEONMAP1(vcvt_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
210 NEONMAP1(vcvt_n_u16_f16, aarch64_neon_vcvtfp2fxu, 0),
211 NEONMAP1(vcvt_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
212 NEONMAP1(vcvt_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
213 NEONMAP1(vcvt_s16_f16, aarch64_neon_fcvtzs, AddRetType | Add1ArgType),
214 NEONMAP1(vcvt_s32_v, aarch64_neon_fcvtzs, AddRetType | Add1ArgType),
215 NEONMAP1(vcvt_s64_v, aarch64_neon_fcvtzs, AddRetType | Add1ArgType),
216 NEONMAP1(vcvt_u16_f16, aarch64_neon_fcvtzu, AddRetType | Add1ArgType),
217 NEONMAP1(vcvt_u32_v, aarch64_neon_fcvtzu, AddRetType | Add1ArgType),
218 NEONMAP1(vcvt_u64_v, aarch64_neon_fcvtzu, AddRetType | Add1ArgType),
219 NEONMAP0(vcvtq_f16_s16),
220 NEONMAP0(vcvtq_f16_u16),
221 NEONMAP0(vcvtq_f32_v),
222 NEONMAP1(vcvtq_n_f16_s16, aarch64_neon_vcvtfxs2fp, 0),
223 NEONMAP1(vcvtq_n_f16_u16, aarch64_neon_vcvtfxu2fp, 0),
224 NEONMAP2(vcvtq_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
225 NEONMAP2(vcvtq_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
226 NEONMAP1(vcvtq_n_s16_f16, aarch64_neon_vcvtfp2fxs, 0),
227 NEONMAP1(vcvtq_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
228 NEONMAP1(vcvtq_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
229 NEONMAP1(vcvtq_n_u16_f16, aarch64_neon_vcvtfp2fxu, 0),
230 NEONMAP1(vcvtq_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
231 NEONMAP1(vcvtq_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
232 NEONMAP1(vcvtq_s16_f16, aarch64_neon_fcvtzs, AddRetType | Add1ArgType),
233 NEONMAP1(vcvtq_s32_v, aarch64_neon_fcvtzs, AddRetType | Add1ArgType),
234 NEONMAP1(vcvtq_s64_v, aarch64_neon_fcvtzs, AddRetType | Add1ArgType),
235 NEONMAP1(vcvtq_u16_f16, aarch64_neon_fcvtzu, AddRetType | Add1ArgType),
236 NEONMAP1(vcvtq_u32_v, aarch64_neon_fcvtzu, AddRetType | Add1ArgType),
237 NEONMAP1(vcvtq_u64_v, aarch64_neon_fcvtzu, AddRetType | Add1ArgType),
238 NEONMAP1(vcvtx_f32_v, aarch64_neon_fcvtxn, AddRetType | Add1ArgType),
239 NEONMAP1(vdot_s32, aarch64_neon_sdot, 0),
240 NEONMAP1(vdot_u32, aarch64_neon_udot, 0),
241 NEONMAP1(vdotq_s32, aarch64_neon_sdot, 0),
242 NEONMAP1(vdotq_u32, aarch64_neon_udot, 0),
243 NEONMAP2(veor3q_s16, aarch64_crypto_eor3u, aarch64_crypto_eor3s, Add1ArgType | UnsignedAlts),
244 NEONMAP2(veor3q_s32, aarch64_crypto_eor3u, aarch64_crypto_eor3s, Add1ArgType | UnsignedAlts),
245 NEONMAP2(veor3q_s64, aarch64_crypto_eor3u, aarch64_crypto_eor3s, Add1ArgType | UnsignedAlts),
246 NEONMAP2(veor3q_s8, aarch64_crypto_eor3u, aarch64_crypto_eor3s, Add1ArgType | UnsignedAlts),
247 NEONMAP2(veor3q_u16, aarch64_crypto_eor3u, aarch64_crypto_eor3s, Add1ArgType | UnsignedAlts),
248 NEONMAP2(veor3q_u32, aarch64_crypto_eor3u, aarch64_crypto_eor3s, Add1ArgType | UnsignedAlts),
249 NEONMAP2(veor3q_u64, aarch64_crypto_eor3u, aarch64_crypto_eor3s, Add1ArgType | UnsignedAlts),
250 NEONMAP2(veor3q_u8, aarch64_crypto_eor3u, aarch64_crypto_eor3s, Add1ArgType | UnsignedAlts),
251 NEONMAP0(vext_v),
252 NEONMAP0(vextq_v),
253 NEONMAP0(vfma_v),
254 NEONMAP0(vfmaq_v),
255 NEONMAP1(vfmlal_high_f16, aarch64_neon_fmlal2, 0),
256 NEONMAP1(vfmlal_low_f16, aarch64_neon_fmlal, 0),
257 NEONMAP1(vfmlalq_high_f16, aarch64_neon_fmlal2, 0),
258 NEONMAP1(vfmlalq_low_f16, aarch64_neon_fmlal, 0),
259 NEONMAP1(vfmlsl_high_f16, aarch64_neon_fmlsl2, 0),
260 NEONMAP1(vfmlsl_low_f16, aarch64_neon_fmlsl, 0),
261 NEONMAP1(vfmlslq_high_f16, aarch64_neon_fmlsl2, 0),
262 NEONMAP1(vfmlslq_low_f16, aarch64_neon_fmlsl, 0),
263 NEONMAP2(vhadd_v, aarch64_neon_uhadd, aarch64_neon_shadd, Add1ArgType | UnsignedAlts),
264 NEONMAP2(vhaddq_v, aarch64_neon_uhadd, aarch64_neon_shadd, Add1ArgType | UnsignedAlts),
265 NEONMAP2(vhsub_v, aarch64_neon_uhsub, aarch64_neon_shsub, Add1ArgType | UnsignedAlts),
266 NEONMAP2(vhsubq_v, aarch64_neon_uhsub, aarch64_neon_shsub, Add1ArgType | UnsignedAlts),
267 NEONMAP1(vld1_x2_v, aarch64_neon_ld1x2, 0),
268 NEONMAP1(vld1_x3_v, aarch64_neon_ld1x3, 0),
269 NEONMAP1(vld1_x4_v, aarch64_neon_ld1x4, 0),
270 NEONMAP1(vld1q_x2_v, aarch64_neon_ld1x2, 0),
271 NEONMAP1(vld1q_x3_v, aarch64_neon_ld1x3, 0),
272 NEONMAP1(vld1q_x4_v, aarch64_neon_ld1x4, 0),
273 NEONMAP1(vmmlaq_f16, aarch64_neon_fmmla, 0),
274 NEONMAP1(vmmlaq_f32_f16, aarch64_neon_fmmla, 0),
275 NEONMAP1(vmmlaq_s32, aarch64_neon_smmla, 0),
276 NEONMAP1(vmmlaq_u32, aarch64_neon_ummla, 0),
277 NEONMAP0(vmovl_v),
278 NEONMAP0(vmovn_v),
279 NEONMAP1(vmul_v, aarch64_neon_pmul, Add1ArgType),
280 NEONMAP1(vmulq_v, aarch64_neon_pmul, Add1ArgType),
281 NEONMAP1(vpadd_v, aarch64_neon_addp, Add1ArgType),
282 NEONMAP2(vpaddl_v, aarch64_neon_uaddlp, aarch64_neon_saddlp, UnsignedAlts),
283 NEONMAP2(vpaddlq_v, aarch64_neon_uaddlp, aarch64_neon_saddlp, UnsignedAlts),
284 NEONMAP1(vpaddq_v, aarch64_neon_addp, Add1ArgType),
285 NEONMAP1(vqabs_v, aarch64_neon_sqabs, Add1ArgType),
286 NEONMAP1(vqabsq_v, aarch64_neon_sqabs, Add1ArgType),
287 NEONMAP2(vqadd_v, aarch64_neon_uqadd, aarch64_neon_sqadd, Add1ArgType | UnsignedAlts),
288 NEONMAP2(vqaddq_v, aarch64_neon_uqadd, aarch64_neon_sqadd, Add1ArgType | UnsignedAlts),
289 NEONMAP2(vqdmlal_v, aarch64_neon_sqdmull, aarch64_neon_sqadd, 0),
290 NEONMAP2(vqdmlsl_v, aarch64_neon_sqdmull, aarch64_neon_sqsub, 0),
291 NEONMAP1(vqdmulh_lane_v, aarch64_neon_sqdmulh_lane, 0),
292 NEONMAP1(vqdmulh_laneq_v, aarch64_neon_sqdmulh_laneq, 0),
293 NEONMAP1(vqdmulh_v, aarch64_neon_sqdmulh, Add1ArgType),
294 NEONMAP1(vqdmulhq_lane_v, aarch64_neon_sqdmulh_lane, 0),
295 NEONMAP1(vqdmulhq_laneq_v, aarch64_neon_sqdmulh_laneq, 0),
296 NEONMAP1(vqdmulhq_v, aarch64_neon_sqdmulh, Add1ArgType),
297 NEONMAP1(vqdmull_v, aarch64_neon_sqdmull, Add1ArgType),
298 NEONMAP2(vqmovn_v, aarch64_neon_uqxtn, aarch64_neon_sqxtn, Add1ArgType | UnsignedAlts),
299 NEONMAP1(vqmovun_v, aarch64_neon_sqxtun, Add1ArgType),
300 NEONMAP1(vqneg_v, aarch64_neon_sqneg, Add1ArgType),
301 NEONMAP1(vqnegq_v, aarch64_neon_sqneg, Add1ArgType),
302 NEONMAP1(vqrdmlah_s16, aarch64_neon_sqrdmlah, Add1ArgType),
303 NEONMAP1(vqrdmlah_s32, aarch64_neon_sqrdmlah, Add1ArgType),
304 NEONMAP1(vqrdmlahq_s16, aarch64_neon_sqrdmlah, Add1ArgType),
305 NEONMAP1(vqrdmlahq_s32, aarch64_neon_sqrdmlah, Add1ArgType),
306 NEONMAP1(vqrdmlsh_s16, aarch64_neon_sqrdmlsh, Add1ArgType),
307 NEONMAP1(vqrdmlsh_s32, aarch64_neon_sqrdmlsh, Add1ArgType),
308 NEONMAP1(vqrdmlshq_s16, aarch64_neon_sqrdmlsh, Add1ArgType),
309 NEONMAP1(vqrdmlshq_s32, aarch64_neon_sqrdmlsh, Add1ArgType),
310 NEONMAP1(vqrdmulh_lane_v, aarch64_neon_sqrdmulh_lane, 0),
311 NEONMAP1(vqrdmulh_laneq_v, aarch64_neon_sqrdmulh_laneq, 0),
312 NEONMAP1(vqrdmulh_v, aarch64_neon_sqrdmulh, Add1ArgType),
313 NEONMAP1(vqrdmulhq_lane_v, aarch64_neon_sqrdmulh_lane, 0),
314 NEONMAP1(vqrdmulhq_laneq_v, aarch64_neon_sqrdmulh_laneq, 0),
315 NEONMAP1(vqrdmulhq_v, aarch64_neon_sqrdmulh, Add1ArgType),
316 NEONMAP2(vqrshl_v, aarch64_neon_uqrshl, aarch64_neon_sqrshl, Add1ArgType | UnsignedAlts),
317 NEONMAP2(vqrshlq_v, aarch64_neon_uqrshl, aarch64_neon_sqrshl, Add1ArgType | UnsignedAlts),
318 NEONMAP2(vqshl_n_v, aarch64_neon_uqshl, aarch64_neon_sqshl, UnsignedAlts),
319 NEONMAP2(vqshl_v, aarch64_neon_uqshl, aarch64_neon_sqshl, Add1ArgType | UnsignedAlts),
320 NEONMAP2(vqshlq_n_v, aarch64_neon_uqshl, aarch64_neon_sqshl,UnsignedAlts),
321 NEONMAP2(vqshlq_v, aarch64_neon_uqshl, aarch64_neon_sqshl, Add1ArgType | UnsignedAlts),
322 NEONMAP1(vqshlu_n_v, aarch64_neon_sqshlu, 0),
323 NEONMAP1(vqshluq_n_v, aarch64_neon_sqshlu, 0),
324 NEONMAP2(vqsub_v, aarch64_neon_uqsub, aarch64_neon_sqsub, Add1ArgType | UnsignedAlts),
325 NEONMAP2(vqsubq_v, aarch64_neon_uqsub, aarch64_neon_sqsub, Add1ArgType | UnsignedAlts),
326 NEONMAP1(vraddhn_v, aarch64_neon_raddhn,
327 Add1ArgType | ArgAsWidenedRetType),
328 NEONMAP1(vrax1q_u64, aarch64_crypto_rax1, 0),
329 NEONMAP2(vrecpe_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
330 NEONMAP2(vrecpeq_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
331 NEONMAP1(vrecps_v, aarch64_neon_frecps, Add1ArgType),
332 NEONMAP1(vrecpsq_v, aarch64_neon_frecps, Add1ArgType),
333 NEONMAP2(vrhadd_v, aarch64_neon_urhadd, aarch64_neon_srhadd, Add1ArgType | UnsignedAlts),
334 NEONMAP2(vrhaddq_v, aarch64_neon_urhadd, aarch64_neon_srhadd, Add1ArgType | UnsignedAlts),
335 NEONMAP1(vrnd32x_f32, aarch64_neon_frint32x, Add1ArgType),
336 NEONMAP1(vrnd32x_f64, aarch64_neon_frint32x, Add1ArgType),
337 NEONMAP1(vrnd32xq_f32, aarch64_neon_frint32x, Add1ArgType),
338 NEONMAP1(vrnd32xq_f64, aarch64_neon_frint32x, Add1ArgType),
339 NEONMAP1(vrnd32z_f32, aarch64_neon_frint32z, Add1ArgType),
340 NEONMAP1(vrnd32z_f64, aarch64_neon_frint32z, Add1ArgType),
341 NEONMAP1(vrnd32zq_f32, aarch64_neon_frint32z, Add1ArgType),
342 NEONMAP1(vrnd32zq_f64, aarch64_neon_frint32z, Add1ArgType),
343 NEONMAP1(vrnd64x_f32, aarch64_neon_frint64x, Add1ArgType),
344 NEONMAP1(vrnd64x_f64, aarch64_neon_frint64x, Add1ArgType),
345 NEONMAP1(vrnd64xq_f32, aarch64_neon_frint64x, Add1ArgType),
346 NEONMAP1(vrnd64xq_f64, aarch64_neon_frint64x, Add1ArgType),
347 NEONMAP1(vrnd64z_f32, aarch64_neon_frint64z, Add1ArgType),
348 NEONMAP1(vrnd64z_f64, aarch64_neon_frint64z, Add1ArgType),
349 NEONMAP1(vrnd64zq_f32, aarch64_neon_frint64z, Add1ArgType),
350 NEONMAP1(vrnd64zq_f64, aarch64_neon_frint64z, Add1ArgType),
351 NEONMAP0(vrndi_v),
352 NEONMAP0(vrndiq_v),
353 NEONMAP2(vrshl_v, aarch64_neon_urshl, aarch64_neon_srshl, Add1ArgType | UnsignedAlts),
354 NEONMAP2(vrshlq_v, aarch64_neon_urshl, aarch64_neon_srshl, Add1ArgType | UnsignedAlts),
355 NEONMAP2(vrshr_n_v, aarch64_neon_urshl, aarch64_neon_srshl, UnsignedAlts),
356 NEONMAP2(vrshrq_n_v, aarch64_neon_urshl, aarch64_neon_srshl, UnsignedAlts),
357 NEONMAP2(vrsqrte_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
358 NEONMAP2(vrsqrteq_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
359 NEONMAP1(vrsqrts_v, aarch64_neon_frsqrts, Add1ArgType),
360 NEONMAP1(vrsqrtsq_v, aarch64_neon_frsqrts, Add1ArgType),
361 NEONMAP1(vrsubhn_v, aarch64_neon_rsubhn,
362 Add1ArgType | ArgAsWidenedRetType),
363 NEONMAP1(vsha1su0q_u32, aarch64_crypto_sha1su0, 0),
364 NEONMAP1(vsha1su1q_u32, aarch64_crypto_sha1su1, 0),
365 NEONMAP1(vsha256h2q_u32, aarch64_crypto_sha256h2, 0),
366 NEONMAP1(vsha256hq_u32, aarch64_crypto_sha256h, 0),
367 NEONMAP1(vsha256su0q_u32, aarch64_crypto_sha256su0, 0),
368 NEONMAP1(vsha256su1q_u32, aarch64_crypto_sha256su1, 0),
369 NEONMAP1(vsha512h2q_u64, aarch64_crypto_sha512h2, 0),
370 NEONMAP1(vsha512hq_u64, aarch64_crypto_sha512h, 0),
371 NEONMAP1(vsha512su0q_u64, aarch64_crypto_sha512su0, 0),
372 NEONMAP1(vsha512su1q_u64, aarch64_crypto_sha512su1, 0),
373 NEONMAP0(vshl_n_v),
374 NEONMAP2(vshl_v, aarch64_neon_ushl, aarch64_neon_sshl, Add1ArgType | UnsignedAlts),
375 NEONMAP0(vshll_n_v),
376 NEONMAP0(vshlq_n_v),
377 NEONMAP2(vshlq_v, aarch64_neon_ushl, aarch64_neon_sshl, Add1ArgType | UnsignedAlts),
378 NEONMAP0(vshr_n_v),
379 NEONMAP0(vshrn_n_v),
380 NEONMAP0(vshrq_n_v),
381 NEONMAP1(vsm3partw1q_u32, aarch64_crypto_sm3partw1, 0),
382 NEONMAP1(vsm3partw2q_u32, aarch64_crypto_sm3partw2, 0),
383 NEONMAP1(vsm3ss1q_u32, aarch64_crypto_sm3ss1, 0),
384 NEONMAP1(vsm3tt1aq_u32, aarch64_crypto_sm3tt1a, 0),
385 NEONMAP1(vsm3tt1bq_u32, aarch64_crypto_sm3tt1b, 0),
386 NEONMAP1(vsm3tt2aq_u32, aarch64_crypto_sm3tt2a, 0),
387 NEONMAP1(vsm3tt2bq_u32, aarch64_crypto_sm3tt2b, 0),
388 NEONMAP1(vsm4ekeyq_u32, aarch64_crypto_sm4ekey, 0),
389 NEONMAP1(vsm4eq_u32, aarch64_crypto_sm4e, 0),
390 NEONMAP1(vst1_x2_v, aarch64_neon_st1x2, 0),
391 NEONMAP1(vst1_x3_v, aarch64_neon_st1x3, 0),
392 NEONMAP1(vst1_x4_v, aarch64_neon_st1x4, 0),
393 NEONMAP1(vst1q_x2_v, aarch64_neon_st1x2, 0),
394 NEONMAP1(vst1q_x3_v, aarch64_neon_st1x3, 0),
395 NEONMAP1(vst1q_x4_v, aarch64_neon_st1x4, 0),
396 NEONMAP0(vsubhn_v),
397 NEONMAP0(vtst_v),
398 NEONMAP0(vtstq_v),
399 NEONMAP1(vusdot_s32, aarch64_neon_usdot, 0),
400 NEONMAP1(vusdotq_s32, aarch64_neon_usdot, 0),
401 NEONMAP1(vusmmlaq_s32, aarch64_neon_usmmla, 0),
402 NEONMAP1(vxarq_u64, aarch64_crypto_xar, 0),
403};
404
405
406// Single-Instruction-Single-Data (SISD) intrinsics.
407//
408// The name is somewhat misleading: not all intrinsics in this table are
409// strictly SISD. While many builtins operate on scalars,
410// * some take vector operands (e.g. reduction builtins such as
411// `vminvq_u16` and `vaddvq_s32`), and
412// * some take both scalar and vector operands (e.g. crypto builtins
413// such as `vsha1cq_u32`).
414//
415// TODO: Either rename this table to better reflect its contents, or
416// restrict it to true SISD intrinsics only.
417const inline ARMNeonVectorIntrinsicInfo AArch64SISDIntrinsicMap[] = {
418 NEONMAP1(vabdd_f64, aarch64_sisd_fabd, Add1ArgType),
419 NEONMAP1(vabds_f32, aarch64_sisd_fabd, Add1ArgType),
420 NEONMAP1(vabsd_s64, aarch64_neon_abs, Add1ArgType),
421 NEONMAP1(vaddlv_s32, aarch64_neon_saddlv, AddRetType | Add1ArgType),
422 NEONMAP1(vaddlv_u32, aarch64_neon_uaddlv, AddRetType | Add1ArgType),
423 NEONMAP1(vaddlvq_s32, aarch64_neon_saddlv, AddRetType | Add1ArgType),
424 NEONMAP1(vaddlvq_u32, aarch64_neon_uaddlv, AddRetType | Add1ArgType),
425 NEONMAP1(vaddv_f32, aarch64_neon_faddv, AddRetType | Add1ArgType),
426 NEONMAP1(vaddv_s16, vector_reduce_add, Add1ArgType),
427 NEONMAP1(vaddv_s32, vector_reduce_add, Add1ArgType),
428 NEONMAP1(vaddv_s8, vector_reduce_add, Add1ArgType),
429 NEONMAP1(vaddv_u16, vector_reduce_add, Add1ArgType),
430 NEONMAP1(vaddv_u32, vector_reduce_add, Add1ArgType),
431 NEONMAP1(vaddv_u8, vector_reduce_add, Add1ArgType),
432 NEONMAP1(vaddvq_f32, aarch64_neon_faddv, AddRetType | Add1ArgType),
433 NEONMAP1(vaddvq_f64, aarch64_neon_faddv, AddRetType | Add1ArgType),
434 NEONMAP1(vaddvq_s16, vector_reduce_add, Add1ArgType),
435 NEONMAP1(vaddvq_s32, vector_reduce_add, Add1ArgType),
436 NEONMAP1(vaddvq_s64, vector_reduce_add, Add1ArgType),
437 NEONMAP1(vaddvq_s8, vector_reduce_add, Add1ArgType),
438 NEONMAP1(vaddvq_u16, vector_reduce_add, Add1ArgType),
439 NEONMAP1(vaddvq_u32, vector_reduce_add, Add1ArgType),
440 NEONMAP1(vaddvq_u64, vector_reduce_add, Add1ArgType),
441 NEONMAP1(vaddvq_u8, vector_reduce_add, Add1ArgType),
442 NEONMAP1(vcaged_f64, aarch64_neon_facge, AddRetType | Add1ArgType),
443 NEONMAP1(vcages_f32, aarch64_neon_facge, AddRetType | Add1ArgType),
444 NEONMAP1(vcagtd_f64, aarch64_neon_facgt, AddRetType | Add1ArgType),
445 NEONMAP1(vcagts_f32, aarch64_neon_facgt, AddRetType | Add1ArgType),
446 NEONMAP1(vcaled_f64, aarch64_neon_facge, AddRetType | Add1ArgType),
447 NEONMAP1(vcales_f32, aarch64_neon_facge, AddRetType | Add1ArgType),
448 NEONMAP1(vcaltd_f64, aarch64_neon_facgt, AddRetType | Add1ArgType),
449 NEONMAP1(vcalts_f32, aarch64_neon_facgt, AddRetType | Add1ArgType),
450 NEONMAP1(vcvtad_s32_f64, aarch64_neon_fcvtas, AddRetType | Add1ArgType),
451 NEONMAP1(vcvtad_s64_f64, aarch64_neon_fcvtas, AddRetType | Add1ArgType),
452 NEONMAP1(vcvtad_u32_f64, aarch64_neon_fcvtau, AddRetType | Add1ArgType),
453 NEONMAP1(vcvtad_u64_f64, aarch64_neon_fcvtau, AddRetType | Add1ArgType),
454 NEONMAP1(vcvtas_s32_f32, aarch64_neon_fcvtas, AddRetType | Add1ArgType),
455 NEONMAP1(vcvtas_s64_f32, aarch64_neon_fcvtas, AddRetType | Add1ArgType),
456 NEONMAP1(vcvtas_u32_f32, aarch64_neon_fcvtau, AddRetType | Add1ArgType),
457 NEONMAP1(vcvtas_u64_f32, aarch64_neon_fcvtau, AddRetType | Add1ArgType),
458 NEONMAP1(vcvtd_n_f64_s64, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType),
459 NEONMAP1(vcvtd_n_f64_u64, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType),
460 NEONMAP1(vcvtd_n_s64_f64, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType),
461 NEONMAP1(vcvtd_n_u64_f64, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType),
462 NEONMAP1(vcvtd_s32_f64, aarch64_neon_fcvtzs, AddRetType | Add1ArgType),
463 NEONMAP1(vcvtd_s64_f64, aarch64_neon_fcvtzs, AddRetType | Add1ArgType),
464 NEONMAP1(vcvtd_u32_f64, aarch64_neon_fcvtzu, AddRetType | Add1ArgType),
465 NEONMAP1(vcvtd_u64_f64, aarch64_neon_fcvtzu, AddRetType | Add1ArgType),
466 NEONMAP0(vcvth_bf16_f32),
467 NEONMAP1(vcvtmd_s32_f64, aarch64_neon_fcvtms, AddRetType | Add1ArgType),
468 NEONMAP1(vcvtmd_s64_f64, aarch64_neon_fcvtms, AddRetType | Add1ArgType),
469 NEONMAP1(vcvtmd_u32_f64, aarch64_neon_fcvtmu, AddRetType | Add1ArgType),
470 NEONMAP1(vcvtmd_u64_f64, aarch64_neon_fcvtmu, AddRetType | Add1ArgType),
471 NEONMAP1(vcvtms_s32_f32, aarch64_neon_fcvtms, AddRetType | Add1ArgType),
472 NEONMAP1(vcvtms_s64_f32, aarch64_neon_fcvtms, AddRetType | Add1ArgType),
473 NEONMAP1(vcvtms_u32_f32, aarch64_neon_fcvtmu, AddRetType | Add1ArgType),
474 NEONMAP1(vcvtms_u64_f32, aarch64_neon_fcvtmu, AddRetType | Add1ArgType),
475 NEONMAP1(vcvtnd_s32_f64, aarch64_neon_fcvtns, AddRetType | Add1ArgType),
476 NEONMAP1(vcvtnd_s64_f64, aarch64_neon_fcvtns, AddRetType | Add1ArgType),
477 NEONMAP1(vcvtnd_u32_f64, aarch64_neon_fcvtnu, AddRetType | Add1ArgType),
478 NEONMAP1(vcvtnd_u64_f64, aarch64_neon_fcvtnu, AddRetType | Add1ArgType),
479 NEONMAP1(vcvtns_s32_f32, aarch64_neon_fcvtns, AddRetType | Add1ArgType),
480 NEONMAP1(vcvtns_s64_f32, aarch64_neon_fcvtns, AddRetType | Add1ArgType),
481 NEONMAP1(vcvtns_u32_f32, aarch64_neon_fcvtnu, AddRetType | Add1ArgType),
482 NEONMAP1(vcvtns_u64_f32, aarch64_neon_fcvtnu, AddRetType | Add1ArgType),
483 NEONMAP1(vcvtpd_s32_f64, aarch64_neon_fcvtps, AddRetType | Add1ArgType),
484 NEONMAP1(vcvtpd_s64_f64, aarch64_neon_fcvtps, AddRetType | Add1ArgType),
485 NEONMAP1(vcvtpd_u32_f64, aarch64_neon_fcvtpu, AddRetType | Add1ArgType),
486 NEONMAP1(vcvtpd_u64_f64, aarch64_neon_fcvtpu, AddRetType | Add1ArgType),
487 NEONMAP1(vcvtps_s32_f32, aarch64_neon_fcvtps, AddRetType | Add1ArgType),
488 NEONMAP1(vcvtps_s64_f32, aarch64_neon_fcvtps, AddRetType | Add1ArgType),
489 NEONMAP1(vcvtps_u32_f32, aarch64_neon_fcvtpu, AddRetType | Add1ArgType),
490 NEONMAP1(vcvtps_u64_f32, aarch64_neon_fcvtpu, AddRetType | Add1ArgType),
491 NEONMAP1(vcvts_n_f32_s32, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType),
492 NEONMAP1(vcvts_n_f32_u32, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType),
493 NEONMAP1(vcvts_n_s32_f32, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType),
494 NEONMAP1(vcvts_n_u32_f32, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType),
495 NEONMAP1(vcvts_s32_f32, aarch64_neon_fcvtzs, AddRetType | Add1ArgType),
496 NEONMAP1(vcvts_s64_f32, aarch64_neon_fcvtzs, AddRetType | Add1ArgType),
497 NEONMAP1(vcvts_u32_f32, aarch64_neon_fcvtzu, AddRetType | Add1ArgType),
498 NEONMAP1(vcvts_u64_f32, aarch64_neon_fcvtzu, AddRetType | Add1ArgType),
499 NEONMAP1(vcvtxd_f32_f64, aarch64_sisd_fcvtxn, 0),
500 NEONMAP1(vmaxnmv_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
501 NEONMAP1(vmaxnmvq_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
502 NEONMAP1(vmaxnmvq_f64, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
503 NEONMAP1(vmaxv_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
504 NEONMAP1(vmaxv_s16, vector_reduce_smax, Add1ArgType),
505 NEONMAP1(vmaxv_s32, vector_reduce_smax, Add1ArgType),
506 NEONMAP1(vmaxv_s8, vector_reduce_smax, Add1ArgType),
507 NEONMAP1(vmaxv_u16, vector_reduce_umax, Add1ArgType),
508 NEONMAP1(vmaxv_u32, vector_reduce_umax, Add1ArgType),
509 NEONMAP1(vmaxv_u8, vector_reduce_umax, Add1ArgType),
510 NEONMAP1(vmaxvq_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
511 NEONMAP1(vmaxvq_f64, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
512 NEONMAP1(vmaxvq_s16, vector_reduce_smax, Add1ArgType),
513 NEONMAP1(vmaxvq_s32, vector_reduce_smax, Add1ArgType),
514 NEONMAP1(vmaxvq_s8, vector_reduce_smax, Add1ArgType),
515 NEONMAP1(vmaxvq_u16, vector_reduce_umax, Add1ArgType),
516 NEONMAP1(vmaxvq_u32, vector_reduce_umax, Add1ArgType),
517 NEONMAP1(vmaxvq_u8, vector_reduce_umax, Add1ArgType),
518 NEONMAP1(vminnmv_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
519 NEONMAP1(vminnmvq_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
520 NEONMAP1(vminnmvq_f64, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
521 NEONMAP1(vminv_f32, aarch64_neon_fminv, AddRetType | Add1ArgType),
522 NEONMAP1(vminv_s16, vector_reduce_smin, Add1ArgType),
523 NEONMAP1(vminv_s32, vector_reduce_smin, Add1ArgType),
524 NEONMAP1(vminv_s8, vector_reduce_smin, Add1ArgType),
525 NEONMAP1(vminv_u16, vector_reduce_umin, Add1ArgType),
526 NEONMAP1(vminv_u32, vector_reduce_umin, Add1ArgType),
527 NEONMAP1(vminv_u8, vector_reduce_umin, Add1ArgType),
528 NEONMAP1(vminvq_f32, aarch64_neon_fminv, AddRetType | Add1ArgType),
529 NEONMAP1(vminvq_f64, aarch64_neon_fminv, AddRetType | Add1ArgType),
530 NEONMAP1(vminvq_s16, vector_reduce_smin, Add1ArgType),
531 NEONMAP1(vminvq_s32, vector_reduce_smin, Add1ArgType),
532 NEONMAP1(vminvq_s8, vector_reduce_smin, Add1ArgType),
533 NEONMAP1(vminvq_u16, vector_reduce_umin, Add1ArgType),
534 NEONMAP1(vminvq_u32, vector_reduce_umin, Add1ArgType),
535 NEONMAP1(vminvq_u8, vector_reduce_umin, Add1ArgType),
536 NEONMAP1(vmull_p64, aarch64_neon_pmull64, 0),
537 NEONMAP1(vmulxd_f64, aarch64_neon_fmulx, Add1ArgType),
538 NEONMAP1(vmulxs_f32, aarch64_neon_fmulx, Add1ArgType),
539 NEONMAP1(vpaddd_s64, vector_reduce_add, Add1ArgType),
540 NEONMAP1(vpaddd_u64, vector_reduce_add, Add1ArgType),
541 NEONMAP1(vpmaxnmqd_f64, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
542 NEONMAP1(vpmaxnms_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
543 NEONMAP1(vpmaxqd_f64, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
544 NEONMAP1(vpmaxs_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
545 NEONMAP1(vpminnmqd_f64, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
546 NEONMAP1(vpminnms_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
547 NEONMAP1(vpminqd_f64, aarch64_neon_fminv, AddRetType | Add1ArgType),
548 NEONMAP1(vpmins_f32, aarch64_neon_fminv, AddRetType | Add1ArgType),
549 NEONMAP1(vqabsb_s8, aarch64_neon_sqabs, Vectorize1ArgType | Use64BitVectors),
550 NEONMAP1(vqabsd_s64, aarch64_neon_sqabs, Add1ArgType),
551 NEONMAP1(vqabsh_s16, aarch64_neon_sqabs, Vectorize1ArgType | Use64BitVectors),
552 NEONMAP1(vqabss_s32, aarch64_neon_sqabs, Add1ArgType),
553 NEONMAP1(vqaddb_s8, aarch64_neon_sqadd, Vectorize1ArgType | Use64BitVectors),
554 NEONMAP1(vqaddb_u8, aarch64_neon_uqadd, Vectorize1ArgType | Use64BitVectors),
555 NEONMAP1(vqaddd_s64, aarch64_neon_sqadd, Add1ArgType),
556 NEONMAP1(vqaddd_u64, aarch64_neon_uqadd, Add1ArgType),
557 NEONMAP1(vqaddh_s16, aarch64_neon_sqadd, Vectorize1ArgType | Use64BitVectors),
558 NEONMAP1(vqaddh_u16, aarch64_neon_uqadd, Vectorize1ArgType | Use64BitVectors),
559 NEONMAP1(vqadds_s32, aarch64_neon_sqadd, Add1ArgType),
560 NEONMAP1(vqadds_u32, aarch64_neon_uqadd, Add1ArgType),
561 NEONMAP1(vqdmulhh_s16, aarch64_neon_sqdmulh, Vectorize1ArgType | Use64BitVectors),
562 NEONMAP1(vqdmulhs_s32, aarch64_neon_sqdmulh, Add1ArgType),
563 NEONMAP1(vqdmullh_s16, aarch64_neon_sqdmull, VectorRet | Use128BitVectors),
564 NEONMAP1(vqdmulls_s32, aarch64_neon_sqdmulls_scalar, 0),
565 NEONMAP1(vqmovnd_s64, aarch64_neon_scalar_sqxtn, AddRetType | Add1ArgType),
566 NEONMAP1(vqmovnd_u64, aarch64_neon_scalar_uqxtn, AddRetType | Add1ArgType),
567 NEONMAP1(vqmovnh_s16, aarch64_neon_sqxtn, VectorRet | Use64BitVectors),
568 NEONMAP1(vqmovnh_u16, aarch64_neon_uqxtn, VectorRet | Use64BitVectors),
569 NEONMAP1(vqmovns_s32, aarch64_neon_sqxtn, VectorRet | Use64BitVectors),
570 NEONMAP1(vqmovns_u32, aarch64_neon_uqxtn, VectorRet | Use64BitVectors),
571 NEONMAP1(vqmovund_s64, aarch64_neon_scalar_sqxtun, AddRetType | Add1ArgType),
572 NEONMAP1(vqmovunh_s16, aarch64_neon_sqxtun, VectorRet | Use64BitVectors),
573 NEONMAP1(vqmovuns_s32, aarch64_neon_sqxtun, VectorRet | Use64BitVectors),
574 NEONMAP1(vqnegb_s8, aarch64_neon_sqneg, Vectorize1ArgType | Use64BitVectors),
575 NEONMAP1(vqnegd_s64, aarch64_neon_sqneg, Add1ArgType),
576 NEONMAP1(vqnegh_s16, aarch64_neon_sqneg, Vectorize1ArgType | Use64BitVectors),
577 NEONMAP1(vqnegs_s32, aarch64_neon_sqneg, Add1ArgType),
578 NEONMAP1(vqrdmlahh_s16, aarch64_neon_sqrdmlah, Vectorize1ArgType | Use64BitVectors),
579 NEONMAP1(vqrdmlahs_s32, aarch64_neon_sqrdmlah, Add1ArgType),
580 NEONMAP1(vqrdmlshh_s16, aarch64_neon_sqrdmlsh, Vectorize1ArgType | Use64BitVectors),
581 NEONMAP1(vqrdmlshs_s32, aarch64_neon_sqrdmlsh, Add1ArgType),
582 NEONMAP1(vqrdmulhh_s16, aarch64_neon_sqrdmulh, Vectorize1ArgType | Use64BitVectors),
583 NEONMAP1(vqrdmulhs_s32, aarch64_neon_sqrdmulh, Add1ArgType),
584 NEONMAP1(vqrshlb_s8, aarch64_neon_sqrshl, Vectorize1ArgType | Use64BitVectors),
585 NEONMAP1(vqrshlb_u8, aarch64_neon_uqrshl, Vectorize1ArgType | Use64BitVectors),
586 NEONMAP1(vqrshld_s64, aarch64_neon_sqrshl, Add1ArgType),
587 NEONMAP1(vqrshld_u64, aarch64_neon_uqrshl, Add1ArgType),
588 NEONMAP1(vqrshlh_s16, aarch64_neon_sqrshl, Vectorize1ArgType | Use64BitVectors),
589 NEONMAP1(vqrshlh_u16, aarch64_neon_uqrshl, Vectorize1ArgType | Use64BitVectors),
590 NEONMAP1(vqrshls_s32, aarch64_neon_sqrshl, Add1ArgType),
591 NEONMAP1(vqrshls_u32, aarch64_neon_uqrshl, Add1ArgType),
592 NEONMAP1(vqrshrnd_n_s64, aarch64_neon_sqrshrn, AddRetType),
593 NEONMAP1(vqrshrnd_n_u64, aarch64_neon_uqrshrn, AddRetType),
594 NEONMAP1(vqrshrnh_n_s16, aarch64_neon_sqrshrn, VectorRet | Use64BitVectors | ArgAsWidenedRetType),
595 NEONMAP1(vqrshrnh_n_u16, aarch64_neon_uqrshrn, VectorRet | Use64BitVectors | ArgAsWidenedRetType),
596 NEONMAP1(vqrshrns_n_s32, aarch64_neon_sqrshrn, VectorRet | Use64BitVectors | ArgAsWidenedRetType),
597 NEONMAP1(vqrshrns_n_u32, aarch64_neon_uqrshrn, VectorRet | Use64BitVectors | ArgAsWidenedRetType),
598 NEONMAP1(vqrshrund_n_s64, aarch64_neon_sqrshrun, AddRetType),
599 NEONMAP1(vqrshrunh_n_s16, aarch64_neon_sqrshrun, VectorRet | Use64BitVectors | ArgAsWidenedRetType),
600 NEONMAP1(vqrshruns_n_s32, aarch64_neon_sqrshrun, VectorRet | Use64BitVectors | ArgAsWidenedRetType),
601 NEONMAP1(vqshlb_n_s8, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors | VectorizeImmArg),
602 NEONMAP1(vqshlb_n_u8, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors | VectorizeImmArg),
603 NEONMAP1(vqshlb_s8, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors),
604 NEONMAP1(vqshlb_u8, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors),
605 NEONMAP1(vqshld_s64, aarch64_neon_sqshl, Add1ArgType),
606 NEONMAP1(vqshld_u64, aarch64_neon_uqshl, Add1ArgType),
607 NEONMAP1(vqshlh_n_s16, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors | VectorizeImmArg),
608 NEONMAP1(vqshlh_n_u16, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors | VectorizeImmArg),
609 NEONMAP1(vqshlh_s16, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors),
610 NEONMAP1(vqshlh_u16, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors),
611 NEONMAP1(vqshls_n_s32, aarch64_neon_sqshl, Add1ArgType),
612 NEONMAP1(vqshls_n_u32, aarch64_neon_uqshl, Add1ArgType),
613 NEONMAP1(vqshls_s32, aarch64_neon_sqshl, Add1ArgType),
614 NEONMAP1(vqshls_u32, aarch64_neon_uqshl, Add1ArgType),
615 NEONMAP1(vqshlub_n_s8, aarch64_neon_sqshlu, Vectorize1ArgType | Use64BitVectors | VectorizeImmArg),
616 NEONMAP1(vqshluh_n_s16, aarch64_neon_sqshlu, Vectorize1ArgType | Use64BitVectors | VectorizeImmArg),
617 NEONMAP1(vqshlus_n_s32, aarch64_neon_sqshlu, Add1ArgType),
618 NEONMAP1(vqshrnd_n_s64, aarch64_neon_sqshrn, AddRetType),
619 NEONMAP1(vqshrnd_n_u64, aarch64_neon_uqshrn, AddRetType),
620 NEONMAP1(vqshrnh_n_s16, aarch64_neon_sqshrn,
621 VectorRet | Use64BitVectors | ArgAsWidenedRetType),
622 NEONMAP1(vqshrnh_n_u16, aarch64_neon_uqshrn,
623 VectorRet | Use64BitVectors | ArgAsWidenedRetType),
624 NEONMAP1(vqshrns_n_s32, aarch64_neon_sqshrn,
625 VectorRet | Use64BitVectors | ArgAsWidenedRetType),
626 NEONMAP1(vqshrns_n_u32, aarch64_neon_uqshrn,
627 VectorRet | Use64BitVectors | ArgAsWidenedRetType),
628 NEONMAP1(vqshrund_n_s64, aarch64_neon_sqshrun, AddRetType),
629 NEONMAP1(vqshrunh_n_s16, aarch64_neon_sqshrun,
630 VectorRet | Use64BitVectors | ArgAsWidenedRetType),
631 NEONMAP1(vqshruns_n_s32, aarch64_neon_sqshrun,
632 VectorRet | Use64BitVectors | ArgAsWidenedRetType),
633 NEONMAP1(vqsubb_s8, aarch64_neon_sqsub, Vectorize1ArgType | Use64BitVectors),
634 NEONMAP1(vqsubb_u8, aarch64_neon_uqsub, Vectorize1ArgType | Use64BitVectors),
635 NEONMAP1(vqsubd_s64, aarch64_neon_sqsub, Add1ArgType),
636 NEONMAP1(vqsubd_u64, aarch64_neon_uqsub, Add1ArgType),
637 NEONMAP1(vqsubh_s16, aarch64_neon_sqsub, Vectorize1ArgType | Use64BitVectors),
638 NEONMAP1(vqsubh_u16, aarch64_neon_uqsub, Vectorize1ArgType | Use64BitVectors),
639 NEONMAP1(vqsubs_s32, aarch64_neon_sqsub, Add1ArgType),
640 NEONMAP1(vqsubs_u32, aarch64_neon_uqsub, Add1ArgType),
641 NEONMAP1(vrecped_f64, aarch64_neon_frecpe, Add1ArgType),
642 NEONMAP1(vrecpes_f32, aarch64_neon_frecpe, Add1ArgType),
643 NEONMAP1(vrecpxd_f64, aarch64_neon_frecpx, Add1ArgType),
644 NEONMAP1(vrecpxs_f32, aarch64_neon_frecpx, Add1ArgType),
645 NEONMAP1(vrshld_s64, aarch64_neon_srshl, Add1ArgType),
646 NEONMAP1(vrshld_u64, aarch64_neon_urshl, Add1ArgType),
647 NEONMAP1(vrsqrted_f64, aarch64_neon_frsqrte, Add1ArgType),
648 NEONMAP1(vrsqrtes_f32, aarch64_neon_frsqrte, Add1ArgType),
649 NEONMAP1(vrsqrtsd_f64, aarch64_neon_frsqrts, Add1ArgType),
650 NEONMAP1(vrsqrtss_f32, aarch64_neon_frsqrts, Add1ArgType),
651 NEONMAP1(vsha1cq_u32, aarch64_crypto_sha1c, 0),
652 NEONMAP1(vsha1h_u32, aarch64_crypto_sha1h, 0),
653 NEONMAP1(vsha1mq_u32, aarch64_crypto_sha1m, 0),
654 NEONMAP1(vsha1pq_u32, aarch64_crypto_sha1p, 0),
655 NEONMAP1(vshld_s64, aarch64_neon_sshl, Add1ArgType),
656 NEONMAP1(vshld_u64, aarch64_neon_ushl, Add1ArgType),
657 NEONMAP1(vslid_n_s64, aarch64_neon_vsli, Vectorize1ArgType),
658 NEONMAP1(vslid_n_u64, aarch64_neon_vsli, Vectorize1ArgType),
659 NEONMAP1(vsqaddb_u8, aarch64_neon_usqadd, Vectorize1ArgType | Use64BitVectors),
660 NEONMAP1(vsqaddd_u64, aarch64_neon_usqadd, Add1ArgType),
661 NEONMAP1(vsqaddh_u16, aarch64_neon_usqadd, Vectorize1ArgType | Use64BitVectors),
662 NEONMAP1(vsqadds_u32, aarch64_neon_usqadd, Add1ArgType),
663 NEONMAP1(vsrid_n_s64, aarch64_neon_vsri, Vectorize1ArgType),
664 NEONMAP1(vsrid_n_u64, aarch64_neon_vsri, Vectorize1ArgType),
665 NEONMAP1(vuqaddb_s8, aarch64_neon_suqadd, Vectorize1ArgType | Use64BitVectors),
666 NEONMAP1(vuqaddd_s64, aarch64_neon_suqadd, Add1ArgType),
667 NEONMAP1(vuqaddh_s16, aarch64_neon_suqadd, Vectorize1ArgType | Use64BitVectors),
668 NEONMAP1(vuqadds_s32, aarch64_neon_suqadd, Add1ArgType),
669 // FP16 scalar intrinisics go here.
670 NEONMAP1(vabdh_f16, aarch64_sisd_fabd, Add1ArgType),
671 NEONMAP1(vcvtah_s32_f16, aarch64_neon_fcvtas, AddRetType | Add1ArgType),
672 NEONMAP1(vcvtah_s64_f16, aarch64_neon_fcvtas, AddRetType | Add1ArgType),
673 NEONMAP1(vcvtah_u32_f16, aarch64_neon_fcvtau, AddRetType | Add1ArgType),
674 NEONMAP1(vcvtah_u64_f16, aarch64_neon_fcvtau, AddRetType | Add1ArgType),
675 NEONMAP1(vcvth_n_f16_s32, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType),
676 NEONMAP1(vcvth_n_f16_s64, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType),
677 NEONMAP1(vcvth_n_f16_u32, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType),
678 NEONMAP1(vcvth_n_f16_u64, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType),
679 NEONMAP1(vcvth_n_s32_f16, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType),
680 NEONMAP1(vcvth_n_s64_f16, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType),
681 NEONMAP1(vcvth_n_u32_f16, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType),
682 NEONMAP1(vcvth_n_u64_f16, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType),
683 NEONMAP1(vcvth_s16_f16, aarch64_neon_fcvtzs, AddRetType | Add1ArgType),
684 NEONMAP1(vcvth_s32_f16, aarch64_neon_fcvtzs, AddRetType | Add1ArgType),
685 NEONMAP1(vcvth_s64_f16, aarch64_neon_fcvtzs, AddRetType | Add1ArgType),
686 NEONMAP1(vcvth_u16_f16, aarch64_neon_fcvtzu, AddRetType | Add1ArgType),
687 NEONMAP1(vcvth_u32_f16, aarch64_neon_fcvtzu, AddRetType | Add1ArgType),
688 NEONMAP1(vcvth_u64_f16, aarch64_neon_fcvtzu, AddRetType | Add1ArgType),
689 NEONMAP1(vcvtmh_s32_f16, aarch64_neon_fcvtms, AddRetType | Add1ArgType),
690 NEONMAP1(vcvtmh_s64_f16, aarch64_neon_fcvtms, AddRetType | Add1ArgType),
691 NEONMAP1(vcvtmh_u32_f16, aarch64_neon_fcvtmu, AddRetType | Add1ArgType),
692 NEONMAP1(vcvtmh_u64_f16, aarch64_neon_fcvtmu, AddRetType | Add1ArgType),
693 NEONMAP1(vcvtnh_s32_f16, aarch64_neon_fcvtns, AddRetType | Add1ArgType),
694 NEONMAP1(vcvtnh_s64_f16, aarch64_neon_fcvtns, AddRetType | Add1ArgType),
695 NEONMAP1(vcvtnh_u32_f16, aarch64_neon_fcvtnu, AddRetType | Add1ArgType),
696 NEONMAP1(vcvtnh_u64_f16, aarch64_neon_fcvtnu, AddRetType | Add1ArgType),
697 NEONMAP1(vcvtph_s32_f16, aarch64_neon_fcvtps, AddRetType | Add1ArgType),
698 NEONMAP1(vcvtph_s64_f16, aarch64_neon_fcvtps, AddRetType | Add1ArgType),
699 NEONMAP1(vcvtph_u32_f16, aarch64_neon_fcvtpu, AddRetType | Add1ArgType),
700 NEONMAP1(vcvtph_u64_f16, aarch64_neon_fcvtpu, AddRetType | Add1ArgType),
701 NEONMAP1(vmulxh_f16, aarch64_neon_fmulx, Add1ArgType),
702 NEONMAP1(vrecpeh_f16, aarch64_neon_frecpe, Add1ArgType),
703 NEONMAP1(vrecpxh_f16, aarch64_neon_frecpx, Add1ArgType),
704 NEONMAP1(vrsqrteh_f16, aarch64_neon_frsqrte, Add1ArgType),
705 NEONMAP1(vrsqrtsh_f16, aarch64_neon_frsqrts, Add1ArgType),
706};
707// clang-format on
708
709} // namespace aarch64
710} // namespace clang
711
712#endif
NEONMAP2
#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier)
Definition AArch64CodeGenUtils.h:113
NEONMAP1
#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier)
Definition AArch64CodeGenUtils.h:109
NEONMAP0
#define NEONMAP0(NameBase)
Definition AArch64CodeGenUtils.h:106
TargetBuiltins.h
Enumerates target-specific builtins in their own namespaces within namespace clang.
clang::aarch64::AArch64SISDIntrinsicMap
const ARMNeonVectorIntrinsicInfo AArch64SISDIntrinsicMap[]
Definition AArch64CodeGenUtils.h:417
clang::aarch64::AArch64SIMDIntrinsicMap
const ARMNeonVectorIntrinsicInfo AArch64SIMDIntrinsicMap[]
Definition AArch64CodeGenUtils.h:119
clang
Top level wrappers for InstallAPI frontend operations.
Definition CalledOnceCheck.h:17
clang::aarch64::AArch64SVEAndSMEVectorIntrinsicInfo
Describes an AArch64 SVE or SME intrinsic.
Definition AArch64CodeGenUtils.h:91
clang::aarch64::AArch64SVEAndSMEVectorIntrinsicInfo::operator<
bool operator<(const AArch64SVEAndSMEVectorIntrinsicInfo &TE) const
Definition AArch64CodeGenUtils.h:99
clang::aarch64::ARMNeonVectorIntrinsicInfo
Describes an ARM or AArch64 NEON intrinsic, or an AArch64 SISD intrinsic.
Definition AArch64CodeGenUtils.h:70
clang::aarch64::ARMNeonVectorIntrinsicInfo::operator<
bool operator<(unsigned RHSBuiltinID) const
Definition AArch64CodeGenUtils.h:77
clang::aarch64::ARMNeonVectorIntrinsicInfo::operator<
bool operator<(const ARMNeonVectorIntrinsicInfo &TE) const
Definition AArch64CodeGenUtils.h:80
Generated on for clang by doxygen 1.14.0